Add relay health preflight job and scheduler deferral handling

KNOWN_ISSUES.md

@@ -30,7 +30,8 @@ Track only **active, reproducible, unresolved** issues.
 - **Area:** Runtime
 - **Symptom:** Multiple scheduler runs fail with `Lock backend error` before prompt handoff/validation.
 - **Trigger/Conditions:** Running scheduler cycles when relay connectivity is unstable or lock backend operations time out; observed in both `daily` and `weekly` task logs.
-- **Workaround:** Retry with explicit platform and inspect reliability report output (`npm run report:lock-reliability`), then validate relay health (`node scripts/agent/check-relay-health.mjs --cadence <daily|weekly>`) before rerunning scheduler.
-- **Impact:** Scheduled agent execution is skipped for affected cycles until lock acquisition succeeds.
-- **Last verified:** 2026-02-14
+- **Workaround:** Run relay preflight (`npm run lock:health -- --cadence <daily|weekly>`) and inspect `task-logs/relay-health/<cadence>.jsonl` for trend/alert data before rerunning scheduler. If scheduler logs `All relays unhealthy preflight`, treat it as an incident signal, defer lock attempts, and escalate relay/network checks (DNS/TLS/connectivity).
+- **Impact:** Scheduled agent execution may be deferred early when all relays are unhealthy, preventing noisy lock retries until relay health recovers.
+- **Related notes:** `docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md`
+- **Last verified:** 2026-02-15
 

README.md

@@ -124,6 +124,7 @@ If you are developing `torch-lock` itself:
 - `npm run lock:check:daily`
 - `npm run lock:check:weekly`
 - `npm run lock:list`
+- `npm run lock:health -- --cadence daily` (relay websocket + publish/read probe; writes history to `task-logs/relay-health/`)
 - `npm run dashboard:serve`
 - `npm test` (run unit tests)
 - `npm run lint` (run linter)

docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md

@@ -0,0 +1,26 @@
+# Relay health preflight job before scheduler lock acquisition
+
+## Context
+- Scheduler runs were intermittently failing during lock acquisition when relay connectivity degraded.
+- Existing preflight checks only validated lock query behavior and did not preserve health history for trend analysis.
+
+## Observation
+- A dedicated relay probe that checks websocket reachability plus publish/read behavior per relay provides a clearer readiness signal before lock acquisition.
+- Persisting each probe result enables windowed success-rate alerts and detection of prolonged all-relay outages.
+
+## Action taken
+- Added `torch-lock health` / `npm run lock:health` to run relay websocket + publish/read probes per relay.
+- Integrated scheduler preflight to call `lock:health` immediately before lock acquisition.
+- Updated scheduler behavior to defer early (status `_deferred`) with reason `All relays unhealthy preflight` and incident metadata when no relay is healthy.
+- Stored probe history in `task-logs/relay-health/<cadence>.jsonl` and evaluated alert thresholds:
+  - all relays down for more than N minutes,
+  - success rate below X% over a rolling time window.
+
+## Validation performed
+- Ran targeted unit tests for relay health threshold evaluation.
+- Ran scheduler flow parity checks to ensure preflight remains before lock acquisition in required ordering.
+
+## Recommendation for next agents
+- Keep `lock:health` enabled for scheduler preflight in automation.
+- Review `task-logs/relay-health/<cadence>.jsonl` during incidents to distinguish transient single-relay flakiness from full relay outages.
+- Tune threshold defaults (`--all-relays-down-minutes`, `--min-success-rate`, `--window-minutes`) per environment reliability goals.

package.json

@@ -16,6 +16,7 @@
     "src/dashboard.mjs",
     "src/roster.mjs",
     "src/torch-config.mjs",
+    "src/relay-health.mjs",
     "src/prompts/",
     "src/docs/",
     "scripts/check-file-size.mjs",
@@ -32,6 +33,7 @@
     "lock:check:weekly": "node bin/torch-lock.mjs check --cadence weekly",
     "lock:list": "node bin/torch-lock.mjs list",
     "lock:lock": "node bin/torch-lock.mjs lock",
+    "lock:health": "node bin/torch-lock.mjs health",
     "dashboard:serve": "node bin/torch-lock.mjs dashboard",
     "build": "node build.mjs",
     "lint": "eslint .",

scripts/agent/check-relay-health.mjs

@@ -1,10 +1,14 @@
 #!/usr/bin/env node
-import { getRelays, getNamespace, getQueryTimeoutMs } from '../../src/torch-config.mjs';
-import { queryLocks } from '../../src/lock-ops.mjs';
-import { todayDateStr } from '../../src/utils.mjs';
+import { runRelayHealthCheck } from '../../src/relay-health.mjs';
 
 function parseArgs(argv) {
-  const args = { cadence: 'daily' };
+  const args = {
+    cadence: 'daily',
+    timeoutMs: 6000,
+    allRelaysDownMinutes: 10,
+    minSuccessRate: 0.7,
+    windowMinutes: 60,
+  };
   for (let i = 0; i < argv.length; i += 1) {
     const value = argv[i];
     if (!value.startsWith('--') && !args.cadence) {
@@ -14,93 +18,41 @@ function parseArgs(argv) {
     if (value === '--cadence') {
       args.cadence = argv[i + 1] || args.cadence;
       i += 1;
+    } else if (value === '--timeout-ms') {
+      args.timeoutMs = Number.parseInt(argv[i + 1], 10) || args.timeoutMs;
+      i += 1;
+    } else if (value === '--all-relays-down-minutes') {
+      args.allRelaysDownMinutes = Number.parseInt(argv[i + 1], 10) || args.allRelaysDownMinutes;
+      i += 1;
+    } else if (value === '--min-success-rate') {
+      const parsed = Number.parseFloat(argv[i + 1]);
+      args.minSuccessRate = Number.isFinite(parsed) ? parsed : args.minSuccessRate;
+      i += 1;
+    } else if (value === '--window-minutes') {
+      args.windowMinutes = Number.parseInt(argv[i + 1], 10) || args.windowMinutes;
+      i += 1;
     }
   }
   return args;
 }
 
-function classifyRelayFailure(outputText) {
-  const text = String(outputText || '').toLowerCase();
-  if (!text.trim()) return 'unknown backend error';
-
-  if ((text.includes('relay') || text.includes('query')) && text.includes('timeout')) {
-    return 'relay query timeout';
-  }
-
-  if (
-    text.includes('connection refused')
-    || text.includes('econnrefused')
-    || text.includes('getaddrinfo')
-    || text.includes('enotfound')
-    || text.includes('eai_again')
-    || (text.includes('websocket') && text.includes('dns'))
-  ) {
-    return 'websocket connection refused/dns';
-  }
-
-  if (
-    text.includes('invalid url')
-    || text.includes('malformed')
-    || text.includes('unsupported protocol')
-    || text.includes('must start with ws')
-    || text.includes('invalid relay')
-  ) {
-    return 'malformed relay url/config';
-  }
-
-  return 'unknown backend error';
-}
-
 async function main() {
-  const { cadence } = parseArgs(process.argv.slice(2));
-  if (cadence !== 'daily' && cadence !== 'weekly') {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.cadence !== 'daily' && args.cadence !== 'weekly') {
     console.error('Usage: node scripts/agent/check-relay-health.mjs --cadence <daily|weekly>');
     process.exit(1);
   }
 
-  const relays = getRelays();
-  const namespace = getNamespace();
-  const timeoutMs = getQueryTimeoutMs();
-  const dateStr = todayDateStr();
-
-  try {
-    const locks = await queryLocks(relays, cadence, dateStr, namespace);
-    console.log(JSON.stringify({
-      ok: true,
-      cadence,
-      namespace,
-      timeoutMs,
-      relays,
-      lockCount: locks.length,
-    }));
-    process.exit(0);
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error ?? 'unknown error');
-    const failureCategory = classifyRelayFailure(message);
-    console.log(JSON.stringify({
-      ok: false,
-      cadence,
-      namespace,
-      timeoutMs,
-      relays,
-      failureCategory,
-      error: message,
-    }));
-    process.exit(2);
+  const result = await runRelayHealthCheck(args);
+  if (!result.ok) {
+    result.failureCategory = 'all relays unhealthy';
   }
+  console.log(JSON.stringify(result));
+  process.exit(result.ok ? 0 : 2);
 }
 
 main().catch((error) => {
   const message = error instanceof Error ? error.message : String(error ?? 'unknown error');
-  const relays = getRelays();
-  const namespace = getNamespace();
-  const failureCategory = classifyRelayFailure(message);
-  console.log(JSON.stringify({
-    ok: false,
-    namespace,
-    relays,
-    failureCategory,
-    error: message,
-  }));
+  console.log(JSON.stringify({ ok: false, failureCategory: 'health check failed', error: message }));
   process.exit(2);
 });

scripts/agent/run-scheduler-cycle.mjs

@@ -417,7 +417,7 @@ function createIdempotencyKey({ cadence, selectedAgent, runDate }) {
 }
 
 async function runLockHealthPreflight({ cadence, platform }) {
-  const preflight = await runCommand('node', ['scripts/agent/check-relay-health.mjs', '--cadence', cadence], {
+  const preflight = await runCommand('npm', ['run', 'lock:health', '--', '--cadence', cadence], {
     env: { AGENT_PLATFORM: platform },
   });
   const payload = parseJsonFromOutput(`${preflight.stdout}\n${preflight.stderr}`) || {};
@@ -593,22 +593,32 @@ async function main() {
     if (schedulerConfig.lockHealthPreflight.enabled && !schedulerConfig.lockHealthPreflight.skip) {
       const preflight = await runLockHealthPreflight({ cadence, platform });
       if (preflight.code !== 0) {
+        const allRelaysUnhealthy = Boolean(preflight.payload?.summary?.allRelaysUnhealthy);
+        const incidentSignal = preflight.payload?.incidentSignal || null;
         await writeLog({
           cadence,
           agent: 'scheduler',
-          status: 'failed',
+          status: allRelaysUnhealthy ? 'deferred' : 'failed',
           platform,
-          reason: 'Lock backend unavailable preflight',
-          detail: `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`,
+          reason: allRelaysUnhealthy
+            ? 'All relays unhealthy preflight'
+            : 'Lock backend unavailable preflight',
+          detail: allRelaysUnhealthy
+            ? `Deferred run before lock acquisition: ${incidentSignal?.reason || 'all relays unhealthy'}.`
+            : `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`,
           metadata: {
             failure_class: 'backend_unavailable',
             preflight_failure_category: preflight.failureCategory,
             relay_list: preflight.relayList.join(', ') || '(none)',
             preflight_stderr_excerpt: preflight.stderrExcerpt || '(empty)',
             preflight_stdout_excerpt: preflight.stdoutExcerpt || '(empty)',
+            incident_signal_id: incidentSignal?.id || null,
+            incident_signal_severity: incidentSignal?.severity || null,
+            preflight_alerts: JSON.stringify(preflight.payload?.alerts || []),
+            relay_health_history_path: preflight.payload?.historyPath || null,
           },
         });
-        process.exit(preflight.code || 1);
+        process.exit(allRelaysUnhealthy ? 0 : (preflight.code || 1));
       }
     }
 

src/docs/TORCH.md

@@ -13,6 +13,9 @@ AGENT_PLATFORM=codex node bin/torch-lock.mjs lock --agent docs-agent --cadence d
 
 # List all active locks
 node bin/torch-lock.mjs list
+
+# Probe relay health before scheduler lock acquisition
+node bin/torch-lock.mjs health --cadence daily
 ```
 
 ## Defaults
@@ -77,7 +80,7 @@ Operational note: scheduler handoff commands are treated as required execution s
 
 Scheduler failure classes in task logs:
 
-- `backend_unavailable` — lock backend unavailable preflight failures and lock acquisition backend exit code `2` failures/deferrals.
+- `backend_unavailable` — lock backend unavailable preflight failures and lock acquisition backend exit code `2` failures/deferrals; includes relay health alert metadata and incident signal IDs when all relays are unhealthy.
 - `prompt_validation_error` — prompt/handoff, memory evidence, artifact verification, and validation command failures.
 
 

src/lib.mjs

@@ -32,6 +32,7 @@ import {
 } from './services/memory/index.js';
 import { ExitError } from './errors.mjs';
 import { todayDateStr, nowUnix, getIsoWeekStr } from './utils.mjs';
+import { runRelayHealthCheck } from './relay-health.mjs';
 import fs from 'node:fs/promises';
 import path from 'node:path';
 
@@ -527,6 +528,10 @@ function parseArgs(argv) {
     offset: null,
     retentionMs: null,
     windowMs: null,
+    timeoutMs: null,
+    allRelaysDownMinutes: null,
+    minSuccessRate: null,
+    windowMinutes: null,
   };
   let i = 0;
 
@@ -589,6 +594,22 @@ function parseArgs(argv) {
       args.windowMs = parseInt(argv[++i], 10);
     } else if (arg.startsWith('--window-ms=')) {
       args.windowMs = parseInt(arg.split('=')[1], 10);
+    } else if (arg === '--timeout-ms') {
+      args.timeoutMs = parseInt(argv[++i], 10);
+    } else if (arg.startsWith('--timeout-ms=')) {
+      args.timeoutMs = parseInt(arg.split('=')[1], 10);
+    } else if (arg === '--all-relays-down-minutes') {
+      args.allRelaysDownMinutes = parseInt(argv[++i], 10);
+    } else if (arg.startsWith('--all-relays-down-minutes=')) {
+      args.allRelaysDownMinutes = parseInt(arg.split('=')[1], 10);
+    } else if (arg === '--min-success-rate') {
+      args.minSuccessRate = parseFloat(argv[++i]);
+    } else if (arg.startsWith('--min-success-rate=')) {
+      args.minSuccessRate = parseFloat(arg.split('=')[1]);
+    } else if (arg === '--window-minutes') {
+      args.windowMinutes = parseInt(argv[++i], 10);
+    } else if (arg.startsWith('--window-minutes=')) {
+      args.windowMinutes = parseInt(arg.split('=')[1], 10);
     }
   }
 
@@ -603,6 +624,7 @@ Commands:
   lock      --agent <name> --cadence <daily|weekly> Claim a lock
   complete  --agent <name> --cadence <daily|weekly> Mark task as completed (permanent)
   list      [--cadence <daily|weekly>]             Print active lock table
+  health    --cadence <daily|weekly>               Probe relay websocket + publish/read health
   dashboard [--port <port>]                        Serve the dashboard (default: ${DEFAULT_DASHBOARD_PORT})
   init      [--force]                              Initialize torch/ directory in current project
   update    [--force]                              Update torch/ configuration (backups, merges)
@@ -697,6 +719,26 @@ export async function main(argv) {
         break;
       }
 
+      case 'health': {
+        if (!args.cadence || !VALID_CADENCES.has(args.cadence)) {
+          console.error(`ERROR: --cadence <${[...VALID_CADENCES].join('|')}> is required for health`);
+          throw new ExitError(1, 'Missing cadence');
+        }
+        const result = await runRelayHealthCheck({
+          cadence: args.cadence,
+          ...(Number.isFinite(args.timeoutMs) ? { timeoutMs: args.timeoutMs } : {}),
+          ...(Number.isFinite(args.allRelaysDownMinutes) ? { allRelaysDownMinutes: args.allRelaysDownMinutes } : {}),
+          ...(Number.isFinite(args.minSuccessRate) ? { minSuccessRate: args.minSuccessRate } : {}),
+          ...(Number.isFinite(args.windowMinutes) ? { windowMinutes: args.windowMinutes } : {}),
+        });
+        if (!result.ok) {
+          result.failureCategory = 'all relays unhealthy';
+        }
+        console.log(JSON.stringify(result, null, 2));
+        if (!result.ok) throw new ExitError(2, 'Relay health check failed');
+        break;
+      }
+
       case 'dashboard': {
         await cmdDashboard(args.port);
         break;

src/prompts/scheduler-flow.md

@@ -81,16 +81,17 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co
 5. When lock health preflight is enabled (`scheduler.lockHealthPreflight: true` or env `SCHEDULER_LOCK_HEALTH_PREFLIGHT=1`), verify relay/query health before selecting an agent or calling `lock:lock`:
 
    ```bash
-   node scripts/agent/check-relay-health.mjs --cadence <cadence>
+   npm run lock:health -- --cadence <cadence>
    ```
 
    - Escape hatch: set `SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT=1` to skip this check for local/offline workflows.
-   - If preflight exits non-zero, write `_failed.md` with reason `Lock backend unavailable preflight` and include:
+   - If preflight exits non-zero because every relay is unhealthy, write `_deferred.md` with reason `All relays unhealthy preflight`, include `incident_signal_id`, and stop before lock acquisition.
+   - For other non-zero preflight failures, write `_failed.md` with reason `Lock backend unavailable preflight` and include:
      - `relay_list`
      - `preflight_failure_category`
      - `preflight_stderr_excerpt`
      - `preflight_stdout_excerpt`
-   - Stop the run immediately when preflight fails.
+   - Always include any preflight alert payload (`preflight_alerts`) in the scheduler metadata for operational triage.
 
 6. Find latest cadence log file, derive the previous agent, then choose the next roster agent not in exclusion set:
 

src/relay-health.mjs

@@ -0,0 +1,281 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import WebSocket from 'ws';
+import { finalizeEvent, generateSecretKey, getPublicKey } from 'nostr-tools/pure';
+import { getNamespace, getRelays } from './torch-config.mjs';
+
+const PROBE_KIND = 27235;
+
+function withTimeout(promise, timeoutMs, message) {
+  let timeoutId;
+  const timeoutPromise = new Promise((_, reject) => {
+    timeoutId = setTimeout(() => reject(new Error(message)), timeoutMs);
+  });
+  return Promise.race([promise, timeoutPromise]).finally(() => clearTimeout(timeoutId));
+}
+
+function nowIso() {
+  return new Date().toISOString();
+}
+
+export function summarizeHistory(historyEntries, { nowMs = Date.now(), windowMinutes = 60 } = {}) {
+  const windowStartMs = nowMs - (windowMinutes * 60_000);
+  const recent = historyEntries.filter((entry) => Date.parse(entry.timestamp || '') >= windowStartMs);
+  const relayProbeCount = recent.reduce((sum, entry) => sum + (entry.summary?.totalRelays || 0), 0);
+  const successCount = recent.reduce((sum, entry) => sum + (entry.summary?.healthyRelays || 0), 0);
+  const successRate = relayProbeCount > 0 ? successCount / relayProbeCount : 1;
+
+  let lastHealthyAtMs = null;
+  for (let i = historyEntries.length - 1; i >= 0; i -= 1) {
+    if ((historyEntries[i].summary?.healthyRelays || 0) > 0) {
+      lastHealthyAtMs = Date.parse(historyEntries[i].timestamp || '');
+      break;
+    }
+  }
+
+  const allDownDurationMinutes = lastHealthyAtMs === null
+    ? null
+    : Math.max(0, Math.floor((nowMs - lastHealthyAtMs) / 60_000));
+
+  return {
+    windowMinutes,
+    sampleCount: recent.length,
+    successRate,
+    allDownDurationMinutes,
+  };
+}
+
+export function evaluateAlertThresholds(currentResult, historyEntries, thresholds, { nowMs = Date.now() } = {}) {
+  const stats = summarizeHistory(historyEntries, { nowMs, windowMinutes: thresholds.windowMinutes });
+  const alerts = [];
+
+  if (currentResult.summary.allRelaysUnhealthy && stats.allDownDurationMinutes !== null && stats.allDownDurationMinutes >= thresholds.allRelaysDownMinutes) {
+    alerts.push({
+      type: 'all_relays_down_duration',
+      thresholdMinutes: thresholds.allRelaysDownMinutes,
+      actualMinutes: stats.allDownDurationMinutes,
+      severity: 'critical',
+    });
+  }
+
+  if (stats.sampleCount > 0 && stats.successRate < thresholds.minSuccessRate) {
+    alerts.push({
+      type: 'relay_success_rate_below_threshold',
+      threshold: thresholds.minSuccessRate,
+      actual: Number(stats.successRate.toFixed(4)),
+      windowMinutes: thresholds.windowMinutes,
+      severity: 'warning',
+    });
+  }
+
+  return { alerts, stats };
+}
+
+async function probeWebSocketReachability(relayUrl, timeoutMs) {
+  const startedAt = Date.now();
+  return withTimeout(new Promise((resolve, reject) => {
+    const ws = new WebSocket(relayUrl);
+    let settled = false;
+
+    const finish = (fn) => (value) => {
+      if (settled) return;
+      settled = true;
+      try {
+        ws.close();
+      } catch {
+        // ignore
+      }
+      fn(value);
+    };
+
+    ws.once('open', finish(() => resolve({ ok: true, latencyMs: Date.now() - startedAt })));
+    ws.once('error', finish((err) => reject(err)));
+  }), timeoutMs, `WebSocket reachability timeout after ${timeoutMs}ms`);
+}
+
+async function probePublishRead(relayUrl, namespace, timeoutMs) {
+  const ws = new WebSocket(relayUrl);
+  const sk = generateSecretKey();
+  const pk = getPublicKey(sk);
+  const createdAt = Math.floor(Date.now() / 1000);
+  const nonce = `${Date.now()}-${Math.random().toString(16).slice(2, 10)}`;
+  const probeTag = `${namespace}-health-probe-${nonce}`;
+  const event = finalizeEvent({
+    kind: PROBE_KIND,
+    created_at: createdAt,
+    tags: [
+      ['t', `${namespace}-relay-health`],
+      ['d', probeTag],
+    ],
+    content: JSON.stringify({ type: 'relay_health_probe', nonce, namespace, pubkey: pk }),
+  }, sk);
+
+  const subId = `probe-${nonce}`;
+
+  return withTimeout(new Promise((resolve, reject) => {
+    let okAck = false;
+    let readSeen = false;
+    let settled = false;
+
+    const settle = (fn) => (value) => {
+      if (settled) return;
+      settled = true;
+      try {
+        ws.close();
+      } catch {
+        // ignore
+      }
+      fn(value);
+    };
+
+    ws.once('open', () => {
+      ws.send(JSON.stringify(['EVENT', event]));
+      ws.send(JSON.stringify(['REQ', subId, { ids: [event.id], limit: 1 }]));
+    });
+
+    ws.on('message', (raw) => {
+      let message;
+      try {
+        message = JSON.parse(raw.toString());
+      } catch {
+        return;
+      }
+      if (!Array.isArray(message) || message.length < 2) return;
+      const type = message[0];
+
+      if (type === 'OK' && message[1] === event.id) {
+        okAck = Boolean(message[2]);
+        if (!okAck) {
+          settle(reject)(new Error(`Relay rejected health probe event: ${String(message[3] || 'unknown reason')}`));
+          return;
+        }
+      }
+
+      if (type === 'EVENT' && message[1] === subId && message[2]?.id === event.id) {
+        readSeen = true;
+      }
+
+      if (okAck && readSeen) {
+        ws.send(JSON.stringify(['CLOSE', subId]));
+        settle(resolve)({ ok: true, eventId: event.id });
+      }
+    });
+
+    ws.once('error', settle(reject));
+  }), timeoutMs, `Publish/read probe timeout after ${timeoutMs}ms`);
+}
+
+async function readHistory(historyPath) {
+  try {
+    const raw = await fs.readFile(historyPath, 'utf8');
+    return raw
+      .split(/\r?\n/)
+      .map((line) => line.trim())
+      .filter(Boolean)
+      .map((line) => {
+        try {
+          return JSON.parse(line);
+        } catch {
+          return null;
+        }
+      })
+      .filter((entry) => entry && typeof entry === 'object');
+  } catch (error) {
+    if (error && error.code === 'ENOENT') return [];
+    throw error;
+  }
+}
+
+async function appendHistory(historyPath, entry) {
+  await fs.mkdir(path.dirname(historyPath), { recursive: true });
+  await fs.appendFile(historyPath, `${JSON.stringify(entry)}\n`, 'utf8');
+}
+
+export async function runRelayHealthCheck(options = {}) {
+  const {
+    cadence = 'daily',
+    relays = getRelays(),
+    namespace = getNamespace(),
+    timeoutMs = 6000,
+    historyPath = path.resolve(process.cwd(), 'task-logs', 'relay-health', `${cadence}.jsonl`),
+    allRelaysDownMinutes = 10,
+    minSuccessRate = 0.7,
+    windowMinutes = 60,
+  } = options;
+
+  const relayResults = [];
+  for (const relay of relays) {
+    const relayResult = {
+      relay,
+      websocketReachable: false,
+      publishReadOk: false,
+      healthy: false,
+      checks: {},
+    };
+
+    try {
+      const wsProbe = await probeWebSocketReachability(relay, timeoutMs);
+      relayResult.websocketReachable = true;
+      relayResult.checks.websocket = wsProbe;
+    } catch (error) {
+      relayResult.checks.websocket = { ok: false, error: error instanceof Error ? error.message : String(error) };
+    }
+
+    if (relayResult.websocketReachable) {
+      try {
+        const prProbe = await probePublishRead(relay, namespace, timeoutMs);
+        relayResult.publishReadOk = true;
+        relayResult.checks.publishRead = prProbe;
+      } catch (error) {
+        relayResult.checks.publishRead = { ok: false, error: error instanceof Error ? error.message : String(error) };
+      }
+    }
+
+    relayResult.healthy = relayResult.websocketReachable && relayResult.publishReadOk;
+    relayResults.push(relayResult);
+  }
+
+  const summary = {
+    totalRelays: relayResults.length,
+    healthyRelays: relayResults.filter((relay) => relay.healthy).length,
+  };
+  summary.unhealthyRelays = summary.totalRelays - summary.healthyRelays;
+  summary.allRelaysUnhealthy = summary.healthyRelays === 0;
+
+  const result = {
+    ok: !summary.allRelaysUnhealthy,
+    cadence,
+    namespace,
+    timestamp: nowIso(),
+    relays,
+    relayResults,
+    summary,
+    incidentSignal: null,
+    alerts: [],
+    alertStats: null,
+    historyPath,
+  };
+
+  const historyEntries = await readHistory(historyPath);
+  const prospectiveHistory = [...historyEntries, result];
+  const thresholdEval = evaluateAlertThresholds(result, prospectiveHistory, {
+    allRelaysDownMinutes,
+    minSuccessRate,
+    windowMinutes,
+  });
+  result.alerts = thresholdEval.alerts;
+  result.alertStats = thresholdEval.stats;
+
+  if (summary.allRelaysUnhealthy) {
+    const incidentId = `relay-health-${cadence}-${Date.now()}`;
+    result.incidentSignal = {
+      id: incidentId,
+      reason: 'All relays unhealthy before lock acquisition',
+      severity: 'critical',
+      suggestedAction: 'Defer scheduler run and investigate relay availability before retrying lock acquisition.',
+    };
+  }
+
+  await appendHistory(historyPath, result);
+  return result;
+}

test/relay-health-thresholds.test.mjs

@@ -0,0 +1,50 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { evaluateAlertThresholds } from '../src/relay-health.mjs';
+
+function entry(timestamp, healthyRelays, totalRelays = 3) {
+  return {
+    timestamp,
+    summary: {
+      healthyRelays,
+      totalRelays,
+      allRelaysUnhealthy: healthyRelays === 0,
+    },
+  };
+}
+
+test('raises all-relays-down duration alert after threshold', () => {
+  const now = Date.parse('2026-02-15T12:00:00Z');
+  const history = [
+    entry('2026-02-15T11:30:00Z', 1),
+    entry('2026-02-15T11:45:00Z', 0),
+    entry('2026-02-15T11:55:00Z', 0),
+    entry('2026-02-15T12:00:00Z', 0),
+  ];
+
+  const current = history[history.length - 1];
+  const result = evaluateAlertThresholds(current, history, {
+    allRelaysDownMinutes: 10,
+    minSuccessRate: 0.1,
+    windowMinutes: 60,
+  }, { nowMs: now });
+
+  assert.ok(result.alerts.some((alert) => alert.type === 'all_relays_down_duration'));
+});
+
+test('raises success-rate alert when rolling health falls below threshold', () => {
+  const history = [
+    entry('2026-02-15T11:50:00Z', 3),
+    entry('2026-02-15T11:55:00Z', 0),
+    entry('2026-02-15T12:00:00Z', 0),
+  ];
+  const current = history[history.length - 1];
+
+  const result = evaluateAlertThresholds(current, history, {
+    allRelaysDownMinutes: 999,
+    minSuccessRate: 0.5,
+    windowMinutes: 30,
+  }, { nowMs: Date.parse('2026-02-15T12:00:00Z') });
+
+  assert.ok(result.alerts.some((alert) => alert.type === 'relay_success_rate_below_threshold'));
+});