Add scheduler relay health preflight gate

scripts/agent/check-relay-health.mjs

@@ -0,0 +1,106 @@
+#!/usr/bin/env node
+import { getRelays, getNamespace, getQueryTimeoutMs } from '../../src/torch-config.mjs';
+import { queryLocks } from '../../src/lock-ops.mjs';
+import { todayDateStr } from '../../src/utils.mjs';
+
+function parseArgs(argv) {
+  const args = { cadence: 'daily' };
+  for (let i = 0; i < argv.length; i += 1) {
+    const value = argv[i];
+    if (!value.startsWith('--') && !args.cadence) {
+      args.cadence = value;
+      continue;
+    }
+    if (value === '--cadence') {
+      args.cadence = argv[i + 1] || args.cadence;
+      i += 1;
+    }
+  }
+  return args;
+}
+
+function classifyRelayFailure(outputText) {
+  const text = String(outputText || '').toLowerCase();
+  if (!text.trim()) return 'unknown backend error';
+
+  if ((text.includes('relay') || text.includes('query')) && text.includes('timeout')) {
+    return 'relay query timeout';
+  }
+
+  if (
+    text.includes('connection refused')
+    || text.includes('econnrefused')
+    || text.includes('getaddrinfo')
+    || text.includes('enotfound')
+    || text.includes('eai_again')
+    || (text.includes('websocket') && text.includes('dns'))
+  ) {
+    return 'websocket connection refused/dns';
+  }
+
+  if (
+    text.includes('invalid url')
+    || text.includes('malformed')
+    || text.includes('unsupported protocol')
+    || text.includes('must start with ws')
+    || text.includes('invalid relay')
+  ) {
+    return 'malformed relay url/config';
+  }
+
+  return 'unknown backend error';
+}
+
+async function main() {
+  const { cadence } = parseArgs(process.argv.slice(2));
+  if (cadence !== 'daily' && cadence !== 'weekly') {
+    console.error('Usage: node scripts/agent/check-relay-health.mjs --cadence <daily|weekly>');
+    process.exit(1);
+  }
+
+  const relays = getRelays();
+  const namespace = getNamespace();
+  const timeoutMs = getQueryTimeoutMs();
+  const dateStr = todayDateStr();
+
+  try {
+    const locks = await queryLocks(relays, cadence, dateStr, namespace);
+    console.log(JSON.stringify({
+      ok: true,
+      cadence,
+      namespace,
+      timeoutMs,
+      relays,
+      lockCount: locks.length,
+    }));
+    process.exit(0);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error ?? 'unknown error');
+    const failureCategory = classifyRelayFailure(message);
+    console.log(JSON.stringify({
+      ok: false,
+      cadence,
+      namespace,
+      timeoutMs,
+      relays,
+      failureCategory,
+      error: message,
+    }));
+    process.exit(2);
+  }
+}
+
+main().catch((error) => {
+  const message = error instanceof Error ? error.message : String(error ?? 'unknown error');
+  const relays = getRelays();
+  const namespace = getNamespace();
+  const failureCategory = classifyRelayFailure(message);
+  console.log(JSON.stringify({
+    ok: false,
+    namespace,
+    relays,
+    failureCategory,
+    error: message,
+  }));
+  process.exit(2);
+});

scripts/agent/run-scheduler-cycle.mjs

@@ -100,6 +100,14 @@ function parseNonNegativeInt(value, fallback) {
   return fallback;
 }
 
+function parseBooleanFlag(value, fallback = false) {
+  if (value === undefined || value === null || value === '') return fallback;
+  const normalized = String(value).trim().toLowerCase();
+  if (['1', 'true', 'yes', 'on'].includes(normalized)) return true;
+  if (['0', 'false', 'no', 'off'].includes(normalized)) return false;
+  return fallback;
+}
+
 function redactSensitive(text) {
   if (!text) return '';
   return String(text)
@@ -300,6 +308,12 @@ async function getSchedulerConfig(cadence, { isInteractive }) {
     process.env.SCHEDULER_LOCK_JITTER_MS,
     parseNonNegativeInt(lockRetryRaw.jitterMs, 75),
   );
+  const lockHealthPreflightFromConfig = parseBooleanFlag(scheduler.lockHealthPreflight, false);
+  const lockHealthPreflightEnabled = parseBooleanFlag(
+    process.env.SCHEDULER_LOCK_HEALTH_PREFLIGHT,
+    lockHealthPreflightFromConfig,
+  );
+  const lockHealthPreflightSkip = parseBooleanFlag(process.env.SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT, false);
 
   return {
     firstPrompt: scheduler.firstPromptByCadence?.[cadence] || null,
@@ -313,6 +327,10 @@ async function getSchedulerConfig(cadence, { isInteractive }) {
       backoffMs,
       jitterMs,
     },
+    lockHealthPreflight: {
+      enabled: lockHealthPreflightEnabled,
+      skip: lockHealthPreflightSkip,
+    },
     memoryPolicy: {
       mode,
       retrieveCommand: typeof memoryPolicyRaw.retrieveCommand === 'string' && memoryPolicyRaw.retrieveCommand.trim()
@@ -329,6 +347,27 @@ async function getSchedulerConfig(cadence, { isInteractive }) {
   };
 }
 
+async function runLockHealthPreflight({ cadence, platform }) {
+  const preflight = await runCommand('node', ['scripts/agent/check-relay-health.mjs', '--cadence', cadence], {
+    env: { AGENT_PLATFORM: platform },
+  });
+  const payload = parseJsonFromOutput(`${preflight.stdout}\n${preflight.stderr}`) || {};
+  const relayList = Array.isArray(payload.relays)
+    ? payload.relays.filter((relay) => typeof relay === 'string' && relay.trim()).map((relay) => relay.trim())
+    : [];
+  const combinedOutput = `${preflight.stderr}\n${preflight.stdout}`;
+  const failureCategory = payload.failureCategory || classifyLockBackendError(combinedOutput);
+
+  return {
+    code: preflight.code,
+    payload,
+    relayList,
+    failureCategory,
+    stderrExcerpt: excerptText(preflight.stderr),
+    stdoutExcerpt: excerptText(preflight.stdout),
+  };
+}
+
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -437,6 +476,26 @@ async function main() {
   }
 
   while (true) {
+    if (schedulerConfig.lockHealthPreflight.enabled && !schedulerConfig.lockHealthPreflight.skip) {
+      const preflight = await runLockHealthPreflight({ cadence, platform });
+      if (preflight.code !== 0) {
+        await writeLog({
+          cadence,
+          agent: 'scheduler',
+          status: 'failed',
+          reason: 'Lock backend unavailable preflight',
+          detail: `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`,
+          metadata: {
+            preflight_failure_category: preflight.failureCategory,
+            relay_list: preflight.relayList.join(', ') || '(none)',
+            preflight_stderr_excerpt: preflight.stderrExcerpt || '(empty)',
+            preflight_stdout_excerpt: preflight.stdoutExcerpt || '(empty)',
+          },
+        });
+        process.exit(preflight.code || 1);
+      }
+    }
+
     const checkResult = await runCommand('npm', ['run', `lock:check:${cadence}`]);
     const checkPayload = parseJsonFromOutput(`${checkResult.stdout}\n${checkResult.stderr}`) || {};
     const excluded = Array.isArray(checkPayload.excluded)