Add scheduler relay health preflight gate

scripts/agent/check-relay-health.mjs

@@ -0,0 +1,106 @@
+#!/usr/bin/env node
+import { getRelays, getNamespace, getQueryTimeoutMs } from '../../src/torch-config.mjs';
+import { queryLocks } from '../../src/lock-ops.mjs';
+import { todayDateStr } from '../../src/utils.mjs';
+
+function parseArgs(argv) {
+  const args = { cadence: 'daily' };
+  for (let i = 0; i < argv.length; i += 1) {
+    const value = argv[i];
+    if (!value.startsWith('--') && !args.cadence) {
+      args.cadence = value;
+      continue;
+    }
+    if (value === '--cadence') {
+      args.cadence = argv[i + 1] || args.cadence;
+      i += 1;
+    }
+  }
+  return args;
+}
+
+function classifyRelayFailure(outputText) {
+  const text = String(outputText || '').toLowerCase();
+  if (!text.trim()) return 'unknown backend error';
+
+  if ((text.includes('relay') || text.includes('query')) && text.includes('timeout')) {
+    return 'relay query timeout';
+  }
+
+  if (
+    text.includes('connection refused')
+    || text.includes('econnrefused')
+    || text.includes('getaddrinfo')
+    || text.includes('enotfound')
+    || text.includes('eai_again')
+    || (text.includes('websocket') && text.includes('dns'))
+  ) {
+    return 'websocket connection refused/dns';
+  }
+
+  if (
+    text.includes('invalid url')
+    || text.includes('malformed')
+    || text.includes('unsupported protocol')
+    || text.includes('must start with ws')
+    || text.includes('invalid relay')
+  ) {
+    return 'malformed relay url/config';
+  }
+
+  return 'unknown backend error';
+}
+
+async function main() {
+  const { cadence } = parseArgs(process.argv.slice(2));
+  if (cadence !== 'daily' && cadence !== 'weekly') {
+    console.error('Usage: node scripts/agent/check-relay-health.mjs --cadence <daily|weekly>');
+    process.exit(1);
+  }
+
+  const relays = getRelays();
+  const namespace = getNamespace();
+  const timeoutMs = getQueryTimeoutMs();
+  const dateStr = todayDateStr();
+
+  try {
+    const locks = await queryLocks(relays, cadence, dateStr, namespace);
+    console.log(JSON.stringify({
+      ok: true,
+      cadence,
+      namespace,
+      timeoutMs,
+      relays,
+      lockCount: locks.length,
+    }));
+    process.exit(0);
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error ?? 'unknown error');
+    const failureCategory = classifyRelayFailure(message);
+    console.log(JSON.stringify({
+      ok: false,
+      cadence,
+      namespace,
+      timeoutMs,
+      relays,
+      failureCategory,
+      error: message,
+    }));
+    process.exit(2);
+  }
+}
+
+main().catch((error) => {
+  const message = error instanceof Error ? error.message : String(error ?? 'unknown error');
+  const relays = getRelays();
+  const namespace = getNamespace();
+  const failureCategory = classifyRelayFailure(message);
+  console.log(JSON.stringify({
+    ok: false,
+    namespace,
+    relays,
+    failureCategory,
+    error: message,
+  }));
+  process.exit(2);
+});

scripts/agent/run-scheduler-cycle.mjs

@@ -100,6 +100,14 @@ function parseNonNegativeInt(value, fallback) {
   return fallback;
 }
 
+function parseBooleanFlag(value, fallback = false) {
+  if (value === undefined || value === null || value === '') return fallback;
+  const normalized = String(value).trim().toLowerCase();
+  if (['1', 'true', 'yes', 'on'].includes(normalized)) return true;
+  if (['0', 'false', 'no', 'off'].includes(normalized)) return false;
+  return fallback;
+}
+
 function redactSensitive(text) {
   if (!text) return '';
   return String(text)
@@ -300,6 +308,12 @@ async function getSchedulerConfig(cadence, { isInteractive }) {
     process.env.SCHEDULER_LOCK_JITTER_MS,
     parseNonNegativeInt(lockRetryRaw.jitterMs, 75),
   );
+  const lockHealthPreflightFromConfig = parseBooleanFlag(scheduler.lockHealthPreflight, false);
+  const lockHealthPreflightEnabled = parseBooleanFlag(
+    process.env.SCHEDULER_LOCK_HEALTH_PREFLIGHT,
+    lockHealthPreflightFromConfig,
+  );
+  const lockHealthPreflightSkip = parseBooleanFlag(process.env.SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT, false);
 
   return {
     firstPrompt: scheduler.firstPromptByCadence?.[cadence] || null,
@@ -313,6 +327,10 @@ async function getSchedulerConfig(cadence, { isInteractive }) {
       backoffMs,
       jitterMs,
     },
+    lockHealthPreflight: {
+      enabled: lockHealthPreflightEnabled,
+      skip: lockHealthPreflightSkip,
+    },
     memoryPolicy: {
       mode,
       retrieveCommand: typeof memoryPolicyRaw.retrieveCommand === 'string' && memoryPolicyRaw.retrieveCommand.trim()
@@ -329,6 +347,27 @@ async function getSchedulerConfig(cadence, { isInteractive }) {
   };
 }
 
+async function runLockHealthPreflight({ cadence, platform }) {
+  const preflight = await runCommand('node', ['scripts/agent/check-relay-health.mjs', '--cadence', cadence], {
+    env: { AGENT_PLATFORM: platform },
+  });
+  const payload = parseJsonFromOutput(`${preflight.stdout}\n${preflight.stderr}`) || {};
+  const relayList = Array.isArray(payload.relays)
+    ? payload.relays.filter((relay) => typeof relay === 'string' && relay.trim()).map((relay) => relay.trim())
+    : [];
+  const combinedOutput = `${preflight.stderr}\n${preflight.stdout}`;
+  const failureCategory = payload.failureCategory || classifyLockBackendError(combinedOutput);
+
+  return {
+    code: preflight.code,
+    payload,
+    relayList,
+    failureCategory,
+    stderrExcerpt: excerptText(preflight.stderr),
+    stdoutExcerpt: excerptText(preflight.stdout),
+  };
+}
+
 function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -437,6 +476,26 @@ async function main() {
   }
 
   while (true) {
+    if (schedulerConfig.lockHealthPreflight.enabled && !schedulerConfig.lockHealthPreflight.skip) {
+      const preflight = await runLockHealthPreflight({ cadence, platform });
+      if (preflight.code !== 0) {
+        await writeLog({
+          cadence,
+          agent: 'scheduler',
+          status: 'failed',
+          reason: 'Lock backend unavailable preflight',
+          detail: `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`,
+          metadata: {
+            preflight_failure_category: preflight.failureCategory,
+            relay_list: preflight.relayList.join(', ') || '(none)',
+            preflight_stderr_excerpt: preflight.stderrExcerpt || '(empty)',
+            preflight_stdout_excerpt: preflight.stdoutExcerpt || '(empty)',
+          },
+        });
+        process.exit(preflight.code || 1);
+      }
+    }
+
     const checkResult = await runCommand('npm', ['run', `lock:check:${cadence}`]);
     const checkPayload = parseJsonFromOutput(`${checkResult.stdout}\n${checkResult.stderr}`) || {};
     const excluded = Array.isArray(checkPayload.excluded)

src/prompts/scheduler-flow.md

@@ -78,7 +78,21 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co
    mkdir -p <log_dir>
    ```
 
-5. Find latest cadence log file, derive the previous agent, then choose the next roster agent not in exclusion set:
+5. When lock health preflight is enabled (`scheduler.lockHealthPreflight: true` or env `SCHEDULER_LOCK_HEALTH_PREFLIGHT=1`), verify relay/query health before selecting an agent or calling `lock:lock`:
+
+   ```bash
+   node scripts/agent/check-relay-health.mjs --cadence <cadence>
+   ```
+
+   - Escape hatch: set `SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT=1` to skip this check for local/offline workflows.
+   - If preflight exits non-zero, write `_failed.md` with reason `Lock backend unavailable preflight` and include:
+     - `relay_list`
+     - `preflight_failure_category`
+     - `preflight_stderr_excerpt`
+     - `preflight_stdout_excerpt`
+   - Stop the run immediately when preflight fails.
+
+6. Find latest cadence log file, derive the previous agent, then choose the next roster agent not in exclusion set:
 
    ```bash
    ls -1 <log_dir> | sort | tail -n 1
@@ -102,7 +116,7 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co
      - Iterate offsets `0..len(roster)-1`.
      - Candidate index: `(start_index + offset) mod len(roster)` (wrap-around required).
      - Choose the first candidate whose agent is **not** in `excluded`.
-   - If no candidate is eligible, execute step 6.
+   - If no candidate is eligible, execute step 7.
 
    Worked examples:
 
@@ -122,10 +136,10 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co
      - First candidate is `bug-reproducer-agent` and is eligible.
      - **Selection result: `bug-reproducer-agent`.**
 
-6. If every roster agent is excluded, write a `_failed.md` log with:
+7. If every roster agent is excluded, write a `_failed.md` log with:
    `All roster tasks currently claimed by other agents` and stop.
 
-7. Claim selected agent:
+8. Claim selected agent:
 
    ```bash
    AGENT_PLATFORM=<platform> \
@@ -141,12 +155,12 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co
      - `lock_stdout_excerpt` (redacted stdout snippet)
    - Keep generic reason text for compatibility, but append actionable retry guidance in `detail` using the command from `lock_command`.
 
-8. Execute `<prompt_dir>/<prompt-file>` end-to-end via configured handoff command.
+9. Execute `<prompt_dir>/<prompt-file>` end-to-end via configured handoff command.
 
    - Scheduler automation runs `scheduler.handoffCommandByCadence.<cadence>` with environment variables for cadence/agent/prompt path.
    - If no handoff command is configured for the cadence, write `_failed.md` and stop.
 
-9. Confirm memory contract completion:
+10. Confirm memory contract completion:
 
    - Memory retrieval evidence must exist for this run (output marker and/or artifact file).
    - Memory storage evidence must exist for this run (output marker and/or artifact file).
@@ -168,17 +182,18 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co
    - If `scheduler.memoryPolicyByCadence.<cadence>.mode = required`, missing evidence is a hard failure.
    - If mode is `optional`, log warning context and continue.
 
-10. Verify required run artifacts for the current run window.
+11. Verify required run artifacts for the current run window.
 
     - Scheduler runs `node scripts/agent/verify-run-artifacts.mjs --since <run-start-iso> --check-failure-notes`.
     - If artifact verification exits non-zero: write `_failed.md` and stop.
 
-11. Run repository checks (for example: `npm run lint`).
+12. Run repository checks (for example: `npm run lint`).
 
     - If any validation command exits non-zero: **fail the run immediately**, write `_failed.md` with the failing command and reason, and stop.
-    - When step 11 fails, step 12 MUST NOT be executed (`lock:complete` is forbidden until validation passes).
+    - step 12 MUST NOT be executed (`lock:complete` is forbidden until validation passes).
+    - In current numbering: when step 12 fails, step 13 MUST NOT run.
 
-12. Publish completion before writing final success log:
+13. Publish completion before writing final success log:
 
     ```bash
     AGENT_PLATFORM=<platform> \
@@ -187,15 +202,15 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co
 
     (Equivalent invocation is allowed: `torch-lock complete --agent <agent-name> --cadence <cadence>`.)
 
-    - Exit `0`: completion published successfully; continue to step 13.
+    - Exit `0`: completion published successfully; continue to step 14.
     - Exit non-zero: **fail the run**, write `_failed.md` with a clear reason that completion publish failed and retry guidance (for example: `Retry npm run lock:complete -- --agent <agent-name> --cadence <cadence> after verifying relay connectivity`), then stop.
 
-13. Create final task log only after step 12 succeeds (scheduler-owned):
+14. Create final task log only after step 13 succeeds (scheduler-owned):
 
     - `_completed.md` MUST be created only after completion publish succeeds.
-    - `_failed.md` is required when step 10, step 11, or step 12 fails, and should include the failure reason and next retry action.
+    - `_failed.md` is required when step 11, step 12, or step 13 fails, and should include the failure reason and next retry action.
 
-14. Commit/push behavior is delegated outside this scheduler script.
+15. Commit/push behavior is delegated outside this scheduler script.
 
     - `scripts/agent/run-scheduler-cycle.mjs` does **not** run `git commit` or `git push`.
     - If commit/push is required for your workflow, perform it in the configured handoff agent command or a separate orchestration step.

src/prompts/weekly-scheduler.md

@@ -13,6 +13,13 @@ Canonical roster source: `src/prompts/roster.json` (`weekly` key).
 - `branch_prefix`: `agents/weekly/`
 - `prompt_dir`: `src/prompts/weekly/`
 
+
+## Lock Backend Preflight Behavior
+
+- When `scheduler.lockHealthPreflight` is enabled (or `SCHEDULER_LOCK_HEALTH_PREFLIGHT=1`), run `node scripts/agent/check-relay-health.mjs --cadence weekly` before agent selection/lock attempt.
+- On preflight failure, scheduler must write a `_failed.md` entry with reason `Lock backend unavailable preflight`, including relay list + classified preflight failure metadata, and stop.
+- Escape hatch for local/offline workflows: `SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT=1`.
+
 ## Weekly Roster
 
 | # | Agent Name | Prompt File |

test/run-scheduler-cycle-memory-policy.test.mjs

@@ -32,7 +32,13 @@ async function runNode(scriptPath, args, { cwd, env }) {
   });
 }
 
-async function setupFixture({ memoryPolicy, lockShellBody = '', roster = ['agent-a'] }) {
+async function setupFixture({
+  memoryPolicy,
+  lockShellBody = '',
+  roster = ['agent-a'],
+  lockHealthPreflight = undefined,
+  preflightScript = '#!/usr/bin/env node\nconsole.log(JSON.stringify({ ok: true, relays: ["wss://relay.test"] }));\nprocess.exit(0);\n',
+}) {
   const root = await fs.mkdtemp(path.join(os.tmpdir(), 'scheduler-memory-policy-'));
   const scriptsDir = path.join(root, 'scripts', 'agent');
   const binDir = path.join(root, 'bin');
@@ -48,6 +54,7 @@ async function setupFixture({ memoryPolicy, lockShellBody = '', roster = ['agent
     '#!/usr/bin/env node\nprocess.exit(0);\n',
     'utf8',
   );
+  await fs.writeFile(path.join(scriptsDir, 'check-relay-health.mjs'), preflightScript, 'utf8');
 
   await fs.writeFile(path.join(root, 'src', 'prompts', 'roster.json'), JSON.stringify({ daily: roster }, null, 2));
   for (const agent of roster) {
@@ -62,6 +69,7 @@ async function setupFixture({ memoryPolicy, lockShellBody = '', roster = ['agent
         handoffCommandByCadence: { daily: 'echo HANDOFF_OK' },
         validationCommandsByCadence: { daily: ['true'] },
         memoryPolicyByCadence: { daily: memoryPolicy },
+        ...(lockHealthPreflight === undefined ? {} : { lockHealthPreflight }),
       },
     }, null, 2),
     'utf8',
@@ -182,6 +190,55 @@ test('records backend failure metadata when lock command exits with code 2', { c
   assert.match(failedBody, /Retry AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily/);
 });
 
+
+
+test('skips lock health preflight by default when not enabled', { concurrency: false }, async () => {
+  const fixture = await setupFixture({
+    memoryPolicy: { mode: 'optional' },
+    preflightScript: '#!/usr/bin/env node\nconsole.error("preflight should not run");\nprocess.exit(9);\n',
+  });
+
+  const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
+    cwd: fixture.root,
+    env: fixture.env,
+  });
+
+  assert.equal(result.code, 0, `stdout: ${result.stdout}\nstderr: ${result.stderr}`);
+  const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily'));
+  assert.ok(logs.some((name) => name.endsWith('__completed.md')));
+});
+
+test('fails scheduler with preflight metadata when lock health preflight is enabled', { concurrency: false }, async () => {
+  const fixture = await setupFixture({
+    memoryPolicy: { mode: 'optional' },
+    lockHealthPreflight: true,
+    preflightScript: `#!/usr/bin/env node
+console.log(JSON.stringify({
+  ok: false,
+  relays: ['wss://relay-1.test', 'wss://relay-2.test'],
+  failureCategory: 'relay query timeout',
+  error: 'Relay query timed out'
+}));
+process.exit(2);
+`,
+  });
+
+  const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
+    cwd: fixture.root,
+    env: fixture.env,
+  });
+
+  assert.equal(result.code, 2, `stdout: ${result.stdout}\nstderr: ${result.stderr}`);
+
+  const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily'));
+  const failedLog = logs.find((name) => name.endsWith('__failed.md'));
+  assert.ok(failedLog);
+  const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8');
+
+  assert.match(failedBody, /reason: Lock backend unavailable preflight/);
+  assert.match(failedBody, /preflight_failure_category: 'relay query timeout'/);
+  assert.match(failedBody, /relay_list: 'wss:\/\/relay-1.test, wss:\/\/relay-2.test'/);
+});
 test('retries lock acquisition for backend error and succeeds on a later attempt', { concurrency: false }, async () => {
   const fixture = await setupFixture({
     memoryPolicy: { mode: 'optional' },

test/scheduler-ratchet.test.mjs

@@ -123,11 +123,12 @@ describe('Scheduler Ratchet Logic (Log Checking)', () => {
 
 
 describe('Scheduler cycle ordering guarantees', () => {
-  it('keeps lock-check → lock-acquire → handoff → artifact-verify → validation → complete → completed-log order', async () => {
+  it('keeps optional preflight before lock-acquire and preserves handoff/artifact/validation/complete ordering', async () => {
     const { readFile } = await import('node:fs/promises');
     const source = await readFile(new URL('../scripts/agent/run-scheduler-cycle.mjs', import.meta.url), 'utf8');
 
     const order = [
+      "const preflight = await runLockHealthPreflight({ cadence, platform });",
       "const checkResult = await runCommand('npm', ['run', `lock:check:${cadence}`]);",
       "const lockAttempt = await acquireLockWithRetry({",
       "if (schedulerConfig.handoffCommand)",