diff --git a/scripts/agent/check-relay-health.mjs b/scripts/agent/check-relay-health.mjs new file mode 100644 index 0000000..b132760 --- /dev/null +++ b/scripts/agent/check-relay-health.mjs @@ -0,0 +1,106 @@ +#!/usr/bin/env node +import { getRelays, getNamespace, getQueryTimeoutMs } from '../../src/torch-config.mjs'; +import { queryLocks } from '../../src/lock-ops.mjs'; +import { todayDateStr } from '../../src/utils.mjs'; + +function parseArgs(argv) { + const args = { cadence: 'daily' }; + for (let i = 0; i < argv.length; i += 1) { + const value = argv[i]; + if (!value.startsWith('--') && !args.cadence) { + args.cadence = value; + continue; + } + if (value === '--cadence') { + args.cadence = argv[i + 1] || args.cadence; + i += 1; + } + } + return args; +} + +function classifyRelayFailure(outputText) { + const text = String(outputText || '').toLowerCase(); + if (!text.trim()) return 'unknown backend error'; + + if ((text.includes('relay') || text.includes('query')) && text.includes('timeout')) { + return 'relay query timeout'; + } + + if ( + text.includes('connection refused') + || text.includes('econnrefused') + || text.includes('getaddrinfo') + || text.includes('enotfound') + || text.includes('eai_again') + || (text.includes('websocket') && text.includes('dns')) + ) { + return 'websocket connection refused/dns'; + } + + if ( + text.includes('invalid url') + || text.includes('malformed') + || text.includes('unsupported protocol') + || text.includes('must start with ws') + || text.includes('invalid relay') + ) { + return 'malformed relay url/config'; + } + + return 'unknown backend error'; +} + +async function main() { + const { cadence } = parseArgs(process.argv.slice(2)); + if (cadence !== 'daily' && cadence !== 'weekly') { + console.error('Usage: node scripts/agent/check-relay-health.mjs --cadence '); + process.exit(1); + } + + const relays = getRelays(); + const namespace = getNamespace(); + const timeoutMs = getQueryTimeoutMs(); + const dateStr = todayDateStr(); + + try { + const locks = await queryLocks(relays, cadence, dateStr, namespace); + console.log(JSON.stringify({ + ok: true, + cadence, + namespace, + timeoutMs, + relays, + lockCount: locks.length, + })); + process.exit(0); + } catch (error) { + const message = error instanceof Error ? error.message : String(error ?? 'unknown error'); + const failureCategory = classifyRelayFailure(message); + console.log(JSON.stringify({ + ok: false, + cadence, + namespace, + timeoutMs, + relays, + failureCategory, + error: message, + })); + process.exit(2); + } +} + +main().catch((error) => { + const message = error instanceof Error ? error.message : String(error ?? 'unknown error'); + const relays = getRelays(); + const namespace = getNamespace(); + const failureCategory = classifyRelayFailure(message); + console.log(JSON.stringify({ + ok: false, + namespace, + relays, + failureCategory, + error: message, + })); + process.exit(2); +}); diff --git a/scripts/agent/run-scheduler-cycle.mjs b/scripts/agent/run-scheduler-cycle.mjs index 338c08d..8171225 100644 --- a/scripts/agent/run-scheduler-cycle.mjs +++ b/scripts/agent/run-scheduler-cycle.mjs @@ -100,6 +100,14 @@ function parseNonNegativeInt(value, fallback) { return fallback; } +function parseBooleanFlag(value, fallback = false) { + if (value === undefined || value === null || value === '') return fallback; + const normalized = String(value).trim().toLowerCase(); + if (['1', 'true', 'yes', 'on'].includes(normalized)) return true; + if (['0', 'false', 'no', 'off'].includes(normalized)) return false; + return fallback; +} + function redactSensitive(text) { if (!text) return ''; return String(text) @@ -300,6 +308,12 @@ async function getSchedulerConfig(cadence, { isInteractive }) { process.env.SCHEDULER_LOCK_JITTER_MS, parseNonNegativeInt(lockRetryRaw.jitterMs, 75), ); + const lockHealthPreflightFromConfig = parseBooleanFlag(scheduler.lockHealthPreflight, false); + const lockHealthPreflightEnabled = parseBooleanFlag( + process.env.SCHEDULER_LOCK_HEALTH_PREFLIGHT, + lockHealthPreflightFromConfig, + ); + const lockHealthPreflightSkip = parseBooleanFlag(process.env.SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT, false); return { firstPrompt: scheduler.firstPromptByCadence?.[cadence] || null, @@ -313,6 +327,10 @@ async function getSchedulerConfig(cadence, { isInteractive }) { backoffMs, jitterMs, }, + lockHealthPreflight: { + enabled: lockHealthPreflightEnabled, + skip: lockHealthPreflightSkip, + }, memoryPolicy: { mode, retrieveCommand: typeof memoryPolicyRaw.retrieveCommand === 'string' && memoryPolicyRaw.retrieveCommand.trim() @@ -329,6 +347,27 @@ async function getSchedulerConfig(cadence, { isInteractive }) { }; } +async function runLockHealthPreflight({ cadence, platform }) { + const preflight = await runCommand('node', ['scripts/agent/check-relay-health.mjs', '--cadence', cadence], { + env: { AGENT_PLATFORM: platform }, + }); + const payload = parseJsonFromOutput(`${preflight.stdout}\n${preflight.stderr}`) || {}; + const relayList = Array.isArray(payload.relays) + ? payload.relays.filter((relay) => typeof relay === 'string' && relay.trim()).map((relay) => relay.trim()) + : []; + const combinedOutput = `${preflight.stderr}\n${preflight.stdout}`; + const failureCategory = payload.failureCategory || classifyLockBackendError(combinedOutput); + + return { + code: preflight.code, + payload, + relayList, + failureCategory, + stderrExcerpt: excerptText(preflight.stderr), + stdoutExcerpt: excerptText(preflight.stdout), + }; +} + function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -437,6 +476,26 @@ async function main() { } while (true) { + if (schedulerConfig.lockHealthPreflight.enabled && !schedulerConfig.lockHealthPreflight.skip) { + const preflight = await runLockHealthPreflight({ cadence, platform }); + if (preflight.code !== 0) { + await writeLog({ + cadence, + agent: 'scheduler', + status: 'failed', + reason: 'Lock backend unavailable preflight', + detail: `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`, + metadata: { + preflight_failure_category: preflight.failureCategory, + relay_list: preflight.relayList.join(', ') || '(none)', + preflight_stderr_excerpt: preflight.stderrExcerpt || '(empty)', + preflight_stdout_excerpt: preflight.stdoutExcerpt || '(empty)', + }, + }); + process.exit(preflight.code || 1); + } + } + const checkResult = await runCommand('npm', ['run', `lock:check:${cadence}`]); const checkPayload = parseJsonFromOutput(`${checkResult.stdout}\n${checkResult.stderr}`) || {}; const excluded = Array.isArray(checkPayload.excluded) diff --git a/src/prompts/scheduler-flow.md b/src/prompts/scheduler-flow.md index bb852ad..7ff1b36 100644 --- a/src/prompts/scheduler-flow.md +++ b/src/prompts/scheduler-flow.md @@ -78,7 +78,21 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co mkdir -p ``` -5. Find latest cadence log file, derive the previous agent, then choose the next roster agent not in exclusion set: +5. When lock health preflight is enabled (`scheduler.lockHealthPreflight: true` or env `SCHEDULER_LOCK_HEALTH_PREFLIGHT=1`), verify relay/query health before selecting an agent or calling `lock:lock`: + + ```bash + node scripts/agent/check-relay-health.mjs --cadence + ``` + + - Escape hatch: set `SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT=1` to skip this check for local/offline workflows. + - If preflight exits non-zero, write `_failed.md` with reason `Lock backend unavailable preflight` and include: + - `relay_list` + - `preflight_failure_category` + - `preflight_stderr_excerpt` + - `preflight_stdout_excerpt` + - Stop the run immediately when preflight fails. + +6. Find latest cadence log file, derive the previous agent, then choose the next roster agent not in exclusion set: ```bash ls -1 | sort | tail -n 1 @@ -102,7 +116,7 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co - Iterate offsets `0..len(roster)-1`. - Candidate index: `(start_index + offset) mod len(roster)` (wrap-around required). - Choose the first candidate whose agent is **not** in `excluded`. - - If no candidate is eligible, execute step 6. + - If no candidate is eligible, execute step 7. Worked examples: @@ -122,10 +136,10 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co - First candidate is `bug-reproducer-agent` and is eligible. - **Selection result: `bug-reproducer-agent`.** -6. If every roster agent is excluded, write a `_failed.md` log with: +7. If every roster agent is excluded, write a `_failed.md` log with: `All roster tasks currently claimed by other agents` and stop. -7. Claim selected agent: +8. Claim selected agent: ```bash AGENT_PLATFORM= \ @@ -141,12 +155,12 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co - `lock_stdout_excerpt` (redacted stdout snippet) - Keep generic reason text for compatibility, but append actionable retry guidance in `detail` using the command from `lock_command`. -8. Execute `/` end-to-end via configured handoff command. +9. Execute `/` end-to-end via configured handoff command. - Scheduler automation runs `scheduler.handoffCommandByCadence.` with environment variables for cadence/agent/prompt path. - If no handoff command is configured for the cadence, write `_failed.md` and stop. -9. Confirm memory contract completion: +10. Confirm memory contract completion: - Memory retrieval evidence must exist for this run (output marker and/or artifact file). - Memory storage evidence must exist for this run (output marker and/or artifact file). @@ -168,17 +182,18 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co - If `scheduler.memoryPolicyByCadence..mode = required`, missing evidence is a hard failure. - If mode is `optional`, log warning context and continue. -10. Verify required run artifacts for the current run window. +11. Verify required run artifacts for the current run window. - Scheduler runs `node scripts/agent/verify-run-artifacts.mjs --since --check-failure-notes`. - If artifact verification exits non-zero: write `_failed.md` and stop. -11. Run repository checks (for example: `npm run lint`). +12. Run repository checks (for example: `npm run lint`). - If any validation command exits non-zero: **fail the run immediately**, write `_failed.md` with the failing command and reason, and stop. - - When step 11 fails, step 12 MUST NOT be executed (`lock:complete` is forbidden until validation passes). + - step 12 MUST NOT be executed (`lock:complete` is forbidden until validation passes). + - In current numbering: when step 12 fails, step 13 MUST NOT run. -12. Publish completion before writing final success log: +13. Publish completion before writing final success log: ```bash AGENT_PLATFORM= \ @@ -187,15 +202,15 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co (Equivalent invocation is allowed: `torch-lock complete --agent --cadence `.) - - Exit `0`: completion published successfully; continue to step 13. + - Exit `0`: completion published successfully; continue to step 14. - Exit non-zero: **fail the run**, write `_failed.md` with a clear reason that completion publish failed and retry guidance (for example: `Retry npm run lock:complete -- --agent --cadence after verifying relay connectivity`), then stop. -13. Create final task log only after step 12 succeeds (scheduler-owned): +14. Create final task log only after step 13 succeeds (scheduler-owned): - `_completed.md` MUST be created only after completion publish succeeds. - - `_failed.md` is required when step 10, step 11, or step 12 fails, and should include the failure reason and next retry action. + - `_failed.md` is required when step 11, step 12, or step 13 fails, and should include the failure reason and next retry action. -14. Commit/push behavior is delegated outside this scheduler script. +15. Commit/push behavior is delegated outside this scheduler script. - `scripts/agent/run-scheduler-cycle.mjs` does **not** run `git commit` or `git push`. - If commit/push is required for your workflow, perform it in the configured handoff agent command or a separate orchestration step. diff --git a/src/prompts/weekly-scheduler.md b/src/prompts/weekly-scheduler.md index 1d849e5..a0d204a 100644 --- a/src/prompts/weekly-scheduler.md +++ b/src/prompts/weekly-scheduler.md @@ -13,6 +13,13 @@ Canonical roster source: `src/prompts/roster.json` (`weekly` key). - `branch_prefix`: `agents/weekly/` - `prompt_dir`: `src/prompts/weekly/` + +## Lock Backend Preflight Behavior + +- When `scheduler.lockHealthPreflight` is enabled (or `SCHEDULER_LOCK_HEALTH_PREFLIGHT=1`), run `node scripts/agent/check-relay-health.mjs --cadence weekly` before agent selection/lock attempt. +- On preflight failure, scheduler must write a `_failed.md` entry with reason `Lock backend unavailable preflight`, including relay list + classified preflight failure metadata, and stop. +- Escape hatch for local/offline workflows: `SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT=1`. + ## Weekly Roster | # | Agent Name | Prompt File | diff --git a/test/run-scheduler-cycle-memory-policy.test.mjs b/test/run-scheduler-cycle-memory-policy.test.mjs index ee8ff85..67b9f87 100644 --- a/test/run-scheduler-cycle-memory-policy.test.mjs +++ b/test/run-scheduler-cycle-memory-policy.test.mjs @@ -32,7 +32,13 @@ async function runNode(scriptPath, args, { cwd, env }) { }); } -async function setupFixture({ memoryPolicy, lockShellBody = '', roster = ['agent-a'] }) { +async function setupFixture({ + memoryPolicy, + lockShellBody = '', + roster = ['agent-a'], + lockHealthPreflight = undefined, + preflightScript = '#!/usr/bin/env node\nconsole.log(JSON.stringify({ ok: true, relays: ["wss://relay.test"] }));\nprocess.exit(0);\n', +}) { const root = await fs.mkdtemp(path.join(os.tmpdir(), 'scheduler-memory-policy-')); const scriptsDir = path.join(root, 'scripts', 'agent'); const binDir = path.join(root, 'bin'); @@ -48,6 +54,7 @@ async function setupFixture({ memoryPolicy, lockShellBody = '', roster = ['agent '#!/usr/bin/env node\nprocess.exit(0);\n', 'utf8', ); + await fs.writeFile(path.join(scriptsDir, 'check-relay-health.mjs'), preflightScript, 'utf8'); await fs.writeFile(path.join(root, 'src', 'prompts', 'roster.json'), JSON.stringify({ daily: roster }, null, 2)); for (const agent of roster) { @@ -62,6 +69,7 @@ async function setupFixture({ memoryPolicy, lockShellBody = '', roster = ['agent handoffCommandByCadence: { daily: 'echo HANDOFF_OK' }, validationCommandsByCadence: { daily: ['true'] }, memoryPolicyByCadence: { daily: memoryPolicy }, + ...(lockHealthPreflight === undefined ? {} : { lockHealthPreflight }), }, }, null, 2), 'utf8', @@ -182,6 +190,55 @@ test('records backend failure metadata when lock command exits with code 2', { c assert.match(failedBody, /Retry AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily/); }); + + +test('skips lock health preflight by default when not enabled', { concurrency: false }, async () => { + const fixture = await setupFixture({ + memoryPolicy: { mode: 'optional' }, + preflightScript: '#!/usr/bin/env node\nconsole.error("preflight should not run");\nprocess.exit(9);\n', + }); + + const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], { + cwd: fixture.root, + env: fixture.env, + }); + + assert.equal(result.code, 0, `stdout: ${result.stdout}\nstderr: ${result.stderr}`); + const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily')); + assert.ok(logs.some((name) => name.endsWith('__completed.md'))); +}); + +test('fails scheduler with preflight metadata when lock health preflight is enabled', { concurrency: false }, async () => { + const fixture = await setupFixture({ + memoryPolicy: { mode: 'optional' }, + lockHealthPreflight: true, + preflightScript: `#!/usr/bin/env node +console.log(JSON.stringify({ + ok: false, + relays: ['wss://relay-1.test', 'wss://relay-2.test'], + failureCategory: 'relay query timeout', + error: 'Relay query timed out' +})); +process.exit(2); +`, + }); + + const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], { + cwd: fixture.root, + env: fixture.env, + }); + + assert.equal(result.code, 2, `stdout: ${result.stdout}\nstderr: ${result.stderr}`); + + const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily')); + const failedLog = logs.find((name) => name.endsWith('__failed.md')); + assert.ok(failedLog); + const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8'); + + assert.match(failedBody, /reason: Lock backend unavailable preflight/); + assert.match(failedBody, /preflight_failure_category: 'relay query timeout'/); + assert.match(failedBody, /relay_list: 'wss:\/\/relay-1.test, wss:\/\/relay-2.test'/); +}); test('retries lock acquisition for backend error and succeeds on a later attempt', { concurrency: false }, async () => { const fixture = await setupFixture({ memoryPolicy: { mode: 'optional' }, diff --git a/test/scheduler-ratchet.test.mjs b/test/scheduler-ratchet.test.mjs index 08734a4..a8b3fe3 100644 --- a/test/scheduler-ratchet.test.mjs +++ b/test/scheduler-ratchet.test.mjs @@ -123,11 +123,12 @@ describe('Scheduler Ratchet Logic (Log Checking)', () => { describe('Scheduler cycle ordering guarantees', () => { - it('keeps lock-check → lock-acquire → handoff → artifact-verify → validation → complete → completed-log order', async () => { + it('keeps optional preflight before lock-acquire and preserves handoff/artifact/validation/complete ordering', async () => { const { readFile } = await import('node:fs/promises'); const source = await readFile(new URL('../scripts/agent/run-scheduler-cycle.mjs', import.meta.url), 'utf8'); const order = [ + "const preflight = await runLockHealthPreflight({ cadence, platform });", "const checkResult = await runCommand('npm', ['run', `lock:check:${cadence}`]);", "const lockAttempt = await acquireLockWithRetry({", "if (schedulerConfig.handoffCommand)",