diff --git a/scripts/agent/run-scheduler-cycle.mjs b/scripts/agent/run-scheduler-cycle.mjs index dbe68a4..f0e0a29 100644 --- a/scripts/agent/run-scheduler-cycle.mjs +++ b/scripts/agent/run-scheduler-cycle.mjs @@ -4,6 +4,7 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { spawn } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; const VALID_CADENCES = new Set(['daily', 'weekly']); const ALL_EXCLUDED_REASON = 'All roster tasks currently claimed by other agents'; @@ -314,6 +315,18 @@ async function getSchedulerConfig(cadence, { isInteractive }) { lockHealthPreflightFromConfig, ); const lockHealthPreflightSkip = parseBooleanFlag(process.env.SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT, false); + const strictLock = parseBooleanFlag( + process.env.SCHEDULER_STRICT_LOCK, + parseBooleanFlag(scheduler.strict_lock, true), + ); + const degradedLockRetryWindowMs = parseNonNegativeInt( + process.env.SCHEDULER_DEGRADED_LOCK_RETRY_WINDOW_MS, + parseNonNegativeInt(scheduler.degraded_lock_retry_window, 3600000), + ); + const maxDeferrals = parseNonNegativeInt( + process.env.SCHEDULER_MAX_DEFERRALS, + parseNonNegativeInt(scheduler.max_deferrals, 3), + ); return { firstPrompt: scheduler.firstPromptByCadence?.[cadence] || null, @@ -331,6 +344,11 @@ async function getSchedulerConfig(cadence, { isInteractive }) { enabled: lockHealthPreflightEnabled, skip: lockHealthPreflightSkip, }, + lockFailurePolicy: { + strictLock, + degradedLockRetryWindowMs, + maxDeferrals, + }, memoryPolicy: { mode, retrieveCommand: typeof memoryPolicyRaw.retrieveCommand === 'string' && memoryPolicyRaw.retrieveCommand.trim() @@ -347,6 +365,40 @@ async function getSchedulerConfig(cadence, { isInteractive }) { }; } +function getRunDateKey(now = new Date()) { + return now.toISOString().slice(0, 10); +} + +async function readRunState(cadence) { + const statePath = path.resolve(process.cwd(), 'task-logs', cadence, '.scheduler-run-state.json'); + const fallback = { run_date: getRunDateKey(), lock_deferral: null }; + const raw = await readJson(statePath, fallback); + if (!raw || typeof raw !== 'object') { + return { statePath, state: fallback }; + } + + if (raw.run_date !== getRunDateKey()) { + return { statePath, state: fallback }; + } + + return { + statePath, + state: { + run_date: raw.run_date, + lock_deferral: raw.lock_deferral && typeof raw.lock_deferral === 'object' ? raw.lock_deferral : null, + }, + }; +} + +async function writeRunState(statePath, state) { + await fs.mkdir(path.dirname(statePath), { recursive: true }); + await fs.writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, 'utf8'); +} + +function createIdempotencyKey({ cadence, selectedAgent, runDate }) { + return `${cadence}:${selectedAgent}:${runDate}:${randomUUID()}`; +} + async function runLockHealthPreflight({ cadence, platform }) { const preflight = await runCommand('node', ['scripts/agent/check-relay-health.mjs', '--cadence', cadence], { env: { AGENT_PLATFORM: platform }, @@ -372,14 +424,19 @@ function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } -async function acquireLockWithRetry({ selectedAgent, cadence, platform, lockRetry }) { +async function acquireLockWithRetry({ selectedAgent, cadence, platform, lockRetry, idempotencyKey }) { const lockCommandArgs = ['run', 'lock:lock', '--', '--agent', selectedAgent, '--cadence', cadence]; const backoffScheduleMs = []; let attempts = 0; while (true) { attempts += 1; - const result = await runCommand('npm', lockCommandArgs, { env: { AGENT_PLATFORM: platform } }); + const result = await runCommand('npm', lockCommandArgs, { + env: { + AGENT_PLATFORM: platform, + ...(idempotencyKey ? { SCHEDULER_LOCK_IDEMPOTENCY_KEY: idempotencyKey } : {}), + }, + }); if (result.code !== 2) { return { result, attempts, backoffScheduleMs }; @@ -481,6 +538,8 @@ async function main() { } while (true) { + const { statePath: runStatePath, state: schedulerRunState } = await readRunState(cadence); + if (schedulerConfig.lockHealthPreflight.enabled && !schedulerConfig.lockHealthPreflight.skip) { const preflight = await runLockHealthPreflight({ cadence, platform }); if (preflight.code !== 0) { @@ -492,6 +551,7 @@ async function main() { reason: 'Lock backend unavailable preflight', detail: `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`, metadata: { + failure_class: 'backend_unavailable', preflight_failure_category: preflight.failureCategory, relay_list: preflight.relayList.join(', ') || '(none)', preflight_stderr_excerpt: preflight.stderrExcerpt || '(empty)', @@ -543,11 +603,16 @@ async function main() { process.exit(1); } + const deferralForAgent = schedulerRunState.lock_deferral?.selected_agent === selectedAgent + ? schedulerRunState.lock_deferral + : null; + const lockAttempt = await acquireLockWithRetry({ selectedAgent, cadence, platform, lockRetry: schedulerConfig.lockRetry, + idempotencyKey: deferralForAgent?.idempotency_key, }); const lockResult = lockAttempt.result; @@ -562,6 +627,46 @@ async function main() { const stderrExcerpt = excerptText(lockResult.stderr); const stdoutExcerpt = excerptText(lockResult.stdout); const backoffSchedule = lockAttempt.backoffScheduleMs.join(', '); + + const existingDeferral = deferralForAgent || {}; + const firstFailureAt = existingDeferral.first_failure_timestamp || new Date().toISOString(); + const firstFailureMs = parseDateValue(firstFailureAt) || Date.now(); + const nowMs = Date.now(); + const deferralAttemptCount = parseNonNegativeInt(existingDeferral.attempt_count, 0) + 1; + const idempotencyKey = existingDeferral.idempotency_key + || createIdempotencyKey({ cadence, selectedAgent, runDate: schedulerRunState.run_date }); + const withinRetryWindow = nowMs - firstFailureMs <= schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs; + const withinDeferralBudget = deferralAttemptCount <= schedulerConfig.lockFailurePolicy.maxDeferrals; + + if (!schedulerConfig.lockFailurePolicy.strictLock && withinRetryWindow && withinDeferralBudget) { + schedulerRunState.lock_deferral = { + attempt_count: deferralAttemptCount, + first_failure_timestamp: firstFailureAt, + backend_category: backendCategory, + idempotency_key: idempotencyKey, + selected_agent: selectedAgent, + }; + await writeRunState(runStatePath, schedulerRunState); + await writeLog({ + cadence, + agent: selectedAgent, + status: 'deferred', + platform, + reason: 'Lock backend deferred', + detail: `Deferred after lock backend failure (${backendCategory}); retry window active and deferral budget remaining.`, + metadata: { + failure_class: 'backend_unavailable', + deferral_attempt_count: deferralAttemptCount, + deferral_first_failure_timestamp: firstFailureAt, + backend_category: backendCategory, + lock_idempotency_key: idempotencyKey, + }, + }); + process.exit(0); + } + + schedulerRunState.lock_deferral = null; + await writeRunState(runStatePath, schedulerRunState); await writeLog({ cadence, agent: selectedAgent, @@ -570,10 +675,14 @@ async function main() { reason: 'Lock backend error', detail: `Lock backend error (${backendCategory}) after ${lockAttempt.attempts} attempt(s). Retry ${lockCommand} after verifying relay connectivity, relay URLs, and DNS/network status.`, metadata: { + failure_class: 'backend_unavailable', lock_attempts_total: lockAttempt.attempts, lock_backoff_schedule_ms: backoffSchedule || '(none)', backend_category: backendCategory, + deferral_attempt_count: deferralAttemptCount, + deferral_first_failure_timestamp: firstFailureAt, lock_command: lockCommand, + lock_idempotency_key: idempotencyKey, lock_stderr_excerpt: stderrExcerpt || '(empty)', lock_stdout_excerpt: stdoutExcerpt || '(empty)', }, @@ -582,6 +691,8 @@ async function main() { } if (lockResult.code !== 0) { + schedulerRunState.lock_deferral = null; + await writeRunState(runStatePath, schedulerRunState); await writeLog({ cadence, agent: selectedAgent, @@ -593,6 +704,8 @@ async function main() { } const promptPath = path.resolve(process.cwd(), 'src/prompts', cadence, `${selectedAgent}.md`); + schedulerRunState.lock_deferral = null; + await writeRunState(runStatePath, schedulerRunState); const runArtifactSince = new Date().toISOString(); const runStartMs = Date.parse(runArtifactSince); const outputChunks = []; @@ -603,7 +716,15 @@ async function main() { }); outputChunks.push(retrieveResult.stdout, retrieveResult.stderr); if (retrieveResult.code !== 0) { - await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Memory retrieval command failed', detail: schedulerConfig.memoryPolicy.retrieveCommand }); + await writeLog({ + cadence, + agent: selectedAgent, + status: 'failed', + platform, + reason: 'Memory retrieval command failed', + detail: schedulerConfig.memoryPolicy.retrieveCommand, + metadata: { failure_class: 'prompt_validation_error' }, + }); process.exit(retrieveResult.code); } } @@ -614,14 +735,30 @@ async function main() { }); outputChunks.push(handoff.stdout, handoff.stderr); if (handoff.code !== 0) { - await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Prompt/handoff execution failed', detail: 'Handoff callback failed.' }); + await writeLog({ + cadence, + agent: selectedAgent, + status: 'failed', + platform, + reason: 'Prompt/handoff execution failed', + detail: 'Handoff callback failed.', + metadata: { failure_class: 'prompt_validation_error' }, + }); process.exit(handoff.code); } } else { const detail = schedulerConfig.missingHandoffCommandForMode ? 'Missing scheduler handoff command for non-interactive run. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.' : 'No handoff callback configured. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.'; - await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Prompt/handoff execution failed', detail }); + await writeLog({ + cadence, + agent: selectedAgent, + status: 'failed', + platform, + reason: 'Prompt/handoff execution failed', + detail, + metadata: { failure_class: 'prompt_validation_error' }, + }); process.exit(1); } @@ -631,7 +768,15 @@ async function main() { }); outputChunks.push(storeResult.stdout, storeResult.stderr); if (storeResult.code !== 0) { - await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Memory storage command failed', detail: schedulerConfig.memoryPolicy.storeCommand }); + await writeLog({ + cadence, + agent: selectedAgent, + status: 'failed', + platform, + reason: 'Memory storage command failed', + detail: schedulerConfig.memoryPolicy.storeCommand, + metadata: { failure_class: 'prompt_validation_error' }, + }); process.exit(storeResult.code); } } @@ -661,6 +806,7 @@ async function main() { platform, reason: 'Required memory steps not verified', detail: `Missing evidence for: ${missingSteps.join(', ')}`, + metadata: { failure_class: 'prompt_validation_error' }, }); process.exit(1); } @@ -692,6 +838,7 @@ async function main() { platform, reason: 'Missing required run artifacts', detail, + metadata: { failure_class: 'prompt_validation_error' }, }); process.exit(artifactCheck.code); } @@ -701,7 +848,15 @@ async function main() { if (!parts.length) continue; const result = await runCommand(parts[0], parts.slice(1)); if (result.code !== 0) { - await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Validation failed', detail: validation }); + await writeLog({ + cadence, + agent: selectedAgent, + status: 'failed', + platform, + reason: 'Validation failed', + detail: validation, + metadata: { failure_class: 'prompt_validation_error' }, + }); process.exit(result.code); } } diff --git a/src/docs/TORCH.md b/src/docs/TORCH.md index b37d4ef..67813f2 100644 --- a/src/docs/TORCH.md +++ b/src/docs/TORCH.md @@ -65,6 +65,9 @@ Common settings: - `scheduler.firstPromptByCadence.daily` / `.weekly` — first-run scheduler starting agent. - `scheduler.handoffCommandByCadence.daily` / `.weekly` — shell command run after lock acquisition; command must use `SCHEDULER_AGENT`, `SCHEDULER_CADENCE`, and `SCHEDULER_PROMPT_PATH` provided by `scripts/agent/run-scheduler-cycle.mjs`. - `scheduler.paused.daily` / `.weekly` — array of agent names to exclude from scheduler rotation. +- `scheduler.strict_lock` — lock backend policy switch (default: `true`); when `false`, scheduler defers backend-unavailable lock failures before converting the run to failed. +- `scheduler.degraded_lock_retry_window` — non-strict deferral window in milliseconds; backend lock failures outside this window immediately consume failure budget and mark run failed. +- `scheduler.max_deferrals` — max number of non-strict lock deferrals allowed in-window before scheduler records a hard failure. Default first-run daily scheduler prompt is `scheduler-update-agent`. @@ -72,6 +75,11 @@ For weekly repository-fit maintenance, TORCH also includes `src/prompts/weekly/r Operational note: scheduler handoff commands are treated as required execution steps. A non-zero exit code (or missing command) is a hard failure: the scheduler writes a `_failed.md` task log, exits immediately, and does not publish `lock:complete` for that run. +Scheduler failure classes in task logs: + +- `backend_unavailable` — lock backend unavailable preflight failures and lock acquisition backend exit code `2` failures/deferrals. +- `prompt_validation_error` — prompt/handoff, memory evidence, artifact verification, and validation command failures. + ## Lock backend production defaults diff --git a/src/prompts/scheduler-flow.md b/src/prompts/scheduler-flow.md index 1e8d517..bb19c3e 100644 --- a/src/prompts/scheduler-flow.md +++ b/src/prompts/scheduler-flow.md @@ -148,17 +148,24 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co - Exit `0`: lock acquired, continue. - Exit `3`: race lost/already locked, return to step 2. - - Exit `2`: lock backend error, write `_failed.md` with reason `Lock backend error`, and include failure metadata fields: + - Exit `2`: lock backend error. + - If `scheduler.strict_lock` is `false`, defer the run while both budget constraints are still satisfied: + - `degraded_lock_retry_window` (ms) since first backend failure has not elapsed. + - `max_deferrals` has not been exceeded. + - Record deferral metadata in scheduler run state (`attempt_count`, `first_failure_timestamp`, `backend_category`, and preserved idempotency key). + - Otherwise write `_failed.md` with reason `Lock backend error`, and include failure metadata fields: - `backend_category` (classified backend failure category) - `lock_command` (raw lock command for retry) - `lock_stderr_excerpt` (redacted stderr snippet) - `lock_stdout_excerpt` (redacted stdout snippet) + - Include `failure_class: backend_unavailable` for both deferred and failed backend-unavailable lock outcomes. - Keep generic reason text for compatibility, but append actionable retry guidance in `detail` using the command from `lock_command`. 9. Execute `/` end-to-end via configured handoff command. - Scheduler automation runs `scheduler.handoffCommandByCadence.` with environment variables for cadence/agent/prompt path. - If no handoff command is configured for the cadence, write `_failed.md` and stop. + - Prompt/handoff and prompt validation failures should emit `failure_class: prompt_validation_error`. 10. Confirm memory contract completion: diff --git a/test/run-scheduler-cycle-memory-policy.test.mjs b/test/run-scheduler-cycle-memory-policy.test.mjs index 699479d..2cf78df 100644 --- a/test/run-scheduler-cycle-memory-policy.test.mjs +++ b/test/run-scheduler-cycle-memory-policy.test.mjs @@ -37,6 +37,7 @@ async function setupFixture({ lockShellBody = '', roster = ['agent-a'], lockHealthPreflight = undefined, + schedulerPolicy = {}, preflightScript = '#!/usr/bin/env node\nconsole.log(JSON.stringify({ ok: true, relays: ["wss://relay.test"] }));\nprocess.exit(0);\n', }) { const root = await fs.mkdtemp(path.join(os.tmpdir(), 'scheduler-memory-policy-')); @@ -69,6 +70,7 @@ async function setupFixture({ handoffCommandByCadence: { daily: 'echo HANDOFF_OK' }, validationCommandsByCadence: { daily: ['true'] }, memoryPolicyByCadence: { daily: memoryPolicy }, + ...schedulerPolicy, ...(lockHealthPreflight === undefined ? {} : { lockHealthPreflight }), }, }, null, 2), @@ -186,6 +188,7 @@ test('records backend failure metadata when lock command exits with code 2', { c const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8'); assert.match(failedBody, /reason: Lock backend error/); + assert.match(failedBody, /failure_class: 'backend_unavailable'/); assert.match(failedBody, /backend_category: 'publish failed to all relays'/); assert.match(failedBody, /lock_command: 'AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily'/); assert.match(failedBody, /lock_stderr_excerpt: 'publish failed to all relays token=\[REDACTED\] SECRET_KEY=\[REDACTED\]'/); @@ -194,6 +197,149 @@ test('records backend failure metadata when lock command exits with code 2', { c assert.match(failedBody, /platform: 'codex'/); }); +test('defers backend failures in non-strict mode and reuses idempotency key on successful retry', { concurrency: false }, async () => { + const fixture = await setupFixture({ + memoryPolicy: { mode: 'optional' }, + schedulerPolicy: { + strict_lock: false, + degraded_lock_retry_window: 3600000, + max_deferrals: 3, + }, + lockShellBody: ` count_file="$PWD/.lock-count" + key_log="$PWD/.idempotency-keys" + count=0 + if [[ -f "$count_file" ]]; then + count="$(cat "$count_file")" + fi + count=$((count + 1)) + echo "$count" > "$count_file" + echo "\${SCHEDULER_LOCK_IDEMPOTENCY_KEY:-missing}" >> "$key_log" + if [[ "$count" -lt 3 ]]; then + echo 'relay query timeout' >&2 + exit 2 + fi + :`, + }); + + const first = await runNode(fixture.scriptPath, ['--cadence', 'daily'], { + cwd: fixture.root, + env: { + ...fixture.env, + SCHEDULER_LOCK_MAX_RETRIES: '0', + SCHEDULER_LOCK_BACKOFF_MS: '0', + SCHEDULER_LOCK_JITTER_MS: '0', + }, + }); + assert.equal(first.code, 0, `stdout: ${first.stdout}\nstderr: ${first.stderr}`); + + const second = await runNode(fixture.scriptPath, ['--cadence', 'daily'], { + cwd: fixture.root, + env: { + ...fixture.env, + SCHEDULER_LOCK_MAX_RETRIES: '0', + SCHEDULER_LOCK_BACKOFF_MS: '0', + SCHEDULER_LOCK_JITTER_MS: '0', + }, + }); + assert.equal(second.code, 0, `stdout: ${second.stdout}\nstderr: ${second.stderr}`); + + const third = await runNode(fixture.scriptPath, ['--cadence', 'daily'], { + cwd: fixture.root, + env: { + ...fixture.env, + SCHEDULER_LOCK_MAX_RETRIES: '0', + SCHEDULER_LOCK_BACKOFF_MS: '0', + SCHEDULER_LOCK_JITTER_MS: '0', + }, + }); + assert.equal(third.code, 0, `stdout: ${third.stdout}\nstderr: ${third.stderr}`); + + const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily')); + const deferredLog = logs.find((name) => name.endsWith('__deferred.md')); + assert.ok(deferredLog); + const deferredBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', deferredLog), 'utf8'); + assert.match(deferredBody, /failure_class: 'backend_unavailable'/); + assert.match(deferredBody, /deferral_attempt_count: '\d+'/); + + const keyLog = await fs.readFile(path.join(fixture.root, '.idempotency-keys'), 'utf8'); + const keys = keyLog.trim().split(/\r?\n/).filter(Boolean); + assert.equal(keys.length, 3); + assert.equal(keys[0], 'missing'); + assert.notEqual(keys[1], 'missing'); + assert.equal(keys[1], keys[2]); +}); + +test('fails after exceeding non-strict deferral budget', { concurrency: false }, async () => { + const fixture = await setupFixture({ + memoryPolicy: { mode: 'optional' }, + schedulerPolicy: { + strict_lock: false, + degraded_lock_retry_window: 3600000, + max_deferrals: 1, + }, + lockShellBody: ` echo 'publish failed to all relays' >&2 + exit 2`, + }); + + const first = await runNode(fixture.scriptPath, ['--cadence', 'daily'], { + cwd: fixture.root, + env: { + ...fixture.env, + SCHEDULER_LOCK_MAX_RETRIES: '0', + SCHEDULER_LOCK_BACKOFF_MS: '0', + SCHEDULER_LOCK_JITTER_MS: '0', + }, + }); + assert.equal(first.code, 0, `stdout: ${first.stdout}\nstderr: ${first.stderr}`); + + const second = await runNode(fixture.scriptPath, ['--cadence', 'daily'], { + cwd: fixture.root, + env: { + ...fixture.env, + SCHEDULER_LOCK_MAX_RETRIES: '0', + SCHEDULER_LOCK_BACKOFF_MS: '0', + SCHEDULER_LOCK_JITTER_MS: '0', + }, + }); + assert.equal(second.code, 2, `stdout: ${second.stdout}\nstderr: ${second.stderr}`); + + const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily')); + const failedLog = logs.find((name) => name.endsWith('__failed.md')); + assert.ok(failedLog); + const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8'); + assert.match(failedBody, /failure_class: 'backend_unavailable'/); + assert.match(failedBody, /deferral_attempt_count: '2'/); +}); + +test('strict lock mode fails immediately without deferral', { concurrency: false }, async () => { + const fixture = await setupFixture({ + memoryPolicy: { mode: 'optional' }, + schedulerPolicy: { + strict_lock: true, + degraded_lock_retry_window: 3600000, + max_deferrals: 5, + }, + lockShellBody: ` echo 'relay timeout' >&2 + exit 2`, + }); + + const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], { + cwd: fixture.root, + env: { + ...fixture.env, + SCHEDULER_LOCK_MAX_RETRIES: '0', + SCHEDULER_LOCK_BACKOFF_MS: '0', + SCHEDULER_LOCK_JITTER_MS: '0', + }, + }); + assert.equal(result.code, 2, `stdout: ${result.stdout}\nstderr: ${result.stderr}`); + + const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily')); + assert.equal(logs.some((name) => name.endsWith('__deferred.md')), false); + const failedLog = logs.find((name) => name.endsWith('__failed.md')); + assert.ok(failedLog); +}); + test('skips lock health preflight by default when not enabled', { concurrency: false }, async () => {