diff --git a/scripts/agent/run-scheduler-cycle.mjs b/scripts/agent/run-scheduler-cycle.mjs index 147fed3..b12b71f 100644 --- a/scripts/agent/run-scheduler-cycle.mjs +++ b/scripts/agent/run-scheduler-cycle.mjs @@ -8,6 +8,14 @@ import { randomUUID } from 'node:crypto'; const VALID_CADENCES = new Set(['daily', 'weekly']); const ALL_EXCLUDED_REASON = 'All roster tasks currently claimed by other agents'; +const FAILURE_CATEGORY = { + PROMPT_PARSE: 'prompt_parse_error', + PROMPT_SCHEMA: 'prompt_schema_error', + LOCK_BACKEND: 'lock_backend_error', + EXECUTION: 'execution_error', +}; + +const LOCK_INCIDENT_LINK = 'docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md'; function parseArgs(argv) { const args = { cadence: null, platform: process.env.AGENT_PLATFORM || 'codex' }; @@ -141,6 +149,58 @@ function excerptText(text, maxChars = 600) { return `${clean.slice(0, maxChars)}…`; } +function categorizeFailureMetadata(failureCategory, extraMetadata = {}) { + return { + failure_category: failureCategory, + ...extraMetadata, + }; +} + +function formatDurationMs(ms) { + if (!Number.isFinite(ms) || ms < 0) return 'unknown'; + const minutes = Math.round(ms / 60000); + return `${minutes} minute(s)`; +} + +function buildLockBackendRemediation({ cadence, retryWindowMs, maxDeferrals, incidentSignalId = null }) { + const steps = [ + `Retry window: ${formatDurationMs(retryWindowMs)} (max deferrals: ${maxDeferrals}).`, + `Run health check: npm run lock:health -- --cadence ${cadence}`, + `Review incident runbook: ${LOCK_INCIDENT_LINK}`, + ]; + if (incidentSignalId) { + steps.push(`Incident signal: ${incidentSignalId}`); + } + return `Recommended auto-remediation: ${steps.join(' ')}`; +} + +async function validatePromptFile(promptPath) { + let content; + try { + content = await fs.readFile(promptPath, 'utf8'); + } catch (error) { + return { + ok: false, + category: FAILURE_CATEGORY.PROMPT_PARSE, + reason: 'Prompt file parse/read failed', + detail: `Prompt not executed; unable to read prompt file at ${promptPath}: ${error.message}`, + }; + } + + const lines = String(content).split(/\r?\n/).map((line) => line.trim()).filter(Boolean); + const firstLine = lines[0] || ''; + if (!firstLine.startsWith('#')) { + return { + ok: false, + category: FAILURE_CATEGORY.PROMPT_SCHEMA, + reason: 'Prompt file schema validation failed', + detail: `Prompt not executed; expected markdown heading on first non-empty line in ${promptPath}.`, + }; + } + + return { ok: true }; +} + function classifyLockBackendError(outputText) { const text = String(outputText || '').toLowerCase(); if (!text.trim()) return 'unknown backend error'; @@ -604,10 +664,10 @@ async function main() { ? 'All relays unhealthy preflight' : 'Lock backend unavailable preflight', detail: allRelaysUnhealthy - ? `Deferred run before lock acquisition: ${incidentSignal?.reason || 'all relays unhealthy'}.` - : `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`, + ? `Deferred run before lock acquisition: ${incidentSignal?.reason || 'all relays unhealthy'}. Prompt not executed. ${buildLockBackendRemediation({ cadence, retryWindowMs: schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs, maxDeferrals: schedulerConfig.lockFailurePolicy.maxDeferrals, incidentSignalId: incidentSignal?.id || null })}` + : `Preflight failed (${preflight.failureCategory}). Prompt not executed. ${buildLockBackendRemediation({ cadence, retryWindowMs: schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs, maxDeferrals: schedulerConfig.lockFailurePolicy.maxDeferrals, incidentSignalId: incidentSignal?.id || null })}`, metadata: { - failure_class: 'backend_unavailable', + ...categorizeFailureMetadata(FAILURE_CATEGORY.LOCK_BACKEND, { failure_class: 'backend_unavailable' }), preflight_failure_category: preflight.failureCategory, relay_list: preflight.relayList.join(', ') || '(none)', preflight_stderr_excerpt: preflight.stderrExcerpt || '(empty)', @@ -714,9 +774,9 @@ async function main() { status: 'deferred', platform, reason: 'Lock backend deferred', - detail: `Deferred after lock backend failure (${backendCategory}); retry window active and deferral budget remaining.`, + detail: `Deferred after lock backend failure (${backendCategory}); retry window active and deferral budget remaining. Prompt not executed. ${buildLockBackendRemediation({ cadence, retryWindowMs: schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs, maxDeferrals: schedulerConfig.lockFailurePolicy.maxDeferrals })}`, metadata: { - failure_class: 'backend_unavailable', + ...categorizeFailureMetadata(FAILURE_CATEGORY.LOCK_BACKEND, { failure_class: 'backend_unavailable' }), deferral_attempt_count: deferralAttemptCount, deferral_first_failure_timestamp: firstFailureAt, backend_category: backendCategory, @@ -734,9 +794,9 @@ async function main() { status: 'failed', platform, reason: 'Lock backend error', - detail: `Lock backend error (${backendCategory}) after ${lockAttempt.attempts} attempt(s). Retry ${lockCommand} after verifying relay connectivity, relay URLs, and DNS/network status.`, + detail: `Lock backend error (${backendCategory}) after ${lockAttempt.attempts} attempt(s). ${buildLockBackendRemediation({ cadence, retryWindowMs: schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs, maxDeferrals: schedulerConfig.lockFailurePolicy.maxDeferrals })}`, metadata: { - failure_class: 'backend_unavailable', + ...categorizeFailureMetadata(FAILURE_CATEGORY.LOCK_BACKEND, { failure_class: 'backend_unavailable' }), lock_attempts_total: lockAttempt.attempts, lock_backoff_schedule_ms: backoffSchedule || '(none)', lock_correlation_id: diagnosticsSummary.correlationId || lockAttempt.correlationId, @@ -769,6 +829,20 @@ async function main() { } const promptPath = path.resolve(process.cwd(), 'src/prompts', cadence, `${selectedAgent}.md`); + const promptValidation = await validatePromptFile(promptPath); + if (!promptValidation.ok) { + await writeLog({ + cadence, + agent: selectedAgent, + status: 'failed', + platform, + reason: promptValidation.reason, + detail: promptValidation.detail, + metadata: categorizeFailureMetadata(promptValidation.category, { prompt_path: promptPath }), + }); + process.exit(1); + } + schedulerRunState.lock_deferral = null; await writeRunState(runStatePath, schedulerRunState); const runArtifactSince = new Date().toISOString(); @@ -788,7 +862,7 @@ async function main() { platform, reason: 'Memory retrieval command failed', detail: schedulerConfig.memoryPolicy.retrieveCommand, - metadata: { failure_class: 'prompt_validation_error' }, + metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }), }); process.exit(retrieveResult.code); } @@ -807,7 +881,7 @@ async function main() { platform, reason: 'Prompt/handoff execution failed', detail: 'Handoff callback failed.', - metadata: { failure_class: 'prompt_validation_error' }, + metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }), }); process.exit(handoff.code); } @@ -822,7 +896,7 @@ async function main() { platform, reason: 'Prompt/handoff execution failed', detail, - metadata: { failure_class: 'prompt_validation_error' }, + metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }), }); process.exit(1); } @@ -840,7 +914,7 @@ async function main() { platform, reason: 'Memory storage command failed', detail: schedulerConfig.memoryPolicy.storeCommand, - metadata: { failure_class: 'prompt_validation_error' }, + metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }), }); process.exit(storeResult.code); } @@ -871,7 +945,7 @@ async function main() { platform, reason: 'Required memory steps not verified', detail: `Missing evidence for: ${missingSteps.join(', ')}`, - metadata: { failure_class: 'prompt_validation_error' }, + metadata: categorizeFailureMetadata(FAILURE_CATEGORY.PROMPT_SCHEMA, { failure_class: 'prompt_validation_error' }), }); process.exit(1); } @@ -903,7 +977,7 @@ async function main() { platform, reason: 'Missing required run artifacts', detail, - metadata: { failure_class: 'prompt_validation_error' }, + metadata: categorizeFailureMetadata(FAILURE_CATEGORY.PROMPT_SCHEMA, { failure_class: 'prompt_validation_error' }), }); process.exit(artifactCheck.code); } @@ -920,7 +994,7 @@ async function main() { platform, reason: 'Validation failed', detail: validation, - metadata: { failure_class: 'prompt_validation_error' }, + metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }), }); process.exit(result.code); } diff --git a/src/docs/TORCH.md b/src/docs/TORCH.md index c7b563a..9a1e0e5 100644 --- a/src/docs/TORCH.md +++ b/src/docs/TORCH.md @@ -80,8 +80,15 @@ Operational note: scheduler handoff commands are treated as required execution s Scheduler failure classes in task logs: -- `backend_unavailable` — lock backend unavailable preflight failures and lock acquisition backend exit code `2` failures/deferrals; includes relay health alert metadata and incident signal IDs when all relays are unhealthy. -- `prompt_validation_error` — prompt/handoff, memory evidence, artifact verification, and validation command failures. +- `backend_unavailable` — legacy compatibility field for lock backend unavailable failures. +- `prompt_validation_error` — legacy compatibility field for prompt/runtime validation failures. + +Scheduler failure categories in task logs: + +- `lock_backend_error` — lock backend unavailable preflight failures and lock acquisition backend exit code `2` failures/deferrals; includes relay health alert metadata, retry window guidance, health check command, and incident runbook link. +- `prompt_parse_error` — scheduler could not read/parse the selected prompt file; prompt execution is skipped. +- `prompt_schema_error` — selected prompt file or generated run artifacts failed schema/contract checks; prompt run is treated as invalid. +- `execution_error` — runtime execution failures in handoff callbacks, memory commands, or configured validation commands. ## Lock backend production defaults @@ -100,6 +107,11 @@ Validation behavior: - Relay URLs must be absolute `ws://` or `wss://` URLs. - Invalid relay URLs or invalid timeout/count ranges are fatal startup errors. - Lock backend errors include phase (`query:primary`, `query:fallback`, `publish:primary`, `publish:fallback`), relay endpoint, and timeout value used. +- When scheduler preflight fails before lock acquisition, task logs must explicitly state `prompt not executed` to make lock-vs-prompt root cause obvious in UI/CLI summaries. +- Interpret `relay_publish_quorum_failure` (derived from `lock_publish_quorum_failed` telemetry) as quorum not met even after retries/fallbacks. Expected operator actions: + 1. Run `npm run lock:health -- --cadence ` to confirm relay readiness and identify failing relays/reasons. + 2. Review task log metadata (`lock_failure_reason_distribution`, `backend_category`, `lock_correlation_id`) for dominant failure modes (timeouts, DNS, auth, malformed relay URL). + 3. If failures persist past retry window, follow incident runbook `docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md` and escalate relay/network remediation. - Relay health snapshots are emitted periodically and whenever lock publish/query fails; snapshots include success rate, timeout rate, rolling latency, and quarantine state per relay. ## Scheduler lock reliability reporting diff --git a/src/prompts/scheduler-flow.md b/src/prompts/scheduler-flow.md index e64b503..8e9a292 100644 --- a/src/prompts/scheduler-flow.md +++ b/src/prompts/scheduler-flow.md @@ -85,8 +85,8 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co ``` - Escape hatch: set `SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT=1` to skip this check for local/offline workflows. - - If preflight exits non-zero because every relay is unhealthy, write `_deferred.md` with reason `All relays unhealthy preflight`, include `incident_signal_id`, and stop before lock acquisition. - - For other non-zero preflight failures, write `_failed.md` with reason `Lock backend unavailable preflight` and include: + - If preflight exits non-zero because every relay is unhealthy, write `_deferred.md` with reason `All relays unhealthy preflight`, include `incident_signal_id`, set `failure_category: lock_backend_error`, state `prompt not executed`, and stop before lock acquisition. + - For other non-zero preflight failures, write `_failed.md` with reason `Lock backend unavailable preflight`, set `failure_category: lock_backend_error`, state `prompt not executed`, and include: - `relay_list` - `preflight_failure_category` - `preflight_stderr_excerpt` @@ -159,14 +159,17 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co - `lock_command` (raw lock command for retry) - `lock_stderr_excerpt` (redacted stderr snippet) - `lock_stdout_excerpt` (redacted stdout snippet) - - Include `failure_class: backend_unavailable` for both deferred and failed backend-unavailable lock outcomes. + - Include `failure_class: backend_unavailable` and `failure_category: lock_backend_error` for both deferred and failed backend-unavailable lock outcomes. + - Include recommended auto-remediation text in `detail`: retry window, `npm run lock:health -- --cadence `, and incident runbook link. - Keep generic reason text for compatibility, but append actionable retry guidance in `detail` using the command from `lock_command`. 9. Execute `/` end-to-end via configured handoff command. - Scheduler automation runs `scheduler.handoffCommandByCadence.` with environment variables for cadence/agent/prompt path. - If no handoff command is configured for the cadence, write `_failed.md` and stop. - - Prompt/handoff and prompt validation failures should emit `failure_class: prompt_validation_error`. + - Prompt file read/parse failures should emit `failure_category: prompt_parse_error`. + - Prompt schema/contract failures should emit `failure_category: prompt_schema_error`. + - Command/handoff/validation runtime failures should emit `failure_category: execution_error`. 10. Confirm memory contract completion: diff --git a/src/prompts/weekly-scheduler.md b/src/prompts/weekly-scheduler.md index a0d204a..5ab53aa 100644 --- a/src/prompts/weekly-scheduler.md +++ b/src/prompts/weekly-scheduler.md @@ -17,7 +17,7 @@ Canonical roster source: `src/prompts/roster.json` (`weekly` key). ## Lock Backend Preflight Behavior - When `scheduler.lockHealthPreflight` is enabled (or `SCHEDULER_LOCK_HEALTH_PREFLIGHT=1`), run `node scripts/agent/check-relay-health.mjs --cadence weekly` before agent selection/lock attempt. -- On preflight failure, scheduler must write a `_failed.md` entry with reason `Lock backend unavailable preflight`, including relay list + classified preflight failure metadata, and stop. +- On preflight failure, scheduler must write a `_failed.md` entry with reason `Lock backend unavailable preflight`, including relay list + classified preflight failure metadata, `failure_category: lock_backend_error`, and `prompt not executed` summary text before stopping. - Escape hatch for local/offline workflows: `SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT=1`. ## Weekly Roster diff --git a/test/run-scheduler-cycle-memory-policy.test.mjs b/test/run-scheduler-cycle-memory-policy.test.mjs index fb8876e..e0f6a52 100644 --- a/test/run-scheduler-cycle-memory-policy.test.mjs +++ b/test/run-scheduler-cycle-memory-policy.test.mjs @@ -88,6 +88,13 @@ case "$2" in lock:check:daily) echo '{"excluded":[]}' ;; + lock:health) + shift 2 + if [[ "\${1:-}" == "--" ]]; then + shift + fi + node scripts/agent/check-relay-health.mjs "$@" + ;; lock:lock) ${lockShellBody || ' :'} ;; @@ -189,6 +196,7 @@ test('records backend failure metadata when lock command exits with code 2', { c assert.match(failedBody, /reason: Lock backend error/); assert.match(failedBody, /failure_class: 'backend_unavailable'/); + assert.match(failedBody, /failure_category: 'lock_backend_error'/); assert.match(failedBody, /backend_category: 'publish failed to all relays'/); assert.match(failedBody, /lock_command: 'AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily'/); assert.match(failedBody, /lock_stderr_excerpt: 'publish failed to all relays token=\[REDACTED\] SECRET_KEY=\[REDACTED\]'/); @@ -196,7 +204,9 @@ test('records backend failure metadata when lock command exits with code 2', { c assert.match(failedBody, /lock_failure_reason_distribution: '\{\}'/); assert.match(failedBody, /lock_attempt_id: '3'/); assert.match(failedBody, /lock_correlation_id: 'daily:agent-a:/); - assert.match(failedBody, /Retry AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily/); + assert.match(failedBody, /Recommended auto-remediation:/); + assert.match(failedBody, /Run health check: npm run lock:health -- --cadence daily/); + assert.match(failedBody, /Review incident runbook: docs\/agent-handoffs\/learnings\/2026-02-15-relay-health-preflight-job.md/); assert.match(failedBody, /platform: 'codex'/); }); @@ -262,6 +272,7 @@ test('defers backend failures in non-strict mode and reuses idempotency key on s assert.ok(deferredLog); const deferredBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', deferredLog), 'utf8'); assert.match(deferredBody, /failure_class: 'backend_unavailable'/); + assert.match(deferredBody, /failure_category: 'lock_backend_error'/); assert.match(deferredBody, /deferral_attempt_count: '\d+'/); const keyLog = await fs.readFile(path.join(fixture.root, '.idempotency-keys'), 'utf8'); @@ -311,6 +322,7 @@ test('fails after exceeding non-strict deferral budget', { concurrency: false }, assert.ok(failedLog); const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8'); assert.match(failedBody, /failure_class: 'backend_unavailable'/); + assert.match(failedBody, /failure_category: 'lock_backend_error'/); assert.match(failedBody, /deferral_attempt_count: '2'/); }); @@ -389,9 +401,80 @@ process.exit(2); const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8'); assert.match(failedBody, /reason: Lock backend unavailable preflight/); + assert.match(failedBody, /prompt not executed/i); + assert.match(failedBody, /failure_category: 'lock_backend_error'/); assert.match(failedBody, /preflight_failure_category: 'relay query timeout'/); assert.match(failedBody, /relay_list: 'wss:\/\/relay-1.test, wss:\/\/relay-2.test'/); }); + + +test('categorizes unreadable prompt files as prompt_parse_error', { concurrency: false }, async () => { + const fixture = await setupFixture({ + memoryPolicy: { mode: 'optional' }, + }); + + await fs.unlink(path.join(fixture.root, 'src', 'prompts', 'daily', 'agent-a.md')); + + const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], { + cwd: fixture.root, + env: fixture.env, + }); + + assert.equal(result.code, 1, `stdout: ${result.stdout} +stderr: ${result.stderr}`); + const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily')); + const failedLog = logs.find((name) => name.endsWith('__failed.md')); + assert.ok(failedLog); + const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8'); + assert.match(failedBody, /reason: Prompt file parse\/read failed/); + assert.match(failedBody, /failure_category: 'prompt_parse_error'/); +}); + +test('categorizes invalid prompt schema as prompt_schema_error', { concurrency: false }, async () => { + const fixture = await setupFixture({ + memoryPolicy: { mode: 'optional' }, + }); + + await fs.writeFile(path.join(fixture.root, 'src', 'prompts', 'daily', 'agent-a.md'), 'plain text without heading\n', 'utf8'); + + const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], { + cwd: fixture.root, + env: fixture.env, + }); + + assert.equal(result.code, 1, `stdout: ${result.stdout} +stderr: ${result.stderr}`); + const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily')); + const failedLog = logs.find((name) => name.endsWith('__failed.md')); + assert.ok(failedLog); + const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8'); + assert.match(failedBody, /reason: Prompt file schema validation failed/); + assert.match(failedBody, /failure_category: 'prompt_schema_error'/); +}); + +test('categorizes handoff runtime failures as execution_error', { concurrency: false }, async () => { + const fixture = await setupFixture({ + memoryPolicy: { mode: 'optional' }, + schedulerPolicy: { + handoffCommandByCadence: { daily: 'exit 7' }, + }, + }); + + const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], { + cwd: fixture.root, + env: fixture.env, + }); + + assert.equal(result.code, 7, `stdout: ${result.stdout} +stderr: ${result.stderr}`); + const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily')); + const failedLog = logs.find((name) => name.endsWith('__failed.md')); + assert.ok(failedLog); + const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8'); + assert.match(failedBody, /reason: Prompt\/handoff execution failed/); + assert.match(failedBody, /failure_category: 'execution_error'/); +}); + test('retries lock acquisition for backend error and succeeds on a later attempt', { concurrency: false }, async () => { const fixture = await setupFixture({ memoryPolicy: { mode: 'optional' },