Classify scheduler failures and document lock remediation

scripts/agent/run-scheduler-cycle.mjs

@@ -8,6 +8,14 @@ import { randomUUID } from 'node:crypto';
 
 const VALID_CADENCES = new Set(['daily', 'weekly']);
 const ALL_EXCLUDED_REASON = 'All roster tasks currently claimed by other agents';
+const FAILURE_CATEGORY = {
+  PROMPT_PARSE: 'prompt_parse_error',
+  PROMPT_SCHEMA: 'prompt_schema_error',
+  LOCK_BACKEND: 'lock_backend_error',
+  EXECUTION: 'execution_error',
+};
+
+const LOCK_INCIDENT_LINK = 'docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md';
 
 function parseArgs(argv) {
   const args = { cadence: null, platform: process.env.AGENT_PLATFORM || 'codex' };
@@ -141,6 +149,58 @@ function excerptText(text, maxChars = 600) {
   return `${clean.slice(0, maxChars)}…`;
 }
 
+function categorizeFailureMetadata(failureCategory, extraMetadata = {}) {
+  return {
+    failure_category: failureCategory,
+    ...extraMetadata,
+  };
+}
+
+function formatDurationMs(ms) {
+  if (!Number.isFinite(ms) || ms < 0) return 'unknown';
+  const minutes = Math.round(ms / 60000);
+  return `${minutes} minute(s)`;
+}
+
+function buildLockBackendRemediation({ cadence, retryWindowMs, maxDeferrals, incidentSignalId = null }) {
+  const steps = [
+    `Retry window: ${formatDurationMs(retryWindowMs)} (max deferrals: ${maxDeferrals}).`,
+    `Run health check: npm run lock:health -- --cadence ${cadence}`,
+    `Review incident runbook: ${LOCK_INCIDENT_LINK}`,
+  ];
+  if (incidentSignalId) {
+    steps.push(`Incident signal: ${incidentSignalId}`);
+  }
+  return `Recommended auto-remediation: ${steps.join(' ')}`;
+}
+
+async function validatePromptFile(promptPath) {
+  let content;
+  try {
+    content = await fs.readFile(promptPath, 'utf8');
+  } catch (error) {
+    return {
+      ok: false,
+      category: FAILURE_CATEGORY.PROMPT_PARSE,
+      reason: 'Prompt file parse/read failed',
+      detail: `Prompt not executed; unable to read prompt file at ${promptPath}: ${error.message}`,
+    };
+  }
+
+  const lines = String(content).split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+  const firstLine = lines[0] || '';
+  if (!firstLine.startsWith('#')) {
+    return {
+      ok: false,
+      category: FAILURE_CATEGORY.PROMPT_SCHEMA,
+      reason: 'Prompt file schema validation failed',
+      detail: `Prompt not executed; expected markdown heading on first non-empty line in ${promptPath}.`,
+    };
+  }
+
+  return { ok: true };
+}
+
 function classifyLockBackendError(outputText) {
   const text = String(outputText || '').toLowerCase();
   if (!text.trim()) return 'unknown backend error';
@@ -604,10 +664,10 @@ async function main() {
             ? 'All relays unhealthy preflight'
             : 'Lock backend unavailable preflight',
           detail: allRelaysUnhealthy
-            ? `Deferred run before lock acquisition: ${incidentSignal?.reason || 'all relays unhealthy'}.`
+            ? `Deferred run before lock acquisition: ${incidentSignal?.reason || 'all relays unhealthy'}. Prompt not executed. ${buildLockBackendRemediation({ cadence, retryWindowMs: schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs, maxDeferrals: schedulerConfig.lockFailurePolicy.maxDeferrals, incidentSignalId: incidentSignal?.id || null })}`
-            : `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`,
+            : `Preflight failed (${preflight.failureCategory}). Prompt not executed. ${buildLockBackendRemediation({ cadence, retryWindowMs: schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs, maxDeferrals: schedulerConfig.lockFailurePolicy.maxDeferrals, incidentSignalId: incidentSignal?.id || null })}`,
           metadata: {
-            failure_class: 'backend_unavailable',
+            ...categorizeFailureMetadata(FAILURE_CATEGORY.LOCK_BACKEND, { failure_class: 'backend_unavailable' }),
             preflight_failure_category: preflight.failureCategory,
             relay_list: preflight.relayList.join(', ') || '(none)',
             preflight_stderr_excerpt: preflight.stderrExcerpt || '(empty)',
@@ -714,9 +774,9 @@ async function main() {
           status: 'deferred',
           platform,
           reason: 'Lock backend deferred',
-          detail: `Deferred after lock backend failure (${backendCategory}); retry window active and deferral budget remaining.`,
+          detail: `Deferred after lock backend failure (${backendCategory}); retry window active and deferral budget remaining. Prompt not executed. ${buildLockBackendRemediation({ cadence, retryWindowMs: schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs, maxDeferrals: schedulerConfig.lockFailurePolicy.maxDeferrals })}`,
           metadata: {
-            failure_class: 'backend_unavailable',
+            ...categorizeFailureMetadata(FAILURE_CATEGORY.LOCK_BACKEND, { failure_class: 'backend_unavailable' }),
             deferral_attempt_count: deferralAttemptCount,
             deferral_first_failure_timestamp: firstFailureAt,
             backend_category: backendCategory,
@@ -734,9 +794,9 @@ async function main() {
         status: 'failed',
         platform,
         reason: 'Lock backend error',
-        detail: `Lock backend error (${backendCategory}) after ${lockAttempt.attempts} attempt(s). Retry ${lockCommand} after verifying relay connectivity, relay URLs, and DNS/network status.`,
+        detail: `Lock backend error (${backendCategory}) after ${lockAttempt.attempts} attempt(s). ${buildLockBackendRemediation({ cadence, retryWindowMs: schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs, maxDeferrals: schedulerConfig.lockFailurePolicy.maxDeferrals })}`,
         metadata: {
-          failure_class: 'backend_unavailable',
+          ...categorizeFailureMetadata(FAILURE_CATEGORY.LOCK_BACKEND, { failure_class: 'backend_unavailable' }),
           lock_attempts_total: lockAttempt.attempts,
           lock_backoff_schedule_ms: backoffSchedule || '(none)',
           lock_correlation_id: diagnosticsSummary.correlationId || lockAttempt.correlationId,
@@ -769,6 +829,20 @@ async function main() {
     }
 
     const promptPath = path.resolve(process.cwd(), 'src/prompts', cadence, `${selectedAgent}.md`);
+    const promptValidation = await validatePromptFile(promptPath);
+    if (!promptValidation.ok) {
+      await writeLog({
+        cadence,
+        agent: selectedAgent,
+        status: 'failed',
+        platform,
+        reason: promptValidation.reason,
+        detail: promptValidation.detail,
+        metadata: categorizeFailureMetadata(promptValidation.category, { prompt_path: promptPath }),
+      });
+      process.exit(1);
+    }
+
     schedulerRunState.lock_deferral = null;
     await writeRunState(runStatePath, schedulerRunState);
     const runArtifactSince = new Date().toISOString();
@@ -788,7 +862,7 @@ async function main() {
           platform,
           reason: 'Memory retrieval command failed',
           detail: schedulerConfig.memoryPolicy.retrieveCommand,
-          metadata: { failure_class: 'prompt_validation_error' },
+          metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }),
         });
         process.exit(retrieveResult.code);
       }
@@ -807,7 +881,7 @@ async function main() {
           platform,
           reason: 'Prompt/handoff execution failed',
           detail: 'Handoff callback failed.',
-          metadata: { failure_class: 'prompt_validation_error' },
+          metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }),
         });
         process.exit(handoff.code);
       }
@@ -822,7 +896,7 @@ async function main() {
         platform,
         reason: 'Prompt/handoff execution failed',
         detail,
-        metadata: { failure_class: 'prompt_validation_error' },
+        metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }),
       });
       process.exit(1);
     }
@@ -840,7 +914,7 @@ async function main() {
           platform,
           reason: 'Memory storage command failed',
           detail: schedulerConfig.memoryPolicy.storeCommand,
-          metadata: { failure_class: 'prompt_validation_error' },
+          metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }),
         });
         process.exit(storeResult.code);
       }
@@ -871,7 +945,7 @@ async function main() {
         platform,
         reason: 'Required memory steps not verified',
         detail: `Missing evidence for: ${missingSteps.join(', ')}`,
-        metadata: { failure_class: 'prompt_validation_error' },
+        metadata: categorizeFailureMetadata(FAILURE_CATEGORY.PROMPT_SCHEMA, { failure_class: 'prompt_validation_error' }),
       });
       process.exit(1);
     }
@@ -903,7 +977,7 @@ async function main() {
         platform,
         reason: 'Missing required run artifacts',
         detail,
-        metadata: { failure_class: 'prompt_validation_error' },
+        metadata: categorizeFailureMetadata(FAILURE_CATEGORY.PROMPT_SCHEMA, { failure_class: 'prompt_validation_error' }),
       });
       process.exit(artifactCheck.code);
     }
@@ -920,7 +994,7 @@ async function main() {
           platform,
           reason: 'Validation failed',
           detail: validation,
-          metadata: { failure_class: 'prompt_validation_error' },
+          metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }),
         });
         process.exit(result.code);
       }

src/docs/TORCH.md

@@ -80,8 +80,15 @@ Operational note: scheduler handoff commands are treated as required execution s
 
 Scheduler failure classes in task logs:
 
-- `backend_unavailable` — lock backend unavailable preflight failures and lock acquisition backend exit code `2` failures/deferrals; includes relay health alert metadata and incident signal IDs when all relays are unhealthy.
+- `backend_unavailable` — legacy compatibility field for lock backend unavailable failures.
-- `prompt_validation_error` — prompt/handoff, memory evidence, artifact verification, and validation command failures.
+- `prompt_validation_error` — legacy compatibility field for prompt/runtime validation failures.
+
+Scheduler failure categories in task logs:
+
+- `lock_backend_error` — lock backend unavailable preflight failures and lock acquisition backend exit code `2` failures/deferrals; includes relay health alert metadata, retry window guidance, health check command, and incident runbook link.
+- `prompt_parse_error` — scheduler could not read/parse the selected prompt file; prompt execution is skipped.
+- `prompt_schema_error` — selected prompt file or generated run artifacts failed schema/contract checks; prompt run is treated as invalid.
+- `execution_error` — runtime execution failures in handoff callbacks, memory commands, or configured validation commands.
 
 
 ## Lock backend production defaults
@@ -100,6 +107,11 @@ Validation behavior:
 - Relay URLs must be absolute `ws://` or `wss://` URLs.
 - Invalid relay URLs or invalid timeout/count ranges are fatal startup errors.
 - Lock backend errors include phase (`query:primary`, `query:fallback`, `publish:primary`, `publish:fallback`), relay endpoint, and timeout value used.
+- When scheduler preflight fails before lock acquisition, task logs must explicitly state `prompt not executed` to make lock-vs-prompt root cause obvious in UI/CLI summaries.
+- Interpret `relay_publish_quorum_failure` (derived from `lock_publish_quorum_failed` telemetry) as quorum not met even after retries/fallbacks. Expected operator actions:
+  1. Run `npm run lock:health -- --cadence <daily|weekly>` to confirm relay readiness and identify failing relays/reasons.
+  2. Review task log metadata (`lock_failure_reason_distribution`, `backend_category`, `lock_correlation_id`) for dominant failure modes (timeouts, DNS, auth, malformed relay URL).
+  3. If failures persist past retry window, follow incident runbook `docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md` and escalate relay/network remediation.
 - Relay health snapshots are emitted periodically and whenever lock publish/query fails; snapshots include success rate, timeout rate, rolling latency, and quarantine state per relay.
 
 ## Scheduler lock reliability reporting

src/prompts/scheduler-flow.md

@@ -85,8 +85,8 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co
    ```
 
    - Escape hatch: set `SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT=1` to skip this check for local/offline workflows.
-   - If preflight exits non-zero because every relay is unhealthy, write `_deferred.md` with reason `All relays unhealthy preflight`, include `incident_signal_id`, and stop before lock acquisition.
+   - If preflight exits non-zero because every relay is unhealthy, write `_deferred.md` with reason `All relays unhealthy preflight`, include `incident_signal_id`, set `failure_category: lock_backend_error`, state `prompt not executed`, and stop before lock acquisition.
-   - For other non-zero preflight failures, write `_failed.md` with reason `Lock backend unavailable preflight` and include:
+   - For other non-zero preflight failures, write `_failed.md` with reason `Lock backend unavailable preflight`, set `failure_category: lock_backend_error`, state `prompt not executed`, and include:
      - `relay_list`
      - `preflight_failure_category`
      - `preflight_stderr_excerpt`
@@ -159,14 +159,17 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co
      - `lock_command` (raw lock command for retry)
      - `lock_stderr_excerpt` (redacted stderr snippet)
      - `lock_stdout_excerpt` (redacted stdout snippet)
-     - Include `failure_class: backend_unavailable` for both deferred and failed backend-unavailable lock outcomes.
+     - Include `failure_class: backend_unavailable` and `failure_category: lock_backend_error` for both deferred and failed backend-unavailable lock outcomes.
+    - Include recommended auto-remediation text in `detail`: retry window, `npm run lock:health -- --cadence <cadence>`, and incident runbook link.
    - Keep generic reason text for compatibility, but append actionable retry guidance in `detail` using the command from `lock_command`.
 
 9. Execute `<prompt_dir>/<prompt-file>` end-to-end via configured handoff command.
 
    - Scheduler automation runs `scheduler.handoffCommandByCadence.<cadence>` with environment variables for cadence/agent/prompt path.
    - If no handoff command is configured for the cadence, write `_failed.md` and stop.
-   - Prompt/handoff and prompt validation failures should emit `failure_class: prompt_validation_error`.
+   - Prompt file read/parse failures should emit `failure_category: prompt_parse_error`.
+   - Prompt schema/contract failures should emit `failure_category: prompt_schema_error`.
+   - Command/handoff/validation runtime failures should emit `failure_category: execution_error`.
 
 10. Confirm memory contract completion:
 

src/prompts/weekly-scheduler.md

@@ -17,7 +17,7 @@ Canonical roster source: `src/prompts/roster.json` (`weekly` key).
 ## Lock Backend Preflight Behavior
 
 - When `scheduler.lockHealthPreflight` is enabled (or `SCHEDULER_LOCK_HEALTH_PREFLIGHT=1`), run `node scripts/agent/check-relay-health.mjs --cadence weekly` before agent selection/lock attempt.
-- On preflight failure, scheduler must write a `_failed.md` entry with reason `Lock backend unavailable preflight`, including relay list + classified preflight failure metadata, and stop.
+- On preflight failure, scheduler must write a `_failed.md` entry with reason `Lock backend unavailable preflight`, including relay list + classified preflight failure metadata, `failure_category: lock_backend_error`, and `prompt not executed` summary text before stopping.
 - Escape hatch for local/offline workflows: `SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT=1`.
 
 ## Weekly Roster

test/run-scheduler-cycle-memory-policy.test.mjs

@@ -88,6 +88,13 @@ case "$2" in
   lock:check:daily)
     echo '{"excluded":[]}'
     ;;
+  lock:health)
+    shift 2
+    if [[ "\${1:-}" == "--" ]]; then
+      shift
+    fi
+    node scripts/agent/check-relay-health.mjs "$@"
+    ;;
   lock:lock)
 ${lockShellBody || '    :'}
     ;;
@@ -189,6 +196,7 @@ test('records backend failure metadata when lock command exits with code 2', { c
 
   assert.match(failedBody, /reason: Lock backend error/);
   assert.match(failedBody, /failure_class: 'backend_unavailable'/);
+  assert.match(failedBody, /failure_category: 'lock_backend_error'/);
   assert.match(failedBody, /backend_category: 'publish failed to all relays'/);
   assert.match(failedBody, /lock_command: 'AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily'/);
   assert.match(failedBody, /lock_stderr_excerpt: 'publish failed to all relays token=\[REDACTED\] SECRET_KEY=\[REDACTED\]'/);
@@ -196,7 +204,9 @@ test('records backend failure metadata when lock command exits with code 2', { c
   assert.match(failedBody, /lock_failure_reason_distribution: '\{\}'/);
   assert.match(failedBody, /lock_attempt_id: '3'/);
   assert.match(failedBody, /lock_correlation_id: 'daily:agent-a:/);
-  assert.match(failedBody, /Retry AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily/);
+  assert.match(failedBody, /Recommended auto-remediation:/);
+  assert.match(failedBody, /Run health check: npm run lock:health -- --cadence daily/);
+  assert.match(failedBody, /Review incident runbook: docs\/agent-handoffs\/learnings\/2026-02-15-relay-health-preflight-job.md/);
   assert.match(failedBody, /platform: 'codex'/);
 });
 
@@ -262,6 +272,7 @@ test('defers backend failures in non-strict mode and reuses idempotency key on s
   assert.ok(deferredLog);
   const deferredBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', deferredLog), 'utf8');
   assert.match(deferredBody, /failure_class: 'backend_unavailable'/);
+  assert.match(deferredBody, /failure_category: 'lock_backend_error'/);
   assert.match(deferredBody, /deferral_attempt_count: '\d+'/);
 
   const keyLog = await fs.readFile(path.join(fixture.root, '.idempotency-keys'), 'utf8');
@@ -311,6 +322,7 @@ test('fails after exceeding non-strict deferral budget', { concurrency: false },
   assert.ok(failedLog);
   const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8');
   assert.match(failedBody, /failure_class: 'backend_unavailable'/);
+  assert.match(failedBody, /failure_category: 'lock_backend_error'/);
   assert.match(failedBody, /deferral_attempt_count: '2'/);
 });
 
@@ -389,9 +401,80 @@ process.exit(2);
   const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8');
 
   assert.match(failedBody, /reason: Lock backend unavailable preflight/);
+  assert.match(failedBody, /prompt not executed/i);
+  assert.match(failedBody, /failure_category: 'lock_backend_error'/);
   assert.match(failedBody, /preflight_failure_category: 'relay query timeout'/);
   assert.match(failedBody, /relay_list: 'wss:\/\/relay-1.test, wss:\/\/relay-2.test'/);
 });
+
+
+test('categorizes unreadable prompt files as prompt_parse_error', { concurrency: false }, async () => {
+  const fixture = await setupFixture({
+    memoryPolicy: { mode: 'optional' },
+  });
+
+  await fs.unlink(path.join(fixture.root, 'src', 'prompts', 'daily', 'agent-a.md'));
+
+  const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
+    cwd: fixture.root,
+    env: fixture.env,
+  });
+
+  assert.equal(result.code, 1, `stdout: ${result.stdout}
+stderr: ${result.stderr}`);
+  const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily'));
+  const failedLog = logs.find((name) => name.endsWith('__failed.md'));
+  assert.ok(failedLog);
+  const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8');
+  assert.match(failedBody, /reason: Prompt file parse\/read failed/);
+  assert.match(failedBody, /failure_category: 'prompt_parse_error'/);
+});
+
+test('categorizes invalid prompt schema as prompt_schema_error', { concurrency: false }, async () => {
+  const fixture = await setupFixture({
+    memoryPolicy: { mode: 'optional' },
+  });
+
+  await fs.writeFile(path.join(fixture.root, 'src', 'prompts', 'daily', 'agent-a.md'), 'plain text without heading\n', 'utf8');
+
+  const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
+    cwd: fixture.root,
+    env: fixture.env,
+  });
+
+  assert.equal(result.code, 1, `stdout: ${result.stdout}
+stderr: ${result.stderr}`);
+  const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily'));
+  const failedLog = logs.find((name) => name.endsWith('__failed.md'));
+  assert.ok(failedLog);
+  const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8');
+  assert.match(failedBody, /reason: Prompt file schema validation failed/);
+  assert.match(failedBody, /failure_category: 'prompt_schema_error'/);
+});
+
+test('categorizes handoff runtime failures as execution_error', { concurrency: false }, async () => {
+  const fixture = await setupFixture({
+    memoryPolicy: { mode: 'optional' },
+    schedulerPolicy: {
+      handoffCommandByCadence: { daily: 'exit 7' },
+    },
+  });
+
+  const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
+    cwd: fixture.root,
+    env: fixture.env,
+  });
+
+  assert.equal(result.code, 7, `stdout: ${result.stdout}
+stderr: ${result.stderr}`);
+  const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily'));
+  const failedLog = logs.find((name) => name.endsWith('__failed.md'));
+  assert.ok(failedLog);
+  const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8');
+  assert.match(failedBody, /reason: Prompt\/handoff execution failed/);
+  assert.match(failedBody, /failure_category: 'execution_error'/);
+});
+
 test('retries lock acquisition for backend error and succeeds on a later attempt', { concurrency: false }, async () => {
   const fixture = await setupFixture({
     memoryPolicy: { mode: 'optional' },