Improve scheduler lock observability and reliability reporting

KNOWN_ISSUES.md

@@ -24,3 +24,13 @@ Track only **active, reproducible, unresolved** issues.
 - **Workaround:** Run `npm run validate:scheduler` separately and inspect output, or ignore errors for now.
 - **Impact:** CI/local testing blocked.
 - **Last verified:** 2026-02-14
+
+### Recurring scheduler lock backend failures in recent task logs
+- **Status:** Monitoring
+- **Area:** Runtime
+- **Symptom:** Multiple scheduler runs fail with `Lock backend error` before prompt handoff/validation.
+- **Trigger/Conditions:** Running scheduler cycles when relay connectivity is unstable or lock backend operations time out; observed in both `daily` and `weekly` task logs.
+- **Workaround:** Retry with explicit platform and inspect reliability report output (`npm run report:lock-reliability`), then validate relay health (`node scripts/agent/check-relay-health.mjs --cadence <daily|weekly>`) before rerunning scheduler.
+- **Impact:** Scheduled agent execution is skipped for affected cycles until lock acquisition succeeds.
+- **Last verified:** 2026-02-14
+

README.md

@@ -127,11 +127,27 @@ If you are developing `torch-lock` itself:
 - `npm run dashboard:serve`
 - `npm test` (run unit tests)
 - `npm run lint` (run linter)
+- `npm run report:lock-reliability` (aggregate recent scheduler logs into markdown+JSON reliability reports)
 
 
 
 
 
+## Scheduler lock reliability reporting
+
+Use the reporting hook below to inspect lock backend reliability trends by platform/cadence/error category/relay:
+
+```bash
+npm run report:lock-reliability
+```
+
+This writes:
+- `artifacts/lock-reliability/lock-reliability-summary.md`
+- `artifacts/lock-reliability/lock-reliability-summary.json`
+
+Recommended cadence: run this at least weekly (or wire into your CI/cron scheduler) and compare `codex` vs other `platform` values.
+
+
 ## Memory rollout plan
 
 1. Deploy memory schema changes with scheduler jobs disabled (`TORCH_MEMORY_ENABLED=false` or subsystem flags set to `false`).

package.json

@@ -38,7 +38,8 @@
     "lock:complete": "node bin/torch-lock.mjs complete",
     "validate:scheduler": "node scripts/validate-scheduler-roster.mjs && node scripts/validate-prompt-contract.mjs && node scripts/validate-scheduler-flow-parity.mjs",
     "scheduler:daily": "node scripts/agent/run-scheduler-cycle.mjs daily",
-    "scheduler:weekly": "node scripts/agent/run-scheduler-cycle.mjs weekly"
+    "scheduler:weekly": "node scripts/agent/run-scheduler-cycle.mjs weekly",
+    "report:lock-reliability": "node scripts/agent/summarize-lock-reliability.mjs --since-days 30"
   },
   "dependencies": {
     "nostr-tools": "2.19.4",

scripts/agent/run-scheduler-cycle.mjs

@@ -421,10 +421,15 @@ function toYamlScalar(value) {
   return `'${str.replace(/'/g, "''")}'`;
 }
 
-async function writeLog({ cadence, agent, status, reason, detail, metadata = {} }) {
+async function writeLog({ cadence, agent, status, reason, detail, platform, metadata = {} }) {
   const logDir = path.resolve(process.cwd(), 'task-logs', cadence);
   await fs.mkdir(logDir, { recursive: true });
   const file = `${ts()}__${agent}__${status}.md`;
+  const mergedMetadata = {
+    platform: platform || process.env.AGENT_PLATFORM || 'unknown',
+    ...metadata,
+  };
+
   const body = [
     '---',
     `cadence: ${cadence}`,
@@ -432,7 +437,7 @@ async function writeLog({ cadence, agent, status, reason, detail, metadata = {}
     `status: ${status}`,
     `created_at: ${new Date().toISOString()}`,
     `timestamp: ${new Date().toISOString()}`,
-    ...Object.entries(metadata)
+    ...Object.entries(mergedMetadata)
       .filter(([, value]) => value !== undefined && value !== null && value !== '')
       .map(([key, value]) => `${key}: ${toYamlScalar(value)}`),
     '---',
@@ -441,7 +446,7 @@ async function writeLog({ cadence, agent, status, reason, detail, metadata = {}
     '',
     `- reason: ${reason}`,
     detail ? `- detail: ${detail}` : null,
-    ...Object.entries(metadata)
+    ...Object.entries(mergedMetadata)
       .filter(([, value]) => value !== undefined && value !== null && value !== '')
       .map(([key, value]) => `- ${key}: ${value}`),
     '',
@@ -483,6 +488,7 @@ async function main() {
           cadence,
           agent: 'scheduler',
           status: 'failed',
+          platform,
           reason: 'Lock backend unavailable preflight',
           detail: `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`,
           metadata: {
@@ -531,6 +537,7 @@ async function main() {
         cadence,
         agent: 'scheduler',
         status: 'failed',
+        platform,
         reason: ALL_EXCLUDED_REASON,
       });
       process.exit(1);
@@ -559,6 +566,7 @@ async function main() {
         cadence,
         agent: selectedAgent,
         status: 'failed',
+        platform,
         reason: 'Lock backend error',
         detail: `Lock backend error (${backendCategory}) after ${lockAttempt.attempts} attempt(s). Retry ${lockCommand} after verifying relay connectivity, relay URLs, and DNS/network status.`,
         metadata: {
@@ -578,6 +586,7 @@ async function main() {
         cadence,
         agent: selectedAgent,
         status: 'failed',
+        platform,
         reason: 'Failed to acquire lock',
       });
       process.exit(lockResult.code);
@@ -594,7 +603,7 @@ async function main() {
       });
       outputChunks.push(retrieveResult.stdout, retrieveResult.stderr);
       if (retrieveResult.code !== 0) {
-        await writeLog({ cadence, agent: selectedAgent, status: 'failed', reason: 'Memory retrieval command failed', detail: schedulerConfig.memoryPolicy.retrieveCommand });
+        await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Memory retrieval command failed', detail: schedulerConfig.memoryPolicy.retrieveCommand });
         process.exit(retrieveResult.code);
       }
     }
@@ -605,14 +614,14 @@ async function main() {
       });
       outputChunks.push(handoff.stdout, handoff.stderr);
       if (handoff.code !== 0) {
-        await writeLog({ cadence, agent: selectedAgent, status: 'failed', reason: 'Prompt/handoff execution failed', detail: 'Handoff callback failed.' });
+        await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Prompt/handoff execution failed', detail: 'Handoff callback failed.' });
         process.exit(handoff.code);
       }
     } else {
       const detail = schedulerConfig.missingHandoffCommandForMode
         ? 'Missing scheduler handoff command for non-interactive run. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.'
         : 'No handoff callback configured. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.';
-      await writeLog({ cadence, agent: selectedAgent, status: 'failed', reason: 'Prompt/handoff execution failed', detail });
+      await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Prompt/handoff execution failed', detail });
       process.exit(1);
     }
 
@@ -622,7 +631,7 @@ async function main() {
       });
       outputChunks.push(storeResult.stdout, storeResult.stderr);
       if (storeResult.code !== 0) {
-        await writeLog({ cadence, agent: selectedAgent, status: 'failed', reason: 'Memory storage command failed', detail: schedulerConfig.memoryPolicy.storeCommand });
+        await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Memory storage command failed', detail: schedulerConfig.memoryPolicy.storeCommand });
         process.exit(storeResult.code);
       }
     }
@@ -649,6 +658,7 @@ async function main() {
         cadence,
         agent: selectedAgent,
         status: 'failed',
+        platform,
         reason: 'Required memory steps not verified',
         detail: `Missing evidence for: ${missingSteps.join(', ')}`,
       });
@@ -679,6 +689,7 @@ async function main() {
         cadence,
         agent: selectedAgent,
         status: 'failed',
+        platform,
         reason: 'Missing required run artifacts',
         detail,
       });
@@ -690,7 +701,7 @@ async function main() {
       if (!parts.length) continue;
       const result = await runCommand(parts[0], parts.slice(1));
       if (result.code !== 0) {
-        await writeLog({ cadence, agent: selectedAgent, status: 'failed', reason: 'Validation failed', detail: validation });
+        await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Validation failed', detail: validation });
         process.exit(result.code);
       }
     }
@@ -706,12 +717,13 @@ async function main() {
         cadence,
         agent: selectedAgent,
         status: 'failed',
+        platform,
         reason: `Completion publish failed. Retry npm run lock:complete -- --agent ${selectedAgent} --cadence ${cadence} after verifying relay connectivity`,
       });
       process.exit(completeResult.code);
     }
 
-    await writeLog({ cadence, agent: selectedAgent, status: 'completed', reason: 'Scheduler cycle completed successfully' });
+    await writeLog({ cadence, agent: selectedAgent, status: 'completed', platform, reason: 'Scheduler cycle completed successfully' });
     process.exit(0);
   }
 }

scripts/agent/summarize-lock-reliability.mjs

@@ -0,0 +1,242 @@
+#!/usr/bin/env node
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+const VALID_CADENCES = ['daily', 'weekly'];
+
+function parseArgs(argv) {
+  const options = {
+    sinceDays: 14,
+    logRoot: path.resolve(process.cwd(), 'task-logs'),
+    outDir: path.resolve(process.cwd(), 'artifacts', 'lock-reliability'),
+  };
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const token = argv[i];
+    if (token === '--since-days') {
+      options.sinceDays = Number.parseInt(argv[i + 1] || '14', 10);
+      i += 1;
+    } else if (token === '--log-root') {
+      options.logRoot = path.resolve(process.cwd(), argv[i + 1] || 'task-logs');
+      i += 1;
+    } else if (token === '--out-dir') {
+      options.outDir = path.resolve(process.cwd(), argv[i + 1] || path.join('artifacts', 'lock-reliability'));
+      i += 1;
+    }
+  }
+
+  if (!Number.isFinite(options.sinceDays) || options.sinceDays < 0) {
+    throw new Error(`Invalid --since-days value: ${options.sinceDays}`);
+  }
+
+  return options;
+}
+
+function parseFrontmatter(markdown) {
+  const text = String(markdown || '');
+  if (!text.startsWith('---\n')) return {};
+  const end = text.indexOf('\n---', 4);
+  if (end === -1) return {};
+  const frontmatter = text.slice(4, end).split(/\r?\n/);
+  const result = {};
+  for (const line of frontmatter) {
+    const match = line.match(/^([a-zA-Z0-9_]+):\s*(.+)\s*$/);
+    if (!match) continue;
+    const [, key, rawValue] = match;
+    result[key] = rawValue.replace(/^'/, '').replace(/'$/, '').replace(/''/g, "'");
+  }
+  return result;
+}
+
+function inc(map, key) {
+  map.set(key, (map.get(key) || 0) + 1);
+}
+
+function collectRelayEndpoints({ frontmatter, body }) {
+  const relays = new Set();
+  const relayList = frontmatter.relay_list || '';
+  relayList.split(',').map((v) => v.trim()).filter(Boolean).forEach((relay) => relays.add(relay));
+
+  const allText = `${body}\n${Object.values(frontmatter).join('\n')}`;
+  const urlMatches = allText.match(/wss?:\/\/[^\s'`),]+/g) || [];
+  for (const url of urlMatches) {
+    relays.add(url.trim());
+  }
+
+  if (!relays.size) relays.add('unknown');
+  return [...relays];
+}
+
+function getBackendCategory({ frontmatter, body }) {
+  if (frontmatter.backend_category) return frontmatter.backend_category;
+  if (frontmatter.preflight_failure_category) return `preflight:${frontmatter.preflight_failure_category}`;
+
+  const reasonMatch = body.match(/- reason:\s*([^\n]+)/i) || body.match(/reason:\s*([^\n]+)/i);
+  const reason = reasonMatch?.[1]?.trim() || '';
+
+  if (/lock backend/i.test(reason)) return 'lock backend error (uncategorized)';
+  if (/completion publish failed/i.test(reason)) return 'completion publish failed';
+  if (/preflight/i.test(reason)) return 'preflight failed (uncategorized)';
+  return null;
+}
+
+function toSortedObject(counterMap) {
+  return Object.fromEntries([...counterMap.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])));
+}
+
+function asMarkdown(report) {
+  const lines = [];
+  lines.push('# Lock Reliability Summary');
+  lines.push('');
+  lines.push(`- Generated at: ${report.generatedAt}`);
+  lines.push(`- Window: last ${report.sinceDays} day(s)`);
+  lines.push(`- Logs scanned: ${report.logsScanned}`);
+  lines.push(`- Runs considered: ${report.runsConsidered}`);
+  lines.push('');
+
+  lines.push('## Totals');
+  lines.push('');
+  lines.push(`- completed: ${report.statusCounts.completed || 0}`);
+  lines.push(`- failed: ${report.statusCounts.failed || 0}`);
+  lines.push('');
+
+  const sections = [
+    ['By platform', report.byPlatform],
+    ['By cadence', report.byCadence],
+    ['By backend error category', report.byBackendCategory],
+    ['By relay endpoint', report.byRelayEndpoint],
+  ];
+
+  for (const [title, obj] of sections) {
+    lines.push(`## ${title}`);
+    lines.push('');
+    const entries = Object.entries(obj);
+    if (!entries.length) {
+      lines.push('- (none)');
+    } else {
+      for (const [key, value] of entries) {
+        lines.push(`- ${key}: ${value}`);
+      }
+    }
+    lines.push('');
+  }
+
+  if (report.recurringBackendErrors.length) {
+    lines.push('## Recurring backend errors (count >= 2)');
+    lines.push('');
+    for (const entry of report.recurringBackendErrors) {
+      lines.push(`- ${entry.category}: ${entry.count}`);
+    }
+    lines.push('');
+  }
+
+  return lines.join('\n');
+}
+
+async function readLogFile(filePath, cadence, cutoffMs) {
+  const stat = await fs.stat(filePath);
+  if (!stat.isFile() || stat.mtimeMs < cutoffMs) return null;
+
+  const basename = path.basename(filePath);
+  const match = basename.match(/^.+__([^_]+?)__(completed|failed)\.md$/);
+  if (!match) return null;
+
+  const body = await fs.readFile(filePath, 'utf8');
+  const frontmatter = parseFrontmatter(body);
+  const status = frontmatter.status || match[2];
+  const platform = frontmatter.platform || 'unknown';
+  const backendCategory = status === 'failed'
+    ? getBackendCategory({ frontmatter, body })
+    : 'n/a';
+  const relayEndpoints = collectRelayEndpoints({ frontmatter, body });
+
+  return {
+    filePath,
+    cadence: frontmatter.cadence || cadence,
+    status,
+    platform,
+    backendCategory,
+    relayEndpoints,
+  };
+}
+
+async function main() {
+  const options = parseArgs(process.argv.slice(2));
+  const now = Date.now();
+  const cutoffMs = now - (options.sinceDays * 24 * 60 * 60 * 1000);
+
+  const byPlatform = new Map();
+  const byCadence = new Map();
+  const byBackendCategory = new Map();
+  const byRelayEndpoint = new Map();
+  const statusCounts = new Map();
+  const records = [];
+  let logsScanned = 0;
+
+  for (const cadence of VALID_CADENCES) {
+    const cadenceDir = path.join(options.logRoot, cadence);
+    let entries;
+    try {
+      entries = await fs.readdir(cadenceDir);
+    } catch {
+      continue;
+    }
+
+    for (const entry of entries) {
+      const fullPath = path.join(cadenceDir, entry);
+      logsScanned += 1;
+      const record = await readLogFile(fullPath, cadence, cutoffMs);
+      if (!record) continue;
+
+      records.push(record);
+      inc(statusCounts, record.status);
+      inc(byPlatform, record.platform);
+      inc(byCadence, record.cadence);
+      if (record.status === 'failed' && record.backendCategory) {
+        inc(byBackendCategory, record.backendCategory);
+      }
+      for (const relay of record.relayEndpoints) {
+        inc(byRelayEndpoint, relay);
+      }
+    }
+  }
+
+  const recurringBackendErrors = [...byBackendCategory.entries()]
+    .filter(([, count]) => count >= 2)
+    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
+    .map(([category, count]) => ({ category, count }));
+
+  const report = {
+    generatedAt: new Date().toISOString(),
+    sinceDays: options.sinceDays,
+    logRoot: options.logRoot,
+    logsScanned,
+    runsConsidered: records.length,
+    statusCounts: toSortedObject(statusCounts),
+    byPlatform: toSortedObject(byPlatform),
+    byCadence: toSortedObject(byCadence),
+    byBackendCategory: toSortedObject(byBackendCategory),
+    byRelayEndpoint: toSortedObject(byRelayEndpoint),
+    recurringBackendErrors,
+  };
+
+  await fs.mkdir(options.outDir, { recursive: true });
+  const markdown = asMarkdown(report);
+  const markdownPath = path.join(options.outDir, 'lock-reliability-summary.md');
+  const jsonPath = path.join(options.outDir, 'lock-reliability-summary.json');
+
+  await fs.writeFile(markdownPath, markdown, 'utf8');
+  await fs.writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8');
+
+  console.log(JSON.stringify({
+    ok: true,
+    markdown: path.relative(process.cwd(), markdownPath),
+    json: path.relative(process.cwd(), jsonPath),
+    recurringBackendErrors,
+  }));
+}
+
+main().catch((error) => {
+  console.error(error?.stack || error?.message || String(error));
+  process.exit(1);
+});

src/docs/TORCH.md

@@ -64,6 +64,19 @@ For weekly repository-fit maintenance, TORCH also includes `src/prompts/weekly/r
 
 Operational note: scheduler handoff commands are treated as required execution steps. A non-zero exit code (or missing command) is a hard failure: the scheduler writes a `_failed.md` task log, exits immediately, and does not publish `lock:complete` for that run.
 
+## Scheduler lock reliability reporting
+
+Run the lock reliability summary to aggregate recent scheduler outcomes by platform, cadence, backend error category, and relay endpoint:
+
+```bash
+npm run report:lock-reliability
+```
+
+Outputs:
+- `artifacts/lock-reliability/lock-reliability-summary.md`
+- `artifacts/lock-reliability/lock-reliability-summary.json`
+
+
 ## Roster precedence
 
 The lock CLI resolves roster names in this order:

src/prompts/scheduler-flow.md

@@ -209,6 +209,7 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co
 
     - `_completed.md` MUST be created only after completion publish succeeds.
     - `_failed.md` is required when step 11, step 12, or step 13 fails, and should include the failure reason and next retry action.
+    - Include `platform` in frontmatter using `AGENT_PLATFORM` (or the scheduler `--platform` value) for both `_completed.md` and `_failed.md`.
 
 15. Commit/push behavior is delegated outside this scheduler script.
 

test/run-scheduler-cycle-memory-policy.test.mjs

@@ -156,7 +156,10 @@ test('accepts required memory policy when markers or artifacts are produced', {
   assert.equal(result.code, 0, `stdout: ${result.stdout}\nstderr: ${result.stderr}`);
 
   const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily'));
-  assert.ok(logs.some((name) => name.endsWith('__completed.md')));
+  const completedLog = logs.find((name) => name.endsWith('__completed.md'));
+  assert.ok(completedLog);
+  const completedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', completedLog), 'utf8');
+  assert.match(completedBody, /platform: 'codex'/);
 
   await fs.access(path.join(fixture.root, '.scheduler-memory', 'retrieve-daily.ok'));
   await fs.access(path.join(fixture.root, '.scheduler-memory', 'store-daily.ok'));
@@ -188,6 +191,7 @@ test('records backend failure metadata when lock command exits with code 2', { c
   assert.match(failedBody, /lock_stderr_excerpt: 'publish failed to all relays token=\[REDACTED\] SECRET_KEY=\[REDACTED\]'/);
   assert.match(failedBody, /lock_stdout_excerpt: 'websocket: connection refused'/);
   assert.match(failedBody, /Retry AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily/);
+  assert.match(failedBody, /platform: 'codex'/);
 });
 
 

test/summarize-lock-reliability.test.mjs

@@ -0,0 +1,74 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { spawn } from 'node:child_process';
+
+const SCRIPT = path.resolve('scripts/agent/summarize-lock-reliability.mjs');
+
+function runNode(args, cwd) {
+  return new Promise((resolve) => {
+    const child = spawn(process.execPath, [SCRIPT, ...args], { cwd, stdio: ['ignore', 'pipe', 'pipe'] });
+    let stdout = '';
+    let stderr = '';
+    child.stdout.on('data', (chunk) => { stdout += chunk.toString(); });
+    child.stderr.on('data', (chunk) => { stderr += chunk.toString(); });
+    child.on('close', (code) => resolve({ code: code ?? 1, stdout, stderr }));
+  });
+}
+
+test('summarizes recent lock reliability by platform/cadence/backend/relay', async () => {
+  const root = await fs.mkdtemp(path.join(os.tmpdir(), 'lock-reliability-'));
+  await fs.mkdir(path.join(root, 'task-logs', 'daily'), { recursive: true });
+  await fs.mkdir(path.join(root, 'task-logs', 'weekly'), { recursive: true });
+
+  await fs.writeFile(path.join(root, 'task-logs', 'daily', '2026-02-14T00-00-00Z__agent-a__failed.md'), `---
+cadence: daily
+agent: agent-a
+status: failed
+platform: codex
+backend_category: publish failed to all relays
+relay_list: 'wss://relay-1.test, wss://relay-2.test'
+---
+# Scheduler failed
+- reason: Lock backend error
+`, 'utf8');
+
+  await fs.writeFile(path.join(root, 'task-logs', 'weekly', '2026-02-14T00-00-01Z__agent-b__failed.md'), `---
+cadence: weekly
+agent: agent-b
+status: failed
+platform: codex
+preflight_failure_category: relay query timeout
+relay_list: 'wss://relay-1.test'
+---
+# Scheduler failed
+- reason: Lock backend unavailable preflight
+`, 'utf8');
+
+  await fs.writeFile(path.join(root, 'task-logs', 'daily', '2026-02-14T00-00-02Z__agent-c__completed.md'), `---
+cadence: daily
+agent: agent-c
+status: completed
+platform: claude
+---
+# Scheduler completed
+`, 'utf8');
+
+  const result = await runNode(['--since-days', '365', '--out-dir', 'artifacts/reliability'], root);
+  assert.equal(result.code, 0, `stdout=${result.stdout}\nstderr=${result.stderr}`);
+
+  const jsonReport = JSON.parse(await fs.readFile(path.join(root, 'artifacts', 'reliability', 'lock-reliability-summary.json'), 'utf8'));
+  assert.equal(jsonReport.byPlatform.codex, 2);
+  assert.equal(jsonReport.byPlatform.claude, 1);
+  assert.equal(jsonReport.byCadence.daily, 2);
+  assert.equal(jsonReport.byCadence.weekly, 1);
+  assert.equal(jsonReport.byBackendCategory['publish failed to all relays'], 1);
+  assert.equal(jsonReport.byBackendCategory['preflight:relay query timeout'], 1);
+  assert.equal(jsonReport.byRelayEndpoint['wss://relay-1.test'], 2);
+
+  const markdown = await fs.readFile(path.join(root, 'artifacts', 'reliability', 'lock-reliability-summary.md'), 'utf8');
+  assert.match(markdown, /## By platform/);
+  assert.match(markdown, /- codex: 2/);
+});