diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md index b9a55bb..f83ca50 100644 --- a/KNOWN_ISSUES.md +++ b/KNOWN_ISSUES.md @@ -24,3 +24,13 @@ Track only **active, reproducible, unresolved** issues. - **Workaround:** Run `npm run validate:scheduler` separately and inspect output, or ignore errors for now. - **Impact:** CI/local testing blocked. - **Last verified:** 2026-02-14 + +### Recurring scheduler lock backend failures in recent task logs +- **Status:** Monitoring +- **Area:** Runtime +- **Symptom:** Multiple scheduler runs fail with `Lock backend error` before prompt handoff/validation. +- **Trigger/Conditions:** Running scheduler cycles when relay connectivity is unstable or lock backend operations time out; observed in both `daily` and `weekly` task logs. +- **Workaround:** Retry with explicit platform and inspect reliability report output (`npm run report:lock-reliability`), then validate relay health (`node scripts/agent/check-relay-health.mjs --cadence `) before rerunning scheduler. +- **Impact:** Scheduled agent execution is skipped for affected cycles until lock acquisition succeeds. +- **Last verified:** 2026-02-14 + diff --git a/README.md b/README.md index 23ed9b7..8a753ee 100644 --- a/README.md +++ b/README.md @@ -127,11 +127,27 @@ If you are developing `torch-lock` itself: - `npm run dashboard:serve` - `npm test` (run unit tests) - `npm run lint` (run linter) +- `npm run report:lock-reliability` (aggregate recent scheduler logs into markdown+JSON reliability reports) +## Scheduler lock reliability reporting + +Use the reporting hook below to inspect lock backend reliability trends by platform/cadence/error category/relay: + +```bash +npm run report:lock-reliability +``` + +This writes: +- `artifacts/lock-reliability/lock-reliability-summary.md` +- `artifacts/lock-reliability/lock-reliability-summary.json` + +Recommended cadence: run this at least weekly (or wire into your CI/cron scheduler) and compare `codex` vs other `platform` values. + + ## Memory rollout plan 1. Deploy memory schema changes with scheduler jobs disabled (`TORCH_MEMORY_ENABLED=false` or subsystem flags set to `false`). diff --git a/package.json b/package.json index 8c55171..1d3583b 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,8 @@ "lock:complete": "node bin/torch-lock.mjs complete", "validate:scheduler": "node scripts/validate-scheduler-roster.mjs && node scripts/validate-prompt-contract.mjs && node scripts/validate-scheduler-flow-parity.mjs", "scheduler:daily": "node scripts/agent/run-scheduler-cycle.mjs daily", - "scheduler:weekly": "node scripts/agent/run-scheduler-cycle.mjs weekly" + "scheduler:weekly": "node scripts/agent/run-scheduler-cycle.mjs weekly", + "report:lock-reliability": "node scripts/agent/summarize-lock-reliability.mjs --since-days 30" }, "dependencies": { "nostr-tools": "2.19.4", diff --git a/scripts/agent/run-scheduler-cycle.mjs b/scripts/agent/run-scheduler-cycle.mjs index 8171225..dbe68a4 100644 --- a/scripts/agent/run-scheduler-cycle.mjs +++ b/scripts/agent/run-scheduler-cycle.mjs @@ -421,10 +421,15 @@ function toYamlScalar(value) { return `'${str.replace(/'/g, "''")}'`; } -async function writeLog({ cadence, agent, status, reason, detail, metadata = {} }) { +async function writeLog({ cadence, agent, status, reason, detail, platform, metadata = {} }) { const logDir = path.resolve(process.cwd(), 'task-logs', cadence); await fs.mkdir(logDir, { recursive: true }); const file = `${ts()}__${agent}__${status}.md`; + const mergedMetadata = { + platform: platform || process.env.AGENT_PLATFORM || 'unknown', + ...metadata, + }; + const body = [ '---', `cadence: ${cadence}`, @@ -432,7 +437,7 @@ async function writeLog({ cadence, agent, status, reason, detail, metadata = {} `status: ${status}`, `created_at: ${new Date().toISOString()}`, `timestamp: ${new Date().toISOString()}`, - ...Object.entries(metadata) + ...Object.entries(mergedMetadata) .filter(([, value]) => value !== undefined && value !== null && value !== '') .map(([key, value]) => `${key}: ${toYamlScalar(value)}`), '---', @@ -441,7 +446,7 @@ async function writeLog({ cadence, agent, status, reason, detail, metadata = {} '', `- reason: ${reason}`, detail ? `- detail: ${detail}` : null, - ...Object.entries(metadata) + ...Object.entries(mergedMetadata) .filter(([, value]) => value !== undefined && value !== null && value !== '') .map(([key, value]) => `- ${key}: ${value}`), '', @@ -483,6 +488,7 @@ async function main() { cadence, agent: 'scheduler', status: 'failed', + platform, reason: 'Lock backend unavailable preflight', detail: `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`, metadata: { @@ -531,6 +537,7 @@ async function main() { cadence, agent: 'scheduler', status: 'failed', + platform, reason: ALL_EXCLUDED_REASON, }); process.exit(1); @@ -559,6 +566,7 @@ async function main() { cadence, agent: selectedAgent, status: 'failed', + platform, reason: 'Lock backend error', detail: `Lock backend error (${backendCategory}) after ${lockAttempt.attempts} attempt(s). Retry ${lockCommand} after verifying relay connectivity, relay URLs, and DNS/network status.`, metadata: { @@ -578,6 +586,7 @@ async function main() { cadence, agent: selectedAgent, status: 'failed', + platform, reason: 'Failed to acquire lock', }); process.exit(lockResult.code); @@ -594,7 +603,7 @@ async function main() { }); outputChunks.push(retrieveResult.stdout, retrieveResult.stderr); if (retrieveResult.code !== 0) { - await writeLog({ cadence, agent: selectedAgent, status: 'failed', reason: 'Memory retrieval command failed', detail: schedulerConfig.memoryPolicy.retrieveCommand }); + await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Memory retrieval command failed', detail: schedulerConfig.memoryPolicy.retrieveCommand }); process.exit(retrieveResult.code); } } @@ -605,14 +614,14 @@ async function main() { }); outputChunks.push(handoff.stdout, handoff.stderr); if (handoff.code !== 0) { - await writeLog({ cadence, agent: selectedAgent, status: 'failed', reason: 'Prompt/handoff execution failed', detail: 'Handoff callback failed.' }); + await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Prompt/handoff execution failed', detail: 'Handoff callback failed.' }); process.exit(handoff.code); } } else { const detail = schedulerConfig.missingHandoffCommandForMode ? 'Missing scheduler handoff command for non-interactive run. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.' : 'No handoff callback configured. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.'; - await writeLog({ cadence, agent: selectedAgent, status: 'failed', reason: 'Prompt/handoff execution failed', detail }); + await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Prompt/handoff execution failed', detail }); process.exit(1); } @@ -622,7 +631,7 @@ async function main() { }); outputChunks.push(storeResult.stdout, storeResult.stderr); if (storeResult.code !== 0) { - await writeLog({ cadence, agent: selectedAgent, status: 'failed', reason: 'Memory storage command failed', detail: schedulerConfig.memoryPolicy.storeCommand }); + await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Memory storage command failed', detail: schedulerConfig.memoryPolicy.storeCommand }); process.exit(storeResult.code); } } @@ -649,6 +658,7 @@ async function main() { cadence, agent: selectedAgent, status: 'failed', + platform, reason: 'Required memory steps not verified', detail: `Missing evidence for: ${missingSteps.join(', ')}`, }); @@ -679,6 +689,7 @@ async function main() { cadence, agent: selectedAgent, status: 'failed', + platform, reason: 'Missing required run artifacts', detail, }); @@ -690,7 +701,7 @@ async function main() { if (!parts.length) continue; const result = await runCommand(parts[0], parts.slice(1)); if (result.code !== 0) { - await writeLog({ cadence, agent: selectedAgent, status: 'failed', reason: 'Validation failed', detail: validation }); + await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Validation failed', detail: validation }); process.exit(result.code); } } @@ -706,12 +717,13 @@ async function main() { cadence, agent: selectedAgent, status: 'failed', + platform, reason: `Completion publish failed. Retry npm run lock:complete -- --agent ${selectedAgent} --cadence ${cadence} after verifying relay connectivity`, }); process.exit(completeResult.code); } - await writeLog({ cadence, agent: selectedAgent, status: 'completed', reason: 'Scheduler cycle completed successfully' }); + await writeLog({ cadence, agent: selectedAgent, status: 'completed', platform, reason: 'Scheduler cycle completed successfully' }); process.exit(0); } } diff --git a/scripts/agent/summarize-lock-reliability.mjs b/scripts/agent/summarize-lock-reliability.mjs new file mode 100644 index 0000000..e2816cd --- /dev/null +++ b/scripts/agent/summarize-lock-reliability.mjs @@ -0,0 +1,242 @@ +#!/usr/bin/env node +import fs from 'node:fs/promises'; +import path from 'node:path'; + +const VALID_CADENCES = ['daily', 'weekly']; + +function parseArgs(argv) { + const options = { + sinceDays: 14, + logRoot: path.resolve(process.cwd(), 'task-logs'), + outDir: path.resolve(process.cwd(), 'artifacts', 'lock-reliability'), + }; + + for (let i = 0; i < argv.length; i += 1) { + const token = argv[i]; + if (token === '--since-days') { + options.sinceDays = Number.parseInt(argv[i + 1] || '14', 10); + i += 1; + } else if (token === '--log-root') { + options.logRoot = path.resolve(process.cwd(), argv[i + 1] || 'task-logs'); + i += 1; + } else if (token === '--out-dir') { + options.outDir = path.resolve(process.cwd(), argv[i + 1] || path.join('artifacts', 'lock-reliability')); + i += 1; + } + } + + if (!Number.isFinite(options.sinceDays) || options.sinceDays < 0) { + throw new Error(`Invalid --since-days value: ${options.sinceDays}`); + } + + return options; +} + +function parseFrontmatter(markdown) { + const text = String(markdown || ''); + if (!text.startsWith('---\n')) return {}; + const end = text.indexOf('\n---', 4); + if (end === -1) return {}; + const frontmatter = text.slice(4, end).split(/\r?\n/); + const result = {}; + for (const line of frontmatter) { + const match = line.match(/^([a-zA-Z0-9_]+):\s*(.+)\s*$/); + if (!match) continue; + const [, key, rawValue] = match; + result[key] = rawValue.replace(/^'/, '').replace(/'$/, '').replace(/''/g, "'"); + } + return result; +} + +function inc(map, key) { + map.set(key, (map.get(key) || 0) + 1); +} + +function collectRelayEndpoints({ frontmatter, body }) { + const relays = new Set(); + const relayList = frontmatter.relay_list || ''; + relayList.split(',').map((v) => v.trim()).filter(Boolean).forEach((relay) => relays.add(relay)); + + const allText = `${body}\n${Object.values(frontmatter).join('\n')}`; + const urlMatches = allText.match(/wss?:\/\/[^\s'`),]+/g) || []; + for (const url of urlMatches) { + relays.add(url.trim()); + } + + if (!relays.size) relays.add('unknown'); + return [...relays]; +} + +function getBackendCategory({ frontmatter, body }) { + if (frontmatter.backend_category) return frontmatter.backend_category; + if (frontmatter.preflight_failure_category) return `preflight:${frontmatter.preflight_failure_category}`; + + const reasonMatch = body.match(/- reason:\s*([^\n]+)/i) || body.match(/reason:\s*([^\n]+)/i); + const reason = reasonMatch?.[1]?.trim() || ''; + + if (/lock backend/i.test(reason)) return 'lock backend error (uncategorized)'; + if (/completion publish failed/i.test(reason)) return 'completion publish failed'; + if (/preflight/i.test(reason)) return 'preflight failed (uncategorized)'; + return null; +} + +function toSortedObject(counterMap) { + return Object.fromEntries([...counterMap.entries()].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))); +} + +function asMarkdown(report) { + const lines = []; + lines.push('# Lock Reliability Summary'); + lines.push(''); + lines.push(`- Generated at: ${report.generatedAt}`); + lines.push(`- Window: last ${report.sinceDays} day(s)`); + lines.push(`- Logs scanned: ${report.logsScanned}`); + lines.push(`- Runs considered: ${report.runsConsidered}`); + lines.push(''); + + lines.push('## Totals'); + lines.push(''); + lines.push(`- completed: ${report.statusCounts.completed || 0}`); + lines.push(`- failed: ${report.statusCounts.failed || 0}`); + lines.push(''); + + const sections = [ + ['By platform', report.byPlatform], + ['By cadence', report.byCadence], + ['By backend error category', report.byBackendCategory], + ['By relay endpoint', report.byRelayEndpoint], + ]; + + for (const [title, obj] of sections) { + lines.push(`## ${title}`); + lines.push(''); + const entries = Object.entries(obj); + if (!entries.length) { + lines.push('- (none)'); + } else { + for (const [key, value] of entries) { + lines.push(`- ${key}: ${value}`); + } + } + lines.push(''); + } + + if (report.recurringBackendErrors.length) { + lines.push('## Recurring backend errors (count >= 2)'); + lines.push(''); + for (const entry of report.recurringBackendErrors) { + lines.push(`- ${entry.category}: ${entry.count}`); + } + lines.push(''); + } + + return lines.join('\n'); +} + +async function readLogFile(filePath, cadence, cutoffMs) { + const stat = await fs.stat(filePath); + if (!stat.isFile() || stat.mtimeMs < cutoffMs) return null; + + const basename = path.basename(filePath); + const match = basename.match(/^.+__([^_]+?)__(completed|failed)\.md$/); + if (!match) return null; + + const body = await fs.readFile(filePath, 'utf8'); + const frontmatter = parseFrontmatter(body); + const status = frontmatter.status || match[2]; + const platform = frontmatter.platform || 'unknown'; + const backendCategory = status === 'failed' + ? getBackendCategory({ frontmatter, body }) + : 'n/a'; + const relayEndpoints = collectRelayEndpoints({ frontmatter, body }); + + return { + filePath, + cadence: frontmatter.cadence || cadence, + status, + platform, + backendCategory, + relayEndpoints, + }; +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const now = Date.now(); + const cutoffMs = now - (options.sinceDays * 24 * 60 * 60 * 1000); + + const byPlatform = new Map(); + const byCadence = new Map(); + const byBackendCategory = new Map(); + const byRelayEndpoint = new Map(); + const statusCounts = new Map(); + const records = []; + let logsScanned = 0; + + for (const cadence of VALID_CADENCES) { + const cadenceDir = path.join(options.logRoot, cadence); + let entries; + try { + entries = await fs.readdir(cadenceDir); + } catch { + continue; + } + + for (const entry of entries) { + const fullPath = path.join(cadenceDir, entry); + logsScanned += 1; + const record = await readLogFile(fullPath, cadence, cutoffMs); + if (!record) continue; + + records.push(record); + inc(statusCounts, record.status); + inc(byPlatform, record.platform); + inc(byCadence, record.cadence); + if (record.status === 'failed' && record.backendCategory) { + inc(byBackendCategory, record.backendCategory); + } + for (const relay of record.relayEndpoints) { + inc(byRelayEndpoint, relay); + } + } + } + + const recurringBackendErrors = [...byBackendCategory.entries()] + .filter(([, count]) => count >= 2) + .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0])) + .map(([category, count]) => ({ category, count })); + + const report = { + generatedAt: new Date().toISOString(), + sinceDays: options.sinceDays, + logRoot: options.logRoot, + logsScanned, + runsConsidered: records.length, + statusCounts: toSortedObject(statusCounts), + byPlatform: toSortedObject(byPlatform), + byCadence: toSortedObject(byCadence), + byBackendCategory: toSortedObject(byBackendCategory), + byRelayEndpoint: toSortedObject(byRelayEndpoint), + recurringBackendErrors, + }; + + await fs.mkdir(options.outDir, { recursive: true }); + const markdown = asMarkdown(report); + const markdownPath = path.join(options.outDir, 'lock-reliability-summary.md'); + const jsonPath = path.join(options.outDir, 'lock-reliability-summary.json'); + + await fs.writeFile(markdownPath, markdown, 'utf8'); + await fs.writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, 'utf8'); + + console.log(JSON.stringify({ + ok: true, + markdown: path.relative(process.cwd(), markdownPath), + json: path.relative(process.cwd(), jsonPath), + recurringBackendErrors, + })); +} + +main().catch((error) => { + console.error(error?.stack || error?.message || String(error)); + process.exit(1); +}); diff --git a/src/docs/TORCH.md b/src/docs/TORCH.md index e480993..edb0d73 100644 --- a/src/docs/TORCH.md +++ b/src/docs/TORCH.md @@ -64,6 +64,19 @@ For weekly repository-fit maintenance, TORCH also includes `src/prompts/weekly/r Operational note: scheduler handoff commands are treated as required execution steps. A non-zero exit code (or missing command) is a hard failure: the scheduler writes a `_failed.md` task log, exits immediately, and does not publish `lock:complete` for that run. +## Scheduler lock reliability reporting + +Run the lock reliability summary to aggregate recent scheduler outcomes by platform, cadence, backend error category, and relay endpoint: + +```bash +npm run report:lock-reliability +``` + +Outputs: +- `artifacts/lock-reliability/lock-reliability-summary.md` +- `artifacts/lock-reliability/lock-reliability-summary.json` + + ## Roster precedence The lock CLI resolves roster names in this order: diff --git a/src/prompts/scheduler-flow.md b/src/prompts/scheduler-flow.md index 7ff1b36..1e8d517 100644 --- a/src/prompts/scheduler-flow.md +++ b/src/prompts/scheduler-flow.md @@ -209,6 +209,7 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co - `_completed.md` MUST be created only after completion publish succeeds. - `_failed.md` is required when step 11, step 12, or step 13 fails, and should include the failure reason and next retry action. + - Include `platform` in frontmatter using `AGENT_PLATFORM` (or the scheduler `--platform` value) for both `_completed.md` and `_failed.md`. 15. Commit/push behavior is delegated outside this scheduler script. diff --git a/test/run-scheduler-cycle-memory-policy.test.mjs b/test/run-scheduler-cycle-memory-policy.test.mjs index 67b9f87..699479d 100644 --- a/test/run-scheduler-cycle-memory-policy.test.mjs +++ b/test/run-scheduler-cycle-memory-policy.test.mjs @@ -156,7 +156,10 @@ test('accepts required memory policy when markers or artifacts are produced', { assert.equal(result.code, 0, `stdout: ${result.stdout}\nstderr: ${result.stderr}`); const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily')); - assert.ok(logs.some((name) => name.endsWith('__completed.md'))); + const completedLog = logs.find((name) => name.endsWith('__completed.md')); + assert.ok(completedLog); + const completedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', completedLog), 'utf8'); + assert.match(completedBody, /platform: 'codex'/); await fs.access(path.join(fixture.root, '.scheduler-memory', 'retrieve-daily.ok')); await fs.access(path.join(fixture.root, '.scheduler-memory', 'store-daily.ok')); @@ -188,6 +191,7 @@ test('records backend failure metadata when lock command exits with code 2', { c assert.match(failedBody, /lock_stderr_excerpt: 'publish failed to all relays token=\[REDACTED\] SECRET_KEY=\[REDACTED\]'/); assert.match(failedBody, /lock_stdout_excerpt: 'websocket: connection refused'/); assert.match(failedBody, /Retry AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily/); + assert.match(failedBody, /platform: 'codex'/); }); diff --git a/test/summarize-lock-reliability.test.mjs b/test/summarize-lock-reliability.test.mjs new file mode 100644 index 0000000..ec4089d --- /dev/null +++ b/test/summarize-lock-reliability.test.mjs @@ -0,0 +1,74 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; + +const SCRIPT = path.resolve('scripts/agent/summarize-lock-reliability.mjs'); + +function runNode(args, cwd) { + return new Promise((resolve) => { + const child = spawn(process.execPath, [SCRIPT, ...args], { cwd, stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { stdout += chunk.toString(); }); + child.stderr.on('data', (chunk) => { stderr += chunk.toString(); }); + child.on('close', (code) => resolve({ code: code ?? 1, stdout, stderr })); + }); +} + +test('summarizes recent lock reliability by platform/cadence/backend/relay', async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'lock-reliability-')); + await fs.mkdir(path.join(root, 'task-logs', 'daily'), { recursive: true }); + await fs.mkdir(path.join(root, 'task-logs', 'weekly'), { recursive: true }); + + await fs.writeFile(path.join(root, 'task-logs', 'daily', '2026-02-14T00-00-00Z__agent-a__failed.md'), `--- +cadence: daily +agent: agent-a +status: failed +platform: codex +backend_category: publish failed to all relays +relay_list: 'wss://relay-1.test, wss://relay-2.test' +--- +# Scheduler failed +- reason: Lock backend error +`, 'utf8'); + + await fs.writeFile(path.join(root, 'task-logs', 'weekly', '2026-02-14T00-00-01Z__agent-b__failed.md'), `--- +cadence: weekly +agent: agent-b +status: failed +platform: codex +preflight_failure_category: relay query timeout +relay_list: 'wss://relay-1.test' +--- +# Scheduler failed +- reason: Lock backend unavailable preflight +`, 'utf8'); + + await fs.writeFile(path.join(root, 'task-logs', 'daily', '2026-02-14T00-00-02Z__agent-c__completed.md'), `--- +cadence: daily +agent: agent-c +status: completed +platform: claude +--- +# Scheduler completed +`, 'utf8'); + + const result = await runNode(['--since-days', '365', '--out-dir', 'artifacts/reliability'], root); + assert.equal(result.code, 0, `stdout=${result.stdout}\nstderr=${result.stderr}`); + + const jsonReport = JSON.parse(await fs.readFile(path.join(root, 'artifacts', 'reliability', 'lock-reliability-summary.json'), 'utf8')); + assert.equal(jsonReport.byPlatform.codex, 2); + assert.equal(jsonReport.byPlatform.claude, 1); + assert.equal(jsonReport.byCadence.daily, 2); + assert.equal(jsonReport.byCadence.weekly, 1); + assert.equal(jsonReport.byBackendCategory['publish failed to all relays'], 1); + assert.equal(jsonReport.byBackendCategory['preflight:relay query timeout'], 1); + assert.equal(jsonReport.byRelayEndpoint['wss://relay-1.test'], 2); + + const markdown = await fs.readFile(path.join(root, 'artifacts', 'reliability', 'lock-reliability-summary.md'), 'utf8'); + assert.match(markdown, /## By platform/); + assert.match(markdown, /- codex: 2/); +});