From c1d0de071cb0f8aff5a71ad9793b78d672974d02 Mon Sep 17 00:00:00 2001 From: thePR0M3TH3AN <53631862+PR0M3TH3AN@users.noreply.github.com> Date: Sat, 14 Feb 2026 20:01:32 -0500 Subject: [PATCH] test: add scheduler lock failure frontmatter contract coverage --- package.json | 5 +- scripts/agent/run-scheduler-cycle.mjs | 19 +- ...run-scheduler-cycle-memory-policy.test.mjs | 4 +- ...uler-lock-failure-schema.contract.test.mjs | 210 ++++++++++++++++++ test/scheduler-preflight-lock.e2e.test.mjs | 2 +- 5 files changed, 229 insertions(+), 11 deletions(-) create mode 100644 test/scheduler-lock-failure-schema.contract.test.mjs diff --git a/package.json b/package.json index c53d745..f328bfd 100644 --- a/package.json +++ b/package.json @@ -39,11 +39,12 @@ "build": "node build.mjs", "lint": "eslint .", "lock:complete": "node bin/torch-lock.mjs complete", - "validate:scheduler": "node scripts/validate-scheduler-roster.mjs && node scripts/validate-prompt-contract.mjs && node scripts/validate-scheduler-flow-parity.mjs", + "validate:scheduler": "node scripts/validate-scheduler-roster.mjs && node scripts/validate-prompt-contract.mjs && node scripts/validate-scheduler-flow-parity.mjs && npm run validate:scheduler-failure-schema", "scheduler:daily": "node scripts/agent/run-scheduler-cycle.mjs daily", "scheduler:weekly": "node scripts/agent/run-scheduler-cycle.mjs weekly", "report:lock-reliability": "node scripts/agent/summarize-lock-reliability.mjs --since-days 30", - "test:integration:e2e": "node --test test/scheduler-preflight-lock.e2e.test.mjs" + "test:integration:e2e": "node --test test/scheduler-preflight-lock.e2e.test.mjs", + "validate:scheduler-failure-schema": "node --test test/scheduler-lock-failure-schema.contract.test.mjs" }, "dependencies": { "nostr-tools": "2.19.4", diff --git a/scripts/agent/run-scheduler-cycle.mjs b/scripts/agent/run-scheduler-cycle.mjs index 1f4b1ee..ba8ccfc 100644 --- a/scripts/agent/run-scheduler-cycle.mjs +++ b/scripts/agent/run-scheduler-cycle.mjs @@ -203,14 +203,19 @@ async function validatePromptFile(promptPath) { function classifyLockBackendError(outputText) { const text = String(outputText || '').toLowerCase(); - if (!text.trim()) return 'unknown backend error'; + if (!text.trim()) return 'unknown_backend_error'; + + const errorCategoryMatch = text.match(/error_category=([a-z0-9_]+)/); + if (errorCategoryMatch?.[1]) { + return errorCategoryMatch[1]; + } if ((text.includes('relay') || text.includes('query')) && text.includes('timeout')) { - return 'relay query timeout'; + return 'relay_query_timeout'; } if (text.includes('publish failed to all relays') || text.includes('failed to publish to any relay')) { - return 'publish failed to all relays'; + return 'relay_publish_quorum_failure'; } if ( @@ -221,7 +226,7 @@ function classifyLockBackendError(outputText) { || text.includes('eai_again') || (text.includes('websocket') && text.includes('dns')) ) { - return 'websocket connection refused/dns'; + return 'websocket_connection_refused_or_dns'; } if ( @@ -231,10 +236,10 @@ function classifyLockBackendError(outputText) { || text.includes('must start with ws') || text.includes('invalid relay') ) { - return 'malformed relay url/config'; + return 'malformed_relay_url_config'; } - return 'unknown backend error'; + return 'unknown_backend_error'; } async function artifactExistsSince(filePath, sinceMs) { @@ -602,6 +607,8 @@ async function writeLog({ cadence, agent, status, reason, detail, platform, meta `cadence: ${cadence}`, `agent: ${agent}`, `status: ${status}`, + `reason: ${toYamlScalar(reason)}`, + detail ? `detail: ${toYamlScalar(detail)}` : null, `created_at: ${new Date().toISOString()}`, `timestamp: ${new Date().toISOString()}`, ...Object.entries(mergedMetadata) diff --git a/test/run-scheduler-cycle-memory-policy.test.mjs b/test/run-scheduler-cycle-memory-policy.test.mjs index e0f6a52..e10ec06 100644 --- a/test/run-scheduler-cycle-memory-policy.test.mjs +++ b/test/run-scheduler-cycle-memory-policy.test.mjs @@ -197,7 +197,7 @@ test('records backend failure metadata when lock command exits with code 2', { c assert.match(failedBody, /reason: Lock backend error/); assert.match(failedBody, /failure_class: 'backend_unavailable'/); assert.match(failedBody, /failure_category: 'lock_backend_error'/); - assert.match(failedBody, /backend_category: 'publish failed to all relays'/); + assert.match(failedBody, /backend_category: 'relay_publish_quorum_failure'/); assert.match(failedBody, /lock_command: 'AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily'/); assert.match(failedBody, /lock_stderr_excerpt: 'publish failed to all relays token=\[REDACTED\] SECRET_KEY=\[REDACTED\]'/); assert.match(failedBody, /lock_stdout_excerpt: 'websocket: connection refused'/); @@ -533,7 +533,7 @@ test('stops retrying after configured lock backend retries are exhausted', { con assert.match(failedBody, /lock_attempts_total: '2'/); assert.match(failedBody, /lock_backoff_schedule_ms: '0'/); - assert.match(failedBody, /backend_category: 'publish failed to all relays'/); + assert.match(failedBody, /backend_category: 'relay_publish_quorum_failure'/); }); test('does not use backend retry flow for exit code 3 lock conflicts', { concurrency: false }, async () => { diff --git a/test/scheduler-lock-failure-schema.contract.test.mjs b/test/scheduler-lock-failure-schema.contract.test.mjs new file mode 100644 index 0000000..1c63e60 --- /dev/null +++ b/test/scheduler-lock-failure-schema.contract.test.mjs @@ -0,0 +1,210 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; + +const SOURCE_SCRIPT = path.resolve('scripts/agent/run-scheduler-cycle.mjs'); + +const REQUIRED_LOCK_FAILURE_FIELDS = [ + 'status', + 'reason', + 'backend_category', + 'lock_command', + 'lock_stderr_excerpt', + 'lock_stdout_excerpt', + 'detail', +]; + +const BACKEND_CATEGORY_ENUM = new Set([ + 'relay_query_timeout', + 'relay_publish_quorum_failure', + 'websocket_connection_refused_or_dns', + 'malformed_relay_url_config', + 'unknown_backend_error', +]); + +async function runNode(scriptPath, args, { cwd, env }) { + return new Promise((resolve) => { + const child = spawn(process.execPath, [scriptPath, ...args], { + cwd, + env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + + child.stdout.on('data', (chunk) => { + stdout += chunk.toString(); + }); + + child.stderr.on('data', (chunk) => { + stderr += chunk.toString(); + }); + + child.on('close', (code) => { + resolve({ code: code ?? 1, stdout, stderr }); + }); + }); +} + +function parseFrontmatter(markdown) { + const lines = String(markdown).split(/\r?\n/); + if (lines[0] !== '---') return {}; + const result = {}; + + for (let i = 1; i < lines.length; i += 1) { + const line = lines[i]; + if (line === '---') break; + const sep = line.indexOf(':'); + if (sep < 0) continue; + const key = line.slice(0, sep).trim(); + const raw = line.slice(sep + 1).trim(); + result[key] = raw.startsWith("'") && raw.endsWith("'") + ? raw.slice(1, -1).replace(/''/g, "'") + : raw; + } + + return result; +} + +function validateLockBackendFailureFrontmatter(frontmatter) { + for (const key of REQUIRED_LOCK_FAILURE_FIELDS) { + assert.equal(typeof frontmatter[key], 'string', `expected ${key} to be a string`); + assert.ok(frontmatter[key].trim().length > 0, `expected ${key} to be non-empty`); + } + + assert.equal(frontmatter.status, 'failed', 'expected status=failed for lock-backend failure artifact'); + assert.ok( + BACKEND_CATEGORY_ENUM.has(frontmatter.backend_category), + `expected backend_category to be canonical enum, received: ${frontmatter.backend_category}`, + ); +} + +async function setupFixture({ lockShellBody }) { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'scheduler-lock-schema-')); + const scriptsDir = path.join(root, 'scripts', 'agent'); + const binDir = path.join(root, 'bin'); + + await fs.mkdir(path.join(root, 'src', 'prompts', 'daily'), { recursive: true }); + await fs.mkdir(scriptsDir, { recursive: true }); + await fs.mkdir(binDir, { recursive: true }); + + await fs.copyFile(SOURCE_SCRIPT, path.join(scriptsDir, 'run-scheduler-cycle.mjs')); + await fs.writeFile(path.join(scriptsDir, 'verify-run-artifacts.mjs'), '#!/usr/bin/env node\nprocess.exit(0);\n', 'utf8'); + await fs.writeFile(path.join(root, 'src', 'prompts', 'roster.json'), JSON.stringify({ daily: ['agent-a'] }, null, 2), 'utf8'); + await fs.writeFile(path.join(root, 'src', 'prompts', 'daily', 'agent-a.md'), '# agent-a\n', 'utf8'); + await fs.writeFile( + path.join(root, 'torch-config.json'), + JSON.stringify({ + scheduler: { + strict_lock: true, + firstPromptByCadence: { daily: 'agent-a' }, + handoffCommandByCadence: { daily: 'echo HANDOFF_OK' }, + validationCommandsByCadence: { daily: ['true'] }, + memoryPolicyByCadence: { daily: { mode: 'optional' } }, + }, + }, null, 2), + 'utf8', + ); + + await fs.writeFile( + path.join(binDir, 'npm'), + `#!/usr/bin/env bash +set -euo pipefail +if [[ "$1" != "run" ]]; then + exit 1 +fi +case "$2" in + lock:check:daily) + echo '{"excluded":[]}' + ;; + lock:lock) + ${lockShellBody} + ;; + lock:complete|lint) + ;; + *) + ;; +esac +`, + 'utf8', + ); + await fs.chmod(path.join(binDir, 'npm'), 0o755); + + return { + root, + scriptPath: path.join(scriptsDir, 'run-scheduler-cycle.mjs'), + env: { + ...process.env, + PATH: `${binDir}:${process.env.PATH}`, + SCHEDULER_LOCK_MAX_RETRIES: '0', + SCHEDULER_LOCK_BACKOFF_MS: '0', + SCHEDULER_LOCK_JITTER_MS: '0', + }, + }; +} + +async function getFailedFrontmatter(root) { + const logDir = path.join(root, 'task-logs', 'daily'); + const logs = await fs.readdir(logDir); + const failedLog = logs.find((name) => name.endsWith('__failed.md')); + assert.ok(failedLog, `expected __failed.md in ${logDir}`); + const content = await fs.readFile(path.join(logDir, failedLog), 'utf8'); + return parseFrontmatter(content); +} + +test('scheduler lock backend failure artifact matches required frontmatter schema', { concurrency: false }, async () => { + const fixture = await setupFixture({ + lockShellBody: "echo 'lock_publish_quorum_failed error_category=relay_publish_quorum_failure publish failed to all relays' >&2\n echo 'relay publish failed' >&1\n exit 2", + }); + + const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], { + cwd: fixture.root, + env: fixture.env, + }); + + assert.equal(result.code, 2, `stdout: ${result.stdout}\nstderr: ${result.stderr}`); + + const frontmatter = await getFailedFrontmatter(fixture.root); + validateLockBackendFailureFrontmatter(frontmatter); + assert.equal(frontmatter.backend_category, 'relay_publish_quorum_failure'); +}); + +test('schema contract catches missing required lock-backend field', () => { + const frontmatter = { + status: 'failed', + reason: 'Lock backend error', + backend_category: 'relay_publish_quorum_failure', + lock_command: 'npm run lock:lock -- --agent agent-a --cadence daily', + lock_stderr_excerpt: 'relay publish failed', + lock_stdout_excerpt: 'retrying', + detail: 'Prompt not executed.', + }; + + delete frontmatter.lock_stdout_excerpt; + + assert.throws( + () => validateLockBackendFailureFrontmatter(frontmatter), + /expected lock_stdout_excerpt to be a string/, + ); +}); + +test('schema contract catches misnamed lock-backend key', () => { + const frontmatter = { + status: 'failed', + reason: 'Lock backend error', + backend_category: 'relay_publish_quorum_failure', + lock_command: 'npm run lock:lock -- --agent agent-a --cadence daily', + lock_stderr_excerpt: 'relay publish failed', + lock_stdout_excerpt_misnamed: 'retrying', + detail: 'Prompt not executed.', + }; + + assert.throws( + () => validateLockBackendFailureFrontmatter(frontmatter), + /expected lock_stdout_excerpt to be a string/, + ); +}); diff --git a/test/scheduler-preflight-lock.e2e.test.mjs b/test/scheduler-preflight-lock.e2e.test.mjs index 3ee5380..21d7333 100644 --- a/test/scheduler-preflight-lock.e2e.test.mjs +++ b/test/scheduler-preflight-lock.e2e.test.mjs @@ -185,7 +185,7 @@ test('lock preflight e2e: exit code 2 quorum failure persists failed backend sta status: 'failed', failure_category: 'lock_backend_error', failure_class: 'backend_unavailable', - backend_category: 'publish failed to all relays', + backend_category: 'relay_publish_quorum_failure', }, );