diff --git a/dashboard/index.html b/dashboard/index.html index cc640b9..cc15186 100644 --- a/dashboard/index.html +++ b/dashboard/index.html @@ -437,6 +437,7 @@ status: content.status ?? 'unknown', date: content.date ?? dTag.split('/')[3] ?? todayStr(), platform: content.platform ?? 'unknown', + model: content.model ?? null, }; } @@ -585,8 +586,14 @@ left.appendChild(meta); header.appendChild(left); + const right = createElement('div', 'flex flex-col items-end gap-1'); const badge = createElement('span', `inline-flex items-center rounded-full px-2 py-0.5 text-xs font-medium ${ps.bg} ${ps.text}`, ps.label); - header.appendChild(badge); + right.appendChild(badge); + + if (lock.model && lock.model !== 'unknown') { + right.appendChild(createElement('span', 'text-xs text-muted mono opacity-75', lock.model)); + } + header.appendChild(right); card.appendChild(header); // Grid diff --git a/dashboard/styles.css b/dashboard/styles.css index 470bd20..1458a19 100644 --- a/dashboard/styles.css +++ b/dashboard/styles.css @@ -143,6 +143,7 @@ a { color: inherit; } .w-full { width: 100%; } .shrink-0 { flex-shrink: 0; } .opacity-60 { opacity: 0.6; } +.opacity-75 { opacity: 0.75; } .text-center { text-align: center; } .text-xs { font-size: 0.75rem; line-height: 1rem; } diff --git a/scripts/agent/run-scheduler-cycle.mjs b/scripts/agent/run-scheduler-cycle.mjs index b5a65cf..88ea618 100644 --- a/scripts/agent/run-scheduler-cycle.mjs +++ b/scripts/agent/run-scheduler-cycle.mjs @@ -18,7 +18,7 @@ const FAILURE_CATEGORY = { const LOCK_INCIDENT_LINK = 'docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md'; function parseArgs(argv) { - const args = { cadence: null, platform: process.env.AGENT_PLATFORM || 'codex' }; + const args = { cadence: null, platform: process.env.AGENT_PLATFORM || 'codex', model: process.env.AGENT_MODEL || null }; for (let i = 0; i < argv.length; i += 1) { const value = argv[i]; if (!value.startsWith('--') && !args.cadence) { @@ -34,6 +34,10 @@ function parseArgs(argv) { args.platform = argv[i + 1] || args.platform; i += 1; } + if (value === '--model') { + args.model = argv[i + 1] || args.model; + i += 1; + } } return args; } @@ -508,8 +512,11 @@ function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } -async function acquireLockWithRetry({ selectedAgent, cadence, platform, lockRetry, idempotencyKey }) { +async function acquireLockWithRetry({ selectedAgent, cadence, platform, model, lockRetry, idempotencyKey }) { const lockCommandArgs = ['run', 'lock:lock', '--', '--agent', selectedAgent, '--cadence', cadence]; + if (model) { + lockCommandArgs.push('--model', model); + } const backoffScheduleMs = []; let attempts = 0; const correlationId = idempotencyKey || `${cadence}:${selectedAgent}:${getRunDateKey()}`; @@ -519,6 +526,7 @@ async function acquireLockWithRetry({ selectedAgent, cadence, platform, lockRetr const result = await runCommand('npm', lockCommandArgs, { env: { AGENT_PLATFORM: platform, + ...(model ? { AGENT_MODEL: model } : {}), ...(idempotencyKey ? { SCHEDULER_LOCK_IDEMPOTENCY_KEY: idempotencyKey } : {}), SCHEDULER_LOCK_CORRELATION_ID: correlationId, SCHEDULER_LOCK_ATTEMPT_ID: String(attempts), @@ -632,7 +640,7 @@ async function writeLog({ cadence, agent, status, reason, detail, platform, meta } async function main() { - const { cadence, platform } = parseArgs(process.argv.slice(2)); + const { cadence, platform, model } = parseArgs(process.argv.slice(2)); if (!VALID_CADENCES.has(cadence)) { console.error('Usage: node scripts/agent/run-scheduler-cycle.mjs '); process.exit(1); @@ -740,6 +748,7 @@ async function main() { selectedAgent, cadence, platform, + model, lockRetry: schedulerConfig.lockRetry, idempotencyKey: deferralForAgent?.idempotency_key, }); @@ -753,7 +762,8 @@ async function main() { const combinedLockOutput = `${lockResult.stderr}\n${lockResult.stdout}`; const diagnosticsSummary = summarizeLockFailureReasons(combinedLockOutput); const backendCategory = lockAttempt.finalBackendCategory || classifyLockBackendError(combinedLockOutput); - const lockCommand = `AGENT_PLATFORM=${platform} npm run lock:lock -- --agent ${selectedAgent} --cadence ${cadence}`; + const modelPart = model ? ` --model ${model}` : ''; + const lockCommand = `AGENT_PLATFORM=${platform} npm run lock:lock -- --agent ${selectedAgent} --cadence ${cadence}${modelPart}`; const stderrExcerpt = excerptText(lockResult.stderr); const stdoutExcerpt = excerptText(lockResult.stdout); const backoffSchedule = lockAttempt.backoffScheduleMs.join(', '); @@ -860,7 +870,7 @@ async function main() { if (schedulerConfig.memoryPolicy.retrieveCommand) { const retrieveResult = await runCommand('bash', ['-lc', schedulerConfig.memoryPolicy.retrieveCommand], { - env: { AGENT_PLATFORM: platform, SCHEDULER_AGENT: selectedAgent, SCHEDULER_CADENCE: cadence, SCHEDULER_PROMPT_PATH: promptPath }, + env: { AGENT_PLATFORM: platform, ...(model ? { AGENT_MODEL: model } : {}), SCHEDULER_AGENT: selectedAgent, SCHEDULER_CADENCE: cadence, SCHEDULER_PROMPT_PATH: promptPath }, }); outputChunks.push(retrieveResult.stdout, retrieveResult.stderr); if (retrieveResult.code !== 0) { @@ -879,7 +889,7 @@ async function main() { if (schedulerConfig.handoffCommand) { const handoff = await runCommand('bash', ['-lc', schedulerConfig.handoffCommand], { - env: { AGENT_PLATFORM: platform, SCHEDULER_AGENT: selectedAgent, SCHEDULER_CADENCE: cadence, SCHEDULER_PROMPT_PATH: promptPath }, + env: { AGENT_PLATFORM: platform, ...(model ? { AGENT_MODEL: model } : {}), SCHEDULER_AGENT: selectedAgent, SCHEDULER_CADENCE: cadence, SCHEDULER_PROMPT_PATH: promptPath }, }); outputChunks.push(handoff.stdout, handoff.stderr); if (handoff.code !== 0) { @@ -912,7 +922,7 @@ async function main() { if (schedulerConfig.memoryPolicy.storeCommand) { const storeResult = await runCommand('bash', ['-lc', schedulerConfig.memoryPolicy.storeCommand], { - env: { AGENT_PLATFORM: platform, SCHEDULER_AGENT: selectedAgent, SCHEDULER_CADENCE: cadence, SCHEDULER_PROMPT_PATH: promptPath }, + env: { AGENT_PLATFORM: platform, ...(model ? { AGENT_MODEL: model } : {}), SCHEDULER_AGENT: selectedAgent, SCHEDULER_CADENCE: cadence, SCHEDULER_PROMPT_PATH: promptPath }, }); outputChunks.push(storeResult.stdout, storeResult.stderr); if (storeResult.code !== 0) { @@ -1011,8 +1021,8 @@ async function main() { const completeResult = await runCommand( 'npm', - ['run', 'lock:complete', '--', '--agent', selectedAgent, '--cadence', cadence], - { env: { AGENT_PLATFORM: platform } }, + ['run', 'lock:complete', '--', '--agent', selectedAgent, '--cadence', cadence, ...(model ? ['--model', model] : [])], + { env: { AGENT_PLATFORM: platform, ...(model ? { AGENT_MODEL: model } : {}) } }, ); if (completeResult.code !== 0) { diff --git a/scripts/validate-scheduler-flow-parity.mjs b/scripts/validate-scheduler-flow-parity.mjs index be93747..330219a 100644 --- a/scripts/validate-scheduler-flow-parity.mjs +++ b/scripts/validate-scheduler-flow-parity.mjs @@ -43,7 +43,7 @@ const invariants = [ ], }, schedulerSnippets: [ - "runCommand('npm', ['run', `lock:check:${cadence}`])", + "runCommand('npm', ['run', `lock:check:${cadence}`", 'checkPayload.excluded', 'checkPayload.locked', 'checkPayload.paused', diff --git a/src/cli-parser.mjs b/src/cli-parser.mjs index f768a00..2a83ee4 100644 --- a/src/cli-parser.mjs +++ b/src/cli-parser.mjs @@ -31,6 +31,8 @@ export function parseArgs(argv) { allRelaysDownMinutes: null, minSuccessRate: null, windowMinutes: null, + platform: null, + model: null, }; let i = 0; @@ -117,6 +119,14 @@ export function parseArgs(argv) { args.windowMinutes = parseInt(argv[++i], 10); } else if (arg.startsWith('--window-minutes=')) { args.windowMinutes = parseInt(arg.split('=')[1], 10); + } else if (arg === '--platform') { + args.platform = argv[++i]; + } else if (arg.startsWith('--platform=')) { + args.platform = arg.split('=')[1]; + } else if (arg === '--model') { + args.model = argv[++i]; + } else if (arg.startsWith('--model=')) { + args.model = arg.split('=')[1]; } } diff --git a/src/lib.mjs b/src/lib.mjs index 0277108..f70e18f 100644 --- a/src/lib.mjs +++ b/src/lib.mjs @@ -33,7 +33,7 @@ import { unpinMemory, } from './services/memory/index.js'; import { ExitError } from './errors.mjs'; -import { todayDateStr, nowUnix, getIsoWeekStr } from './utils.mjs'; +import { todayDateStr, nowUnix, getIsoWeekStr, detectPlatform } from './utils.mjs'; import { runRelayHealthCheck } from './relay-health.mjs'; import fs from 'node:fs/promises'; import path from 'node:path'; @@ -204,7 +204,10 @@ export async function cmdCheck(cadence, deps = {}) { * @returns {Promise<{status: string, eventId: string}>} * @throws {ExitError} If lock is denied (already locked, completed, or race lost) */ -export async function cmdLock(agent, cadence, dryRun = false, deps = {}) { +export async function cmdLock(agent, cadence, optionsOrDryRun = false, deps = {}) { + const options = typeof optionsOrDryRun === 'object' ? optionsOrDryRun : { dryRun: !!optionsOrDryRun }; + const { dryRun = false, platform = null, model = null } = options; + const { getRelays = _getRelays, getNamespace = _getNamespace, @@ -290,7 +293,8 @@ export async function cmdLock(agent, cadence, dryRun = false, deps = {}) { status: 'started', namespace, date: dateStr, - platform: process.env.AGENT_PLATFORM || 'unknown', + platform: platform || process.env.AGENT_PLATFORM || detectPlatform() || 'unknown', + model: model || process.env.AGENT_MODEL || 'unknown', lockedAt: new Date(now * 1000).toISOString(), expiresAt: new Date(expiresAt * 1000).toISOString(), }), @@ -436,7 +440,10 @@ export async function cmdList(cadence, deps = {}) { * @returns {Promise<{status: string, eventId: string}>} * @throws {ExitError} If no active lock exists for the agent */ -export async function cmdComplete(agent, cadence, dryRun = false, deps = {}) { +export async function cmdComplete(agent, cadence, optionsOrDryRun = false, deps = {}) { + const options = typeof optionsOrDryRun === 'object' ? optionsOrDryRun : { dryRun: !!optionsOrDryRun }; + const { dryRun = false, platform = null, model = null } = options; + const { getRelays = _getRelays, getNamespace = _getNamespace, @@ -500,7 +507,8 @@ export async function cmdComplete(agent, cadence, dryRun = false, deps = {}) { status: 'completed', namespace, date: dateStr, - platform: process.env.AGENT_PLATFORM || 'unknown', + platform: platform || process.env.AGENT_PLATFORM || detectPlatform() || 'unknown', + model: model || process.env.AGENT_MODEL || 'unknown', startedAt: startedAtIso, completedAt: new Date(now * 1000).toISOString(), }), @@ -614,7 +622,11 @@ export async function main(argv) { console.error(`ERROR: --cadence <${[...VALID_CADENCES].join('|')}> is required for lock`); throw new ExitError(1, 'Missing cadence'); } - await cmdLock(args.agent, args.cadence, args.dryRun); + await cmdLock(args.agent, args.cadence, { + dryRun: args.dryRun, + platform: args.platform, + model: args.model + }); break; } @@ -627,7 +639,11 @@ export async function main(argv) { console.error(`ERROR: --cadence <${[...VALID_CADENCES].join('|')}> is required for complete`); throw new ExitError(1, 'Missing cadence'); } - await cmdComplete(args.agent, args.cadence, args.dryRun); + await cmdComplete(args.agent, args.cadence, { + dryRun: args.dryRun, + platform: args.platform, + model: args.model + }); break; } diff --git a/src/utils.mjs b/src/utils.mjs index 0538a3e..4b7c76c 100644 --- a/src/utils.mjs +++ b/src/utils.mjs @@ -27,3 +27,10 @@ export function getIsoWeekStr(dateInput) { return `${date.getUTCFullYear()}-W${String(weekNo).padStart(2, '0')}`; } + +export function detectPlatform() { + if (process.env.JULES_API_KEY || process.env.JULES_SESSION_ID) return 'jules'; + if (process.env.CODEX_API_KEY || process.env.CODEX_SESSION_ID) return 'codex'; + if (process.env.GOOSE_API_KEY || process.env.GOOSE_SESSION_ID) return 'goose'; + return null; +} diff --git a/test/scheduler-ratchet.test.mjs b/test/scheduler-ratchet.test.mjs index a8b3fe3..00fe540 100644 --- a/test/scheduler-ratchet.test.mjs +++ b/test/scheduler-ratchet.test.mjs @@ -129,7 +129,7 @@ describe('Scheduler cycle ordering guarantees', () => { const order = [ "const preflight = await runLockHealthPreflight({ cadence, platform });", - "const checkResult = await runCommand('npm', ['run', `lock:check:${cadence}`]);", + "const checkResult = await runCommand('npm', ['run', `lock:check:${cadence}`", "const lockAttempt = await acquireLockWithRetry({", "if (schedulerConfig.handoffCommand)", "const artifactCheck = await runCommand('node', [", diff --git a/test_output_full.txt b/test_output_full.txt new file mode 100644 index 0000000..4ce136d --- /dev/null +++ b/test_output_full.txt @@ -0,0 +1,1411 @@ + +> torch-lock@0.1.0 test +> npm run validate:scheduler && npm run test:integration:e2e && node --test test/*.test.mjs tests/*.test.js test/cli-parser.test.mjs test/lib.test.mjs + + +> torch-lock@0.1.0 validate:scheduler +> node scripts/validate-scheduler-roster.mjs && node scripts/validate-prompt-contract.mjs && node scripts/validate-scheduler-flow-parity.mjs && npm run validate:scheduler-failure-schema + +Scheduler tables, roster, and prompt filenames are in sync. +Prompt contract validated for all daily/weekly prompts. +Scheduler flow parity validated for prompts and scheduler implementation. + +> torch-lock@0.1.0 validate:scheduler-failure-schema +> node --test test/scheduler-lock-failure-schema.contract.test.mjs + +TAP version 13 +# Subtest: scheduler lock backend failure artifact matches required frontmatter schema +ok 1 - scheduler lock backend failure artifact matches required frontmatter schema + --- + duration_ms: 113.187003 + type: 'test' + ... +# Subtest: schema contract catches missing required lock-backend field +ok 2 - schema contract catches missing required lock-backend field + --- + duration_ms: 2.04682 + type: 'test' + ... +# Subtest: schema contract catches misnamed lock-backend key +ok 3 - schema contract catches misnamed lock-backend key + --- + duration_ms: 0.530155 + type: 'test' + ... +1..3 +# tests 3 +# suites 0 +# pass 3 +# fail 0 +# cancelled 0 +# skipped 0 +# todo 0 +# duration_ms 208.177362 + +> torch-lock@0.1.0 test:integration:e2e +> node --test test/scheduler-preflight-lock.e2e.test.mjs + +TAP version 13 +# Subtest: lock preflight e2e: successful lock writes completed status snapshot +ok 1 - lock preflight e2e: successful lock writes completed status snapshot + --- + duration_ms: 431.690059 + type: 'test' + ... +# Subtest: lock preflight e2e: exit code 2 quorum failure persists failed backend status and prompt-not-started marker +ok 2 - lock preflight e2e: exit code 2 quorum failure persists failed backend status and prompt-not-started marker + --- + duration_ms: 131.767309 + type: 'test' + ... +# Subtest: lock preflight e2e: non-lock failure exits failed without prompt parse/schema classification +ok 3 - lock preflight e2e: non-lock failure exits failed without prompt parse/schema classification + --- + duration_ms: 114.287449 + type: 'test' + ... +1..3 +# tests 3 +# suites 0 +# pass 3 +# fail 0 +# cancelled 0 +# skipped 0 +# todo 0 +# duration_ms 764.828908 +TAP version 13 +# Subtest: parseArgs should parse basic command +ok 1 - parseArgs should parse basic command + --- + duration_ms: 1.620443 + type: 'test' + ... +# Subtest: parseArgs should parse agent and cadence +ok 2 - parseArgs should parse agent and cadence + --- + duration_ms: 0.277377 + type: 'test' + ... +# Subtest: parseArgs should parse equals sign syntax +ok 3 - parseArgs should parse equals sign syntax + --- + duration_ms: 2.057448 + type: 'test' + ... +# Subtest: parseArgs should parse dry-run flag +ok 4 - parseArgs should parse dry-run flag + --- + duration_ms: 0.418567 + type: 'test' + ... +# Subtest: parseArgs should parse port with default +ok 5 - parseArgs should parse port with default + --- + duration_ms: 0.205075 + type: 'test' + ... +# Subtest: parseArgs should parse custom port +ok 6 - parseArgs should parse custom port + --- + duration_ms: 0.184827 + type: 'test' + ... +# Subtest: parseArgs should parse ignore-logs flag +ok 7 - parseArgs should parse ignore-logs flag + --- + duration_ms: 0.328366 + type: 'test' + ... +# Subtest: parseArgs should parse memory flags +ok 8 - parseArgs should parse memory flags + --- + duration_ms: 1.159906 + type: 'test' + ... +# Subtest: parseArgs should parse memory flags with equals +ok 9 - parseArgs should parse memory flags with equals + --- + duration_ms: 0.688999 + type: 'test' + ... +# Subtest: src/lib.mjs + # Subtest: cmdCheck + # Subtest: returns correct structure with empty locks + ok 1 - returns correct structure with empty locks + --- + duration_ms: 2.919645 + type: 'test' + ... + # Subtest: identifies locked agents + ok 2 - identifies locked agents + --- + duration_ms: 0.969385 + type: 'test' + ... + # Subtest: excludes paused agents + ok 3 - excludes paused agents + --- + duration_ms: 0.808782 + type: 'test' + ... + # Subtest: identifies unknown locked agents + ok 4 - identifies unknown locked agents + --- + duration_ms: 0.965329 + type: 'test' + ... + 1..4 + ok 1 - cmdCheck + --- + duration_ms: 7.092631 + type: 'suite' + ... + # Subtest: cmdLock + # Subtest: successfully locks an available agent + ok 1 - successfully locks an available agent + --- + duration_ms: 15.796158 + type: 'test' + ... + # Subtest: fails if agent is not in roster + ok 2 - fails if agent is not in roster + --- + duration_ms: 1.289443 + type: 'test' + ... + # Subtest: fails if agent is already locked + ok 3 - fails if agent is already locked + --- + duration_ms: 0.520591 + type: 'test' + ... + # Subtest: fails if agent is already completed + ok 4 - fails if agent is already completed + --- + duration_ms: 0.579941 + type: 'test' + ... + # Subtest: fails if race check is lost + ok 5 - fails if race check is lost + --- + duration_ms: 7.619904 + type: 'test' + ... + # Subtest: handles race condition with identical timestamps using eventId tie-breaker (we win) + ok 6 - handles race condition with identical timestamps using eventId tie-breaker (we win) + --- + duration_ms: 14.865596 + type: 'test' + ... + # Subtest: handles race condition with identical timestamps using eventId tie-breaker (we lose) + ok 7 - handles race condition with identical timestamps using eventId tie-breaker (we lose) + --- + duration_ms: 2.930522 + type: 'test' + ... + # Subtest: dry run does not publish + ok 8 - dry run does not publish + --- + duration_ms: 0.728273 + type: 'test' + ... + 1..8 + ok 2 - cmdLock + --- + duration_ms: 45.622223 + type: 'suite' + ... + # Subtest: cmdList + # Subtest: lists active locks + ok 1 - lists active locks + --- + duration_ms: 0.843279 + type: 'test' + ... + # Subtest: warns about unknown agents + ok 2 - warns about unknown agents + --- + duration_ms: 0.342592 + type: 'test' + ... + # Subtest: handles no locks + ok 3 - handles no locks + --- + duration_ms: 0.40154 + type: 'test' + ... + 1..3 + ok 3 - cmdList + --- + duration_ms: 1.802133 + type: 'suite' + ... + # Subtest: cmdComplete + # Subtest: successfully completes an active lock + ok 1 - successfully completes an active lock + --- + duration_ms: 1.018085 + type: 'test' + ... + # Subtest: fails if no active lock found + ok 2 - fails if no active lock found + --- + duration_ms: 0.359386 + type: 'test' + ... + # Subtest: detects already completed task + ok 3 - detects already completed task + --- + duration_ms: 0.433073 + type: 'test' + ... + 1..3 + ok 4 - cmdComplete + --- + duration_ms: 1.989418 + type: 'suite' + ... + 1..4 +ok 10 - src/lib.mjs + --- + duration_ms: 57.546812 + type: 'suite' + ... +# Published to 1/3 relays (required=1, timeout=25ms) +# Published to 2/2 relays (required=2, timeout=25ms) +# Published to 2/3 relays (required=2, timeout=25ms) +# (node:10747) ExperimentalWarning: The MockTimers API is an experimental feature and might change at any time +# (Use `node --trace-warnings ...` to show where the warning was created) +# Subtest: ci-resilience lock acquisition under unstable relays + # Subtest: recovers after all relays are down for an initial outage window + ok 1 - recovers after all relays are down for an initial outage window + --- + duration_ms: 21.97099 + type: 'test' + ... + # Subtest: survives intermittent timeout spikes and reaches quorum with bounded retries + ok 2 - survives intermittent timeout spikes and reaches quorum with bounded retries + --- + duration_ms: 3.762778 + type: 'test' + ... + # Subtest: handles staggered relay recovery and never exceeds retry budget + ok 3 - handles staggered relay recovery and never exceeds retry budget + --- + duration_ms: 2.945008 + type: 'test' + ... + 1..3 +ok 11 - ci-resilience lock acquisition under unstable relays + --- + duration_ms: 30.460506 + type: 'suite' + ... +# Published to 1/1 relays (required=1, timeout=200ms) +# Published to 1/1 relays (required=1, timeout=200ms) +# (node:10764) ExperimentalWarning: The MockTimers API is an experimental feature and might change at any time +# (Use `node --trace-warnings ...` to show where the warning was created) +# Subtest: publishLock retry behavior + # Subtest: succeeds immediately on first attempt without retries + ok 1 - succeeds immediately on first attempt without retries + --- + duration_ms: 5.49731 + type: 'test' + ... + # Subtest: retries a transient failure then succeeds within retry budget with bounded jitter + ok 2 - retries a transient failure then succeeds within retry budget with bounded jitter + --- + duration_ms: 2.514908 + type: 'test' + ... + # Subtest: returns terminal relay_publish_quorum_failure after exhausting retry budget + ok 3 - returns terminal relay_publish_quorum_failure after exhausting retry budget + --- + duration_ms: 4.142038 + type: 'test' + ... + # Subtest: short-circuits non-retryable failures without extra attempts + ok 4 - short-circuits non-retryable failures without extra attempts + --- + duration_ms: 1.895161 + type: 'test' + ... + 1..4 +ok 12 - publishLock retry behavior + --- + duration_ms: 15.809506 + type: 'suite' + ... +# Published to 2/2 relays (required=1, timeout=200ms) +# {"event":"lock_publish_quorum_met","correlationId":"none","attemptId":"1","publishAttempt":1,"successCount":1,"relayAttemptedCount":2,"requiredSuccesses":1,"timeoutMs":200,"retryTimeline":[{"publishAttempt":1,"successCount":1,"relayAttemptedCount":2,"elapsedMs":1}],"totalElapsedMs":1} +# Published to 1/2 relays (required=1, timeout=200ms) +# {"event":"lock_publish_quorum_met","correlationId":"none","attemptId":"1","publishAttempt":1,"successCount":1,"relayAttemptedCount":2,"requiredSuccesses":1,"timeoutMs":200,"retryTimeline":[{"publishAttempt":1,"successCount":1,"relayAttemptedCount":2,"elapsedMs":0}],"totalElapsedMs":0} +# Published to 1/2 relays (required=1, timeout=200ms) +# {"event":"lock_publish_quorum_met","correlationId":"none","attemptId":"1","publishAttempt":1,"successCount":1,"relayAttemptedCount":2,"requiredSuccesses":1,"timeoutMs":200,"retryTimeline":[{"publishAttempt":1,"successCount":1,"relayAttemptedCount":2,"elapsedMs":1}],"totalElapsedMs":1} +# Published to 1/2 relays (required=1, timeout=200ms) +# {"event":"lock_publish_quorum_met","correlationId":"none","attemptId":"1","publishAttempt":1,"successCount":1,"relayAttemptedCount":2,"requiredSuccesses":1,"timeoutMs":200,"retryTimeline":[{"publishAttempt":1,"successCount":1,"relayAttemptedCount":2,"elapsedMs":0}],"totalElapsedMs":0} +# Published to 1/2 relays (required=1, timeout=200ms) +# {"event":"lock_publish_quorum_met","correlationId":"none","attemptId":"1","publishAttempt":1,"successCount":1,"relayAttemptedCount":1,"requiredSuccesses":1,"timeoutMs":200,"retryTimeline":[{"publishAttempt":1,"successCount":1,"relayAttemptedCount":1,"elapsedMs":0}],"totalElapsedMs":0} +# Published to 1/1 relays (required=1, timeout=200ms) +# {"event":"lock_publish_quorum_met","correlationId":"none","attemptId":"1","publishAttempt":1,"successCount":1,"relayAttemptedCount":1,"requiredSuccesses":1,"timeoutMs":200,"retryTimeline":[{"publishAttempt":1,"successCount":1,"relayAttemptedCount":1,"elapsedMs":0}],"totalElapsedMs":0} +# Published to 1/1 relays (required=1, timeout=200ms) +# {"event":"lock_publish_quorum_met","correlationId":"none","attemptId":"1","publishAttempt":1,"successCount":1,"relayAttemptedCount":3,"requiredSuccesses":1,"timeoutMs":200,"retryTimeline":[{"publishAttempt":1,"successCount":1,"relayAttemptedCount":3,"elapsedMs":0}],"totalElapsedMs":0} +# Published to 1/3 relays (required=1, timeout=200ms) +# Subtest: lock-ops + # Subtest: parseLockEvent + # Subtest: parses a valid lock event correctly + ok 1 - parses a valid lock event correctly + --- + duration_ms: 1.724025 + type: 'test' + ... + 1..1 + ok 1 - parseLockEvent + --- + duration_ms: 2.821259 + type: 'suite' + ... + # Subtest: queryLocks + # Subtest: falls back to fallback relays when primary query fails + ok 1 - falls back to fallback relays when primary query fails + --- + duration_ms: 16.303634 + type: 'test' + ... + 1..1 + ok 2 - queryLocks + --- + duration_ms: 16.570043 + type: 'suite' + ... + # Subtest: publishLock + # Subtest: retries transient failures and succeeds on a later attempt + ok 1 - retries transient failures and succeeds on a later attempt + --- + duration_ms: 7.211161 + type: 'test' + ... + # Subtest: fails after retry budget is exhausted for persistent transient failures + ok 2 - fails after retry budget is exhausted for persistent transient failures + --- + duration_ms: 2.64143 + type: 'test' + ... + # Subtest: fails without retry for persistent validation failures + ok 3 - fails without retry for persistent validation failures + --- + duration_ms: 1.182058 + type: 'test' + ... + # Subtest: includes per-relay reason categories and retry metadata in failure diagnostics + ok 4 - includes per-relay reason categories and retry metadata in failure diagnostics + --- + duration_ms: 2.589672 + type: 'test' + ... + # Subtest: returns success when mixed relay outcomes still satisfy quorum + ok 5 - returns success when mixed relay outcomes still satisfy quorum + --- + duration_ms: 1.498691 + type: 'test' + ... + # Subtest: uses fallback relays to satisfy min success quorum + ok 6 - uses fallback relays to satisfy min success quorum + --- + duration_ms: 1.41078 + type: 'test' + ... + # Subtest: quarantines repeatedly failing relays and still reaches quorum with one healthy relay + ok 7 - quarantines repeatedly failing relays and still reaches quorum with one healthy relay + --- + duration_ms: 2.300327 + type: 'test' + ... + # Subtest: uses fallback and min active pool to reintroduce quarantined relay when needed + ok 8 - uses fallback and min active pool to reintroduce quarantined relay when needed + --- + duration_ms: 1.210666 + type: 'test' + ... + 1..8 + ok 3 - publishLock + --- + duration_ms: 20.954015 + type: 'suite' + ... + 1..3 +ok 13 - lock-ops + --- + duration_ms: 41.436378 + type: 'suite' + ... +# memory_telemetry { +# event: 'memory:ingested', +# payload: { +# memory_id: '2117ead5-3b9d-4844-b481-1ef5f24b4751', +# agent_id: 'agent-telemetry', +# dedupe_key: 'e7ded6e3f76233804d735eaa761caac09e07587175c70721167158477ef2428e', +# chunk_events: 1, +# window_start: 0, +# window_end: 1771186401441 +# }, +# ts: 1771186401448 +# } +# memory_metric { +# metric: 'memory_ingested_total', +# payload: { count: 1 }, +# ts: 1771186401449 +# } +# memory_metric { +# metric: 'memory_retrieved_total', +# payload: { count: 1, cache_hit: false }, +# ts: 1771186401450 +# } +# memory_telemetry { +# event: 'memory:pruned', +# payload: { pruned_count: 1, retention_ms: 1000, dry_run: true }, +# ts: 1771186401452 +# } +# memory_metric { +# metric: 'memory_pruned_total', +# payload: { count: 0, retention_ms: 1000, dry_run: true }, +# ts: 1771186401452 +# } +# memory_telemetry { +# event: 'memory:prune_skipped', +# payload: { reason: 'flag_disabled', retention_ms: 1000 }, +# ts: 1771186401452 +# } +# Subtest: listMemories, inspectMemory, pinMemory, and unpinMemory support admin hooks +ok 14 - listMemories, inspectMemory, pinMemory, and unpinMemory support admin hooks + --- + duration_ms: 3.004044 + type: 'test' + ... +# Subtest: triggerPruneDryRun returns candidate metadata without deleting records +ok 15 - triggerPruneDryRun returns candidate metadata without deleting records + --- + duration_ms: 0.499711 + type: 'test' + ... +# Subtest: memoryStats reports counts/rates and retrieval telemetry remains redaction-safe +ok 16 - memoryStats reports counts/rates and retrieval telemetry remains redaction-safe + --- + duration_ms: 10.910469 + type: 'test' + ... +# Subtest: runPruneCycle respects prune feature-flag modes +ok 17 - runPruneCycle respects prune feature-flag modes + --- + duration_ms: 1.466885 + type: 'test' + ... +# Subtest: stores runtime events in signer/session namespace and clears scope +ok 18 - stores runtime events in signer/session namespace and clears scope + --- + duration_ms: 2.84762 + type: 'test' + ... +# Subtest: expires runtime events and records expiration metrics +ok 19 - expires runtime events and records expiration metrics + --- + duration_ms: 10.99748 + type: 'test' + ... +# Subtest: enforces LRU bounds and tracks eviction metrics +ok 20 - enforces LRU bounds and tracks eviction metrics + --- + duration_ms: 0.733276 + type: 'test' + ... +# Subtest: blocks decrypted/session-sensitive data from durable promotion without sanitizer pass +ok 21 - blocks decrypted/session-sensitive data from durable promotion without sanitizer pass + --- + duration_ms: 0.393937 + type: 'test' + ... +# Subtest: createEmbedderAdapter selects in-memory backend and supports vector CRUD +ok 22 - createEmbedderAdapter selects in-memory backend and supports vector CRUD + --- + duration_ms: 2.86485 + type: 'test' + ... +# Subtest: createEmbedderAdapter reads backend from environment +ok 23 - createEmbedderAdapter reads backend from environment + --- + duration_ms: 0.489315 + type: 'test' + ... +# Subtest: memory feature flags parse defaults and canary allow-lists +ok 24 - memory feature flags parse defaults and canary allow-lists + --- + duration_ms: 1.434389 + type: 'test' + ... +# Subtest: memory prune flag supports off, dry-run, and active modes +ok 25 - memory prune flag supports off, dry-run, and active modes + --- + duration_ms: 0.33523 + type: 'test' + ... +# Subtest: ingestMemoryWindow gathers sources, redacts pii, chunks, embeds, links, and emits telemetry +ok 26 - ingestMemoryWindow gathers sources, redacts pii, chunks, embeds, links, and emits telemetry + --- + duration_ms: 6.394873 + type: 'test' + ... +# Subtest: ingestMemoryWindow dedupes overlapping windows by hashed content + agent + timestamp bucket +ok 27 - ingestMemoryWindow dedupes overlapping windows by hashed content + agent + timestamp bucket + --- + duration_ms: 1.126878 + type: 'test' + ... +# Subtest: createLifecyclePlan respects delete safety constraints and logs reasons +ok 28 - createLifecyclePlan respects delete safety constraints and logs reasons + --- + duration_ms: 2.742536 + type: 'test' + ... +# Subtest: createLifecyclePlan condenses near-duplicates and archives merged source records +ok 29 - createLifecyclePlan condenses near-duplicates and archives merged source records + --- + duration_ms: 1.079231 + type: 'test' + ... +# Subtest: applyLifecycleActions executes keep/archive/delete and merge markers +ok 30 - applyLifecycleActions executes keep/archive/delete and merge markers + --- + duration_ms: 1.36098 + type: 'test' + ... +# Subtest: filterAndRankMemories applies metadata filters and composite score +ok 31 - filterAndRankMemories applies metadata filters and composite score + --- + duration_ms: 2.950958 + type: 'test' + ... +# Subtest: formatMemoriesForPrompt returns prompt blocks and enforces token budget trimming +ok 32 - formatMemoriesForPrompt returns prompt blocks and enforces token budget trimming + --- + duration_ms: 2.739927 + type: 'test' + ... +# Subtest: isMemoryRetrievalEnabled supports global toggle and agent canary list +ok 33 - isMemoryRetrievalEnabled supports global toggle and agent canary list + --- + duration_ms: 0.448947 + type: 'test' + ... +# Subtest: renderPromptWithMemoryContext injects bounded memory block before base prompt +ok 34 - renderPromptWithMemoryContext injects bounded memory block before base prompt + --- + duration_ms: 0.829988 + type: 'test' + ... +# Subtest: createDbAdvisoryLockProvider acquires and releases postgres advisory locks +ok 35 - createDbAdvisoryLockProvider acquires and releases postgres advisory locks + --- + duration_ms: 2.589181 + type: 'test' + ... +# Subtest: startMemoryMaintenanceScheduler runs all jobs and emits metrics including retries +ok 36 - startMemoryMaintenanceScheduler runs all jobs and emits metrics including retries + --- + duration_ms: 26.977701 + type: 'test' + ... +# Subtest: startMemoryMaintenanceScheduler skips jobs when feature flags disable memory +ok 37 - startMemoryMaintenanceScheduler skips jobs when feature flags disable memory + --- + duration_ms: 11.361744 + type: 'test' + ... +# Subtest: normalizeMemoryItem applies durable schema defaults and coercions +ok 38 - normalizeMemoryItem applies durable schema defaults and coercions + --- + duration_ms: 2.791732 + type: 'test' + ... +# Subtest: validateMemoryItem reports field-level errors +ok 39 - validateMemoryItem reports field-level errors + --- + duration_ms: 0.562119 + type: 'test' + ... +# Subtest: ingestEvents rejects invalid writes and logs structured validation details +ok 40 - ingestEvents rejects invalid writes and logs structured validation details + --- + duration_ms: 6.509995 + type: 'test' + ... +# Subtest: loadMemoryPromptTemplates loads all memory prompt templates from disk +ok 41 - loadMemoryPromptTemplates loads all memory prompt templates from disk + --- + duration_ms: 2.92964 + type: 'test' + ... +# Subtest: summarizeEvents retries with repair prompt when first response is malformed JSON +ok 42 - summarizeEvents retries with repair prompt when first response is malformed JSON + --- + duration_ms: 2.465327 + type: 'test' + ... +# Subtest: summarizeEvents falls back to deterministic minimal summary with conservative importance when parsing fails twice +ok 43 - summarizeEvents falls back to deterministic minimal summary with conservative importance when parsing fails twice + --- + duration_ms: 1.808332 + type: 'test' + ... +# Subtest: CLI Smoke Test + # Subtest: should print usage when no args provided + ok 1 - should print usage when no args provided + --- + duration_ms: 182.814575 + type: 'test' + ... + # Subtest: should fail when checking without cadence + ok 2 - should fail when checking without cadence + --- + duration_ms: 166.438326 + type: 'test' + ... + # Subtest: should include paused agents in check output + ok 3 - should include paused agents in check output + --- + duration_ms: 1195.679046 + type: 'test' + ... + 1..3 +ok 44 - CLI Smoke Test + --- + duration_ms: 1546.840524 + type: 'suite' + ... +# Initializing torch configuration in /app/test-ops-MYawHJ/project1/torch... +# Created torch/roster.json +# Created torch/META_PROMPTS.md +# Created torch/scheduler-flow.md +# Created torch/daily-scheduler.md +# Created torch/weekly-scheduler.md +# Created torch/scripts/check-file-size.mjs +# Created torch/scripts/check-innerhtml.mjs +# Created 22 files in torch/prompts/daily/ +# Created 21 files in torch/prompts/weekly/ +# Created torch-config.json with namespace "torch-908041f9" +# Initialization complete. +# You can now customize the files in torch/ and torch/prompts/. +# Initializing torch configuration in /app/test-ops-MYawHJ/project2/torch... +# Initializing torch configuration in /app/test-ops-MYawHJ/project3/torch... +# Created torch/roster.json +# Created torch/META_PROMPTS.md +# Created torch/scheduler-flow.md +# Created torch/daily-scheduler.md +# Created torch/weekly-scheduler.md +# Created torch/scripts/check-file-size.mjs +# Created torch/scripts/check-innerhtml.mjs +# Created 22 files in torch/prompts/daily/ +# Created 21 files in torch/prompts/weekly/ +# Created torch-config.json with namespace "torch-4743a45e" +# Initialization complete. +# You can now customize the files in torch/ and torch/prompts/. +# Updating torch configuration in /app/test-ops-MYawHJ/project3/torch... +# Creating backup at torch/_backups/backup_2026-02-15T20-13-21-831Z... +# Updating static files... +# Updated META_PROMPTS.md +# Updated scheduler-flow.md +# Updated daily-scheduler.md +# Updated weekly-scheduler.md +# Skipped roster.json (preserved) +# Updating scripts... +# Skipped check-file-size.mjs (preserved) +# Skipped check-innerhtml.mjs (preserved) +# Updating prompts... +# daily/: 0 added, 0 updated, 22 preserved +# weekly/: 0 added, 0 updated, 21 preserved +# Update complete. +# Initializing torch configuration in /app/test-ops-MYawHJ/project4/torch... +# Created torch/roster.json +# Created torch/META_PROMPTS.md +# Created torch/scheduler-flow.md +# Created torch/daily-scheduler.md +# Created torch/weekly-scheduler.md +# Created torch/scripts/check-file-size.mjs +# Created torch/scripts/check-innerhtml.mjs +# Created 22 files in torch/prompts/daily/ +# Created 21 files in torch/prompts/weekly/ +# Created torch-config.json with namespace "torch-e3cdf2f1" +# Initialization complete. +# You can now customize the files in torch/ and torch/prompts/. +# Updating torch configuration in /app/test-ops-MYawHJ/project4/torch... +# Creating backup at torch/_backups/backup_2026-02-15T20-13-21-853Z... +# Updating static files... +# Updated META_PROMPTS.md +# Updated scheduler-flow.md +# Updated daily-scheduler.md +# Updated weekly-scheduler.md +# Overwrote roster.json (forced) +# Updating scripts... +# Updated check-file-size.mjs (forced) +# Updated check-innerhtml.mjs (forced) +# Updating prompts... +# daily/: 0 added, 22 updated, 0 preserved +# weekly/: 0 added, 21 updated, 0 preserved +# Update complete. +# Initializing torch configuration in /app/test-ops-MYawHJ/project5/torch... +# Created torch/roster.json +# Created torch/META_PROMPTS.md +# Created torch/scheduler-flow.md +# Created torch/daily-scheduler.md +# Created torch/weekly-scheduler.md +# Created torch/scripts/check-file-size.mjs +# Created torch/scripts/check-innerhtml.mjs +# Created 22 files in torch/prompts/daily/ +# Created 21 files in torch/prompts/weekly/ +# Created torch-config.json with namespace "torch-2384b7ee" +# Initialization complete. +# You can now customize the files in torch/ and torch/prompts/. +# Updating torch configuration in /app/test-ops-MYawHJ/project5/torch... +# Creating backup at torch/_backups/backup_2026-02-15T20-13-21-882Z... +# Updating static files... +# Updated META_PROMPTS.md +# Updated scheduler-flow.md +# Updated daily-scheduler.md +# Updated weekly-scheduler.md +# Skipped roster.json (preserved) +# Updating scripts... +# Skipped check-file-size.mjs (preserved) +# Skipped check-innerhtml.mjs (preserved) +# Updating prompts... +# daily/: 0 added, 0 updated, 22 preserved +# weekly/: 0 added, 0 updated, 21 preserved +# Update complete. +# Initializing torch configuration in /app/test-ops-MYawHJ/project_roster/torch... +# Created torch/roster.json +# Created torch/META_PROMPTS.md +# Created torch/scheduler-flow.md +# Created torch/daily-scheduler.md +# Created torch/weekly-scheduler.md +# Created torch/scripts/check-file-size.mjs +# Created torch/scripts/check-innerhtml.mjs +# Created 22 files in torch/prompts/daily/ +# Created 21 files in torch/prompts/weekly/ +# Created torch-config.json with namespace "torch-54b8ac08" +# Initialization complete. +# You can now customize the files in torch/ and torch/prompts/. +# Checking locks: namespace=torch-54b8ac08, cadence=daily, date=2026-02-15 +# Relays: wss://relay.damus.io, wss://nos.lol, wss://relay.primal.net +# {"event":"relay_health_snapshot","reason":"query:periodic","relays":[{"relay":"wss://nos.lol","score":0.6,"sampleSize":0,"successRate":0.5,"timeoutRate":0,"averageLatencyMs":null,"failureStreak":0,"quarantined":false,"quarantineRemainingMs":0,"cooldownMs":30000,"lastResultAt":null},{"relay":"wss://relay.damus.io","score":0.6,"sampleSize":0,"successRate":0.5,"timeoutRate":0,"averageLatencyMs":null,"failureStreak":0,"quarantined":false,"quarantineRemainingMs":0,"cooldownMs":30000,"lastResultAt":null},{"relay":"wss://relay.nostr.band","score":0.6,"sampleSize":0,"successRate":0.5,"timeoutRate":0,"averageLatencyMs":null,"failureStreak":0,"quarantined":false,"quarantineRemainingMs":0,"cooldownMs":30000,"lastResultAt":null},{"relay":"wss://relay.primal.net","score":0.6,"sampleSize":0,"successRate":0.5,"timeoutRate":0,"averageLatencyMs":null,"failureStreak":0,"quarantined":false,"quarantineRemainingMs":0,"cooldownMs":30000,"lastResultAt":null},{"relay":"wss://relay.snort.social","score":0.6,"sampleSize":0,"successRate":0.5,"timeoutRate":0,"averageLatencyMs":null,"failureStreak":0,"quarantined":false,"quarantineRemainingMs":0,"cooldownMs":30000,"lastResultAt":null}]} +# Initializing torch configuration in /app/test-ops-MYawHJ/project_config_init/torch... +# Created torch/roster.json +# Created torch/META_PROMPTS.md +# Created torch/scheduler-flow.md +# Created torch/daily-scheduler.md +# Created torch/weekly-scheduler.md +# Created torch/scripts/check-file-size.mjs +# Created torch/scripts/check-innerhtml.mjs +# Created 22 files in torch/prompts/daily/ +# Created 21 files in torch/prompts/weekly/ +# Created torch-config.json with namespace "torch-efa899f0" +# Initialization complete. +# You can now customize the files in torch/ and torch/prompts/. +# Subtest: cmdInit creates directory structure and files +ok 45 - cmdInit creates directory structure and files + --- + duration_ms: 17.36701 + type: 'test' + ... +# Subtest: cmdInit fails if directory exists without force +ok 46 - cmdInit fails if directory exists without force + --- + duration_ms: 2.237348 + type: 'test' + ... +# Subtest: cmdUpdate preserves modified prompt +ok 47 - cmdUpdate preserves modified prompt + --- + duration_ms: 21.311809 + type: 'test' + ... +# Subtest: cmdUpdate overwrites prompt with force +ok 48 - cmdUpdate overwrites prompt with force + --- + duration_ms: 29.879284 + type: 'test' + ... +# Subtest: cmdUpdate creates backup +ok 49 - cmdUpdate creates backup + --- + duration_ms: 20.097993 + type: 'test' + ... +# Subtest: torch-lock check respects local roster +ok 50 - torch-lock check respects local roster + --- + duration_ms: 1015.011829 + type: 'test' + ... +# Subtest: cmdInit creates torch-config.json with random namespace +ok 51 - cmdInit creates torch-config.json with random namespace + --- + duration_ms: 10.937543 + type: 'test' + ... +# Published to 1/3 relays (required=1, timeout=80ms) +# Subtest: relay fanout quorum integration + # Subtest: passes with required=1 when one relay succeeds and two fail + ok 1 - passes with required=1 when one relay succeeds and two fail + --- + duration_ms: 133.403363 + type: 'test' + ... + # Subtest: fails with quorum error for required=1 when all relays fail + ok 2 - fails with quorum error for required=1 when all relays fail + --- + duration_ms: 93.429309 + type: 'test' + ... + # Subtest: fails with required=2 when only one relay succeeds + ok 3 - fails with required=2 when only one relay succeeds + --- + duration_ms: 90.435224 + type: 'test' + ... +# Published to 2/3 relays (required=2, timeout=80ms) + # Subtest: passes when delayed successes reach quorum before timeout + ok 4 - passes when delayed successes reach quorum before timeout + --- + duration_ms: 89.852649 + type: 'test' + ... + 1..4 +ok 52 - relay fanout quorum integration + --- + duration_ms: 408.877143 + type: 'suite' + ... +# Subtest: raises all-relays-down duration alert after threshold +ok 53 - raises all-relays-down duration alert after threshold + --- + duration_ms: 1.714447 + type: 'test' + ... +# Subtest: raises success-rate alert when rolling health falls below threshold +ok 54 - raises success-rate alert when rolling health falls below threshold + --- + duration_ms: 0.525892 + type: 'test' + ... +# Subtest: fails required memory policy when retrieval/storage evidence is missing +ok 55 - fails required memory policy when retrieval/storage evidence is missing + --- + duration_ms: 937.45108 + type: 'test' + ... +# Subtest: accepts required memory policy when markers or artifacts are produced +ok 56 - accepts required memory policy when markers or artifacts are produced + --- + duration_ms: 984.200775 + type: 'test' + ... +# Subtest: records backend failure metadata when lock command exits with code 2 +not ok 57 - records backend failure metadata when lock command exits with code 2 + --- + duration_ms: 1007.754548 + type: 'test' + location: '/app/test/run-scheduler-cycle-memory-policy.test.mjs:198:1' + failureType: 'testCodeFailure' + error: |- + The input did not match the regular expression /lock_command: 'AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily'/. Input: + + '---\n' + + 'cadence: daily\n' + + 'agent: agent-a\n' + + 'status: failed\n' + + "reason: 'Lock backend error'\n" + + "detail: 'Lock backend error (relay_publish_quorum_failure) after 3 attempt(s). Prompt not executed. Recommended auto-remediation: Likely backend/relay connectivity issue during lock acquisition. Retry window: 60 minute(s) (max deferrals: 3). Retry command: npm run scheduler:daily Run health check: npm run lock:health -- --cadence daily Review incident runbook: docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md'\n" + + 'created_at: 2026-02-15T20:13:26.081Z\n' + + 'timestamp: 2026-02-15T20:13:26.081Z\n' + + "platform: 'codex'\n" + + "failure_category: 'lock_backend_error'\n" + + "failure_class: 'backend_unavailable'\n" + + "lock_attempts_total: '3'\n" + + "lock_backoff_schedule_ms: '303, 565'\n" + + "lock_correlation_id: 'daily:agent-a:2026-02-15'\n" + + "lock_attempt_id: '3'\n" + + "lock_failure_reason_distribution: '{}'\n" + + "backend_category: 'relay_publish_quorum_failure'\n" + + "deferral_attempt_count: '1'\n" + + "deferral_first_failure_timestamp: '2026-02-15T20:13:26.076Z'\n" + + "lock_command: 'AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily '\n" + + "lock_idempotency_key: 'daily:agent-a:2026-02-15:fabb960c-72be-4427-85fa-a9df9051fbce'\n" + + "lock_stderr_excerpt: 'publish failed to all relays token=[REDACTED] SECRET_KEY=[REDACTED]'\n" + + "lock_stdout_excerpt: 'websocket: connection refused'\n" + + '---\n' + + '# Scheduler failed\n' + + '- reason: Lock backend error\n' + + '- detail: Lock backend error (relay_publish_quorum_failure) after 3 attempt(s). Prompt not executed. Recommended auto-remediation: Likely backend/relay connectivity issue during lock acquisition. Retry window: 60 minute(s) (max deferrals: 3). Retry command: npm run scheduler:daily Run health check: npm run lock:health -- --cadence daily Review incident runbook: docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md\n' + + '- platform: codex\n' + + '- failure_category: lock_backend_error\n' + + '- failure_class: backend_unavailable\n' + + '- lock_attempts_total: 3\n' + + '- lock_backoff_schedule_ms: 303, 565\n' + + '- lock_correlation_id: daily:agent-a:2026-02-15\n' + + '- lock_attempt_id: 3\n' + + '- lock_failure_reason_distribution: {}\n' + + '- backend_category: relay_publish_quorum_failure\n' + + '- deferral_attempt_count: 1\n' + + '- deferral_first_failure_timestamp: 2026-02-15T20:13:26.076Z\n' + + '- lock_command: AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily \n' + + '- lock_idempotency_key: daily:agent-a:2026-02-15:fabb960c-72be-4427-85fa-a9df9051fbce\n' + + '- lock_stderr_excerpt: publish failed to all relays token=[REDACTED] SECRET_KEY=[REDACTED]\n' + + '- lock_stdout_excerpt: websocket: connection refused' + + code: 'ERR_ASSERTION' + name: 'AssertionError' + expected: + actual: |- + --- + cadence: daily + agent: agent-a + status: failed + reason: 'Lock backend error' + detail: 'Lock backend error (relay_publish_quorum_failure) after 3 attempt(s). Prompt not executed. Recommended auto-remediation: Likely backend/relay connectivity issue during lock acquisition. Retry window: 60 minute(s) (max deferrals: 3). Retry command: npm run scheduler:daily Run health check: npm run lock:health -- --cadence daily Review incident runbook: docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md' + created_at: 2026-02-15T20:13:26.081Z + timestamp: 2026-02-15T20:13:26.081Z + platform: 'codex' + failure_category: 'lock_backend_error' + failure_class: 'backend_unavailable' + lock_attempts_total: '3' + lock_backoff_schedule_ms: '303, 565' + lock_correlation_id: 'daily:agent-a:2026-02-15' + lock_attempt_id: '3' + lock_failure_reason_distribution: '{}' + backend_category: 'relay_publish_quorum_failure' + deferral_attempt_count: '1' + deferral_first_failure_timestamp: '2026-02-15T20:13:26.076Z' + lock_command: 'AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily ' + lock_idempotency_key: 'daily:agent-a:2026-02-15:fabb960c-72be-4427-85fa-a9df9051fbce' + lock_stderr_excerpt: 'publish failed to all relays token=[REDACTED] SECRET_KEY=[REDACTED]' + lock_stdout_excerpt: 'websocket: connection refused' + --- + # Scheduler failed + - reason: Lock backend error + - detail: Lock backend error (relay_publish_quorum_failure) after 3 attempt(s). Prompt not executed. Recommended auto-remediation: Likely backend/relay connectivity issue during lock acquisition. Retry window: 60 minute(s) (max deferrals: 3). Retry command: npm run scheduler:daily Run health check: npm run lock:health -- --cadence daily Review incident runbook: docs/agent-handoffs/learnings/2026-02-15-relay-health-preflight-job.md + - platform: codex + - failure_category: lock_backend_error + - failure_class: backend_unavailable + - lock_attempts_total: 3 + - lock_backoff_schedule_ms: 303, 565 + - lock_correlation_id: daily:agent-a:2026-02-15 + - lock_attempt_id: 3 + - lock_failure_reason_distribution: {} + - backend_category: relay_publish_quorum_failure + - deferral_attempt_count: 1 + - deferral_first_failure_timestamp: 2026-02-15T20:13:26.076Z + - lock_command: AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily + - lock_idempotency_key: daily:agent-a:2026-02-15:fabb960c-72be-4427-85fa-a9df9051fbce + - lock_stderr_excerpt: publish failed to all relays token=[REDACTED] SECRET_KEY=[REDACTED] + - lock_stdout_excerpt: websocket: connection refused + operator: 'match' + stack: |- + TestContext. (file:///app/test/run-scheduler-cycle-memory-policy.test.mjs:222:10) + async Test.run (node:internal/test_runner/test:1054:7) + async Test.processPendingSubtests (node:internal/test_runner/test:744:7) + ... +# Subtest: defers backend failures in non-strict mode and reuses idempotency key on successful retry +ok 58 - defers backend failures in non-strict mode and reuses idempotency key on successful retry + --- + duration_ms: 912.672612 + type: 'test' + ... +# Subtest: fails after exceeding non-strict deferral budget +ok 59 - fails after exceeding non-strict deferral budget + --- + duration_ms: 248.626971 + type: 'test' + ... +# Subtest: strict lock mode fails immediately without deferral +ok 60 - strict lock mode fails immediately without deferral + --- + duration_ms: 106.189072 + type: 'test' + ... +# Subtest: skips lock health preflight by default when not enabled +ok 61 - skips lock health preflight by default when not enabled + --- + duration_ms: 406.810088 + type: 'test' + ... +# Subtest: fails scheduler with preflight metadata when lock health preflight is enabled +ok 62 - fails scheduler with preflight metadata when lock health preflight is enabled + --- + duration_ms: 134.668686 + type: 'test' + ... +# Subtest: categorizes unreadable prompt files as prompt_parse_error +ok 63 - categorizes unreadable prompt files as prompt_parse_error + --- + duration_ms: 95.881715 + type: 'test' + ... +# Subtest: categorizes invalid prompt schema as prompt_schema_error +ok 64 - categorizes invalid prompt schema as prompt_schema_error + --- + duration_ms: 96.324439 + type: 'test' + ... +# Subtest: categorizes handoff runtime failures as execution_error +ok 65 - categorizes handoff runtime failures as execution_error + --- + duration_ms: 367.972373 + type: 'test' + ... +# Subtest: retries lock acquisition for backend error and succeeds on a later attempt +ok 66 - retries lock acquisition for backend error and succeeds on a later attempt + --- + duration_ms: 429.659053 + type: 'test' + ... +# Subtest: stops retrying after configured lock backend retries are exhausted +ok 67 - stops retrying after configured lock backend retries are exhausted + --- + duration_ms: 107.180645 + type: 'test' + ... +# Subtest: does not use backend retry flow for exit code 3 lock conflicts +ok 68 - does not use backend retry flow for exit code 3 lock conflicts + --- + duration_ms: 452.544617 + type: 'test' + ... +# Subtest: runs selected scheduler prompt through configured runner command +ok 69 - runs selected scheduler prompt through configured runner command + --- + duration_ms: 363.385434 + type: 'test' + ... +# Subtest: propagates non-zero runner exit code +ok 70 - propagates non-zero runner exit code + --- + duration_ms: 381.323533 + type: 'test' + ... +# Subtest: scheduler lock backend failure artifact matches required frontmatter schema +ok 71 - scheduler lock backend failure artifact matches required frontmatter schema + --- + duration_ms: 118.780177 + type: 'test' + ... +# Subtest: schema contract catches missing required lock-backend field +ok 72 - schema contract catches missing required lock-backend field + --- + duration_ms: 2.472888 + type: 'test' + ... +# Subtest: schema contract catches misnamed lock-backend key +ok 73 - schema contract catches misnamed lock-backend key + --- + duration_ms: 0.61455 + type: 'test' + ... +# Subtest: lock preflight e2e: successful lock writes completed status snapshot +ok 74 - lock preflight e2e: successful lock writes completed status snapshot + --- + duration_ms: 423.832617 + type: 'test' + ... +# Subtest: lock preflight e2e: exit code 2 quorum failure persists failed backend status and prompt-not-started marker +ok 75 - lock preflight e2e: exit code 2 quorum failure persists failed backend status and prompt-not-started marker + --- + duration_ms: 102.67988 + type: 'test' + ... +# Subtest: lock preflight e2e: non-lock failure exits failed without prompt parse/schema classification +ok 76 - lock preflight e2e: non-lock failure exits failed without prompt parse/schema classification + --- + duration_ms: 93.074425 + type: 'test' + ... +# Subtest: Scheduler Ratchet Logic (Log Checking) + # Subtest: excludes agents completed today (daily) + ok 1 - excludes agents completed today (daily) + --- + duration_ms: 2.682381 + type: 'test' + ... + # Subtest: excludes agents completed this week (weekly) + ok 2 - excludes agents completed this week (weekly) + --- + duration_ms: 0.583067 + type: 'test' + ... + # Subtest: combines locked and completed agents in exclusion list + ok 3 - combines locked and completed agents in exclusion list + --- + duration_ms: 0.625284 + type: 'test' + ... + # Subtest: ignores logs when --ignore-logs is set + ok 4 - ignores logs when --ignore-logs is set + --- + duration_ms: 0.360757 + type: 'test' + ... + # Subtest: handles missing log directory gracefully + ok 5 - handles missing log directory gracefully + --- + duration_ms: 0.41839 + type: 'test' + ... + 1..5 +ok 77 - Scheduler Ratchet Logic (Log Checking) + --- + duration_ms: 6.213062 + type: 'suite' + ... +# Subtest: Scheduler cycle ordering guarantees + # Subtest: keeps optional preflight before lock-acquire and preserves handoff/artifact/validation/complete ordering + ok 1 - keeps optional preflight before lock-acquire and preserves handoff/artifact/validation/complete ordering + --- + duration_ms: 13.095275 + type: 'test' + ... + # Subtest: includes lock backend failure metadata and classifier checkpoints + ok 2 - includes lock backend failure metadata and classifier checkpoints + --- + duration_ms: 2.476803 + type: 'test' + ... + 1..2 +ok 78 - Scheduler cycle ordering guarantees + --- + duration_ms: 16.171833 + type: 'suite' + ... +# Subtest: summarizes recent lock reliability by platform/cadence/backend/relay +ok 79 - summarizes recent lock reliability by platform/cadence/backend/relay + --- + duration_ms: 90.135355 + type: 'test' + ... +# Subtest: loadTorchConfig (Unit Tests with Mocked FS) + # Subtest: loads valid config correctly + ok 1 - loads valid config correctly + --- + duration_ms: 3.698182 + type: 'test' + ... + # Subtest: returns default config when file is missing + ok 2 - returns default config when file is missing + --- + duration_ms: 0.560655 + type: 'test' + ... + # Subtest: throws error on malformed JSON + ok 3 - throws error on malformed JSON + --- + duration_ms: 0.904083 + type: 'test' + ... + # Subtest: caches the config and ignores subsequent fs calls + ok 4 - caches the config and ignores subsequent fs calls + --- + duration_ms: 0.735535 + type: 'test' + ... + 1..4 +ok 80 - loadTorchConfig (Unit Tests with Mocked FS) + --- + duration_ms: 7.728387 + type: 'suite' + ... +# Subtest: torch-config + # Subtest: getTorchConfigPath + # Subtest: returns default path when env var is not set + ok 1 - returns default path when env var is not set + --- + duration_ms: 1.038823 + type: 'test' + ... + # Subtest: returns custom path when env var is set + ok 2 - returns custom path when env var is set + --- + duration_ms: 0.373229 + type: 'test' + ... + 1..2 + ok 1 - getTorchConfigPath + --- + duration_ms: 2.906623 + type: 'suite' + ... + # Subtest: parseTorchConfig + # Subtest: parses new lock backend knobs + ok 1 - parses new lock backend knobs + --- + duration_ms: 1.386898 + type: 'test' + ... + 1..1 + ok 2 - parseTorchConfig + --- + duration_ms: 1.560858 + type: 'suite' + ... + # Subtest: loadTorchConfig + # Subtest: throws fatal error for invalid relay URL + ok 1 - throws fatal error for invalid relay URL + --- + duration_ms: 1.653095 + type: 'test' + ... + # Subtest: throws fatal error for invalid timeout range + ok 2 - throws fatal error for invalid timeout range + --- + duration_ms: 0.918183 + type: 'test' + ... + # Subtest: throws error for malformed JSON + ok 3 - throws error for malformed JSON + --- + duration_ms: 0.676975 + type: 'test' + ... + 1..3 + ok 3 - loadTorchConfig + --- + duration_ms: 4.025461 + type: 'suite' + ... + # Subtest: backend getters + # Subtest: returns default relays + ok 1 - returns default relays + --- + duration_ms: 0.716481 + type: 'test' + ... + # Subtest: validates env knob values + ok 2 - validates env knob values + --- + duration_ms: 0.906231 + type: 'test' + ... + 1..2 + ok 4 - backend getters + --- + duration_ms: 3.323674 + type: 'suite' + ... + 1..4 +ok 81 - torch-config + --- + duration_ms: 13.000003 + type: 'suite' + ... +# Subtest: Date Utilities + # Subtest: getIsoWeekStr + # Subtest: returns correct ISO week for simple cases + ok 1 - returns correct ISO week for simple cases + --- + duration_ms: 1.40434 + type: 'test' + ... + # Subtest: handles year boundaries correctly + ok 2 - handles year boundaries correctly + --- + duration_ms: 0.231904 + type: 'test' + ... + # Subtest: handles leap years + ok 3 - handles leap years + --- + duration_ms: 1.523179 + type: 'test' + ... + # Subtest: handles empty input (defaults to today) + ok 4 - handles empty input (defaults to today) + --- + duration_ms: 0.382726 + type: 'test' + ... + # Subtest: handles invalid input gracefully + ok 5 - handles invalid input gracefully + --- + duration_ms: 0.409621 + type: 'test' + ... + 1..5 + ok 1 - getIsoWeekStr + --- + duration_ms: 5.224416 + type: 'suite' + ... + 1..1 +ok 82 - Date Utilities + --- + duration_ms: 5.840199 + type: 'suite' + ... +# Subtest: passes when artifacts include metadata and failure IDs map to unresolved known issue entries +ok 83 - passes when artifacts include metadata and failure IDs map to unresolved known issue entries + --- + duration_ms: 85.064858 + type: 'test' + ... +# Subtest: fails when required metadata is missing from artifacts +ok 84 - fails when required metadata is missing from artifacts + --- + duration_ms: 74.757197 + type: 'test' + ... +# Subtest: fails when failure identifiers are not cross-linked to known issues or incidents +ok 85 - fails when failure identifiers are not cross-linked to known issues or incidents + --- + duration_ms: 75.580496 + type: 'test' + ... +# Subtest: schema validation covers required fields, coercion, and rejection paths +ok 86 - schema validation covers required fields, coercion, and rejection paths + --- + duration_ms: 3.908932 + type: 'test' + ... +# Subtest: summarizer parser accepts valid json and recovers after malformed json response +ok 87 - summarizer parser accepts valid json and recovers after malformed json response + --- + duration_ms: 5.087917 + type: 'test' + ... +# Subtest: ingest integration persists memory records and links embeddings from event batches +ok 88 - ingest integration persists memory records and links embeddings from event batches + --- + duration_ms: 2.860986 + type: 'test' + ... +# Subtest: retrieval ranking blends semantic, importance, and recency signals +ok 89 - retrieval ranking blends semantic, importance, and recency signals + --- + duration_ms: 1.096624 + type: 'test' + ... +# Subtest: pruner policy protects pins, merges duplicates, and picks archive/delete outcomes +ok 90 - pruner policy protects pins, merges duplicates, and picks archive/delete outcomes + --- + duration_ms: 1.781503 + type: 'test' + ... +# Subtest: scheduler smoke test handles mocked clock and lock contention +ok 91 - scheduler smoke test handles mocked clock and lock contention + --- + duration_ms: 32.326914 + type: 'test' + ... +1..91 +# tests 146 +# suites 23 +# pass 145 +# fail 1 +# cancelled 0 +# skipped 0 +# todo 0 +# duration_ms 8379.403849