Add lock deferral policy for scheduler backend failures

This commit is contained in:
thePR0M3TH3AN committed 2026-02-14 19:23:37 -05:00
1 parent 430558558e
commit 9aad0d933d
4 files changed
+324 -8

No files matched your search

+162 -7
View File
@@ -4,6 +4,7 @@
import fs from 'node:fs/promises';
import path from 'node:path';
import { spawn } from 'node:child_process';
import { randomUUID } from 'node:crypto';
const VALID_CADENCES = new Set(['daily', 'weekly']);
const ALL_EXCLUDED_REASON = 'All roster tasks currently claimed by other agents';
@@ -314,6 +315,18 @@ async function getSchedulerConfig(cadence, { isInteractive }) {
lockHealthPreflightFromConfig,
);
const lockHealthPreflightSkip = parseBooleanFlag(process.env.SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT, false);
const strictLock = parseBooleanFlag(
process.env.SCHEDULER_STRICT_LOCK,
parseBooleanFlag(scheduler.strict_lock, true),
);
const degradedLockRetryWindowMs = parseNonNegativeInt(
process.env.SCHEDULER_DEGRADED_LOCK_RETRY_WINDOW_MS,
parseNonNegativeInt(scheduler.degraded_lock_retry_window, 3600000),
);
const maxDeferrals = parseNonNegativeInt(
process.env.SCHEDULER_MAX_DEFERRALS,
parseNonNegativeInt(scheduler.max_deferrals, 3),
);
return {
firstPrompt: scheduler.firstPromptByCadence?.[cadence] || null,
@@ -331,6 +344,11 @@ async function getSchedulerConfig(cadence, { isInteractive }) {
enabled: lockHealthPreflightEnabled,
skip: lockHealthPreflightSkip,
},
lockFailurePolicy: {
strictLock,
degradedLockRetryWindowMs,
maxDeferrals,
},
memoryPolicy: {
mode,
retrieveCommand: typeof memoryPolicyRaw.retrieveCommand === 'string' && memoryPolicyRaw.retrieveCommand.trim()
@@ -347,6 +365,40 @@ async function getSchedulerConfig(cadence, { isInteractive }) {
};
}
function getRunDateKey(now = new Date()) {
return now.toISOString().slice(0, 10);
}
async function readRunState(cadence) {
const statePath = path.resolve(process.cwd(), 'task-logs', cadence, '.scheduler-run-state.json');
const fallback = { run_date: getRunDateKey(), lock_deferral: null };
const raw = await readJson(statePath, fallback);
if (!raw || typeof raw !== 'object') {
return { statePath, state: fallback };
}
if (raw.run_date !== getRunDateKey()) {
return { statePath, state: fallback };
}
return {
statePath,
state: {
run_date: raw.run_date,
lock_deferral: raw.lock_deferral && typeof raw.lock_deferral === 'object' ? raw.lock_deferral : null,
},
};
}
async function writeRunState(statePath, state) {
await fs.mkdir(path.dirname(statePath), { recursive: true });
await fs.writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, 'utf8');
}
function createIdempotencyKey({ cadence, selectedAgent, runDate }) {
return `${cadence}:${selectedAgent}:${runDate}:${randomUUID()}`;
}
async function runLockHealthPreflight({ cadence, platform }) {
const preflight = await runCommand('node', ['scripts/agent/check-relay-health.mjs', '--cadence', cadence], {
env: { AGENT_PLATFORM: platform },
@@ -372,14 +424,19 @@ function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function acquireLockWithRetry({ selectedAgent, cadence, platform, lockRetry }) {
async function acquireLockWithRetry({ selectedAgent, cadence, platform, lockRetry, idempotencyKey }) {
const lockCommandArgs = ['run', 'lock:lock', '--', '--agent', selectedAgent, '--cadence', cadence];
const backoffScheduleMs = [];
let attempts = 0;
while (true) {
attempts += 1;
const result = await runCommand('npm', lockCommandArgs, { env: { AGENT_PLATFORM: platform } });
const result = await runCommand('npm', lockCommandArgs, {
env: {
AGENT_PLATFORM: platform,
...(idempotencyKey ? { SCHEDULER_LOCK_IDEMPOTENCY_KEY: idempotencyKey } : {}),
},
});
if (result.code !== 2) {
return { result, attempts, backoffScheduleMs };
@@ -481,6 +538,8 @@ async function main() {
}
while (true) {
const { statePath: runStatePath, state: schedulerRunState } = await readRunState(cadence);
if (schedulerConfig.lockHealthPreflight.enabled && !schedulerConfig.lockHealthPreflight.skip) {
const preflight = await runLockHealthPreflight({ cadence, platform });
if (preflight.code !== 0) {
@@ -492,6 +551,7 @@ async function main() {
reason: 'Lock backend unavailable preflight',
detail: `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`,
metadata: {
failure_class: 'backend_unavailable',
preflight_failure_category: preflight.failureCategory,
relay_list: preflight.relayList.join(', ') || '(none)',
preflight_stderr_excerpt: preflight.stderrExcerpt || '(empty)',
@@ -543,11 +603,16 @@ async function main() {
process.exit(1);
}
const deferralForAgent = schedulerRunState.lock_deferral?.selected_agent === selectedAgent
? schedulerRunState.lock_deferral
: null;
const lockAttempt = await acquireLockWithRetry({
selectedAgent,
cadence,
platform,
lockRetry: schedulerConfig.lockRetry,
idempotencyKey: deferralForAgent?.idempotency_key,
});
const lockResult = lockAttempt.result;
@@ -562,6 +627,46 @@ async function main() {
const stderrExcerpt = excerptText(lockResult.stderr);
const stdoutExcerpt = excerptText(lockResult.stdout);
const backoffSchedule = lockAttempt.backoffScheduleMs.join(', ');
const existingDeferral = deferralForAgent || {};
const firstFailureAt = existingDeferral.first_failure_timestamp || new Date().toISOString();
const firstFailureMs = parseDateValue(firstFailureAt) || Date.now();
const nowMs = Date.now();
const deferralAttemptCount = parseNonNegativeInt(existingDeferral.attempt_count, 0) + 1;
const idempotencyKey = existingDeferral.idempotency_key
|| createIdempotencyKey({ cadence, selectedAgent, runDate: schedulerRunState.run_date });
const withinRetryWindow = nowMs - firstFailureMs <= schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs;
const withinDeferralBudget = deferralAttemptCount <= schedulerConfig.lockFailurePolicy.maxDeferrals;
if (!schedulerConfig.lockFailurePolicy.strictLock && withinRetryWindow && withinDeferralBudget) {
schedulerRunState.lock_deferral = {
attempt_count: deferralAttemptCount,
first_failure_timestamp: firstFailureAt,
backend_category: backendCategory,
idempotency_key: idempotencyKey,
selected_agent: selectedAgent,
};
await writeRunState(runStatePath, schedulerRunState);
await writeLog({
cadence,
agent: selectedAgent,
status: 'deferred',
platform,
reason: 'Lock backend deferred',
detail: `Deferred after lock backend failure (${backendCategory}); retry window active and deferral budget remaining.`,
metadata: {
failure_class: 'backend_unavailable',
deferral_attempt_count: deferralAttemptCount,
deferral_first_failure_timestamp: firstFailureAt,
backend_category: backendCategory,
lock_idempotency_key: idempotencyKey,
},
});
process.exit(0);
}
schedulerRunState.lock_deferral = null;
await writeRunState(runStatePath, schedulerRunState);
await writeLog({
cadence,
agent: selectedAgent,
@@ -570,10 +675,14 @@ async function main() {
reason: 'Lock backend error',
detail: `Lock backend error (${backendCategory}) after ${lockAttempt.attempts} attempt(s). Retry ${lockCommand} after verifying relay connectivity, relay URLs, and DNS/network status.`,
metadata: {
failure_class: 'backend_unavailable',
lock_attempts_total: lockAttempt.attempts,
lock_backoff_schedule_ms: backoffSchedule || '(none)',
backend_category: backendCategory,
deferral_attempt_count: deferralAttemptCount,
deferral_first_failure_timestamp: firstFailureAt,
lock_command: lockCommand,
lock_idempotency_key: idempotencyKey,
lock_stderr_excerpt: stderrExcerpt || '(empty)',
lock_stdout_excerpt: stdoutExcerpt || '(empty)',
},
@@ -582,6 +691,8 @@ async function main() {
}
if (lockResult.code !== 0) {
schedulerRunState.lock_deferral = null;
await writeRunState(runStatePath, schedulerRunState);
await writeLog({
cadence,
agent: selectedAgent,
@@ -593,6 +704,8 @@ async function main() {
}
const promptPath = path.resolve(process.cwd(), 'src/prompts', cadence, `${selectedAgent}.md`);
schedulerRunState.lock_deferral = null;
await writeRunState(runStatePath, schedulerRunState);
const runArtifactSince = new Date().toISOString();
const runStartMs = Date.parse(runArtifactSince);
const outputChunks = [];
@@ -603,7 +716,15 @@ async function main() {
});
outputChunks.push(retrieveResult.stdout, retrieveResult.stderr);
if (retrieveResult.code !== 0) {
await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Memory retrieval command failed', detail: schedulerConfig.memoryPolicy.retrieveCommand });
await writeLog({
cadence,
agent: selectedAgent,
status: 'failed',
platform,
reason: 'Memory retrieval command failed',
detail: schedulerConfig.memoryPolicy.retrieveCommand,
metadata: { failure_class: 'prompt_validation_error' },
});
process.exit(retrieveResult.code);
}
}
@@ -614,14 +735,30 @@ async function main() {
});
outputChunks.push(handoff.stdout, handoff.stderr);
if (handoff.code !== 0) {
await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Prompt/handoff execution failed', detail: 'Handoff callback failed.' });
await writeLog({
cadence,
agent: selectedAgent,
status: 'failed',
platform,
reason: 'Prompt/handoff execution failed',
detail: 'Handoff callback failed.',
metadata: { failure_class: 'prompt_validation_error' },
});
process.exit(handoff.code);
}
} else {
const detail = schedulerConfig.missingHandoffCommandForMode
? 'Missing scheduler handoff command for non-interactive run. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.'
: 'No handoff callback configured. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.';
await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Prompt/handoff execution failed', detail });
await writeLog({
cadence,
agent: selectedAgent,
status: 'failed',
platform,
reason: 'Prompt/handoff execution failed',
detail,
metadata: { failure_class: 'prompt_validation_error' },
});
process.exit(1);
}
@@ -631,7 +768,15 @@ async function main() {
});
outputChunks.push(storeResult.stdout, storeResult.stderr);
if (storeResult.code !== 0) {
await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Memory storage command failed', detail: schedulerConfig.memoryPolicy.storeCommand });
await writeLog({
cadence,
agent: selectedAgent,
status: 'failed',
platform,
reason: 'Memory storage command failed',
detail: schedulerConfig.memoryPolicy.storeCommand,
metadata: { failure_class: 'prompt_validation_error' },
});
process.exit(storeResult.code);
}
}
@@ -661,6 +806,7 @@ async function main() {
platform,
reason: 'Required memory steps not verified',
detail: `Missing evidence for: ${missingSteps.join(', ')}`,
metadata: { failure_class: 'prompt_validation_error' },
});
process.exit(1);
}
@@ -692,6 +838,7 @@ async function main() {
platform,
reason: 'Missing required run artifacts',
detail,
metadata: { failure_class: 'prompt_validation_error' },
});
process.exit(artifactCheck.code);
}
@@ -701,7 +848,15 @@ async function main() {
if (!parts.length) continue;
const result = await runCommand(parts[0], parts.slice(1));
if (result.code !== 0) {
await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Validation failed', detail: validation });
await writeLog({
cadence,
agent: selectedAgent,
status: 'failed',
platform,
reason: 'Validation failed',
detail: validation,
metadata: { failure_class: 'prompt_validation_error' },
});
process.exit(result.code);
}
}
+8
View File
@@ -65,6 +65,9 @@ Common settings:
- `scheduler.firstPromptByCadence.daily` / `.weekly` — first-run scheduler starting agent.
- `scheduler.handoffCommandByCadence.daily` / `.weekly` — shell command run after lock acquisition; command must use `SCHEDULER_AGENT`, `SCHEDULER_CADENCE`, and `SCHEDULER_PROMPT_PATH` provided by `scripts/agent/run-scheduler-cycle.mjs`.
- `scheduler.paused.daily` / `.weekly` — array of agent names to exclude from scheduler rotation.
- `scheduler.strict_lock` — lock backend policy switch (default: `true`); when `false`, scheduler defers backend-unavailable lock failures before converting the run to failed.
- `scheduler.degraded_lock_retry_window` — non-strict deferral window in milliseconds; backend lock failures outside this window immediately consume failure budget and mark run failed.
- `scheduler.max_deferrals` — max number of non-strict lock deferrals allowed in-window before scheduler records a hard failure.
Default first-run daily scheduler prompt is `scheduler-update-agent`.
@@ -72,6 +75,11 @@ For weekly repository-fit maintenance, TORCH also includes `src/prompts/weekly/r
Operational note: scheduler handoff commands are treated as required execution steps. A non-zero exit code (or missing command) is a hard failure: the scheduler writes a `_failed.md` task log, exits immediately, and does not publish `lock:complete` for that run.
Scheduler failure classes in task logs:
- `backend_unavailable` — lock backend unavailable preflight failures and lock acquisition backend exit code `2` failures/deferrals.
- `prompt_validation_error` — prompt/handoff, memory evidence, artifact verification, and validation command failures.
## Lock backend production defaults
+8 -1
View File
@@ -148,17 +148,24 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co
- Exit `0`: lock acquired, continue.
- Exit `3`: race lost/already locked, return to step 2.
- Exit `2`: lock backend error, write `_failed.md` with reason `Lock backend error`, and include failure metadata fields:
- Exit `2`: lock backend error.
- If `scheduler.strict_lock` is `false`, defer the run while both budget constraints are still satisfied:
- `degraded_lock_retry_window` (ms) since first backend failure has not elapsed.
- `max_deferrals` has not been exceeded.
- Record deferral metadata in scheduler run state (`attempt_count`, `first_failure_timestamp`, `backend_category`, and preserved idempotency key).
- Otherwise write `_failed.md` with reason `Lock backend error`, and include failure metadata fields:
- `backend_category` (classified backend failure category)
- `lock_command` (raw lock command for retry)
- `lock_stderr_excerpt` (redacted stderr snippet)
- `lock_stdout_excerpt` (redacted stdout snippet)
- Include `failure_class: backend_unavailable` for both deferred and failed backend-unavailable lock outcomes.
- Keep generic reason text for compatibility, but append actionable retry guidance in `detail` using the command from `lock_command`.
9. Execute `<prompt_dir>/<prompt-file>` end-to-end via configured handoff command.
- Scheduler automation runs `scheduler.handoffCommandByCadence.<cadence>` with environment variables for cadence/agent/prompt path.
- If no handoff command is configured for the cadence, write `_failed.md` and stop.
- Prompt/handoff and prompt validation failures should emit `failure_class: prompt_validation_error`.
10. Confirm memory contract completion:
@@ -37,6 +37,7 @@ async function setupFixture({
lockShellBody = '',
roster = ['agent-a'],
lockHealthPreflight = undefined,
schedulerPolicy = {},
preflightScript = '#!/usr/bin/env node\nconsole.log(JSON.stringify({ ok: true, relays: ["wss://relay.test"] }));\nprocess.exit(0);\n',
}) {
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'scheduler-memory-policy-'));
@@ -69,6 +70,7 @@ async function setupFixture({
handoffCommandByCadence: { daily: 'echo HANDOFF_OK' },
validationCommandsByCadence: { daily: ['true'] },
memoryPolicyByCadence: { daily: memoryPolicy },
...schedulerPolicy,
...(lockHealthPreflight === undefined ? {} : { lockHealthPreflight }),
},
}, null, 2),
@@ -186,6 +188,7 @@ test('records backend failure metadata when lock command exits with code 2', { c
const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8');
assert.match(failedBody, /reason: Lock backend error/);
assert.match(failedBody, /failure_class: 'backend_unavailable'/);
assert.match(failedBody, /backend_category: 'publish failed to all relays'/);
assert.match(failedBody, /lock_command: 'AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily'/);
assert.match(failedBody, /lock_stderr_excerpt: 'publish failed to all relays token=\[REDACTED\] SECRET_KEY=\[REDACTED\]'/);
@@ -194,6 +197,149 @@ test('records backend failure metadata when lock command exits with code 2', { c
assert.match(failedBody, /platform: 'codex'/);
});
test('defers backend failures in non-strict mode and reuses idempotency key on successful retry', { concurrency: false }, async () => {
const fixture = await setupFixture({
memoryPolicy: { mode: 'optional' },
schedulerPolicy: {
strict_lock: false,
degraded_lock_retry_window: 3600000,
max_deferrals: 3,
},
lockShellBody: ` count_file="$PWD/.lock-count"
key_log="$PWD/.idempotency-keys"
count=0
if [[ -f "$count_file" ]]; then
count="$(cat "$count_file")"
fi
count=$((count + 1))
echo "$count" > "$count_file"
echo "\${SCHEDULER_LOCK_IDEMPOTENCY_KEY:-missing}" >> "$key_log"
if [[ "$count" -lt 3 ]]; then
echo 'relay query timeout' >&2
exit 2
fi
:`,
});
const first = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
cwd: fixture.root,
env: {
...fixture.env,
SCHEDULER_LOCK_MAX_RETRIES: '0',
SCHEDULER_LOCK_BACKOFF_MS: '0',
SCHEDULER_LOCK_JITTER_MS: '0',
},
});
assert.equal(first.code, 0, `stdout: ${first.stdout}\nstderr: ${first.stderr}`);
const second = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
cwd: fixture.root,
env: {
...fixture.env,
SCHEDULER_LOCK_MAX_RETRIES: '0',
SCHEDULER_LOCK_BACKOFF_MS: '0',
SCHEDULER_LOCK_JITTER_MS: '0',
},
});
assert.equal(second.code, 0, `stdout: ${second.stdout}\nstderr: ${second.stderr}`);
const third = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
cwd: fixture.root,
env: {
...fixture.env,
SCHEDULER_LOCK_MAX_RETRIES: '0',
SCHEDULER_LOCK_BACKOFF_MS: '0',
SCHEDULER_LOCK_JITTER_MS: '0',
},
});
assert.equal(third.code, 0, `stdout: ${third.stdout}\nstderr: ${third.stderr}`);
const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily'));
const deferredLog = logs.find((name) => name.endsWith('__deferred.md'));
assert.ok(deferredLog);
const deferredBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', deferredLog), 'utf8');
assert.match(deferredBody, /failure_class: 'backend_unavailable'/);
assert.match(deferredBody, /deferral_attempt_count: '\d+'/);
const keyLog = await fs.readFile(path.join(fixture.root, '.idempotency-keys'), 'utf8');
const keys = keyLog.trim().split(/\r?\n/).filter(Boolean);
assert.equal(keys.length, 3);
assert.equal(keys[0], 'missing');
assert.notEqual(keys[1], 'missing');
assert.equal(keys[1], keys[2]);
});
test('fails after exceeding non-strict deferral budget', { concurrency: false }, async () => {
const fixture = await setupFixture({
memoryPolicy: { mode: 'optional' },
schedulerPolicy: {
strict_lock: false,
degraded_lock_retry_window: 3600000,
max_deferrals: 1,
},
lockShellBody: ` echo 'publish failed to all relays' >&2
exit 2`,
});
const first = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
cwd: fixture.root,
env: {
...fixture.env,
SCHEDULER_LOCK_MAX_RETRIES: '0',
SCHEDULER_LOCK_BACKOFF_MS: '0',
SCHEDULER_LOCK_JITTER_MS: '0',
},
});
assert.equal(first.code, 0, `stdout: ${first.stdout}\nstderr: ${first.stderr}`);
const second = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
cwd: fixture.root,
env: {
...fixture.env,
SCHEDULER_LOCK_MAX_RETRIES: '0',
SCHEDULER_LOCK_BACKOFF_MS: '0',
SCHEDULER_LOCK_JITTER_MS: '0',
},
});
assert.equal(second.code, 2, `stdout: ${second.stdout}\nstderr: ${second.stderr}`);
const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily'));
const failedLog = logs.find((name) => name.endsWith('__failed.md'));
assert.ok(failedLog);
const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8');
assert.match(failedBody, /failure_class: 'backend_unavailable'/);
assert.match(failedBody, /deferral_attempt_count: '2'/);
});
test('strict lock mode fails immediately without deferral', { concurrency: false }, async () => {
const fixture = await setupFixture({
memoryPolicy: { mode: 'optional' },
schedulerPolicy: {
strict_lock: true,
degraded_lock_retry_window: 3600000,
max_deferrals: 5,
},
lockShellBody: ` echo 'relay timeout' >&2
exit 2`,
});
const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
cwd: fixture.root,
env: {
...fixture.env,
SCHEDULER_LOCK_MAX_RETRIES: '0',
SCHEDULER_LOCK_BACKOFF_MS: '0',
SCHEDULER_LOCK_JITTER_MS: '0',
},
});
assert.equal(result.code, 2, `stdout: ${result.stdout}\nstderr: ${result.stderr}`);
const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily'));
assert.equal(logs.some((name) => name.endsWith('__deferred.md')), false);
const failedLog = logs.find((name) => name.endsWith('__failed.md'));
assert.ok(failedLog);
});
test('skips lock health preflight by default when not enabled', { concurrency: false }, async () => {