mirror of
https://github.com/PR0M3TH3AN/TORCH.git
synced 2026-10-27 07:51:52 +00:00
Add lock deferral policy for scheduler backend failures
This commit is contained in:
1 parent
430558558e
commit
9aad0d933d
4 files changed
+324
-8
No files matched your search
@@ -4,6 +4,7 @@
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
|
||||
const VALID_CADENCES = new Set(['daily', 'weekly']);
|
||||
const ALL_EXCLUDED_REASON = 'All roster tasks currently claimed by other agents';
|
||||
@@ -314,6 +315,18 @@ async function getSchedulerConfig(cadence, { isInteractive }) {
|
||||
lockHealthPreflightFromConfig,
|
||||
);
|
||||
const lockHealthPreflightSkip = parseBooleanFlag(process.env.SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT, false);
|
||||
const strictLock = parseBooleanFlag(
|
||||
process.env.SCHEDULER_STRICT_LOCK,
|
||||
parseBooleanFlag(scheduler.strict_lock, true),
|
||||
);
|
||||
const degradedLockRetryWindowMs = parseNonNegativeInt(
|
||||
process.env.SCHEDULER_DEGRADED_LOCK_RETRY_WINDOW_MS,
|
||||
parseNonNegativeInt(scheduler.degraded_lock_retry_window, 3600000),
|
||||
);
|
||||
const maxDeferrals = parseNonNegativeInt(
|
||||
process.env.SCHEDULER_MAX_DEFERRALS,
|
||||
parseNonNegativeInt(scheduler.max_deferrals, 3),
|
||||
);
|
||||
|
||||
return {
|
||||
firstPrompt: scheduler.firstPromptByCadence?.[cadence] || null,
|
||||
@@ -331,6 +344,11 @@ async function getSchedulerConfig(cadence, { isInteractive }) {
|
||||
enabled: lockHealthPreflightEnabled,
|
||||
skip: lockHealthPreflightSkip,
|
||||
},
|
||||
lockFailurePolicy: {
|
||||
strictLock,
|
||||
degradedLockRetryWindowMs,
|
||||
maxDeferrals,
|
||||
},
|
||||
memoryPolicy: {
|
||||
mode,
|
||||
retrieveCommand: typeof memoryPolicyRaw.retrieveCommand === 'string' && memoryPolicyRaw.retrieveCommand.trim()
|
||||
@@ -347,6 +365,40 @@ async function getSchedulerConfig(cadence, { isInteractive }) {
|
||||
};
|
||||
}
|
||||
|
||||
function getRunDateKey(now = new Date()) {
|
||||
return now.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
async function readRunState(cadence) {
|
||||
const statePath = path.resolve(process.cwd(), 'task-logs', cadence, '.scheduler-run-state.json');
|
||||
const fallback = { run_date: getRunDateKey(), lock_deferral: null };
|
||||
const raw = await readJson(statePath, fallback);
|
||||
if (!raw || typeof raw !== 'object') {
|
||||
return { statePath, state: fallback };
|
||||
}
|
||||
|
||||
if (raw.run_date !== getRunDateKey()) {
|
||||
return { statePath, state: fallback };
|
||||
}
|
||||
|
||||
return {
|
||||
statePath,
|
||||
state: {
|
||||
run_date: raw.run_date,
|
||||
lock_deferral: raw.lock_deferral && typeof raw.lock_deferral === 'object' ? raw.lock_deferral : null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function writeRunState(statePath, state) {
|
||||
await fs.mkdir(path.dirname(statePath), { recursive: true });
|
||||
await fs.writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, 'utf8');
|
||||
}
|
||||
|
||||
function createIdempotencyKey({ cadence, selectedAgent, runDate }) {
|
||||
return `${cadence}:${selectedAgent}:${runDate}:${randomUUID()}`;
|
||||
}
|
||||
|
||||
async function runLockHealthPreflight({ cadence, platform }) {
|
||||
const preflight = await runCommand('node', ['scripts/agent/check-relay-health.mjs', '--cadence', cadence], {
|
||||
env: { AGENT_PLATFORM: platform },
|
||||
@@ -372,14 +424,19 @@ function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function acquireLockWithRetry({ selectedAgent, cadence, platform, lockRetry }) {
|
||||
async function acquireLockWithRetry({ selectedAgent, cadence, platform, lockRetry, idempotencyKey }) {
|
||||
const lockCommandArgs = ['run', 'lock:lock', '--', '--agent', selectedAgent, '--cadence', cadence];
|
||||
const backoffScheduleMs = [];
|
||||
let attempts = 0;
|
||||
|
||||
while (true) {
|
||||
attempts += 1;
|
||||
const result = await runCommand('npm', lockCommandArgs, { env: { AGENT_PLATFORM: platform } });
|
||||
const result = await runCommand('npm', lockCommandArgs, {
|
||||
env: {
|
||||
AGENT_PLATFORM: platform,
|
||||
...(idempotencyKey ? { SCHEDULER_LOCK_IDEMPOTENCY_KEY: idempotencyKey } : {}),
|
||||
},
|
||||
});
|
||||
|
||||
if (result.code !== 2) {
|
||||
return { result, attempts, backoffScheduleMs };
|
||||
@@ -481,6 +538,8 @@ async function main() {
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const { statePath: runStatePath, state: schedulerRunState } = await readRunState(cadence);
|
||||
|
||||
if (schedulerConfig.lockHealthPreflight.enabled && !schedulerConfig.lockHealthPreflight.skip) {
|
||||
const preflight = await runLockHealthPreflight({ cadence, platform });
|
||||
if (preflight.code !== 0) {
|
||||
@@ -492,6 +551,7 @@ async function main() {
|
||||
reason: 'Lock backend unavailable preflight',
|
||||
detail: `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`,
|
||||
metadata: {
|
||||
failure_class: 'backend_unavailable',
|
||||
preflight_failure_category: preflight.failureCategory,
|
||||
relay_list: preflight.relayList.join(', ') || '(none)',
|
||||
preflight_stderr_excerpt: preflight.stderrExcerpt || '(empty)',
|
||||
@@ -543,11 +603,16 @@ async function main() {
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const deferralForAgent = schedulerRunState.lock_deferral?.selected_agent === selectedAgent
|
||||
? schedulerRunState.lock_deferral
|
||||
: null;
|
||||
|
||||
const lockAttempt = await acquireLockWithRetry({
|
||||
selectedAgent,
|
||||
cadence,
|
||||
platform,
|
||||
lockRetry: schedulerConfig.lockRetry,
|
||||
idempotencyKey: deferralForAgent?.idempotency_key,
|
||||
});
|
||||
const lockResult = lockAttempt.result;
|
||||
|
||||
@@ -562,6 +627,46 @@ async function main() {
|
||||
const stderrExcerpt = excerptText(lockResult.stderr);
|
||||
const stdoutExcerpt = excerptText(lockResult.stdout);
|
||||
const backoffSchedule = lockAttempt.backoffScheduleMs.join(', ');
|
||||
|
||||
const existingDeferral = deferralForAgent || {};
|
||||
const firstFailureAt = existingDeferral.first_failure_timestamp || new Date().toISOString();
|
||||
const firstFailureMs = parseDateValue(firstFailureAt) || Date.now();
|
||||
const nowMs = Date.now();
|
||||
const deferralAttemptCount = parseNonNegativeInt(existingDeferral.attempt_count, 0) + 1;
|
||||
const idempotencyKey = existingDeferral.idempotency_key
|
||||
|| createIdempotencyKey({ cadence, selectedAgent, runDate: schedulerRunState.run_date });
|
||||
const withinRetryWindow = nowMs - firstFailureMs <= schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs;
|
||||
const withinDeferralBudget = deferralAttemptCount <= schedulerConfig.lockFailurePolicy.maxDeferrals;
|
||||
|
||||
if (!schedulerConfig.lockFailurePolicy.strictLock && withinRetryWindow && withinDeferralBudget) {
|
||||
schedulerRunState.lock_deferral = {
|
||||
attempt_count: deferralAttemptCount,
|
||||
first_failure_timestamp: firstFailureAt,
|
||||
backend_category: backendCategory,
|
||||
idempotency_key: idempotencyKey,
|
||||
selected_agent: selectedAgent,
|
||||
};
|
||||
await writeRunState(runStatePath, schedulerRunState);
|
||||
await writeLog({
|
||||
cadence,
|
||||
agent: selectedAgent,
|
||||
status: 'deferred',
|
||||
platform,
|
||||
reason: 'Lock backend deferred',
|
||||
detail: `Deferred after lock backend failure (${backendCategory}); retry window active and deferral budget remaining.`,
|
||||
metadata: {
|
||||
failure_class: 'backend_unavailable',
|
||||
deferral_attempt_count: deferralAttemptCount,
|
||||
deferral_first_failure_timestamp: firstFailureAt,
|
||||
backend_category: backendCategory,
|
||||
lock_idempotency_key: idempotencyKey,
|
||||
},
|
||||
});
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
schedulerRunState.lock_deferral = null;
|
||||
await writeRunState(runStatePath, schedulerRunState);
|
||||
await writeLog({
|
||||
cadence,
|
||||
agent: selectedAgent,
|
||||
@@ -570,10 +675,14 @@ async function main() {
|
||||
reason: 'Lock backend error',
|
||||
detail: `Lock backend error (${backendCategory}) after ${lockAttempt.attempts} attempt(s). Retry ${lockCommand} after verifying relay connectivity, relay URLs, and DNS/network status.`,
|
||||
metadata: {
|
||||
failure_class: 'backend_unavailable',
|
||||
lock_attempts_total: lockAttempt.attempts,
|
||||
lock_backoff_schedule_ms: backoffSchedule || '(none)',
|
||||
backend_category: backendCategory,
|
||||
deferral_attempt_count: deferralAttemptCount,
|
||||
deferral_first_failure_timestamp: firstFailureAt,
|
||||
lock_command: lockCommand,
|
||||
lock_idempotency_key: idempotencyKey,
|
||||
lock_stderr_excerpt: stderrExcerpt || '(empty)',
|
||||
lock_stdout_excerpt: stdoutExcerpt || '(empty)',
|
||||
},
|
||||
@@ -582,6 +691,8 @@ async function main() {
|
||||
}
|
||||
|
||||
if (lockResult.code !== 0) {
|
||||
schedulerRunState.lock_deferral = null;
|
||||
await writeRunState(runStatePath, schedulerRunState);
|
||||
await writeLog({
|
||||
cadence,
|
||||
agent: selectedAgent,
|
||||
@@ -593,6 +704,8 @@ async function main() {
|
||||
}
|
||||
|
||||
const promptPath = path.resolve(process.cwd(), 'src/prompts', cadence, `${selectedAgent}.md`);
|
||||
schedulerRunState.lock_deferral = null;
|
||||
await writeRunState(runStatePath, schedulerRunState);
|
||||
const runArtifactSince = new Date().toISOString();
|
||||
const runStartMs = Date.parse(runArtifactSince);
|
||||
const outputChunks = [];
|
||||
@@ -603,7 +716,15 @@ async function main() {
|
||||
});
|
||||
outputChunks.push(retrieveResult.stdout, retrieveResult.stderr);
|
||||
if (retrieveResult.code !== 0) {
|
||||
await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Memory retrieval command failed', detail: schedulerConfig.memoryPolicy.retrieveCommand });
|
||||
await writeLog({
|
||||
cadence,
|
||||
agent: selectedAgent,
|
||||
status: 'failed',
|
||||
platform,
|
||||
reason: 'Memory retrieval command failed',
|
||||
detail: schedulerConfig.memoryPolicy.retrieveCommand,
|
||||
metadata: { failure_class: 'prompt_validation_error' },
|
||||
});
|
||||
process.exit(retrieveResult.code);
|
||||
}
|
||||
}
|
||||
@@ -614,14 +735,30 @@ async function main() {
|
||||
});
|
||||
outputChunks.push(handoff.stdout, handoff.stderr);
|
||||
if (handoff.code !== 0) {
|
||||
await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Prompt/handoff execution failed', detail: 'Handoff callback failed.' });
|
||||
await writeLog({
|
||||
cadence,
|
||||
agent: selectedAgent,
|
||||
status: 'failed',
|
||||
platform,
|
||||
reason: 'Prompt/handoff execution failed',
|
||||
detail: 'Handoff callback failed.',
|
||||
metadata: { failure_class: 'prompt_validation_error' },
|
||||
});
|
||||
process.exit(handoff.code);
|
||||
}
|
||||
} else {
|
||||
const detail = schedulerConfig.missingHandoffCommandForMode
|
||||
? 'Missing scheduler handoff command for non-interactive run. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.'
|
||||
: 'No handoff callback configured. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.';
|
||||
await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Prompt/handoff execution failed', detail });
|
||||
await writeLog({
|
||||
cadence,
|
||||
agent: selectedAgent,
|
||||
status: 'failed',
|
||||
platform,
|
||||
reason: 'Prompt/handoff execution failed',
|
||||
detail,
|
||||
metadata: { failure_class: 'prompt_validation_error' },
|
||||
});
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
@@ -631,7 +768,15 @@ async function main() {
|
||||
});
|
||||
outputChunks.push(storeResult.stdout, storeResult.stderr);
|
||||
if (storeResult.code !== 0) {
|
||||
await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Memory storage command failed', detail: schedulerConfig.memoryPolicy.storeCommand });
|
||||
await writeLog({
|
||||
cadence,
|
||||
agent: selectedAgent,
|
||||
status: 'failed',
|
||||
platform,
|
||||
reason: 'Memory storage command failed',
|
||||
detail: schedulerConfig.memoryPolicy.storeCommand,
|
||||
metadata: { failure_class: 'prompt_validation_error' },
|
||||
});
|
||||
process.exit(storeResult.code);
|
||||
}
|
||||
}
|
||||
@@ -661,6 +806,7 @@ async function main() {
|
||||
platform,
|
||||
reason: 'Required memory steps not verified',
|
||||
detail: `Missing evidence for: ${missingSteps.join(', ')}`,
|
||||
metadata: { failure_class: 'prompt_validation_error' },
|
||||
});
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -692,6 +838,7 @@ async function main() {
|
||||
platform,
|
||||
reason: 'Missing required run artifacts',
|
||||
detail,
|
||||
metadata: { failure_class: 'prompt_validation_error' },
|
||||
});
|
||||
process.exit(artifactCheck.code);
|
||||
}
|
||||
@@ -701,7 +848,15 @@ async function main() {
|
||||
if (!parts.length) continue;
|
||||
const result = await runCommand(parts[0], parts.slice(1));
|
||||
if (result.code !== 0) {
|
||||
await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Validation failed', detail: validation });
|
||||
await writeLog({
|
||||
cadence,
|
||||
agent: selectedAgent,
|
||||
status: 'failed',
|
||||
platform,
|
||||
reason: 'Validation failed',
|
||||
detail: validation,
|
||||
metadata: { failure_class: 'prompt_validation_error' },
|
||||
});
|
||||
process.exit(result.code);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,6 +65,9 @@ Common settings:
|
||||
- `scheduler.firstPromptByCadence.daily` / `.weekly` — first-run scheduler starting agent.
|
||||
- `scheduler.handoffCommandByCadence.daily` / `.weekly` — shell command run after lock acquisition; command must use `SCHEDULER_AGENT`, `SCHEDULER_CADENCE`, and `SCHEDULER_PROMPT_PATH` provided by `scripts/agent/run-scheduler-cycle.mjs`.
|
||||
- `scheduler.paused.daily` / `.weekly` — array of agent names to exclude from scheduler rotation.
|
||||
- `scheduler.strict_lock` — lock backend policy switch (default: `true`); when `false`, scheduler defers backend-unavailable lock failures before converting the run to failed.
|
||||
- `scheduler.degraded_lock_retry_window` — non-strict deferral window in milliseconds; backend lock failures outside this window immediately consume failure budget and mark run failed.
|
||||
- `scheduler.max_deferrals` — max number of non-strict lock deferrals allowed in-window before scheduler records a hard failure.
|
||||
|
||||
Default first-run daily scheduler prompt is `scheduler-update-agent`.
|
||||
|
||||
@@ -72,6 +75,11 @@ For weekly repository-fit maintenance, TORCH also includes `src/prompts/weekly/r
|
||||
|
||||
Operational note: scheduler handoff commands are treated as required execution steps. A non-zero exit code (or missing command) is a hard failure: the scheduler writes a `_failed.md` task log, exits immediately, and does not publish `lock:complete` for that run.
|
||||
|
||||
Scheduler failure classes in task logs:
|
||||
|
||||
- `backend_unavailable` — lock backend unavailable preflight failures and lock acquisition backend exit code `2` failures/deferrals.
|
||||
- `prompt_validation_error` — prompt/handoff, memory evidence, artifact verification, and validation command failures.
|
||||
|
||||
|
||||
## Lock backend production defaults
|
||||
|
||||
|
||||
@@ -148,17 +148,24 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co
|
||||
|
||||
- Exit `0`: lock acquired, continue.
|
||||
- Exit `3`: race lost/already locked, return to step 2.
|
||||
- Exit `2`: lock backend error, write `_failed.md` with reason `Lock backend error`, and include failure metadata fields:
|
||||
- Exit `2`: lock backend error.
|
||||
- If `scheduler.strict_lock` is `false`, defer the run while both budget constraints are still satisfied:
|
||||
- `degraded_lock_retry_window` (ms) since first backend failure has not elapsed.
|
||||
- `max_deferrals` has not been exceeded.
|
||||
- Record deferral metadata in scheduler run state (`attempt_count`, `first_failure_timestamp`, `backend_category`, and preserved idempotency key).
|
||||
- Otherwise write `_failed.md` with reason `Lock backend error`, and include failure metadata fields:
|
||||
- `backend_category` (classified backend failure category)
|
||||
- `lock_command` (raw lock command for retry)
|
||||
- `lock_stderr_excerpt` (redacted stderr snippet)
|
||||
- `lock_stdout_excerpt` (redacted stdout snippet)
|
||||
- Include `failure_class: backend_unavailable` for both deferred and failed backend-unavailable lock outcomes.
|
||||
- Keep generic reason text for compatibility, but append actionable retry guidance in `detail` using the command from `lock_command`.
|
||||
|
||||
9. Execute `<prompt_dir>/<prompt-file>` end-to-end via configured handoff command.
|
||||
|
||||
- Scheduler automation runs `scheduler.handoffCommandByCadence.<cadence>` with environment variables for cadence/agent/prompt path.
|
||||
- If no handoff command is configured for the cadence, write `_failed.md` and stop.
|
||||
- Prompt/handoff and prompt validation failures should emit `failure_class: prompt_validation_error`.
|
||||
|
||||
10. Confirm memory contract completion:
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ async function setupFixture({
|
||||
lockShellBody = '',
|
||||
roster = ['agent-a'],
|
||||
lockHealthPreflight = undefined,
|
||||
schedulerPolicy = {},
|
||||
preflightScript = '#!/usr/bin/env node\nconsole.log(JSON.stringify({ ok: true, relays: ["wss://relay.test"] }));\nprocess.exit(0);\n',
|
||||
}) {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'scheduler-memory-policy-'));
|
||||
@@ -69,6 +70,7 @@ async function setupFixture({
|
||||
handoffCommandByCadence: { daily: 'echo HANDOFF_OK' },
|
||||
validationCommandsByCadence: { daily: ['true'] },
|
||||
memoryPolicyByCadence: { daily: memoryPolicy },
|
||||
...schedulerPolicy,
|
||||
...(lockHealthPreflight === undefined ? {} : { lockHealthPreflight }),
|
||||
},
|
||||
}, null, 2),
|
||||
@@ -186,6 +188,7 @@ test('records backend failure metadata when lock command exits with code 2', { c
|
||||
const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8');
|
||||
|
||||
assert.match(failedBody, /reason: Lock backend error/);
|
||||
assert.match(failedBody, /failure_class: 'backend_unavailable'/);
|
||||
assert.match(failedBody, /backend_category: 'publish failed to all relays'/);
|
||||
assert.match(failedBody, /lock_command: 'AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily'/);
|
||||
assert.match(failedBody, /lock_stderr_excerpt: 'publish failed to all relays token=\[REDACTED\] SECRET_KEY=\[REDACTED\]'/);
|
||||
@@ -194,6 +197,149 @@ test('records backend failure metadata when lock command exits with code 2', { c
|
||||
assert.match(failedBody, /platform: 'codex'/);
|
||||
});
|
||||
|
||||
test('defers backend failures in non-strict mode and reuses idempotency key on successful retry', { concurrency: false }, async () => {
|
||||
const fixture = await setupFixture({
|
||||
memoryPolicy: { mode: 'optional' },
|
||||
schedulerPolicy: {
|
||||
strict_lock: false,
|
||||
degraded_lock_retry_window: 3600000,
|
||||
max_deferrals: 3,
|
||||
},
|
||||
lockShellBody: ` count_file="$PWD/.lock-count"
|
||||
key_log="$PWD/.idempotency-keys"
|
||||
count=0
|
||||
if [[ -f "$count_file" ]]; then
|
||||
count="$(cat "$count_file")"
|
||||
fi
|
||||
count=$((count + 1))
|
||||
echo "$count" > "$count_file"
|
||||
echo "\${SCHEDULER_LOCK_IDEMPOTENCY_KEY:-missing}" >> "$key_log"
|
||||
if [[ "$count" -lt 3 ]]; then
|
||||
echo 'relay query timeout' >&2
|
||||
exit 2
|
||||
fi
|
||||
:`,
|
||||
});
|
||||
|
||||
const first = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
|
||||
cwd: fixture.root,
|
||||
env: {
|
||||
...fixture.env,
|
||||
SCHEDULER_LOCK_MAX_RETRIES: '0',
|
||||
SCHEDULER_LOCK_BACKOFF_MS: '0',
|
||||
SCHEDULER_LOCK_JITTER_MS: '0',
|
||||
},
|
||||
});
|
||||
assert.equal(first.code, 0, `stdout: ${first.stdout}\nstderr: ${first.stderr}`);
|
||||
|
||||
const second = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
|
||||
cwd: fixture.root,
|
||||
env: {
|
||||
...fixture.env,
|
||||
SCHEDULER_LOCK_MAX_RETRIES: '0',
|
||||
SCHEDULER_LOCK_BACKOFF_MS: '0',
|
||||
SCHEDULER_LOCK_JITTER_MS: '0',
|
||||
},
|
||||
});
|
||||
assert.equal(second.code, 0, `stdout: ${second.stdout}\nstderr: ${second.stderr}`);
|
||||
|
||||
const third = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
|
||||
cwd: fixture.root,
|
||||
env: {
|
||||
...fixture.env,
|
||||
SCHEDULER_LOCK_MAX_RETRIES: '0',
|
||||
SCHEDULER_LOCK_BACKOFF_MS: '0',
|
||||
SCHEDULER_LOCK_JITTER_MS: '0',
|
||||
},
|
||||
});
|
||||
assert.equal(third.code, 0, `stdout: ${third.stdout}\nstderr: ${third.stderr}`);
|
||||
|
||||
const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily'));
|
||||
const deferredLog = logs.find((name) => name.endsWith('__deferred.md'));
|
||||
assert.ok(deferredLog);
|
||||
const deferredBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', deferredLog), 'utf8');
|
||||
assert.match(deferredBody, /failure_class: 'backend_unavailable'/);
|
||||
assert.match(deferredBody, /deferral_attempt_count: '\d+'/);
|
||||
|
||||
const keyLog = await fs.readFile(path.join(fixture.root, '.idempotency-keys'), 'utf8');
|
||||
const keys = keyLog.trim().split(/\r?\n/).filter(Boolean);
|
||||
assert.equal(keys.length, 3);
|
||||
assert.equal(keys[0], 'missing');
|
||||
assert.notEqual(keys[1], 'missing');
|
||||
assert.equal(keys[1], keys[2]);
|
||||
});
|
||||
|
||||
test('fails after exceeding non-strict deferral budget', { concurrency: false }, async () => {
|
||||
const fixture = await setupFixture({
|
||||
memoryPolicy: { mode: 'optional' },
|
||||
schedulerPolicy: {
|
||||
strict_lock: false,
|
||||
degraded_lock_retry_window: 3600000,
|
||||
max_deferrals: 1,
|
||||
},
|
||||
lockShellBody: ` echo 'publish failed to all relays' >&2
|
||||
exit 2`,
|
||||
});
|
||||
|
||||
const first = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
|
||||
cwd: fixture.root,
|
||||
env: {
|
||||
...fixture.env,
|
||||
SCHEDULER_LOCK_MAX_RETRIES: '0',
|
||||
SCHEDULER_LOCK_BACKOFF_MS: '0',
|
||||
SCHEDULER_LOCK_JITTER_MS: '0',
|
||||
},
|
||||
});
|
||||
assert.equal(first.code, 0, `stdout: ${first.stdout}\nstderr: ${first.stderr}`);
|
||||
|
||||
const second = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
|
||||
cwd: fixture.root,
|
||||
env: {
|
||||
...fixture.env,
|
||||
SCHEDULER_LOCK_MAX_RETRIES: '0',
|
||||
SCHEDULER_LOCK_BACKOFF_MS: '0',
|
||||
SCHEDULER_LOCK_JITTER_MS: '0',
|
||||
},
|
||||
});
|
||||
assert.equal(second.code, 2, `stdout: ${second.stdout}\nstderr: ${second.stderr}`);
|
||||
|
||||
const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily'));
|
||||
const failedLog = logs.find((name) => name.endsWith('__failed.md'));
|
||||
assert.ok(failedLog);
|
||||
const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8');
|
||||
assert.match(failedBody, /failure_class: 'backend_unavailable'/);
|
||||
assert.match(failedBody, /deferral_attempt_count: '2'/);
|
||||
});
|
||||
|
||||
test('strict lock mode fails immediately without deferral', { concurrency: false }, async () => {
|
||||
const fixture = await setupFixture({
|
||||
memoryPolicy: { mode: 'optional' },
|
||||
schedulerPolicy: {
|
||||
strict_lock: true,
|
||||
degraded_lock_retry_window: 3600000,
|
||||
max_deferrals: 5,
|
||||
},
|
||||
lockShellBody: ` echo 'relay timeout' >&2
|
||||
exit 2`,
|
||||
});
|
||||
|
||||
const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
|
||||
cwd: fixture.root,
|
||||
env: {
|
||||
...fixture.env,
|
||||
SCHEDULER_LOCK_MAX_RETRIES: '0',
|
||||
SCHEDULER_LOCK_BACKOFF_MS: '0',
|
||||
SCHEDULER_LOCK_JITTER_MS: '0',
|
||||
},
|
||||
});
|
||||
assert.equal(result.code, 2, `stdout: ${result.stdout}\nstderr: ${result.stderr}`);
|
||||
|
||||
const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily'));
|
||||
assert.equal(logs.some((name) => name.endsWith('__deferred.md')), false);
|
||||
const failedLog = logs.find((name) => name.endsWith('__failed.md'));
|
||||
assert.ok(failedLog);
|
||||
});
|
||||
|
||||
|
||||
|
||||
test('skips lock health preflight by default when not enabled', { concurrency: false }, async () => {
|
||||
|
||||
Reference in new issue
Block a user