mirror of
https://github.com/PR0M3TH3AN/TORCH.git
synced 2026-11-06 21:01:54 +00:00
Add lock deferral policy for scheduler backend failures
This commit is contained in:
1 parent
430558558e
commit
9aad0d933d
4 files changed
+324
-8
No files matched your search
@@ -4,6 +4,7 @@
|
|||||||
import fs from 'node:fs/promises';
|
import fs from 'node:fs/promises';
|
||||||
import path from 'node:path';
|
import path from 'node:path';
|
||||||
import { spawn } from 'node:child_process';
|
import { spawn } from 'node:child_process';
|
||||||
|
import { randomUUID } from 'node:crypto';
|
||||||
|
|
||||||
const VALID_CADENCES = new Set(['daily', 'weekly']);
|
const VALID_CADENCES = new Set(['daily', 'weekly']);
|
||||||
const ALL_EXCLUDED_REASON = 'All roster tasks currently claimed by other agents';
|
const ALL_EXCLUDED_REASON = 'All roster tasks currently claimed by other agents';
|
||||||
@@ -314,6 +315,18 @@ async function getSchedulerConfig(cadence, { isInteractive }) {
|
|||||||
lockHealthPreflightFromConfig,
|
lockHealthPreflightFromConfig,
|
||||||
);
|
);
|
||||||
const lockHealthPreflightSkip = parseBooleanFlag(process.env.SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT, false);
|
const lockHealthPreflightSkip = parseBooleanFlag(process.env.SCHEDULER_SKIP_LOCK_HEALTH_PREFLIGHT, false);
|
||||||
|
const strictLock = parseBooleanFlag(
|
||||||
|
process.env.SCHEDULER_STRICT_LOCK,
|
||||||
|
parseBooleanFlag(scheduler.strict_lock, true),
|
||||||
|
);
|
||||||
|
const degradedLockRetryWindowMs = parseNonNegativeInt(
|
||||||
|
process.env.SCHEDULER_DEGRADED_LOCK_RETRY_WINDOW_MS,
|
||||||
|
parseNonNegativeInt(scheduler.degraded_lock_retry_window, 3600000),
|
||||||
|
);
|
||||||
|
const maxDeferrals = parseNonNegativeInt(
|
||||||
|
process.env.SCHEDULER_MAX_DEFERRALS,
|
||||||
|
parseNonNegativeInt(scheduler.max_deferrals, 3),
|
||||||
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
firstPrompt: scheduler.firstPromptByCadence?.[cadence] || null,
|
firstPrompt: scheduler.firstPromptByCadence?.[cadence] || null,
|
||||||
@@ -331,6 +344,11 @@ async function getSchedulerConfig(cadence, { isInteractive }) {
|
|||||||
enabled: lockHealthPreflightEnabled,
|
enabled: lockHealthPreflightEnabled,
|
||||||
skip: lockHealthPreflightSkip,
|
skip: lockHealthPreflightSkip,
|
||||||
},
|
},
|
||||||
|
lockFailurePolicy: {
|
||||||
|
strictLock,
|
||||||
|
degradedLockRetryWindowMs,
|
||||||
|
maxDeferrals,
|
||||||
|
},
|
||||||
memoryPolicy: {
|
memoryPolicy: {
|
||||||
mode,
|
mode,
|
||||||
retrieveCommand: typeof memoryPolicyRaw.retrieveCommand === 'string' && memoryPolicyRaw.retrieveCommand.trim()
|
retrieveCommand: typeof memoryPolicyRaw.retrieveCommand === 'string' && memoryPolicyRaw.retrieveCommand.trim()
|
||||||
@@ -347,6 +365,40 @@ async function getSchedulerConfig(cadence, { isInteractive }) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getRunDateKey(now = new Date()) {
|
||||||
|
return now.toISOString().slice(0, 10);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readRunState(cadence) {
|
||||||
|
const statePath = path.resolve(process.cwd(), 'task-logs', cadence, '.scheduler-run-state.json');
|
||||||
|
const fallback = { run_date: getRunDateKey(), lock_deferral: null };
|
||||||
|
const raw = await readJson(statePath, fallback);
|
||||||
|
if (!raw || typeof raw !== 'object') {
|
||||||
|
return { statePath, state: fallback };
|
||||||
|
}
|
||||||
|
|
||||||
|
if (raw.run_date !== getRunDateKey()) {
|
||||||
|
return { statePath, state: fallback };
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
statePath,
|
||||||
|
state: {
|
||||||
|
run_date: raw.run_date,
|
||||||
|
lock_deferral: raw.lock_deferral && typeof raw.lock_deferral === 'object' ? raw.lock_deferral : null,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeRunState(statePath, state) {
|
||||||
|
await fs.mkdir(path.dirname(statePath), { recursive: true });
|
||||||
|
await fs.writeFile(statePath, `${JSON.stringify(state, null, 2)}\n`, 'utf8');
|
||||||
|
}
|
||||||
|
|
||||||
|
function createIdempotencyKey({ cadence, selectedAgent, runDate }) {
|
||||||
|
return `${cadence}:${selectedAgent}:${runDate}:${randomUUID()}`;
|
||||||
|
}
|
||||||
|
|
||||||
async function runLockHealthPreflight({ cadence, platform }) {
|
async function runLockHealthPreflight({ cadence, platform }) {
|
||||||
const preflight = await runCommand('node', ['scripts/agent/check-relay-health.mjs', '--cadence', cadence], {
|
const preflight = await runCommand('node', ['scripts/agent/check-relay-health.mjs', '--cadence', cadence], {
|
||||||
env: { AGENT_PLATFORM: platform },
|
env: { AGENT_PLATFORM: platform },
|
||||||
@@ -372,14 +424,19 @@ function sleep(ms) {
|
|||||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function acquireLockWithRetry({ selectedAgent, cadence, platform, lockRetry }) {
|
async function acquireLockWithRetry({ selectedAgent, cadence, platform, lockRetry, idempotencyKey }) {
|
||||||
const lockCommandArgs = ['run', 'lock:lock', '--', '--agent', selectedAgent, '--cadence', cadence];
|
const lockCommandArgs = ['run', 'lock:lock', '--', '--agent', selectedAgent, '--cadence', cadence];
|
||||||
const backoffScheduleMs = [];
|
const backoffScheduleMs = [];
|
||||||
let attempts = 0;
|
let attempts = 0;
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
attempts += 1;
|
attempts += 1;
|
||||||
const result = await runCommand('npm', lockCommandArgs, { env: { AGENT_PLATFORM: platform } });
|
const result = await runCommand('npm', lockCommandArgs, {
|
||||||
|
env: {
|
||||||
|
AGENT_PLATFORM: platform,
|
||||||
|
...(idempotencyKey ? { SCHEDULER_LOCK_IDEMPOTENCY_KEY: idempotencyKey } : {}),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
if (result.code !== 2) {
|
if (result.code !== 2) {
|
||||||
return { result, attempts, backoffScheduleMs };
|
return { result, attempts, backoffScheduleMs };
|
||||||
@@ -481,6 +538,8 @@ async function main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
while (true) {
|
while (true) {
|
||||||
|
const { statePath: runStatePath, state: schedulerRunState } = await readRunState(cadence);
|
||||||
|
|
||||||
if (schedulerConfig.lockHealthPreflight.enabled && !schedulerConfig.lockHealthPreflight.skip) {
|
if (schedulerConfig.lockHealthPreflight.enabled && !schedulerConfig.lockHealthPreflight.skip) {
|
||||||
const preflight = await runLockHealthPreflight({ cadence, platform });
|
const preflight = await runLockHealthPreflight({ cadence, platform });
|
||||||
if (preflight.code !== 0) {
|
if (preflight.code !== 0) {
|
||||||
@@ -492,6 +551,7 @@ async function main() {
|
|||||||
reason: 'Lock backend unavailable preflight',
|
reason: 'Lock backend unavailable preflight',
|
||||||
detail: `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`,
|
detail: `Preflight failed (${preflight.failureCategory}). Retry scheduler after verifying relay connectivity, relay URLs, and DNS/network status.`,
|
||||||
metadata: {
|
metadata: {
|
||||||
|
failure_class: 'backend_unavailable',
|
||||||
preflight_failure_category: preflight.failureCategory,
|
preflight_failure_category: preflight.failureCategory,
|
||||||
relay_list: preflight.relayList.join(', ') || '(none)',
|
relay_list: preflight.relayList.join(', ') || '(none)',
|
||||||
preflight_stderr_excerpt: preflight.stderrExcerpt || '(empty)',
|
preflight_stderr_excerpt: preflight.stderrExcerpt || '(empty)',
|
||||||
@@ -543,11 +603,16 @@ async function main() {
|
|||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const deferralForAgent = schedulerRunState.lock_deferral?.selected_agent === selectedAgent
|
||||||
|
? schedulerRunState.lock_deferral
|
||||||
|
: null;
|
||||||
|
|
||||||
const lockAttempt = await acquireLockWithRetry({
|
const lockAttempt = await acquireLockWithRetry({
|
||||||
selectedAgent,
|
selectedAgent,
|
||||||
cadence,
|
cadence,
|
||||||
platform,
|
platform,
|
||||||
lockRetry: schedulerConfig.lockRetry,
|
lockRetry: schedulerConfig.lockRetry,
|
||||||
|
idempotencyKey: deferralForAgent?.idempotency_key,
|
||||||
});
|
});
|
||||||
const lockResult = lockAttempt.result;
|
const lockResult = lockAttempt.result;
|
||||||
|
|
||||||
@@ -562,6 +627,46 @@ async function main() {
|
|||||||
const stderrExcerpt = excerptText(lockResult.stderr);
|
const stderrExcerpt = excerptText(lockResult.stderr);
|
||||||
const stdoutExcerpt = excerptText(lockResult.stdout);
|
const stdoutExcerpt = excerptText(lockResult.stdout);
|
||||||
const backoffSchedule = lockAttempt.backoffScheduleMs.join(', ');
|
const backoffSchedule = lockAttempt.backoffScheduleMs.join(', ');
|
||||||
|
|
||||||
|
const existingDeferral = deferralForAgent || {};
|
||||||
|
const firstFailureAt = existingDeferral.first_failure_timestamp || new Date().toISOString();
|
||||||
|
const firstFailureMs = parseDateValue(firstFailureAt) || Date.now();
|
||||||
|
const nowMs = Date.now();
|
||||||
|
const deferralAttemptCount = parseNonNegativeInt(existingDeferral.attempt_count, 0) + 1;
|
||||||
|
const idempotencyKey = existingDeferral.idempotency_key
|
||||||
|
|| createIdempotencyKey({ cadence, selectedAgent, runDate: schedulerRunState.run_date });
|
||||||
|
const withinRetryWindow = nowMs - firstFailureMs <= schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs;
|
||||||
|
const withinDeferralBudget = deferralAttemptCount <= schedulerConfig.lockFailurePolicy.maxDeferrals;
|
||||||
|
|
||||||
|
if (!schedulerConfig.lockFailurePolicy.strictLock && withinRetryWindow && withinDeferralBudget) {
|
||||||
|
schedulerRunState.lock_deferral = {
|
||||||
|
attempt_count: deferralAttemptCount,
|
||||||
|
first_failure_timestamp: firstFailureAt,
|
||||||
|
backend_category: backendCategory,
|
||||||
|
idempotency_key: idempotencyKey,
|
||||||
|
selected_agent: selectedAgent,
|
||||||
|
};
|
||||||
|
await writeRunState(runStatePath, schedulerRunState);
|
||||||
|
await writeLog({
|
||||||
|
cadence,
|
||||||
|
agent: selectedAgent,
|
||||||
|
status: 'deferred',
|
||||||
|
platform,
|
||||||
|
reason: 'Lock backend deferred',
|
||||||
|
detail: `Deferred after lock backend failure (${backendCategory}); retry window active and deferral budget remaining.`,
|
||||||
|
metadata: {
|
||||||
|
failure_class: 'backend_unavailable',
|
||||||
|
deferral_attempt_count: deferralAttemptCount,
|
||||||
|
deferral_first_failure_timestamp: firstFailureAt,
|
||||||
|
backend_category: backendCategory,
|
||||||
|
lock_idempotency_key: idempotencyKey,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
schedulerRunState.lock_deferral = null;
|
||||||
|
await writeRunState(runStatePath, schedulerRunState);
|
||||||
await writeLog({
|
await writeLog({
|
||||||
cadence,
|
cadence,
|
||||||
agent: selectedAgent,
|
agent: selectedAgent,
|
||||||
@@ -570,10 +675,14 @@ async function main() {
|
|||||||
reason: 'Lock backend error',
|
reason: 'Lock backend error',
|
||||||
detail: `Lock backend error (${backendCategory}) after ${lockAttempt.attempts} attempt(s). Retry ${lockCommand} after verifying relay connectivity, relay URLs, and DNS/network status.`,
|
detail: `Lock backend error (${backendCategory}) after ${lockAttempt.attempts} attempt(s). Retry ${lockCommand} after verifying relay connectivity, relay URLs, and DNS/network status.`,
|
||||||
metadata: {
|
metadata: {
|
||||||
|
failure_class: 'backend_unavailable',
|
||||||
lock_attempts_total: lockAttempt.attempts,
|
lock_attempts_total: lockAttempt.attempts,
|
||||||
lock_backoff_schedule_ms: backoffSchedule || '(none)',
|
lock_backoff_schedule_ms: backoffSchedule || '(none)',
|
||||||
backend_category: backendCategory,
|
backend_category: backendCategory,
|
||||||
|
deferral_attempt_count: deferralAttemptCount,
|
||||||
|
deferral_first_failure_timestamp: firstFailureAt,
|
||||||
lock_command: lockCommand,
|
lock_command: lockCommand,
|
||||||
|
lock_idempotency_key: idempotencyKey,
|
||||||
lock_stderr_excerpt: stderrExcerpt || '(empty)',
|
lock_stderr_excerpt: stderrExcerpt || '(empty)',
|
||||||
lock_stdout_excerpt: stdoutExcerpt || '(empty)',
|
lock_stdout_excerpt: stdoutExcerpt || '(empty)',
|
||||||
},
|
},
|
||||||
@@ -582,6 +691,8 @@ async function main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (lockResult.code !== 0) {
|
if (lockResult.code !== 0) {
|
||||||
|
schedulerRunState.lock_deferral = null;
|
||||||
|
await writeRunState(runStatePath, schedulerRunState);
|
||||||
await writeLog({
|
await writeLog({
|
||||||
cadence,
|
cadence,
|
||||||
agent: selectedAgent,
|
agent: selectedAgent,
|
||||||
@@ -593,6 +704,8 @@ async function main() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const promptPath = path.resolve(process.cwd(), 'src/prompts', cadence, `${selectedAgent}.md`);
|
const promptPath = path.resolve(process.cwd(), 'src/prompts', cadence, `${selectedAgent}.md`);
|
||||||
|
schedulerRunState.lock_deferral = null;
|
||||||
|
await writeRunState(runStatePath, schedulerRunState);
|
||||||
const runArtifactSince = new Date().toISOString();
|
const runArtifactSince = new Date().toISOString();
|
||||||
const runStartMs = Date.parse(runArtifactSince);
|
const runStartMs = Date.parse(runArtifactSince);
|
||||||
const outputChunks = [];
|
const outputChunks = [];
|
||||||
@@ -603,7 +716,15 @@ async function main() {
|
|||||||
});
|
});
|
||||||
outputChunks.push(retrieveResult.stdout, retrieveResult.stderr);
|
outputChunks.push(retrieveResult.stdout, retrieveResult.stderr);
|
||||||
if (retrieveResult.code !== 0) {
|
if (retrieveResult.code !== 0) {
|
||||||
await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Memory retrieval command failed', detail: schedulerConfig.memoryPolicy.retrieveCommand });
|
await writeLog({
|
||||||
|
cadence,
|
||||||
|
agent: selectedAgent,
|
||||||
|
status: 'failed',
|
||||||
|
platform,
|
||||||
|
reason: 'Memory retrieval command failed',
|
||||||
|
detail: schedulerConfig.memoryPolicy.retrieveCommand,
|
||||||
|
metadata: { failure_class: 'prompt_validation_error' },
|
||||||
|
});
|
||||||
process.exit(retrieveResult.code);
|
process.exit(retrieveResult.code);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -614,14 +735,30 @@ async function main() {
|
|||||||
});
|
});
|
||||||
outputChunks.push(handoff.stdout, handoff.stderr);
|
outputChunks.push(handoff.stdout, handoff.stderr);
|
||||||
if (handoff.code !== 0) {
|
if (handoff.code !== 0) {
|
||||||
await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Prompt/handoff execution failed', detail: 'Handoff callback failed.' });
|
await writeLog({
|
||||||
|
cadence,
|
||||||
|
agent: selectedAgent,
|
||||||
|
status: 'failed',
|
||||||
|
platform,
|
||||||
|
reason: 'Prompt/handoff execution failed',
|
||||||
|
detail: 'Handoff callback failed.',
|
||||||
|
metadata: { failure_class: 'prompt_validation_error' },
|
||||||
|
});
|
||||||
process.exit(handoff.code);
|
process.exit(handoff.code);
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
const detail = schedulerConfig.missingHandoffCommandForMode
|
const detail = schedulerConfig.missingHandoffCommandForMode
|
||||||
? 'Missing scheduler handoff command for non-interactive run. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.'
|
? 'Missing scheduler handoff command for non-interactive run. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.'
|
||||||
: 'No handoff callback configured. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.';
|
: 'No handoff callback configured. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.';
|
||||||
await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Prompt/handoff execution failed', detail });
|
await writeLog({
|
||||||
|
cadence,
|
||||||
|
agent: selectedAgent,
|
||||||
|
status: 'failed',
|
||||||
|
platform,
|
||||||
|
reason: 'Prompt/handoff execution failed',
|
||||||
|
detail,
|
||||||
|
metadata: { failure_class: 'prompt_validation_error' },
|
||||||
|
});
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -631,7 +768,15 @@ async function main() {
|
|||||||
});
|
});
|
||||||
outputChunks.push(storeResult.stdout, storeResult.stderr);
|
outputChunks.push(storeResult.stdout, storeResult.stderr);
|
||||||
if (storeResult.code !== 0) {
|
if (storeResult.code !== 0) {
|
||||||
await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Memory storage command failed', detail: schedulerConfig.memoryPolicy.storeCommand });
|
await writeLog({
|
||||||
|
cadence,
|
||||||
|
agent: selectedAgent,
|
||||||
|
status: 'failed',
|
||||||
|
platform,
|
||||||
|
reason: 'Memory storage command failed',
|
||||||
|
detail: schedulerConfig.memoryPolicy.storeCommand,
|
||||||
|
metadata: { failure_class: 'prompt_validation_error' },
|
||||||
|
});
|
||||||
process.exit(storeResult.code);
|
process.exit(storeResult.code);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -661,6 +806,7 @@ async function main() {
|
|||||||
platform,
|
platform,
|
||||||
reason: 'Required memory steps not verified',
|
reason: 'Required memory steps not verified',
|
||||||
detail: `Missing evidence for: ${missingSteps.join(', ')}`,
|
detail: `Missing evidence for: ${missingSteps.join(', ')}`,
|
||||||
|
metadata: { failure_class: 'prompt_validation_error' },
|
||||||
});
|
});
|
||||||
process.exit(1);
|
process.exit(1);
|
||||||
}
|
}
|
||||||
@@ -692,6 +838,7 @@ async function main() {
|
|||||||
platform,
|
platform,
|
||||||
reason: 'Missing required run artifacts',
|
reason: 'Missing required run artifacts',
|
||||||
detail,
|
detail,
|
||||||
|
metadata: { failure_class: 'prompt_validation_error' },
|
||||||
});
|
});
|
||||||
process.exit(artifactCheck.code);
|
process.exit(artifactCheck.code);
|
||||||
}
|
}
|
||||||
@@ -701,7 +848,15 @@ async function main() {
|
|||||||
if (!parts.length) continue;
|
if (!parts.length) continue;
|
||||||
const result = await runCommand(parts[0], parts.slice(1));
|
const result = await runCommand(parts[0], parts.slice(1));
|
||||||
if (result.code !== 0) {
|
if (result.code !== 0) {
|
||||||
await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Validation failed', detail: validation });
|
await writeLog({
|
||||||
|
cadence,
|
||||||
|
agent: selectedAgent,
|
||||||
|
status: 'failed',
|
||||||
|
platform,
|
||||||
|
reason: 'Validation failed',
|
||||||
|
detail: validation,
|
||||||
|
metadata: { failure_class: 'prompt_validation_error' },
|
||||||
|
});
|
||||||
process.exit(result.code);
|
process.exit(result.code);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -65,6 +65,9 @@ Common settings:
|
|||||||
- `scheduler.firstPromptByCadence.daily` / `.weekly` — first-run scheduler starting agent.
|
- `scheduler.firstPromptByCadence.daily` / `.weekly` — first-run scheduler starting agent.
|
||||||
- `scheduler.handoffCommandByCadence.daily` / `.weekly` — shell command run after lock acquisition; command must use `SCHEDULER_AGENT`, `SCHEDULER_CADENCE`, and `SCHEDULER_PROMPT_PATH` provided by `scripts/agent/run-scheduler-cycle.mjs`.
|
- `scheduler.handoffCommandByCadence.daily` / `.weekly` — shell command run after lock acquisition; command must use `SCHEDULER_AGENT`, `SCHEDULER_CADENCE`, and `SCHEDULER_PROMPT_PATH` provided by `scripts/agent/run-scheduler-cycle.mjs`.
|
||||||
- `scheduler.paused.daily` / `.weekly` — array of agent names to exclude from scheduler rotation.
|
- `scheduler.paused.daily` / `.weekly` — array of agent names to exclude from scheduler rotation.
|
||||||
|
- `scheduler.strict_lock` — lock backend policy switch (default: `true`); when `false`, scheduler defers backend-unavailable lock failures before converting the run to failed.
|
||||||
|
- `scheduler.degraded_lock_retry_window` — non-strict deferral window in milliseconds; backend lock failures outside this window immediately consume failure budget and mark run failed.
|
||||||
|
- `scheduler.max_deferrals` — max number of non-strict lock deferrals allowed in-window before scheduler records a hard failure.
|
||||||
|
|
||||||
Default first-run daily scheduler prompt is `scheduler-update-agent`.
|
Default first-run daily scheduler prompt is `scheduler-update-agent`.
|
||||||
|
|
||||||
@@ -72,6 +75,11 @@ For weekly repository-fit maintenance, TORCH also includes `src/prompts/weekly/r
|
|||||||
|
|
||||||
Operational note: scheduler handoff commands are treated as required execution steps. A non-zero exit code (or missing command) is a hard failure: the scheduler writes a `_failed.md` task log, exits immediately, and does not publish `lock:complete` for that run.
|
Operational note: scheduler handoff commands are treated as required execution steps. A non-zero exit code (or missing command) is a hard failure: the scheduler writes a `_failed.md` task log, exits immediately, and does not publish `lock:complete` for that run.
|
||||||
|
|
||||||
|
Scheduler failure classes in task logs:
|
||||||
|
|
||||||
|
- `backend_unavailable` — lock backend unavailable preflight failures and lock acquisition backend exit code `2` failures/deferrals.
|
||||||
|
- `prompt_validation_error` — prompt/handoff, memory evidence, artifact verification, and validation command failures.
|
||||||
|
|
||||||
|
|
||||||
## Lock backend production defaults
|
## Lock backend production defaults
|
||||||
|
|
||||||
|
|||||||
@@ -148,17 +148,24 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co
|
|||||||
|
|
||||||
- Exit `0`: lock acquired, continue.
|
- Exit `0`: lock acquired, continue.
|
||||||
- Exit `3`: race lost/already locked, return to step 2.
|
- Exit `3`: race lost/already locked, return to step 2.
|
||||||
- Exit `2`: lock backend error, write `_failed.md` with reason `Lock backend error`, and include failure metadata fields:
|
- Exit `2`: lock backend error.
|
||||||
|
- If `scheduler.strict_lock` is `false`, defer the run while both budget constraints are still satisfied:
|
||||||
|
- `degraded_lock_retry_window` (ms) since first backend failure has not elapsed.
|
||||||
|
- `max_deferrals` has not been exceeded.
|
||||||
|
- Record deferral metadata in scheduler run state (`attempt_count`, `first_failure_timestamp`, `backend_category`, and preserved idempotency key).
|
||||||
|
- Otherwise write `_failed.md` with reason `Lock backend error`, and include failure metadata fields:
|
||||||
- `backend_category` (classified backend failure category)
|
- `backend_category` (classified backend failure category)
|
||||||
- `lock_command` (raw lock command for retry)
|
- `lock_command` (raw lock command for retry)
|
||||||
- `lock_stderr_excerpt` (redacted stderr snippet)
|
- `lock_stderr_excerpt` (redacted stderr snippet)
|
||||||
- `lock_stdout_excerpt` (redacted stdout snippet)
|
- `lock_stdout_excerpt` (redacted stdout snippet)
|
||||||
|
- Include `failure_class: backend_unavailable` for both deferred and failed backend-unavailable lock outcomes.
|
||||||
- Keep generic reason text for compatibility, but append actionable retry guidance in `detail` using the command from `lock_command`.
|
- Keep generic reason text for compatibility, but append actionable retry guidance in `detail` using the command from `lock_command`.
|
||||||
|
|
||||||
9. Execute `<prompt_dir>/<prompt-file>` end-to-end via configured handoff command.
|
9. Execute `<prompt_dir>/<prompt-file>` end-to-end via configured handoff command.
|
||||||
|
|
||||||
- Scheduler automation runs `scheduler.handoffCommandByCadence.<cadence>` with environment variables for cadence/agent/prompt path.
|
- Scheduler automation runs `scheduler.handoffCommandByCadence.<cadence>` with environment variables for cadence/agent/prompt path.
|
||||||
- If no handoff command is configured for the cadence, write `_failed.md` and stop.
|
- If no handoff command is configured for the cadence, write `_failed.md` and stop.
|
||||||
|
- Prompt/handoff and prompt validation failures should emit `failure_class: prompt_validation_error`.
|
||||||
|
|
||||||
10. Confirm memory contract completion:
|
10. Confirm memory contract completion:
|
||||||
|
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ async function setupFixture({
|
|||||||
lockShellBody = '',
|
lockShellBody = '',
|
||||||
roster = ['agent-a'],
|
roster = ['agent-a'],
|
||||||
lockHealthPreflight = undefined,
|
lockHealthPreflight = undefined,
|
||||||
|
schedulerPolicy = {},
|
||||||
preflightScript = '#!/usr/bin/env node\nconsole.log(JSON.stringify({ ok: true, relays: ["wss://relay.test"] }));\nprocess.exit(0);\n',
|
preflightScript = '#!/usr/bin/env node\nconsole.log(JSON.stringify({ ok: true, relays: ["wss://relay.test"] }));\nprocess.exit(0);\n',
|
||||||
}) {
|
}) {
|
||||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'scheduler-memory-policy-'));
|
const root = await fs.mkdtemp(path.join(os.tmpdir(), 'scheduler-memory-policy-'));
|
||||||
@@ -69,6 +70,7 @@ async function setupFixture({
|
|||||||
handoffCommandByCadence: { daily: 'echo HANDOFF_OK' },
|
handoffCommandByCadence: { daily: 'echo HANDOFF_OK' },
|
||||||
validationCommandsByCadence: { daily: ['true'] },
|
validationCommandsByCadence: { daily: ['true'] },
|
||||||
memoryPolicyByCadence: { daily: memoryPolicy },
|
memoryPolicyByCadence: { daily: memoryPolicy },
|
||||||
|
...schedulerPolicy,
|
||||||
...(lockHealthPreflight === undefined ? {} : { lockHealthPreflight }),
|
...(lockHealthPreflight === undefined ? {} : { lockHealthPreflight }),
|
||||||
},
|
},
|
||||||
}, null, 2),
|
}, null, 2),
|
||||||
@@ -186,6 +188,7 @@ test('records backend failure metadata when lock command exits with code 2', { c
|
|||||||
const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8');
|
const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8');
|
||||||
|
|
||||||
assert.match(failedBody, /reason: Lock backend error/);
|
assert.match(failedBody, /reason: Lock backend error/);
|
||||||
|
assert.match(failedBody, /failure_class: 'backend_unavailable'/);
|
||||||
assert.match(failedBody, /backend_category: 'publish failed to all relays'/);
|
assert.match(failedBody, /backend_category: 'publish failed to all relays'/);
|
||||||
assert.match(failedBody, /lock_command: 'AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily'/);
|
assert.match(failedBody, /lock_command: 'AGENT_PLATFORM=codex npm run lock:lock -- --agent agent-a --cadence daily'/);
|
||||||
assert.match(failedBody, /lock_stderr_excerpt: 'publish failed to all relays token=\[REDACTED\] SECRET_KEY=\[REDACTED\]'/);
|
assert.match(failedBody, /lock_stderr_excerpt: 'publish failed to all relays token=\[REDACTED\] SECRET_KEY=\[REDACTED\]'/);
|
||||||
@@ -194,6 +197,149 @@ test('records backend failure metadata when lock command exits with code 2', { c
|
|||||||
assert.match(failedBody, /platform: 'codex'/);
|
assert.match(failedBody, /platform: 'codex'/);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('defers backend failures in non-strict mode and reuses idempotency key on successful retry', { concurrency: false }, async () => {
|
||||||
|
const fixture = await setupFixture({
|
||||||
|
memoryPolicy: { mode: 'optional' },
|
||||||
|
schedulerPolicy: {
|
||||||
|
strict_lock: false,
|
||||||
|
degraded_lock_retry_window: 3600000,
|
||||||
|
max_deferrals: 3,
|
||||||
|
},
|
||||||
|
lockShellBody: ` count_file="$PWD/.lock-count"
|
||||||
|
key_log="$PWD/.idempotency-keys"
|
||||||
|
count=0
|
||||||
|
if [[ -f "$count_file" ]]; then
|
||||||
|
count="$(cat "$count_file")"
|
||||||
|
fi
|
||||||
|
count=$((count + 1))
|
||||||
|
echo "$count" > "$count_file"
|
||||||
|
echo "\${SCHEDULER_LOCK_IDEMPOTENCY_KEY:-missing}" >> "$key_log"
|
||||||
|
if [[ "$count" -lt 3 ]]; then
|
||||||
|
echo 'relay query timeout' >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
:`,
|
||||||
|
});
|
||||||
|
|
||||||
|
const first = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
|
||||||
|
cwd: fixture.root,
|
||||||
|
env: {
|
||||||
|
...fixture.env,
|
||||||
|
SCHEDULER_LOCK_MAX_RETRIES: '0',
|
||||||
|
SCHEDULER_LOCK_BACKOFF_MS: '0',
|
||||||
|
SCHEDULER_LOCK_JITTER_MS: '0',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(first.code, 0, `stdout: ${first.stdout}\nstderr: ${first.stderr}`);
|
||||||
|
|
||||||
|
const second = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
|
||||||
|
cwd: fixture.root,
|
||||||
|
env: {
|
||||||
|
...fixture.env,
|
||||||
|
SCHEDULER_LOCK_MAX_RETRIES: '0',
|
||||||
|
SCHEDULER_LOCK_BACKOFF_MS: '0',
|
||||||
|
SCHEDULER_LOCK_JITTER_MS: '0',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(second.code, 0, `stdout: ${second.stdout}\nstderr: ${second.stderr}`);
|
||||||
|
|
||||||
|
const third = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
|
||||||
|
cwd: fixture.root,
|
||||||
|
env: {
|
||||||
|
...fixture.env,
|
||||||
|
SCHEDULER_LOCK_MAX_RETRIES: '0',
|
||||||
|
SCHEDULER_LOCK_BACKOFF_MS: '0',
|
||||||
|
SCHEDULER_LOCK_JITTER_MS: '0',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(third.code, 0, `stdout: ${third.stdout}\nstderr: ${third.stderr}`);
|
||||||
|
|
||||||
|
const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily'));
|
||||||
|
const deferredLog = logs.find((name) => name.endsWith('__deferred.md'));
|
||||||
|
assert.ok(deferredLog);
|
||||||
|
const deferredBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', deferredLog), 'utf8');
|
||||||
|
assert.match(deferredBody, /failure_class: 'backend_unavailable'/);
|
||||||
|
assert.match(deferredBody, /deferral_attempt_count: '\d+'/);
|
||||||
|
|
||||||
|
const keyLog = await fs.readFile(path.join(fixture.root, '.idempotency-keys'), 'utf8');
|
||||||
|
const keys = keyLog.trim().split(/\r?\n/).filter(Boolean);
|
||||||
|
assert.equal(keys.length, 3);
|
||||||
|
assert.equal(keys[0], 'missing');
|
||||||
|
assert.notEqual(keys[1], 'missing');
|
||||||
|
assert.equal(keys[1], keys[2]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('fails after exceeding non-strict deferral budget', { concurrency: false }, async () => {
|
||||||
|
const fixture = await setupFixture({
|
||||||
|
memoryPolicy: { mode: 'optional' },
|
||||||
|
schedulerPolicy: {
|
||||||
|
strict_lock: false,
|
||||||
|
degraded_lock_retry_window: 3600000,
|
||||||
|
max_deferrals: 1,
|
||||||
|
},
|
||||||
|
lockShellBody: ` echo 'publish failed to all relays' >&2
|
||||||
|
exit 2`,
|
||||||
|
});
|
||||||
|
|
||||||
|
const first = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
|
||||||
|
cwd: fixture.root,
|
||||||
|
env: {
|
||||||
|
...fixture.env,
|
||||||
|
SCHEDULER_LOCK_MAX_RETRIES: '0',
|
||||||
|
SCHEDULER_LOCK_BACKOFF_MS: '0',
|
||||||
|
SCHEDULER_LOCK_JITTER_MS: '0',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(first.code, 0, `stdout: ${first.stdout}\nstderr: ${first.stderr}`);
|
||||||
|
|
||||||
|
const second = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
|
||||||
|
cwd: fixture.root,
|
||||||
|
env: {
|
||||||
|
...fixture.env,
|
||||||
|
SCHEDULER_LOCK_MAX_RETRIES: '0',
|
||||||
|
SCHEDULER_LOCK_BACKOFF_MS: '0',
|
||||||
|
SCHEDULER_LOCK_JITTER_MS: '0',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(second.code, 2, `stdout: ${second.stdout}\nstderr: ${second.stderr}`);
|
||||||
|
|
||||||
|
const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily'));
|
||||||
|
const failedLog = logs.find((name) => name.endsWith('__failed.md'));
|
||||||
|
assert.ok(failedLog);
|
||||||
|
const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8');
|
||||||
|
assert.match(failedBody, /failure_class: 'backend_unavailable'/);
|
||||||
|
assert.match(failedBody, /deferral_attempt_count: '2'/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('strict lock mode fails immediately without deferral', { concurrency: false }, async () => {
|
||||||
|
const fixture = await setupFixture({
|
||||||
|
memoryPolicy: { mode: 'optional' },
|
||||||
|
schedulerPolicy: {
|
||||||
|
strict_lock: true,
|
||||||
|
degraded_lock_retry_window: 3600000,
|
||||||
|
max_deferrals: 5,
|
||||||
|
},
|
||||||
|
lockShellBody: ` echo 'relay timeout' >&2
|
||||||
|
exit 2`,
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], {
|
||||||
|
cwd: fixture.root,
|
||||||
|
env: {
|
||||||
|
...fixture.env,
|
||||||
|
SCHEDULER_LOCK_MAX_RETRIES: '0',
|
||||||
|
SCHEDULER_LOCK_BACKOFF_MS: '0',
|
||||||
|
SCHEDULER_LOCK_JITTER_MS: '0',
|
||||||
|
},
|
||||||
|
});
|
||||||
|
assert.equal(result.code, 2, `stdout: ${result.stdout}\nstderr: ${result.stderr}`);
|
||||||
|
|
||||||
|
const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily'));
|
||||||
|
assert.equal(logs.some((name) => name.endsWith('__deferred.md')), false);
|
||||||
|
const failedLog = logs.find((name) => name.endsWith('__failed.md'));
|
||||||
|
assert.ok(failedLog);
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
test('skips lock health preflight by default when not enabled', { concurrency: false }, async () => {
|
test('skips lock health preflight by default when not enabled', { concurrency: false }, async () => {
|
||||||
|
|||||||
Reference in new issue
Block a user