mirror of
https://github.com/PR0M3TH3AN/TORCH.git
synced 2026-10-27 16:01:54 +00:00
google-labs-jules[bot]andPR0M3TH3AN
74950ca1a9
Extracted logic into scheduler-config.mjs, scheduler-logger.mjs, and scheduler-state.mjs to improve maintainability and reduce file size (1160 -> 827 lines). Corrected module imports and verified via linting and dry-run usage. Co-authored-by: PR0M3TH3AN <[email protected]>
827 lines
33 KiB
JavaScript
827 lines
33 KiB
JavaScript
#!/usr/bin/env node
|
|
/**
|
|
* run-scheduler-cycle.mjs — TORCH Agent Scheduler
|
|
*
|
|
* Orchestrates one full scheduling cycle for a given cadence (daily|weekly).
|
|
* Selects the next eligible agent from the roster, acquires a distributed Nostr
|
|
* lock, executes the agent's prompt via a configured handoff command, verifies
|
|
* required artifacts and memory evidence, then publishes completion.
|
|
*
|
|
* This script owns the full lifecycle for a single agent slot per invocation:
|
|
* lock acquisition → prompt execution → artifact verification → lock:complete.
|
|
* Spawned agents MUST NOT call lock:complete themselves.
|
|
*
|
|
* Main flow (mirrors src/prompts/scheduler-flow.md numbered MUST steps):
|
|
* 1. Parse cadence (daily|weekly), platform, and model from CLI args.
|
|
* 2. Load roster from src/prompts/roster.json.
|
|
* 3. Read AGENTS.md policy file (best-effort, non-fatal if missing).
|
|
* 4. [loop] Read cadence run-state (deferral tracking for the current UTC day).
|
|
* 5. [loop] Optional lock health preflight — fail/defer if all relays unhealthy.
|
|
* 6. [loop] Run lock:check to build the exclusion set (locked + paused + completed).
|
|
* 7. [loop] Apply local time-window guard (24 h daily / 7 d weekly) to exclusion set.
|
|
* 8. [loop] Round-robin select next eligible agent from roster minus exclusion set.
|
|
* 9. [loop] If no agent eligible → exit (cycle-saturated or all-excluded).
|
|
* 10. [loop] Acquire lock with retry. exit 3 → back to step 4; exit 2 → defer or fail.
|
|
* 11. [loop] Validate prompt file (readable, correct format).
|
|
* 12. [loop] Clear deferral state. Run memory retrieve command (if configured).
|
|
* 13. [loop] Run handoff command — hard fail if missing or exits non-zero.
|
|
* 14. [loop] Run memory store command (if configured).
|
|
* 15. [loop] Verify memory evidence (markers/artifact files).
|
|
* 16. [loop] Verify run artifacts via verify-run-artifacts.mjs.
|
|
* 17. [loop] Run validation commands (default: npm run lint).
|
|
* 18. [loop] Publish lock:complete — hard fail if relay error.
|
|
* 19. [loop] Write completed task log and print run summary. Exit 0.
|
|
*
|
|
* Key invariants:
|
|
* - lock:complete is called ONLY after all validation gates pass (step 18 is last gate).
|
|
* - Completed task log is written ONLY after lock:complete succeeds.
|
|
* - Lock exit code 3 (race lost) causes a loop restart to pick the next agent.
|
|
* - Memory mode 'required' turns missing evidence into a hard failure.
|
|
*/
|
|
// Source of truth: numbered MUST steps 2 and 4-16 in src/prompts/scheduler-flow.md are
|
|
// implemented by this script; step 3 (policy-file read) is best-effort and non-fatal.
|
|
import fs from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
|
|
import {
|
|
runCommand,
|
|
parseJsonFromOutput,
|
|
readJson,
|
|
parseNonNegativeInt,
|
|
excerptText,
|
|
buildRecentlyRunExclusionSet,
|
|
parseDateValue,
|
|
parseFrontmatterCreatedAt,
|
|
parseFrontmatterAgent,
|
|
parseTimestampFromFilename,
|
|
isStrictSchedulerLogFilename,
|
|
parseAgentFromFilename,
|
|
} from './scheduler-utils.mjs';
|
|
|
|
import {
|
|
classifyLockBackendError,
|
|
buildLockBackendRemediation,
|
|
runLockHealthPreflight,
|
|
acquireLockWithRetry,
|
|
summarizeLockFailureReasons,
|
|
} from './scheduler-lock.mjs';
|
|
|
|
import {
|
|
VALID_CADENCES,
|
|
parseArgs,
|
|
getSchedulerConfig,
|
|
} from './scheduler-config.mjs';
|
|
|
|
import {
|
|
ts,
|
|
writeLog,
|
|
exitWithSummary,
|
|
categorizeFailureMetadata,
|
|
} from './scheduler-logger.mjs';
|
|
|
|
import {
|
|
readRunState,
|
|
writeRunState,
|
|
createIdempotencyKey,
|
|
} from './scheduler-state.mjs';
|
|
|
|
const ALL_EXCLUDED_REASON = 'All roster tasks currently claimed by other agents';
|
|
const FAILURE_CATEGORY = {
|
|
PROMPT_PARSE: 'prompt_parse_error',
|
|
PROMPT_SCHEMA: 'prompt_schema_error',
|
|
LOCK_BACKEND: 'lock_backend_error',
|
|
EXECUTION: 'execution_error',
|
|
};
|
|
|
|
/**
|
|
* Validates that a prompt file is readable and contains a valid markdown heading
|
|
* or blockquote as its first non-empty line.
|
|
*
|
|
* This is a lightweight schema check — it does not parse frontmatter or validate
|
|
* the full prompt contract (that is handled by validate-prompt-contract.mjs).
|
|
*
|
|
* @param {string} promptPath - Absolute path to the prompt markdown file.
|
|
* @returns {Promise<{ok: true}|{ok: false, category: string, reason: string, detail: string}>}
|
|
*/
|
|
async function validatePromptFile(promptPath) {
|
|
let content;
|
|
try {
|
|
content = await fs.readFile(promptPath, 'utf8');
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
category: FAILURE_CATEGORY.PROMPT_PARSE,
|
|
reason: 'Prompt file parse/read failed',
|
|
detail: `Prompt not executed; unable to read prompt file at ${promptPath}: ${error.message}`,
|
|
};
|
|
}
|
|
|
|
const lines = String(content).split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
|
|
const firstLine = lines[0] || '';
|
|
if (!firstLine.startsWith('#') && !firstLine.startsWith('>')) {
|
|
return {
|
|
ok: false,
|
|
category: FAILURE_CATEGORY.PROMPT_SCHEMA,
|
|
reason: 'Prompt file schema validation failed',
|
|
detail: `Prompt not executed; expected markdown heading or blockquote on first non-empty line in ${promptPath}.`,
|
|
};
|
|
}
|
|
|
|
return { ok: true };
|
|
}
|
|
|
|
/**
|
|
* Checks whether a file exists and was modified at or after `sinceMs`.
|
|
* Used to verify that memory evidence artifacts were produced during the current run.
|
|
*
|
|
* @param {string|null} filePath - Relative or absolute path to check.
|
|
* @param {number} sinceMs - Epoch milliseconds threshold (run start time).
|
|
* @returns {Promise<boolean>}
|
|
*/
|
|
async function artifactExistsSince(filePath, sinceMs) {
|
|
if (!filePath) return false;
|
|
const resolved = path.isAbsolute(filePath)
|
|
? filePath
|
|
: path.resolve(process.cwd(), filePath);
|
|
try {
|
|
const stat = await fs.stat(resolved);
|
|
return stat.isFile() && stat.mtimeMs >= sinceMs;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Verifies that a memory pipeline step (retrieve or store) produced observable evidence.
|
|
* Evidence can be either a deterministic output marker string OR a qualifying artifact file.
|
|
* A step is considered complete when either form of evidence is present.
|
|
*
|
|
* @param {Object} params
|
|
* @param {string} params.name - Step name ('retrieve'|'store') for logging.
|
|
* @param {string[]} params.markers - Substrings to search for in combined command output.
|
|
* @param {string[]} params.artifacts - Relative paths of artifact files to check.
|
|
* @param {string} params.outputText - Combined stdout+stderr from all commands this run.
|
|
* @param {number} params.sinceMs - Run start epoch ms; artifact files must be newer.
|
|
* @returns {Promise<{name: string, markerMatched: boolean, artifactMatched: boolean, complete: boolean}>}
|
|
*/
|
|
async function verifyMemoryStep({ name, markers, artifacts, outputText, sinceMs }) {
|
|
const markerMatched = markers.some((marker) => outputText.includes(marker));
|
|
const artifactResults = await Promise.all(artifacts.map((artifact) => artifactExistsSince(artifact, sinceMs)));
|
|
const artifactMatched = artifactResults.some(Boolean);
|
|
|
|
return {
|
|
name,
|
|
markerMatched,
|
|
artifactMatched,
|
|
complete: markerMatched || artifactMatched,
|
|
};
|
|
}
|
|
|
|
|
|
/**
|
|
* Finds the most recent valid scheduler task log file in `logDir`.
|
|
* "Valid" means the filename passes isStrictSchedulerLogFilename() AND the file
|
|
* contains a parseable timestamp (from frontmatter created_at or the filename itself).
|
|
*
|
|
* Files with unparseable timestamps are skipped with a console warning.
|
|
* The effective timestamp for sorting is derived from frontmatter first, then filename.
|
|
*
|
|
* @param {string} logDir - Absolute path to the cadence log directory (e.g. task-logs/daily/).
|
|
* @returns {Promise<string|null>} Filename (not path) of the latest log, or null if none.
|
|
*/
|
|
async function getLatestFile(logDir) {
|
|
await fs.mkdir(logDir, { recursive: true });
|
|
const entries = await fs.readdir(logDir, { withFileTypes: true });
|
|
const files = entries
|
|
.filter((entry) => entry.isFile())
|
|
.map((entry) => entry.name)
|
|
.filter((filename) => isStrictSchedulerLogFilename(filename))
|
|
.sort((a, b) => b.localeCompare(a));
|
|
|
|
const results = await Promise.all(
|
|
files.map(async (filename) => {
|
|
const filePath = path.join(logDir, filename);
|
|
const content = await fs.readFile(filePath, 'utf8').catch(() => '');
|
|
const createdAtMs = parseDateValue(parseFrontmatterCreatedAt(content));
|
|
const fileTimestampMs = parseTimestampFromFilename(filename);
|
|
const effectiveMs = createdAtMs ?? fileTimestampMs;
|
|
|
|
if (!effectiveMs) {
|
|
console.warn(`[scheduler] Ignoring invalid log timestamp in ${filename}; checking next candidate.`);
|
|
return null;
|
|
}
|
|
return { filename, effectiveMs };
|
|
}),
|
|
);
|
|
|
|
let latest = null;
|
|
|
|
for (const result of results) {
|
|
if (!result) continue;
|
|
if (!latest || result.effectiveMs > latest.effectiveMs) {
|
|
latest = result;
|
|
}
|
|
}
|
|
|
|
return latest?.filename || null;
|
|
}
|
|
|
|
/**
|
|
* Selects the next eligible agent from the roster using round-robin scheduling.
|
|
*
|
|
* Selection algorithm:
|
|
* - If `previousAgent` is in the roster, start scanning from the next index (wrap-around).
|
|
* - Otherwise, start from `firstPrompt` index (if configured) or index 0.
|
|
* - Returns the first candidate not present in `excludedSet`.
|
|
* - Returns null if every roster agent is excluded.
|
|
*
|
|
* @param {Object} params
|
|
* @param {string[]} params.roster - Ordered array of agent names for this cadence.
|
|
* @param {Set<string>} params.excludedSet - Agents to skip (locked, paused, completed, time-excluded).
|
|
* @param {string|null} params.previousAgent - Agent that ran most recently (from latest log).
|
|
* @param {string|null} params.firstPrompt - Configured first-run starting agent name.
|
|
* @returns {string|null} Selected agent name, or null if roster is fully excluded.
|
|
*/
|
|
function selectNextAgent({ roster, excludedSet, previousAgent, firstPrompt }) {
|
|
const previousIndex = roster.indexOf(previousAgent);
|
|
const firstIndex = roster.indexOf(firstPrompt);
|
|
const startIndex = previousIndex >= 0
|
|
? (previousIndex + 1) % roster.length
|
|
: (firstIndex >= 0 ? firstIndex : 0);
|
|
|
|
for (let offset = 0; offset < roster.length; offset += 1) {
|
|
const candidate = roster[(startIndex + offset) % roster.length];
|
|
if (!excludedSet.has(candidate)) {
|
|
return candidate;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function main() {
|
|
const { cadence, platform, model } = parseArgs(process.argv.slice(2));
|
|
if (!VALID_CADENCES.has(cadence)) {
|
|
console.error('Usage: node scripts/agent/run-scheduler-cycle.mjs <daily|weekly>');
|
|
process.exit(1);
|
|
}
|
|
|
|
const roster = (await readJson(path.resolve(process.cwd(), 'src/prompts/roster.json'), {}))[cadence] || [];
|
|
if (!Array.isArray(roster) || roster.length === 0) {
|
|
console.error(`No roster entries for cadence ${cadence}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
const isInteractive = Boolean(process.stdin.isTTY && process.stdout.isTTY);
|
|
const schedulerConfig = await getSchedulerConfig(cadence, { isInteractive });
|
|
const logDir = path.resolve(process.cwd(), 'task-logs', cadence);
|
|
|
|
const agentsPath = path.resolve(process.cwd(), 'AGENTS.md');
|
|
try {
|
|
const agentsContent = await fs.readFile(agentsPath, 'utf8');
|
|
process.stdout.write(`${agentsContent}\n`);
|
|
} catch {
|
|
console.log('No AGENTS.md found; continuing');
|
|
}
|
|
|
|
while (true) {
|
|
const { statePath: runStatePath, state: schedulerRunState } = await readRunState(cadence);
|
|
|
|
if (schedulerConfig.lockHealthPreflight.enabled && !schedulerConfig.lockHealthPreflight.skip) {
|
|
const preflight = await runLockHealthPreflight({ cadence, platform });
|
|
if (preflight.code !== 0) {
|
|
const allRelaysUnhealthy = Boolean(preflight.payload?.summary?.allRelaysUnhealthy);
|
|
const incidentSignal = preflight.payload?.incidentSignal || null;
|
|
await writeLog({
|
|
cadence,
|
|
agent: 'scheduler',
|
|
status: allRelaysUnhealthy ? 'deferred' : 'failed',
|
|
platform,
|
|
reason: allRelaysUnhealthy
|
|
? 'All relays unhealthy preflight'
|
|
: 'Lock backend unavailable preflight',
|
|
detail: allRelaysUnhealthy
|
|
? `Deferred run before lock acquisition: ${incidentSignal?.reason || 'all relays unhealthy'}. Prompt not executed. ${buildLockBackendRemediation({ cadence, retryWindowMs: schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs, maxDeferrals: schedulerConfig.lockFailurePolicy.maxDeferrals, incidentSignalId: incidentSignal?.id || null })}`
|
|
: `Preflight failed (${preflight.failureCategory}). Prompt not executed. ${buildLockBackendRemediation({ cadence, retryWindowMs: schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs, maxDeferrals: schedulerConfig.lockFailurePolicy.maxDeferrals, incidentSignalId: incidentSignal?.id || null })}`,
|
|
metadata: {
|
|
...categorizeFailureMetadata(FAILURE_CATEGORY.LOCK_BACKEND, { failure_class: 'backend_unavailable' }),
|
|
preflight_failure_category: preflight.failureCategory,
|
|
relay_list: preflight.relayList.join(', ') || '(none)',
|
|
preflight_stderr_excerpt: preflight.stderrExcerpt || '(empty)',
|
|
preflight_stdout_excerpt: preflight.stdoutExcerpt || '(empty)',
|
|
incident_signal_id: incidentSignal?.id || null,
|
|
incident_signal_severity: incidentSignal?.severity || null,
|
|
preflight_alerts: JSON.stringify(preflight.payload?.alerts || []),
|
|
relay_health_history_path: preflight.payload?.historyPath || null,
|
|
},
|
|
});
|
|
await exitWithSummary(allRelaysUnhealthy ? 0 : (preflight.code || 1), {
|
|
status: allRelaysUnhealthy ? 'deferred' : 'failed',
|
|
agent: 'scheduler',
|
|
promptPath: null,
|
|
reason: allRelaysUnhealthy ? 'All relays unhealthy preflight' : 'Lock backend unavailable preflight',
|
|
platform,
|
|
});
|
|
}
|
|
}
|
|
|
|
// Step 6: Build exclusion set from relay state. Prefer the top-level `excluded`
|
|
// field (computed by cmdCheck), falling back to the union of locked/paused/completed.
|
|
const checkResult = await runCommand('npm', ['run', `lock:check:${cadence}`, '--', '--json', '--quiet']);
|
|
const checkPayload = parseJsonFromOutput(`${checkResult.stdout}\n${checkResult.stderr}`) || {};
|
|
const excluded = Array.isArray(checkPayload.excluded)
|
|
? checkPayload.excluded
|
|
: [...new Set([
|
|
...(Array.isArray(checkPayload.locked) ? checkPayload.locked : []),
|
|
...(Array.isArray(checkPayload.paused) ? checkPayload.paused : []),
|
|
...(Array.isArray(checkPayload.completed) ? checkPayload.completed : []),
|
|
])];
|
|
const excludedSet = new Set(excluded);
|
|
|
|
// Time-window guard: exclude agents that already ran within the cadence window
|
|
// (24 h for daily, 7 d for weekly). This catches the cross-midnight edge case
|
|
// where the relay's date-scoped lock has rolled over but the run was recent.
|
|
const windowLabel = cadence === 'weekly' ? '7-day' : '24-hour';
|
|
const recentlyRunSet = await buildRecentlyRunExclusionSet(logDir, cadence);
|
|
const newlyTimeExcluded = [...recentlyRunSet].filter((agent) => !excludedSet.has(agent));
|
|
for (const agent of recentlyRunSet) {
|
|
excludedSet.add(agent);
|
|
}
|
|
if (newlyTimeExcluded.length > 0) {
|
|
console.log(`[scheduler] Time-window guard (${windowLabel}): ${newlyTimeExcluded.join(', ')} excluded — ran within window`);
|
|
}
|
|
|
|
const latestFile = await getLatestFile(logDir);
|
|
let previousAgent = null;
|
|
|
|
if (latestFile) {
|
|
const latestPath = path.join(logDir, latestFile);
|
|
const content = await fs.readFile(latestPath, 'utf8').catch(() => '');
|
|
previousAgent = parseFrontmatterAgent(content) || parseAgentFromFilename(latestFile);
|
|
if (!roster.includes(previousAgent)) {
|
|
previousAgent = null;
|
|
}
|
|
}
|
|
|
|
const selectedAgent = selectNextAgent({
|
|
roster,
|
|
excludedSet,
|
|
previousAgent,
|
|
firstPrompt: schedulerConfig.firstPrompt,
|
|
});
|
|
|
|
if (!selectedAgent) {
|
|
// Cycle saturation: every roster agent has a recent local log within the
|
|
// window. This is the expected steady state once a full rotation completes.
|
|
// Exit 0 so the triggering process (cron, CI) does not treat it as an error.
|
|
const allCycleSaturated = roster.every((agent) => recentlyRunSet.has(agent));
|
|
if (allCycleSaturated) {
|
|
const reason = `Cycle complete: all ${roster.length} agents ran within the ${windowLabel} window`;
|
|
console.log(`[scheduler] ${reason}. Nothing to do.`);
|
|
await exitWithSummary(0, {
|
|
status: 'skipped',
|
|
agent: 'scheduler',
|
|
promptPath: null,
|
|
reason,
|
|
platform,
|
|
});
|
|
}
|
|
|
|
await writeLog({
|
|
cadence,
|
|
agent: 'scheduler',
|
|
status: 'failed',
|
|
platform,
|
|
reason: ALL_EXCLUDED_REASON,
|
|
});
|
|
await exitWithSummary(1, {
|
|
status: 'failed',
|
|
agent: 'scheduler',
|
|
promptPath: null,
|
|
reason: ALL_EXCLUDED_REASON,
|
|
platform,
|
|
});
|
|
}
|
|
|
|
const deferralForAgent = schedulerRunState.lock_deferral?.selected_agent === selectedAgent
|
|
? schedulerRunState.lock_deferral
|
|
: null;
|
|
|
|
const lockAttempt = await acquireLockWithRetry({
|
|
selectedAgent,
|
|
cadence,
|
|
platform,
|
|
model,
|
|
lockRetry: schedulerConfig.lockRetry,
|
|
idempotencyKey: deferralForAgent?.idempotency_key,
|
|
});
|
|
const lockResult = lockAttempt.result;
|
|
|
|
// Lock exit 3: race lost — another agent claimed this slot between our check
|
|
// and our publish. Restart the loop to pick the next available agent.
|
|
if (lockResult.code === 3) {
|
|
continue;
|
|
}
|
|
|
|
if (lockResult.code === 2) {
|
|
const combinedLockOutput = `${lockResult.stderr}\n${lockResult.stdout}`;
|
|
const diagnosticsSummary = summarizeLockFailureReasons(combinedLockOutput);
|
|
const backendCategory = lockAttempt.finalBackendCategory || classifyLockBackendError(combinedLockOutput);
|
|
const modelPart = model ? ` --model ${model}` : '';
|
|
const lockCommand = `AGENT_PLATFORM=${platform} npm run lock:lock -- --agent ${selectedAgent} --cadence ${cadence}${modelPart}`;
|
|
const stderrExcerpt = excerptText(lockResult.stderr);
|
|
const stdoutExcerpt = excerptText(lockResult.stdout);
|
|
const backoffSchedule = lockAttempt.backoffScheduleMs.join(', ');
|
|
|
|
const existingDeferral = deferralForAgent || {};
|
|
const firstFailureAt = existingDeferral.first_failure_timestamp || new Date().toISOString();
|
|
const firstFailureMs = parseDateValue(firstFailureAt) || Date.now();
|
|
const nowMs = Date.now();
|
|
const deferralAttemptCount = parseNonNegativeInt(existingDeferral.attempt_count, 0) + 1;
|
|
const idempotencyKey = existingDeferral.idempotency_key
|
|
|| createIdempotencyKey({ cadence, selectedAgent, runDate: schedulerRunState.run_date });
|
|
const withinRetryWindow = nowMs - firstFailureMs <= schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs;
|
|
const withinDeferralBudget = deferralAttemptCount <= schedulerConfig.lockFailurePolicy.maxDeferrals;
|
|
|
|
if (!schedulerConfig.lockFailurePolicy.strictLock && withinRetryWindow && withinDeferralBudget) {
|
|
schedulerRunState.lock_deferral = {
|
|
attempt_count: deferralAttemptCount,
|
|
first_failure_timestamp: firstFailureAt,
|
|
backend_category: backendCategory,
|
|
idempotency_key: idempotencyKey,
|
|
selected_agent: selectedAgent,
|
|
};
|
|
await writeRunState(runStatePath, schedulerRunState);
|
|
await writeLog({
|
|
cadence,
|
|
agent: selectedAgent,
|
|
status: 'deferred',
|
|
platform,
|
|
reason: 'Lock backend deferred',
|
|
detail: `Deferred after lock backend failure (${backendCategory}); retry window active and deferral budget remaining. Prompt not executed. ${buildLockBackendRemediation({ cadence, retryWindowMs: schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs, maxDeferrals: schedulerConfig.lockFailurePolicy.maxDeferrals })}`,
|
|
metadata: {
|
|
...categorizeFailureMetadata(FAILURE_CATEGORY.LOCK_BACKEND, { failure_class: 'backend_unavailable' }),
|
|
deferral_attempt_count: deferralAttemptCount,
|
|
deferral_first_failure_timestamp: firstFailureAt,
|
|
backend_category: backendCategory,
|
|
lock_idempotency_key: idempotencyKey,
|
|
},
|
|
});
|
|
await exitWithSummary(0, {
|
|
status: 'deferred',
|
|
agent: selectedAgent,
|
|
promptPath: null,
|
|
reason: 'Lock backend deferred',
|
|
platform,
|
|
detail: `Lock backend failure (${backendCategory})`,
|
|
});
|
|
}
|
|
|
|
schedulerRunState.lock_deferral = null;
|
|
await writeRunState(runStatePath, schedulerRunState);
|
|
await writeLog({
|
|
cadence,
|
|
agent: selectedAgent,
|
|
status: 'failed',
|
|
platform,
|
|
reason: 'Lock backend error',
|
|
detail: `Lock backend error (${backendCategory}) after ${lockAttempt.attempts} attempt(s). Prompt not executed. ${buildLockBackendRemediation({ cadence, retryWindowMs: schedulerConfig.lockFailurePolicy.degradedLockRetryWindowMs, maxDeferrals: schedulerConfig.lockFailurePolicy.maxDeferrals })}`,
|
|
metadata: {
|
|
...categorizeFailureMetadata(FAILURE_CATEGORY.LOCK_BACKEND, { failure_class: 'backend_unavailable' }),
|
|
lock_attempts_total: lockAttempt.attempts,
|
|
lock_backoff_schedule_ms: backoffSchedule || '(none)',
|
|
lock_correlation_id: diagnosticsSummary.correlationId || lockAttempt.correlationId,
|
|
lock_attempt_id: diagnosticsSummary.attemptId || String(lockAttempt.attempts),
|
|
lock_total_retry_timeline_ms: diagnosticsSummary.totalElapsedMs,
|
|
lock_failure_reason_distribution: JSON.stringify(diagnosticsSummary.reasonDistribution),
|
|
backend_category: backendCategory,
|
|
deferral_attempt_count: deferralAttemptCount,
|
|
deferral_first_failure_timestamp: firstFailureAt,
|
|
lock_command: lockCommand,
|
|
lock_idempotency_key: idempotencyKey,
|
|
lock_stderr_excerpt: stderrExcerpt || '(empty)',
|
|
lock_stdout_excerpt: stdoutExcerpt || '(empty)',
|
|
},
|
|
});
|
|
await exitWithSummary(2, {
|
|
status: 'failed',
|
|
agent: selectedAgent,
|
|
promptPath: null,
|
|
reason: 'Lock backend error',
|
|
platform,
|
|
detail: `Lock backend error (${backendCategory})`,
|
|
});
|
|
}
|
|
|
|
if (lockResult.code !== 0) {
|
|
schedulerRunState.lock_deferral = null;
|
|
await writeRunState(runStatePath, schedulerRunState);
|
|
await writeLog({
|
|
cadence,
|
|
agent: selectedAgent,
|
|
status: 'failed',
|
|
platform,
|
|
reason: 'Failed to acquire lock',
|
|
});
|
|
await exitWithSummary(lockResult.code, {
|
|
status: 'failed',
|
|
agent: selectedAgent,
|
|
promptPath: null,
|
|
reason: 'Failed to acquire lock',
|
|
platform,
|
|
});
|
|
}
|
|
|
|
const promptPath = path.resolve(process.cwd(), 'src/prompts', cadence, `${selectedAgent}.md`);
|
|
const promptValidation = await validatePromptFile(promptPath);
|
|
if (!promptValidation.ok) {
|
|
await writeLog({
|
|
cadence,
|
|
agent: selectedAgent,
|
|
status: 'failed',
|
|
platform,
|
|
reason: promptValidation.reason,
|
|
detail: promptValidation.detail,
|
|
metadata: categorizeFailureMetadata(promptValidation.category, { prompt_path: promptPath }),
|
|
});
|
|
await exitWithSummary(1, {
|
|
status: 'failed',
|
|
agent: selectedAgent,
|
|
promptPath,
|
|
reason: promptValidation.reason,
|
|
detail: promptValidation.detail,
|
|
platform,
|
|
});
|
|
}
|
|
|
|
schedulerRunState.lock_deferral = null;
|
|
await writeRunState(runStatePath, schedulerRunState);
|
|
const runArtifactSince = new Date().toISOString();
|
|
const runArtifactTimestamp = ts();
|
|
const runStartMs = Date.parse(runArtifactSince);
|
|
const outputChunks = [];
|
|
|
|
// Pre-create the memory update file path and inject it as SCHEDULER_MEMORY_FILE
|
|
// so the spawned agent can write learnings without computing the path itself.
|
|
const memoryDir = path.resolve(process.cwd(), 'memory-updates');
|
|
await fs.mkdir(memoryDir, { recursive: true });
|
|
const memoryFile = path.join(memoryDir, `${ts()}__${selectedAgent}.md`);
|
|
const schedulerEnv = {
|
|
AGENT_PLATFORM: platform,
|
|
...(model ? { AGENT_MODEL: model } : {}),
|
|
SCHEDULER_AGENT: selectedAgent,
|
|
SCHEDULER_CADENCE: cadence,
|
|
SCHEDULER_PROMPT_PATH: promptPath,
|
|
SCHEDULER_MEMORY_FILE: memoryFile,
|
|
SCHEDULER_RUN_TIMESTAMP: runArtifactTimestamp,
|
|
};
|
|
|
|
if (schedulerConfig.memoryPolicy.retrieveCommand) {
|
|
const retrieveResult = await runCommand('bash', ['-lc', schedulerConfig.memoryPolicy.retrieveCommand], {
|
|
env: schedulerEnv,
|
|
});
|
|
outputChunks.push(retrieveResult.stdout, retrieveResult.stderr);
|
|
if (retrieveResult.code !== 0) {
|
|
await writeLog({
|
|
cadence,
|
|
agent: selectedAgent,
|
|
status: 'failed',
|
|
platform,
|
|
reason: 'Memory retrieval command failed',
|
|
detail: schedulerConfig.memoryPolicy.retrieveCommand,
|
|
metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }),
|
|
});
|
|
await exitWithSummary(retrieveResult.code, {
|
|
status: 'failed',
|
|
agent: selectedAgent,
|
|
promptPath,
|
|
reason: 'Memory retrieval command failed',
|
|
detail: schedulerConfig.memoryPolicy.retrieveCommand,
|
|
memoryFile,
|
|
platform,
|
|
});
|
|
}
|
|
}
|
|
|
|
if (schedulerConfig.handoffCommand) {
|
|
const handoff = await runCommand('bash', ['-lc', schedulerConfig.handoffCommand], {
|
|
env: schedulerEnv,
|
|
});
|
|
outputChunks.push(handoff.stdout, handoff.stderr);
|
|
if (handoff.code !== 0) {
|
|
await writeLog({
|
|
cadence,
|
|
agent: selectedAgent,
|
|
status: 'failed',
|
|
platform,
|
|
reason: 'Prompt/handoff execution failed',
|
|
detail: 'Handoff callback failed.',
|
|
metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }),
|
|
});
|
|
await exitWithSummary(handoff.code, {
|
|
status: 'failed',
|
|
agent: selectedAgent,
|
|
promptPath,
|
|
reason: 'Prompt/handoff execution failed',
|
|
detail: 'Handoff callback failed',
|
|
memoryFile,
|
|
platform,
|
|
});
|
|
}
|
|
} else {
|
|
const detail = schedulerConfig.missingHandoffCommandForMode
|
|
? 'Missing scheduler handoff command for non-interactive run. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.'
|
|
: 'No handoff callback configured. Set scheduler.handoffCommandByCadence.daily|weekly in torch-config.json.';
|
|
await writeLog({
|
|
cadence,
|
|
agent: selectedAgent,
|
|
status: 'failed',
|
|
platform,
|
|
reason: 'Prompt/handoff execution failed',
|
|
detail,
|
|
metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }),
|
|
});
|
|
await exitWithSummary(1, {
|
|
status: 'failed',
|
|
agent: selectedAgent,
|
|
promptPath,
|
|
reason: 'Prompt/handoff execution failed',
|
|
detail,
|
|
memoryFile,
|
|
platform,
|
|
});
|
|
}
|
|
|
|
if (schedulerConfig.memoryPolicy.storeCommand) {
|
|
const storeResult = await runCommand('bash', ['-lc', schedulerConfig.memoryPolicy.storeCommand], {
|
|
env: schedulerEnv,
|
|
});
|
|
outputChunks.push(storeResult.stdout, storeResult.stderr);
|
|
if (storeResult.code !== 0) {
|
|
await writeLog({
|
|
cadence,
|
|
agent: selectedAgent,
|
|
status: 'failed',
|
|
platform,
|
|
reason: 'Memory storage command failed',
|
|
detail: schedulerConfig.memoryPolicy.storeCommand,
|
|
metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }),
|
|
});
|
|
await exitWithSummary(storeResult.code, {
|
|
status: 'failed',
|
|
agent: selectedAgent,
|
|
promptPath,
|
|
reason: 'Memory storage command failed',
|
|
detail: schedulerConfig.memoryPolicy.storeCommand,
|
|
memoryFile,
|
|
platform,
|
|
});
|
|
}
|
|
}
|
|
|
|
const memoryOutput = outputChunks.join('\n');
|
|
const retrieveCheck = await verifyMemoryStep({
|
|
name: 'retrieve',
|
|
markers: schedulerConfig.memoryPolicy.retrieveSuccessMarkers,
|
|
artifacts: schedulerConfig.memoryPolicy.retrieveArtifacts,
|
|
outputText: memoryOutput,
|
|
sinceMs: runStartMs,
|
|
});
|
|
const storeCheck = await verifyMemoryStep({
|
|
name: 'store',
|
|
markers: schedulerConfig.memoryPolicy.storeSuccessMarkers,
|
|
artifacts: schedulerConfig.memoryPolicy.storeArtifacts,
|
|
outputText: memoryOutput,
|
|
sinceMs: runStartMs,
|
|
});
|
|
|
|
// mode='required' → missing evidence is a hard failure; mode='optional' → warning only.
|
|
const missingSteps = [retrieveCheck, storeCheck].filter((step) => !step.complete).map((step) => step.name);
|
|
if (missingSteps.length && schedulerConfig.memoryPolicy.mode === 'required') {
|
|
await writeLog({
|
|
cadence,
|
|
agent: selectedAgent,
|
|
status: 'failed',
|
|
platform,
|
|
reason: 'Required memory steps not verified',
|
|
detail: `Missing evidence for: ${missingSteps.join(', ')}`,
|
|
metadata: categorizeFailureMetadata(FAILURE_CATEGORY.PROMPT_SCHEMA, { failure_class: 'prompt_validation_error' }),
|
|
});
|
|
await exitWithSummary(1, {
|
|
status: 'failed',
|
|
agent: selectedAgent,
|
|
promptPath,
|
|
reason: 'Required memory steps not verified',
|
|
detail: `Missing evidence for: ${missingSteps.join(', ')}`,
|
|
memoryFile,
|
|
platform,
|
|
});
|
|
}
|
|
|
|
if (missingSteps.length) {
|
|
console.warn(`[scheduler] Optional memory evidence missing for ${missingSteps.join(', ')}.`);
|
|
}
|
|
|
|
const artifactCheck = await runCommand('node', [
|
|
'scripts/agent/verify-run-artifacts.mjs',
|
|
'--since',
|
|
runArtifactSince,
|
|
'--agent',
|
|
selectedAgent,
|
|
'--cadence',
|
|
cadence,
|
|
'--prompt-path',
|
|
promptPath,
|
|
'--run-start',
|
|
runArtifactSince,
|
|
'--check-failure-notes',
|
|
]);
|
|
if (artifactCheck.code !== 0) {
|
|
const detail = artifactCheck.stderr.trim() || artifactCheck.stdout.trim() || 'Artifact verification failed.';
|
|
await writeLog({
|
|
cadence,
|
|
agent: selectedAgent,
|
|
status: 'failed',
|
|
platform,
|
|
reason: 'Missing required run artifacts',
|
|
detail,
|
|
metadata: categorizeFailureMetadata(FAILURE_CATEGORY.PROMPT_SCHEMA, { failure_class: 'prompt_validation_error' }),
|
|
});
|
|
await exitWithSummary(artifactCheck.code, {
|
|
status: 'failed',
|
|
agent: selectedAgent,
|
|
promptPath,
|
|
reason: 'Missing required run artifacts',
|
|
detail,
|
|
memoryFile,
|
|
platform,
|
|
});
|
|
}
|
|
|
|
for (const validation of schedulerConfig.validationCommands) {
|
|
const parts = validation.split(' ').filter(Boolean);
|
|
if (!parts.length) continue;
|
|
const result = await runCommand(parts[0], parts.slice(1));
|
|
if (result.code !== 0) {
|
|
await writeLog({
|
|
cadence,
|
|
agent: selectedAgent,
|
|
status: 'failed',
|
|
platform,
|
|
reason: 'Validation failed',
|
|
detail: validation,
|
|
metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }),
|
|
});
|
|
await exitWithSummary(result.code, {
|
|
status: 'failed',
|
|
agent: selectedAgent,
|
|
promptPath,
|
|
reason: 'Validation failed',
|
|
detail: validation,
|
|
memoryFile,
|
|
platform,
|
|
});
|
|
}
|
|
}
|
|
|
|
const completeResult = await runCommand(
|
|
'npm',
|
|
['run', 'lock:complete', '--', '--agent', selectedAgent, '--cadence', cadence, ...(model ? ['--model', model] : [])],
|
|
{ env: { AGENT_PLATFORM: platform, ...(model ? { AGENT_MODEL: model } : {}) } },
|
|
);
|
|
|
|
if (completeResult.code !== 0) {
|
|
await writeLog({
|
|
cadence,
|
|
agent: selectedAgent,
|
|
status: 'failed',
|
|
platform,
|
|
reason: `Completion publish failed. Retry npm run lock:complete -- --agent ${selectedAgent} --cadence ${cadence} after verifying relay connectivity`,
|
|
});
|
|
await exitWithSummary(completeResult.code, {
|
|
status: 'failed',
|
|
agent: selectedAgent,
|
|
promptPath,
|
|
reason: 'Completion publish failed',
|
|
detail: 'Retry npm run lock:complete after verifying relay connectivity',
|
|
memoryFile,
|
|
platform,
|
|
});
|
|
}
|
|
|
|
await writeLog({ cadence, agent: selectedAgent, status: 'completed', platform, reason: 'Scheduler cycle completed successfully' });
|
|
await exitWithSummary(0, {
|
|
status: 'completed',
|
|
agent: selectedAgent,
|
|
promptPath,
|
|
reason: 'Scheduler cycle completed successfully',
|
|
memoryFile,
|
|
platform,
|
|
});
|
|
}
|
|
}
|
|
|
|
main().catch(async (err) => {
|
|
console.error(err);
|
|
process.exit(1);
|
|
});
|