Merge commit '5e7db85bee1b07108ee03c49a2ba040d20cf55c4' into torch/work-integration

# Conflicts:
#	docs/TEST_INTEGRITY.md
This commit is contained in:
thePR0M3TH3AN committed 2026-10-03 16:51:24 -04:00
commit d96f252098
37 files changed
+3436 -53

No files matched your search

+18 -3
View File
@@ -31,15 +31,28 @@ function commandRecord(command, args, cwd) {
return { command, args, cwd, mutatesRuntime: true };
}
function mcpArgs(mcp) {
function mcpRuntimeEnvironmentNames(env, platform) {
const runtimeDirectory = env?.XDG_RUNTIME_DIR;
if (platform !== 'linux' || typeof runtimeDirectory !== 'string'
|| !/^\/run\/user\/[1-9]\d*$/.test(runtimeDirectory)) {
return [];
}
return ['XDG_RUNTIME_DIR'];
}
function mcpArgs(mcp, { env, platform } = {}) {
if (!mcp) return [];
if (typeof mcp.name !== 'string' || !/^[a-z0-9_-]+$/.test(mcp.name)
|| typeof mcp.command !== 'string' || !Array.isArray(mcp.args)) {
throw new TorchError('Codex MCP launch configuration is invalid', { code: 'INVALID_RUNTIME_INPUT' });
}
const environmentNames = mcpRuntimeEnvironmentNames(env, platform);
return [
'-c', `mcp_servers.${mcp.name}.command=${JSON.stringify(mcp.command)}`,
'-c', `mcp_servers.${mcp.name}.args=${JSON.stringify(mcp.args)}`,
...(environmentNames.length
? ['-c', `mcp_servers.${mcp.name}.env_vars=${JSON.stringify(environmentNames)}`]
: []),
];
}
@@ -169,12 +182,14 @@ export function diagnoseCodexStartupFailure({ stdout, stderr } = {}) {
export class CodexRuntimeAdapter {
constructor({
env = process.env,
platform = process.platform,
runner = null,
executable = 'codex',
nodeExecutable = process.execPath,
} = {}) {
this.name = 'codex';
this.env = env;
this.platform = platform;
this.runner = runner;
this.executable = executable;
this.nodeExecutable = nodeExecutable;
@@ -241,7 +256,7 @@ export class CodexRuntimeAdapter {
const cwd = text(worktree, 'worktree');
const args = [
'--cd', cwd, ...launchPolicyArgs(launchPolicy ?? this.configuration.launchPolicy),
...mcpArgs(mcp),
...mcpArgs(mcp, { env: this.env, platform: this.platform }),
...((reasoning ?? this.configuration.reasoning) ? ['-c', `model_reasoning_effort=${JSON.stringify(text(reasoning ?? this.configuration.reasoning, 'reasoning'))}`] : []),
'exec', '--json',
...optionalModelArgs(model ?? this.configuration.model),
@@ -266,7 +281,7 @@ export class CodexRuntimeAdapter {
: '';
const args = [
'--cd', cwd, ...launchPolicyArgs(launchPolicy ?? this.configuration.launchPolicy),
...mcpArgs(mcp),
...mcpArgs(mcp, { env: this.env, platform: this.platform }),
...((reasoning ?? this.configuration.reasoning) ? ['-c', `model_reasoning_effort=${JSON.stringify(text(reasoning ?? this.configuration.reasoning, 'reasoning'))}`] : []),
'exec', 'resume', '--json',
...optionalModelArgs(model ?? this.configuration.model), sessionId, `${text(message, 'message')}${currentInstructions}`,
+55 -1
View File
@@ -14,7 +14,7 @@ import { startConsole } from './console/server.mjs';
import { classifyRecoverability, createLocalCanonical, planLocalCanonical } from './canonical/local.mjs';
import { openControlPlane } from './control-plane/service.mjs';
import { fetchCanonicalObjects, planCanonicalFetch } from './canonical/fetch.mjs';
import { spawnSync } from 'node:child_process';
import { spawn as spawnProcess, spawnSync } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import {
mkdirSync, mkdtempSync, renameSync, rmSync, statSync, writeFileSync,
@@ -70,6 +70,8 @@ import {
} from './design/architect.mjs';
import { FleetEvolutionService } from './evolution/service.mjs';
import { HierarchyEvolutionService } from './evolution/hierarchy.mjs';
import { issueStoppedExecutorEvidence } from './runtime/stopped-executor-evidence.mjs';
import { configureContinuation, planContinuation, runContinuation } from './runtime/continuation.mjs';
import { ConvergenceService } from './convergence/service.mjs';
import {
attachForge, detachForge, forgeStatus, planForgeAttach, planForgeDetach,
@@ -95,6 +97,8 @@ Usage:
torch install --restore [--dry-run] [--yes] [--json]
torch setup [--parent <path>] [--dry-run] [--yes] [--json]
torch up [--fresh] [--only <id,id>] [--dry-run] [--yes] [--json]
torch recover-runtime --area <id> --unit <service> --invocation <id> [--expected-session <id> --expected-updated-at <time> --yes] [--json]
torch continuation <status|enable|pause|tick> [--max-turns-per-day <1-48>] [--today-max-turns <1-96>] [--concurrency <1-7>] [--limit <1-7>] [--yes] [--json]
torch down [--dry-run] [--yes] [--json]
torch capture [--json]
torch detach [--dry-run] [--yes] [--json]
@@ -576,6 +580,56 @@ export async function runCli(argv = process.argv.slice(2), {
process.stdout.write(HELP);
return 0;
}
if (command === 'continuation') {
const control = openControlPlane({ repositoryRoot: optionValue(argv, '--repo') ?? cwd, env });
try {
const operation = argv[1] ?? 'status';
const backlog = new BacklogService({ repositoryRoot: control.repositoryRoot, controlPlane: control });
if (operation === 'status' || !argv.includes('--yes') || argv.includes('--dry-run')) {
print(planContinuation(control, backlog), { json }); return 0;
}
if (operation === 'enable' || operation === 'pause') {
print(configureContinuation(control, { actorId: 'owner', enabled: operation === 'enable',
maxTurnsPerDay: optionValue(argv, '--max-turns-per-day') === undefined ? undefined : Number(optionValue(argv, '--max-turns-per-day')),
todayMaxTurns: optionValue(argv, '--today-max-turns') === undefined ? undefined : Number(optionValue(argv, '--today-max-turns')),
maxConcurrency: optionValue(argv, '--concurrency') === undefined ? undefined : Number(optionValue(argv, '--concurrency')) }), { json }); return 0;
}
if (operation !== 'tick') throw new TorchError('Unknown continuation operation.', { code: 'UNKNOWN_COMMAND' });
print(await runContinuation(control, backlog, { actorId: 'owner', limit: optionValue(argv, '--limit') === undefined ? undefined : Number(optionValue(argv, '--limit')),
launch: areaId => new Promise(resolveTurn => {
const child = spawnProcess(process.execPath, [fileURLToPath(new URL('../bin/torch.mjs', import.meta.url)),
'up', '--only', areaId, '--yes', '--json'], { cwd: control.repositoryRoot, env,
stdio: ['ignore', 'pipe', 'pipe'] });
child.stdout.resume(); child.stderr.resume();
let failed = false;
child.on('error', () => { failed = true; });
child.on('close', (code, signal) => resolveTurn(failed || signal ? null : code));
}) }), { json });
return 0;
} finally { control.close(); }
}
if (command === 'recover-runtime') {
const control = openControlPlane({ repositoryRoot: optionValue(argv, '--repo') ?? cwd, env });
try {
const areaId = optionValue(argv, '--area');
const issued = issueStoppedExecutorEvidence({ controlPlane: control, actorId: 'owner', areaId,
unit: optionValue(argv, '--unit'), invocationId: optionValue(argv, '--invocation') });
if (!argv.includes('--yes') || argv.includes('--dry-run')) {
print({ ...issued.plan, requiresApproval: true }, { json });
return 0;
}
const expected = issued.plan.expectedIdentity;
if (optionValue(argv, '--expected-session') !== expected.runtimeSessionId
|| optionValue(argv, '--expected-updated-at') !== expected.updatedAt) {
throw new TorchError('Apply requires the exact session and updated timestamp from the recovery preview.', {
code: 'RUNTIME_RECOVERY_SNAPSHOT_CONFLICT',
});
}
print(control.recoverOwnerStoppedUnknownIdentity({ actorId: 'owner', areaId,
expectedIdentity: expected, evidence: issued.evidence }), { json });
return 0;
} finally { control.close(); }
}
if (command === 'runtimes') {
const operation = argv[1] ?? 'list';
if (operation === 'update') {
+32 -1
View File
@@ -7,6 +7,7 @@ import { readInstallManifest } from '../kernel/install.mjs';
import { projectStatePath } from '../kernel/paths.mjs';
import { loadProjectConfig } from '../kernel/config.mjs';
import { organizationGraphFromConfig } from '../kernel/organization.mjs';
import { consumeStoppedExecutorEvidence } from '../runtime/stopped-executor-evidence.mjs';
export const PRESENCE_STATES = Object.freeze([
'starting', 'working', 'waiting', 'idle', 'stale', 'stopping', 'offline',
@@ -459,6 +460,36 @@ export class ControlPlane {
return { messageId: id, recipient: recipientId, acknowledgedAt };
}
recoverOwnerStoppedUnknownIdentity({ actorId, areaId, expectedIdentity, evidence } = {}) {
const owner = this.assertOwnerActor(actorId);
const id = this.assertIdentity(areaId);
if (!expectedIdentity || ['state', 'runtime', 'runtimeSessionId', 'updatedAt'].some(key => !expectedIdentity[key])) {
throw new TorchError('Recovery requires the complete previewed identity snapshot.', { code: 'RUNTIME_RECOVERY_SNAPSHOT_REQUIRED' });
}
this.database.exec('BEGIN IMMEDIATE');
try {
const current = this.identity(id);
if (current.state !== 'working' || !current.summary?.includes('Runtime executor completion is unknown')
|| ['state', 'runtime', 'runtimeSessionId', 'updatedAt'].some(key => current[key] !== expectedIdentity[key])) {
throw new TorchError('Identity changed since recovery was reviewed.', { code: 'RUNTIME_RECOVERY_SNAPSHOT_CONFLICT' });
}
const facts = consumeStoppedExecutorEvidence(evidence, { projectId: this.projectId, repositoryRoot: this.repositoryRoot, identity: current });
const now = this.clock().toISOString();
const result = this.database.prepare(`UPDATE identities SET state = 'offline', summary = ?, heartbeat_at = ?, updated_at = ?
WHERE area_id = ? AND state = ? AND runtime = ? AND runtime_session_id = ? AND updated_at = ?`)
.run('Owner verified stopped legacy executor; prior outcome remains unknown.', now, now,
id, current.state, current.runtime, current.runtimeSessionId, current.updatedAt);
if (result.changes !== 1) throw new TorchError('Recovery lost its identity snapshot.', { code: 'RUNTIME_RECOVERY_SNAPSHOT_CONFLICT' });
const audit = this.auditOwnerAction({ actorId: owner, operation: 'runtime.legacy-unknown.recover-stopped',
entityType: 'identity', entityId: id, details: { previousIdentity: expectedIdentity, ...facts } });
this.database.exec('COMMIT');
return { identity: this.identity(id), auditId: audit.id, priorOutcome: 'unknown', sessionsStarted: false, mutationPerformed: true };
} catch (error) {
this.database.exec('ROLLBACK');
throw error;
}
}
reportStatus({ areaId, state, summary, runtime, runtimeSessionId, task } = {}) {
const id = this.assertIdentity(areaId);
const normalizedState = requiredText(state, 'state');
@@ -751,7 +782,7 @@ export class ControlPlane {
auditOwnerAction({ actorId, operation, entityType, entityId = null, details = null } = {}) {
const actor = this.assertOwnerActor(actorId);
if (!['artifact.feedback', 'backlog.priority', 'backlog.create', 'agent.request', 'approval.decide', 'approval.notify', 'forge.sync', 'canonical.fetch', 'profile.change', 'schedule.launcher.install', 'schedule.launcher.reconcile', 'schedule.wake.recover', 'hierarchy.conclude-pilot', 'check.recover-prepared'].includes(operation)) {
if (!['artifact.feedback', 'backlog.priority', 'backlog.create', 'agent.request', 'approval.decide', 'approval.notify', 'forge.sync', 'canonical.fetch', 'profile.change', 'schedule.launcher.install', 'schedule.launcher.reconcile', 'schedule.wake.recover', 'hierarchy.conclude-pilot', 'check.recover-prepared', 'runtime.legacy-unknown.recover-stopped', 'runtime.continuation.configure'].includes(operation)) {
throw new TorchError('This owner audit surface does not authorize the requested operation', {
code: 'OWNER_OPERATION_UNSUPPORTED', details: { operation },
});
+14 -1
View File
@@ -16,6 +16,8 @@ import { assessBacklogHealth, backlogObservedCommitDistance } from '../backlog/s
import { observeTaskActivity } from '../backlog/activity.mjs';
import { ScheduleLauncherService } from '../schedules/launcher.mjs';
import { hierarchyOrder } from '../runtime/hierarchy-order.mjs';
import { observeRuntimeTurn } from '../runtime/turn-guard.mjs';
import { observeContinuation } from '../runtime/continuation.mjs';
function json(path) {
return JSON.parse(readFileSync(path, 'utf8'));
@@ -403,6 +405,7 @@ export function observeProject({ repositoryRoot, env = process.env, now = () =>
metrics: ['cachedInput', 'uncachedInput', 'cacheCreation', 'cacheRead', 'compactions', 'resumedPromptBytes', 'costPerVerifiedItem'],
},
};
result.continuation = observeContinuation({ stateRoot, tasks: result.backlog, now });
result.backlogHealth = backlogHealth(repository.root, result.backlog, result.agents, now);
result.backlogActivity = observeBacklogActivity({ repositoryRoot: repository.root,
tasks: result.backlog, config, manifest, now: now() });
@@ -417,11 +420,21 @@ export function observeProject({ repositoryRoot, env = process.env, now = () =>
`).all().map((row) => [row.area_id, row]));
result.agents = result.agents.map((agent) => {
const row = live.get(agent.areaId);
return row ? {
const reported = row ? {
...agent, state: row.state, runtime: row.runtime ?? agent.runtime,
runtimeSessionId: row.runtime_session_id, summary: row.summary,
task: row.task, heartbeatAt: row.heartbeat_at,
} : agent;
const executor = observeRuntimeTurn({ stateRoot, repositoryRoot: repository.root,
projectId: manifest.projectId, areaId: agent.areaId, now });
const staleIdle = ['idle', 'offline'].includes(reported.state);
return { ...reported, reportedState: reported.state, reportedSummary: reported.summary, executor,
...(executor.state === 'active' && staleIdle ? {
state: executor.phase,
summary: `Live runtime executor is ${executor.phase} (last reported state: ${reported.state}).`,
} : executor.state === 'unknown' && staleIdle ? {
state: 'unknown', summary: 'Runtime turn guard exists, but its executor could not be verified.',
} : {}) };
});
}
if (hasTable(database, 'schedule_wake_reservations')) {
+178
View File
@@ -0,0 +1,178 @@
import { createHash, randomUUID } from 'node:crypto';
import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmdirSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { TorchError } from '../kernel/errors.mjs';
function statePath(control) { return join(control.stateRoot, 'continuation.json'); }
function occupiedAreas(control) {
const path = join(control.stateRoot, 'sessions', 'turn-guards');
return existsSync(path) ? readdirSync(path, { withFileTypes: true }).filter(d => d.isDirectory()).map(d => d.name) : [];
}
function taskKey(tasks) { return JSON.stringify(tasks.map(t => [t.id, t.revision]).sort((a, b) => a[0].localeCompare(b[0]))); }
function dailyCap(state, day) {
return state.dailyOverride?.day === day ? Math.max(state.maxTurnsPerDay, state.dailyOverride.maxTurns) : state.maxTurnsPerDay;
}
function read(control) {
const policyPath = join(control.stateRoot, 'continuation-policy.json');
const state = { enabled: false, maxTurnsPerDay: 12, maxConcurrency: 3, areas: [], day: null, attempts: 0, handled: {}, progress: {},
...(existsSync(statePath(control)) ? JSON.parse(readFileSync(statePath(control), 'utf8')) : {}),
...(existsSync(policyPath) ? JSON.parse(readFileSync(policyPath, 'utf8')) : {}) };
if (typeof state.enabled !== 'boolean' || !Number.isInteger(state.maxTurnsPerDay)
|| state.maxTurnsPerDay < 1 || state.maxTurnsPerDay > 48 || !Array.isArray(state.areas)
|| !Number.isInteger(state.maxConcurrency) || state.maxConcurrency < 1 || state.maxConcurrency > 7
|| !Number.isInteger(state.attempts) || state.attempts < 0 || !state.handled || typeof state.handled !== 'object'
|| !state.progress || typeof state.progress !== 'object'
|| (state.dailyOverride != null && (!/^\d{4}-\d{2}-\d{2}$/.test(state.dailyOverride.day)
|| !Number.isInteger(state.dailyOverride.maxTurns) || state.dailyOverride.maxTurns < 1 || state.dailyOverride.maxTurns > 96))) {
throw new TorchError('Continuation policy or attempt ledger is invalid; launch refused.', { code: 'CONTINUATION_POLICY_INVALID' });
}
return state;
}
function save(control, state, policy = false) {
const path = policy ? join(control.stateRoot, 'continuation-policy.json') : statePath(control);
const temporary = `${path}.${randomUUID()}.tmp`;
const value = policy ? { enabled: state.enabled, maxTurnsPerDay: state.maxTurnsPerDay, maxConcurrency: state.maxConcurrency, areas: state.areas, dailyOverride: state.dailyOverride ?? null }
: { day: state.day, attempts: state.attempts, handled: state.handled, progress: state.progress };
writeFileSync(temporary, JSON.stringify(value), { flag: 'wx', mode: 0o600 });
renameSync(temporary, path);
}
export function configureContinuation(control, { actorId, enabled, maxTurnsPerDay, maxConcurrency, todayMaxTurns } = {}) {
control.assertOwnerActor(actorId);
const previous = read(control);
maxTurnsPerDay ??= previous.maxTurnsPerDay;
maxConcurrency ??= previous.maxConcurrency;
if (typeof enabled !== 'boolean' || !Number.isInteger(maxTurnsPerDay) || maxTurnsPerDay < 1 || maxTurnsPerDay > 48
|| !Number.isInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 7
|| (todayMaxTurns !== undefined && (!Number.isInteger(todayMaxTurns) || todayMaxTurns < maxTurnsPerDay || todayMaxTurns > 96))) {
throw new TorchError('Continuation requires explicit enable/pause, 1–48 daily turns, 1–7 concurrent turns and an optional same-day total between the daily cap and 96.', { code: 'CONTINUATION_POLICY_INVALID' });
}
const state = { ...previous, enabled, maxTurnsPerDay, maxConcurrency, areas: control.listAgents().map(a => a.areaId) };
if (todayMaxTurns !== undefined) state.dailyOverride = { day: control.clock().toISOString().slice(0, 10), maxTurns: todayMaxTurns };
save(control, state, true);
control.auditOwnerAction({ actorId, operation: 'runtime.continuation.configure', entityType: 'project', entityId: control.projectId,
details: { enabled, maxTurnsPerDay, maxConcurrency, dailyOverride: state.dailyOverride ?? null, areas: state.areas } });
return { enabled, maxTurnsPerDay, maxConcurrency, dailyOverride: state.dailyOverride ?? null, areas: state.areas, sessionsStarted: false };
}
export function observeContinuation({ stateRoot, tasks = [], now = () => new Date() } = {}) {
try {
const state = read({ stateRoot });
const day = now().toISOString().slice(0, 10);
const attempts = state.day === day ? state.attempts : 0;
const cap = dailyCap(state, day);
const remainingTurns = Math.max(0, cap - attempts);
const held = Object.entries(state.progress).filter(([area, progress]) => {
const active = tasks.filter(t => t.owner === area && ['assigned', 'in_progress'].includes(t.state));
return active.length && progress?.status === 0 && progress.noProgress >= 2 && progress.taskKey === taskKey(active);
}).map(([areaId]) => ({ areaId, reason: 'no-progress-needs-coordination' }));
return { available: existsSync(join(stateRoot, 'continuation-policy.json')), enabled: state.enabled,
day, dayTimezone: 'UTC', attempts, maxTurnsPerDay: cap, configuredMaxTurnsPerDay: state.maxTurnsPerDay,
dailyOverride: state.dailyOverride ?? null, maxConcurrency: state.maxConcurrency, remainingTurns,
stopReason: !state.enabled ? 'paused' : remainingTurns === 0 ? 'daily-turn-cap' : null,
capacityHeldBy: occupiedAreas({ stateRoot }), held,
budgetKind: 'automatic-turn-cap', manualTurnsIncluded: false, tokenOrCostGovernance: 'unknown',
timerInstallationVerified: false, mutationPerformed: false };
} catch {
return { available: false, stopReason: 'policy-or-ledger-unreadable', mutationPerformed: false };
}
}
export function planContinuation(control, backlog, { at = new Date(), limit } = {}) {
const state = read(control);
const day = at.toISOString().slice(0, 10);
const attempts = state.day === day ? state.attempts : 0;
const cap = dailyCap(state, day);
if (limit !== undefined && (!Number.isInteger(limit) || limit < 1 || limit > 7)) throw new TorchError('Continuation limit must be 1–7.', { code: 'CONTINUATION_POLICY_INVALID' });
const capacity = Math.min(limit ?? state.maxConcurrency, state.maxConcurrency);
const metadata = JSON.parse(readFileSync(join(control.stateRoot, 'project.json'), 'utf8'));
if (!state.enabled || metadata.detachedAt || attempts >= cap) {
return { enabled: state.enabled, maxConcurrency: capacity, reason: metadata.detachedAt ? 'detached' : !state.enabled ? 'paused' : 'daily-turn-cap', candidates: [], attempts };
}
const tasks = backlog.list();
const candidates = [];
const held = [];
for (const areaId of state.areas) {
const agent = control.getAgent(areaId);
if (!['idle', 'offline', 'waiting'].includes(agent.state)
|| existsSync(join(control.stateRoot, 'sessions', 'turn-guards', areaId))) continue;
const messages = control.readMessages({ recipient: areaId, unacknowledgedOnly: true, limit: 1000 });
const active = tasks.filter(t => t.owner === areaId && ['assigned', 'in_progress'].includes(t.state)
&& (t.dependencies ?? []).every(id => tasks.find(t => t.id === id)?.state === 'completed'));
const previous = state.handled[areaId] ?? {};
const freshMessages = messages.filter(m => !previous.messageIds?.includes(m.id));
const freshTasks = active.filter(t => previous.taskRevisions?.[t.id] !== t.revision);
const progress = state.progress[areaId];
const continueWork = active.length > 0 && progress?.status === 0
&& progress.taskKey === taskKey(active) && progress.noProgress < 2;
if (!freshMessages.length && !freshTasks.length && !continueWork) {
if (active.length && progress?.noProgress >= 2) held.push({ areaId, reason: 'no-progress-needs-coordination', taskIds: active.map(t => t.id) });
continue;
}
const signal = { messageIds: messages.map(m => m.id), taskRevisions: Object.fromEntries(active.map(t => [t.id, t.revision])) };
const fingerprint = createHash('sha256').update(JSON.stringify(signal)).digest('hex');
candidates.push({ areaId, fingerprint, signal, workKey: taskKey(active),
messageCount: freshMessages.length, taskIds: active.map(t => t.id), continueWork });
}
const ordinal = area => state.handled[area]?.dispatchedDay === day ? state.handled[area].dispatchedOrdinal ?? 0 : 0;
candidates.sort((a, b) => ordinal(a.areaId) - ordinal(b.areaId)
|| Number(b.areaId === 'session-manager') - Number(a.areaId === 'session-manager'));
return { enabled: true, maxConcurrency: capacity, day, attempts, remainingTurns: cap - attempts,
occupiedAreas: occupiedAreas(control), held,
candidates: candidates.slice(0, Math.min(capacity, cap - attempts)), mutationPerformed: false };
}
export async function runContinuation(control, backlog, { actorId, launch, at, clock = () => at ?? new Date(), limit } = {}) {
control.assertOwnerActor(actorId);
if (typeof launch !== 'function') throw new TorchError('Continuation needs an authorized launcher.', { code: 'RUNTIME_EXECUTION_NOT_AUTHORIZED' });
const lock = join(control.stateRoot, 'continuation.lock');
try { mkdirSync(lock); } catch (error) {
if (error.code === 'EEXIST') return { reason: 'controller-active-or-unreconciled', launched: [] };
throw error;
}
try {
const pending = new Map();
const launched = [];
let reason = null;
while (true) {
// Refill immediately after EACH completion. Pending promises and physical
// guards both count, including manual executors outside this controller.
let current = planContinuation(control, backlog, { at: clock(), limit });
let occupied = new Set([...occupiedAreas(control), ...pending.keys()]);
while (occupied.size < current.maxConcurrency && current.enabled && !current.reason) {
const candidate = current.candidates.find(c => !occupied.has(c.areaId));
if (!candidate) break;
const state = read(control);
if (state.day !== current.day) { state.day = current.day; state.attempts = 0; }
state.attempts++;
state.handled[candidate.areaId] = { ...candidate.signal,
dispatchedDay: current.day, dispatchedOrdinal: state.attempts };
save(control, state); // Reserve before launch; unknown attempts are never refunded.
let result;
try { result = launch(candidate.areaId); } catch { result = null; }
pending.set(candidate.areaId, Promise.resolve(result).catch(() => null)
.then(status => ({ areaId: candidate.areaId, status, candidate })));
current = planContinuation(control, backlog, { at: clock(), limit });
occupied = new Set([...occupiedAreas(control), ...pending.keys()]);
}
reason = current.reason ?? (occupied.size >= current.maxConcurrency ? 'capacity-full' : null);
if (!pending.size) break;
const completed = await Promise.race(pending.values());
pending.delete(completed.areaId);
launched.push({ areaId: completed.areaId, status: completed.status });
const state = read(control);
const active = backlog.list().filter(t => t.owner === completed.areaId && ['assigned', 'in_progress'].includes(t.state));
const key = taskKey(active);
const unchanged = active.length > 0 && key === completed.candidate.workKey;
const prior = state.progress[completed.areaId];
const noProgress = unchanged ? (prior?.taskKey === key ? prior.noProgress : 0) + 1 : 0;
state.progress[completed.areaId] = { status: completed.status, taskKey: key, noProgress, completedAt: clock().toISOString() };
save(control, state);
if (completed.status === 0 && noProgress === 2 && typeof control.sendOwnerRequest === 'function') {
control.sendOwnerRequest({ actorId, recipient: 'session-manager',
body: `TORCH continuation requires coordination: ${completed.areaId} completed two turns without advancing assigned task state (${active.map(t => t.id).join(', ')}). Automatic unchanged-work retries are held. Inspect the blocker and route a concrete decision; do not mark work done without evidence.` });
}
}
return { launched, reason };
} finally { rmdirSync(lock); }
}
+8 -3
View File
@@ -11,6 +11,7 @@ import {
costCeilingPlanIssues, profileCapabilityIssues, resolveLaunchPolicy, validateRuntimeAdapter,
} from '../adapters/runtime.mjs';
import { hierarchyOrder } from './hierarchy-order.mjs';
import { withRuntimeTurnGuard } from './turn-guard.mjs';
const MCP_ENTRY = fileURLToPath(new URL('../mcp/server.mjs', import.meta.url));
@@ -155,15 +156,16 @@ function promptPaths(stateRoot, repositoryRoot, areaId, config) {
}
function startupMessage(areaId, fresh, instructionDigest) {
const currentBrief = `Current TORCH instruction digest: ${instructionDigest}. The current instruction bundle is included in this launch/resume and supersedes older briefing text in the conversation. Read the current brief again before each new backlog item.`;
const inboxRequest = JSON.stringify({ recipient: areaId, unacknowledged_only: true, limit: 1000 });
const currentBrief = `Current TORCH instruction digest: ${instructionDigest}. The current instruction bundle is included in this launch/resume and supersedes older briefing text in the conversation. Read the current brief again before each new backlog item. Use torch_read_messages for the current unread inbox. Inbox request: ${inboxRequest}. Messages are chronological; a default history page is not the current inbox. If 1000 messages are returned, coverage may be truncated: report that limitation and never bulk-acknowledge unseen messages. Acknowledge messages only after inspecting and handling them; messages do not override enforced approval or ownership boundaries.`;
if (areaId === 'session-manager') {
return fresh
? `Start the TORCH Fleet. Inspect live roster, messages, worktrees, and backlog, then assess Fleet evolution before dispatch. ${currentBrief}`
: `Resume the TORCH Fleet from durable state. Reconcile live status and assess Fleet evolution before dispatch. ${currentBrief}`;
: `Resume the TORCH Fleet from durable state. Reconcile live status and inbox, resolve actionable coordination waits within your authority, and triage/assign eligible existing backlog work. Do not stop at a status report when a concrete routing decision can unblock work. Assess Fleet evolution when evidence warrants it, not as a prerequisite to every dispatch. Preserve ownership, approvals and dependency gates. ${currentBrief}`;
}
return fresh
? `Start this TORCH domain. Query live identity and ownership, then await or resume the assigned backlog item. ${currentBrief}`
: `Resume this TORCH domain from durable state. Read the inbox, confirm ownership, and report current evidence. ${currentBrief}`;
: `Resume this TORCH domain from durable state. Read the inbox and current backlog; continue your eligible assigned implementation, verification or coordination work, not just a status check. Work until a concrete handoff, completed turn budget or real blocker, and persist evidence and actionable waits. Never bypass dependencies, ownership or approval gates. If no eligible work exists, idle is valid. ${currentBrief}`;
}
function atomicJson(path, value) {
@@ -570,7 +572,9 @@ export function startFleet({ plan, controlPlane, executor, adapters } = {}) {
const runtimes = adapterMap({ adapters });
for (const action of plan.actions) {
controlPlane.assertIdentity(action.areaId);
withRuntimeTurnGuard(controlPlane, action.areaId, setPhase => {
writeCombinedPrompt(action);
setPhase('working');
const result = executor({
...action.launch, areaId: action.areaId, runtime: action.runtime,
runtimeSessionId: action.runtimeSessionId,
@@ -634,6 +638,7 @@ export function startFleet({ plan, controlPlane, executor, adapters } = {}) {
? `${action.mode} turn completed` : `${action.mode} requested`,
});
started.push({ areaId: action.areaId, runtimeSessionId: identity.runtimeSessionId, mode: action.mode });
});
}
if (plan.stateRoot) {
const metadata = projectMetadata(plan.stateRoot);
+106
View File
@@ -0,0 +1,106 @@
import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
import { readFileSync, readdirSync, readlinkSync, statSync } from 'node:fs';
import { basename, join, resolve } from 'node:path';
import { TorchError } from '../kernel/errors.mjs';
const issued = new WeakMap();
const TTL_MS = 60_000;
function refuse(message) {
throw new TorchError(message, { code: 'RUNTIME_RECOVERY_UNVERIFIED' });
}
export function readStoppedUnit(unit) {
const result = spawnSync('systemctl', ['--user', 'show', unit,
'--property=Id,ActiveState,SubState,MainPID,ControlPID,InvocationID,ExecStart,WorkingDirectory,ControlGroup,ExecMainExitTimestamp'],
{ encoding: 'utf8', timeout: 5000, maxBuffer: 64 * 1024 });
if (result.error || result.signal || result.status !== 0) refuse('Cannot verify the owner-selected service invocation.');
return Object.fromEntries(result.stdout.trim().split('\n').map(line => {
const index = line.indexOf('=');
return [line.slice(0, index), line.slice(index + 1)];
}));
}
export function readNativeProcesses() {
if (process.platform !== 'linux') refuse('Legacy process inspection is supported only on Linux.');
const processes = [];
for (const name of readdirSync('/proc').filter(value => /^\d+$/.test(value))) {
const directory = join('/proc', name);
try {
if (statSync(directory).uid !== process.getuid()) continue;
const argv = readFileSync(join(directory, 'cmdline'), 'utf8').split('\0').filter(Boolean);
if (!argv.length) continue;
const native = argv.some(value => /^(?:codex|claude|pi)(?:\.js)?$/.test(basename(value)));
if (native) processes.push({ pid: Number(name), argv, cwd: readlinkSync(join(directory, 'cwd')) });
} catch (error) {
if (error.code !== 'ENOENT' && error.code !== 'ESRCH') refuse('Native process visibility is incomplete; recovery refused.');
}
}
return processes;
}
function verify(record) {
const unit = record.readUnit(record.unit);
const prefix = `torch-${record.projectId}-`;
if (typeof record.unit !== 'string' || !record.unit.startsWith(prefix) || !/^[A-Za-z0-9_.-]+\.service$/.test(record.unit)
|| unit.Id !== record.unit || !/^[a-f0-9]{32}$/.test(unit.InvocationID ?? '')
|| unit.InvocationID !== record.invocationId
|| !['failed', 'inactive'].includes(unit.ActiveState)
|| !['failed', 'dead'].includes(unit.SubState)
|| unit.MainPID !== '0' || unit.ControlPID !== '0' || unit.ControlGroup !== ''
|| resolve(unit.WorkingDirectory || '/') !== record.repositoryRoot) {
refuse('The selected project service is not a verified stopped invocation.');
}
// Explicit legacy operator binding, not a claim of cryptographic session provenance.
// Exact area token and terminal timestamp must agree with the retained unknown row.
if (!(unit.ExecStart ?? '').split(/[\s;]+/).includes(record.identity.areaId)
|| !Number.isFinite(Date.parse(unit.ExecMainExitTimestamp))
|| Math.abs(Date.parse(unit.ExecMainExitTimestamp) - Date.parse(record.identity.updatedAt)) > 5000) {
refuse('Service area or terminal timestamp does not match the retained unknown identity.');
}
for (const processRecord of record.readProcesses()) {
if (processRecord.argv.includes(record.identity.runtimeSessionId)
|| processRecord.argv.includes(record.worktree)
|| resolve(processRecord.cwd) === record.worktree) {
refuse('A native process still references the selected session or workspace.');
}
}
return { unit: record.unit, invocationId: unit.InvocationID,
terminalAt: new Date(Date.parse(unit.ExecMainExitTimestamp)).toISOString(),
outcome: 'unknown', verification: 'owner-selected-legacy-invocation' };
}
export function issueStoppedExecutorEvidence({ controlPlane, actorId, areaId, unit, invocationId,
readUnit = readStoppedUnit, readProcesses = readNativeProcesses, now = () => Date.now() } = {}) {
controlPlane.assertOwnerActor(actorId);
const identity = controlPlane.getAgent(areaId);
if (identity.state !== 'working' || !identity.runtimeSessionId || !identity.updatedAt
|| !identity.summary?.includes('Runtime executor completion is unknown')) {
refuse('Only an exact retained working/unknown identity can use legacy recovery.');
}
const worktree = controlPlane.manifest.external?.find(entry => entry.type === 'worktree' && entry.area === areaId)?.path;
if (!worktree) refuse('No registered workspace exists for this identity.');
const record = { identity: Object.freeze({ areaId, state: identity.state, runtime: identity.runtime,
runtimeSessionId: identity.runtimeSessionId, updatedAt: identity.updatedAt }),
unit, invocationId, projectId: controlPlane.projectId, repositoryRoot: resolve(controlPlane.repositoryRoot),
worktree: resolve(worktree), readUnit, readProcesses, now, issuedAt: now() };
const facts = verify(record);
const evidence = Object.freeze({});
issued.set(evidence, record);
return { evidence, plan: { areaId, expectedIdentity: record.identity, ...facts,
mutationPerformed: false, effect: 'Recover presence to offline; preserve task, session and unknown outcome. No provider launch.' } };
}
export function consumeStoppedExecutorEvidence(evidence, { projectId, repositoryRoot, identity } = {}) {
const record = issued.get(evidence);
if (!record) refuse('Recovery requires fresh Runtime-issued evidence, not caller text or a Boolean.');
issued.delete(evidence);
if (record.projectId !== projectId || record.repositoryRoot !== resolve(repositoryRoot)
|| record.now() < record.issuedAt
|| record.now() - record.issuedAt > TTL_MS
|| ['areaId', 'state', 'runtime', 'runtimeSessionId', 'updatedAt'].some(key => record.identity[key] !== identity[key])) {
refuse('Recovery evidence is stale, expired or belongs to another identity.');
}
const facts = verify(record); // Recheck physical state immediately before the transactional mutation.
return { ...facts, digest: createHash('sha256').update(JSON.stringify({ identity: record.identity, ...facts })).digest('hex') };
}
+82
View File
@@ -0,0 +1,82 @@
import { randomUUID } from 'node:crypto';
import { mkdirSync, readFileSync, readlinkSync, renameSync, rmdirSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
import { basename, join, resolve } from 'node:path';
import { TorchError } from '../kernel/errors.mjs';
function nativeProcess(pid) {
if (process.platform !== 'linux') throw new Error('Process observation unavailable');
const directory = `/proc/${pid}`;
const stat = readFileSync(join(directory, 'stat'), 'utf8');
const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
return { uid: statSync(directory).uid, state: fields[0], startTicks: fields[19],
cwd: readlinkSync(join(directory, 'cwd')),
argv: readFileSync(join(directory, 'cmdline'), 'utf8').split('\0').filter(Boolean) };
}
// Observational only: never changes assignments, heartbeat, authority or launch eligibility.
export function observeRuntimeTurn({ stateRoot, repositoryRoot, projectId, areaId,
inspectProcess = nativeProcess, uid = process.getuid?.(), now = () => new Date() } = {}) {
if (!/^[a-z0-9][a-z0-9-]*$/.test(areaId)) return { state: 'unknown', reason: 'invalid-identity' };
let record;
try { record = JSON.parse(readFileSync(join(stateRoot, 'sessions', 'turn-guards', areaId, 'owner.json'), 'utf8')); }
catch (error) { return { state: error.code === 'ENOENT' ? 'inactive' : 'unknown', reason: 'guard-unavailable' }; }
if (record.projectId !== projectId || record.areaId !== areaId || !Number.isSafeInteger(record.pid)
|| record.pid < 1 || typeof record.nonce !== 'string' || !record.nonce) {
return { state: 'unknown', reason: 'guard-invalid' };
}
try {
const live = inspectProcess(record.pid);
if (live.uid !== uid || ['Z', 'X'].includes(live.state)) return { state: 'unknown', reason: 'executor-not-live' };
if (record.startTicks) {
if (record.startTicks !== live.startTicks) return { state: 'unknown', reason: 'executor-replaced' };
} else {
// Existing pre-observation guards: require an exact identity-bound TORCH up process.
const only = live.argv.indexOf('--only');
if (resolve(live.cwd) !== resolve(repositoryRoot) || !live.argv.includes('up')
|| only < 0 || live.argv[only + 1] !== areaId
|| !live.argv.some(arg => ['torch.mjs', 'torch'].includes(basename(arg)))) {
return { state: 'unknown', reason: 'legacy-executor-unverified' };
}
}
return { state: 'active', phase: record.phase === 'starting' ? 'starting' : 'working',
observedAt: now().toISOString(), source: 'live-process-and-turn-guard' };
} catch { return { state: 'unknown', reason: 'executor-unobservable' }; }
}
// One physical executor per identity, including the interval after an agent
// reports idle but before its harness actually exits. Crashed guards never expire.
export function withRuntimeTurnGuard(controlPlane, areaId, run) {
controlPlane.assertIdentity(areaId);
if (!/^[a-z0-9][a-z0-9-]*$/.test(areaId)) throw new TorchError('Invalid runtime guard identity.', { code: 'INVALID_RUNTIME_INPUT' });
const parent = join(controlPlane.stateRoot, 'sessions', 'turn-guards');
mkdirSync(parent, { recursive: true });
const directory = join(parent, areaId);
try { mkdirSync(directory); } catch (error) {
if (error.code === 'EEXIST') throw new TorchError('An executor already holds this identity turn guard.', {
code: 'RUNTIME_TURN_ACTIVE', details: { areaId },
});
throw error;
}
const path = join(directory, 'owner.json');
const nonce = randomUUID();
let startTicks;
try { startTicks = nativeProcess(process.pid).startTicks; } catch { /* Unavailable hosts remain observationally unknown. */ }
const record = { nonce, pid: process.pid, areaId, projectId: controlPlane.projectId, startTicks, phase: 'starting' };
writeFileSync(path, JSON.stringify(record), { flag: 'wx', mode: 0o600 });
const setPhase = phase => {
if (!['starting', 'working'].includes(phase)) throw new TorchError('Invalid runtime phase.', { code: 'INVALID_RUNTIME_INPUT' });
const current = JSON.parse(readFileSync(path, 'utf8'));
if (current.nonce !== nonce || current.pid !== process.pid) throw new TorchError('Runtime guard changed.', { code: 'RUNTIME_TURN_ACTIVE' });
const temporary = `${path}.${nonce}.tmp`;
writeFileSync(temporary, JSON.stringify({ ...record, phase }), { flag: 'wx', mode: 0o600 });
renameSync(temporary, path);
};
try { return run(setPhase); } finally {
// Delete only the exact guard created by this invocation; preserve foreign/changed guards.
const record = JSON.parse(readFileSync(path, 'utf8'));
if (record.nonce === nonce && record.pid === process.pid) {
unlinkSync(path);
rmdirSync(directory);
}
}
}