TASK-125dd886-6859-4e3c-a0c7-abbfde9eac24: enforce atomic runtime reservations

This commit is contained in:
thePR0M3TH3AN committed 2026-10-01 05:13:42 -04:00
1 parent 28c941d1ec
commit ff285ffd29
4 files changed
+667 -3

No files matched your search

+13
View File
@@ -451,6 +451,19 @@ failure. Planning alone never invokes a provider. Legacy
`session_manager.start_last: true` remains accepted in existing configs but is
ignored in favor of the active organization graph.
## ADR-024: Runtime reservation schema compatibility
**Status:** accepted.
Runtime reservation leases require control-plane schema version 3 or later.
An engine may migrate state that predates reservations, but it must refuse a
reservation-bearing database relabelled by an older engine and must refuse a
database with a newer schema. Runtime and Release must keep the reservation-
capable engine active for any project that has reservation state; rollback to
an older launcher is unsupported until an explicit, recovery-tested migration
exists. This is a compatibility boundary, not permission to install, activate,
or start a runtime.
## Consequences
These decisions keep the first usable release local-first and provider-
+65
View File
@@ -1,5 +1,43 @@
# Test Integrity Notes
## 2026-10-01 — Atomic runtime identity reservations
```yaml
test_integrity_note:
change_type: new_tests
scenarios:
- id: SCN-runtime-launch-reservation
given: A fresh or resumed identity with a durable launch-attempt token and executor-issued lifecycle evidence
when: A launch is reserved, held as unknown, settled, or reconciled
then: Attempts cannot replay, known provider IDs persist, only reservation-bound evidence changes state, and unknown launches remain held
- id: SCN-runtime-launch-contention
given: Independent manual and scheduled-dispatch contender processes connected to the same SQLite project state
when: Both request a fresh reservation for one identity at the same release boundary
then: Exactly one durable reservation exists and the other contender observes the active-reservation refusal
- id: SCN-runtime-launch-schema-compatibility
given: Fresh, migratable legacy, downgraded, and newer local control-plane databases
when: The reservation-capable engine opens their persistent state
then: It migrates only safe older state and refuses both an old-engine relabel with retained leases and a newer unsupported schema
observable_outcomes:
- Durable reservation token, state, expected and captured provider identity, and stable refusal codes
- Persisted identity state after rejected resume, unknown hold, terminal settlement, and trusted stopped-process reconciliation
- Exactly one persisted reservation across independent process connections
- Recorded schema version and stable refusal codes for downgrade or newer-engine incompatibility
determinism_controls:
- Disposable Git and SQLite fixture, fixed evidence timestamps, and injected executor provenance boundary
- Parent IPC releases two local child processes only after both report readiness; no provider calls, network, retries, sleeps, or timeout assertions
anti_cheat_rationale:
prevents:
- Reusing a settled lifecycle token to dispatch a second provider
- Clearing or replacing a known provider ID during a fresh reservation or rejected resume
- Releasing an unknown launch from a caller-supplied boolean, free-form text, or forged process proof
- Passing a same-process-only lock test while separate dispatch connections can double-reserve
- Relabelling reservation-bearing state after an older engine has ignored its lease guard
relaxation:
did_relax_any_assertion: false
if_true_explain_spec_basis: ""
```
## 2026-09-30 — Bounded routine coordination
```yaml
@@ -2531,3 +2569,30 @@ test_integrity_note:
did_relax_any_assertion: false
if_true_explain_spec_basis: ""
```
# Runtime launch reservations (2026-10-01)
```yaml
test_integrity_note:
change_type: new_tests
scenarios:
- id: SCN-runtime-launch-reservation
given: "An installed deterministic project with fresh and resumed runtime launch attempts"
when: "A launch reserves, is held without terminal evidence, settles with attributable evidence, or presents a mismatched resumed ID or attempt"
then: "Exactly one per-project identity reservation blocks contenders; fresh launches may bind only a post-dispatch captured ID; unknown launches remain guarded; mismatches fail closed; only explicit reconciliation or terminal evidence releases presence"
observable_outcomes:
- "Durable reservation rows, refusal codes, audit events and externally observable identity presence"
- "Fresh null provider IDs, captured ID binding, guarded unknown state and offline terminal failure"
determinism_controls:
- "Disposable Git and XDG fixtures with direct local control-plane calls"
- "Fixed attempt and provider IDs; no timers, retries, sleeps, real providers or wake execution"
anti_cheat_rationale:
prevents:
- "Creating duplicate starts while a fresh provider ID is not yet capturable"
- "Treating unknown completion or elapsed time as permission to restart"
- "Binding a captured provider ID to a different resumed attempt"
- "Hiding provider or wake invocation behind reservation acquisition"
relaxation:
did_relax_any_assertion: false
if_true_explain_spec_basis: ""
```
+365 -2
View File
@@ -17,6 +17,8 @@ const MESSAGE_KINDS = new Set([
'approval-request', 'approval-decision',
]);
const CONTROL_PLANE_SCHEMA_VERSION = 3;
function readJson(path, code) {
try {
return JSON.parse(readFileSync(path, 'utf8'));
@@ -39,6 +41,40 @@ function optionalText(value, name) {
return requiredText(value, name);
}
function requiredRecord(value, name) {
if (!value || typeof value !== 'object' || Array.isArray(value)) {
throw new TorchError(`${name} must be a structured record`, {
code: 'INVALID_RUNTIME_LAUNCH_EVIDENCE', details: { field: name },
});
}
return value;
}
function requiredEvidenceField(record, field) {
if (!Object.hasOwn(record, field)) {
throw new TorchError(`Runtime launch evidence requires ${field}`, {
code: 'INVALID_RUNTIME_LAUNCH_EVIDENCE', details: { field },
});
}
return record[field];
}
function boundedEvidenceText(value, name) {
const text = requiredText(value, name);
if (text.length > 240) {
throw new TorchError(`${name} exceeds the bounded executor-evidence limit`, {
code: 'INVALID_RUNTIME_LAUNCH_EVIDENCE', details: { field: name, maximum: 240 },
});
}
return text;
}
function boundedEvidenceValue(value, name) {
if (value === null) return null;
if (typeof value === 'number' && Number.isFinite(value)) return value;
return boundedEvidenceText(value, name);
}
function normalizeReferences(references = {}) {
return {
task: optionalText(references.task, 'references.task'),
@@ -132,6 +168,28 @@ function rowToMessage(row) {
}
function initializeSchema(database) {
const currentVersion = database.prepare('PRAGMA user_version').get().user_version;
const reservationTable = database.prepare(`
SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'runtime_launch_reservations'
`).get();
if (currentVersion > CONTROL_PLANE_SCHEMA_VERSION) {
throw new TorchError('Local control-plane state requires a newer engine', {
code: 'CONTROL_PLANE_SCHEMA_NEWER_THAN_ENGINE',
details: { currentVersion, supportedVersion: CONTROL_PLANE_SCHEMA_VERSION },
});
}
if (currentVersion < CONTROL_PLANE_SCHEMA_VERSION && reservationTable) {
throw new TorchError('Local runtime reservation state was opened by an older engine and cannot be relabelled', {
code: 'CONTROL_PLANE_SCHEMA_DOWNGRADE_DETECTED',
details: { currentVersion, requiredVersion: CONTROL_PLANE_SCHEMA_VERSION },
});
}
if (currentVersion === CONTROL_PLANE_SCHEMA_VERSION && !reservationTable) {
throw new TorchError('Local control-plane schema is incomplete for its recorded version', {
code: 'CONTROL_PLANE_SCHEMA_CORRUPT',
details: { currentVersion, missingTable: 'runtime_launch_reservations' },
});
}
database.exec(`
PRAGMA journal_mode = WAL;
PRAGMA foreign_keys = ON;
@@ -213,21 +271,46 @@ function initializeSchema(database) {
decided_by TEXT,
decision_note TEXT
);
CREATE TABLE IF NOT EXISTS runtime_launch_reservations (
id TEXT PRIMARY KEY,
project_id TEXT NOT NULL,
area_id TEXT NOT NULL,
runtime TEXT NOT NULL,
attempt_id TEXT NOT NULL,
mode TEXT NOT NULL CHECK (mode IN ('fresh', 'resume')),
expected_runtime_session_id TEXT,
captured_runtime_session_id TEXT,
state TEXT NOT NULL CHECK (state IN ('starting', 'held', 'succeeded', 'failed', 'reconciled-stopped')),
evidence TEXT,
created_at TEXT NOT NULL,
settled_at TEXT
);
CREATE INDEX IF NOT EXISTS approval_requests_pending_approver
ON approval_requests(project_id, approver_id, status, created_at, id);
CREATE INDEX IF NOT EXISTS approval_requests_pending_requester
ON approval_requests(project_id, requester_id, status, created_at, id);
CREATE INDEX IF NOT EXISTS messages_recipient_created
ON messages(recipient_id, created_at, id);
PRAGMA user_version = 2;
CREATE INDEX IF NOT EXISTS runtime_launch_reservations_active_identity
ON runtime_launch_reservations(project_id, area_id, state, created_at, id);
CREATE UNIQUE INDEX IF NOT EXISTS runtime_launch_reservations_unique_attempt
ON runtime_launch_reservations(project_id, area_id, attempt_id);
PRAGMA user_version = ${CONTROL_PLANE_SCHEMA_VERSION};
`);
}
export class ControlPlane {
constructor({ repositoryRoot, env = process.env, clock = () => new Date(), idFactory = randomUUID } = {}) {
constructor({
repositoryRoot,
env = process.env,
clock = () => new Date(),
idFactory = randomUUID,
verifyRuntimeLaunchEvidence = () => false,
} = {}) {
this.repositoryRoot = resolve(repositoryRoot ?? process.cwd());
this.clock = clock;
this.idFactory = idFactory;
this.verifyRuntimeLaunchEvidence = verifyRuntimeLaunchEvidence;
this.manifest = readInstallManifest(this.repositoryRoot);
this.config = loadProjectConfig(this.repositoryRoot);
this.roster = readJson(join(this.repositoryRoot, '.torch', 'roster.yaml'), 'INVALID_TORCH_ROSTER');
@@ -485,6 +568,286 @@ export class ControlPlane {
return this.identity(id);
}
runtimeLaunchReservations({ areaId } = {}) {
const clauses = ['project_id = ?'];
const values = [this.projectId];
if (areaId !== undefined) {
clauses.push('area_id = ?');
values.push(this.assertIdentity(areaId));
}
return this.database.prepare(`
SELECT * FROM runtime_launch_reservations WHERE ${clauses.join(' AND ')}
ORDER BY created_at ASC, id ASC
`).all(...values).map((row) => ({
id: row.id, projectId: row.project_id, areaId: row.area_id, runtime: row.runtime,
attemptId: row.attempt_id, mode: row.mode,
expectedRuntimeSessionId: row.expected_runtime_session_id ?? null,
capturedRuntimeSessionId: row.captured_runtime_session_id ?? null,
state: row.state, evidence: row.evidence ?? null, createdAt: row.created_at,
settledAt: row.settled_at ?? null,
}));
}
runtimeLaunchEvidence({ evidence, reservation, outcome, terminal, kind }) {
const record = requiredRecord(evidence, 'evidence');
const source = boundedEvidenceText(record.source, 'evidence.source');
const evidenceAttemptId = boundedEvidenceText(record.attemptId, 'evidence.attemptId');
const evidenceReservationId = boundedEvidenceText(record.reservationId, 'evidence.reservationId');
const observedAt = boundedEvidenceText(record.observedAt, 'evidence.observedAt');
if (source !== 'runtime-executor'
|| evidenceAttemptId !== reservation.attempt_id
|| evidenceReservationId !== reservation.id) {
throw new TorchError('Runtime executor evidence is not attributable to this reservation', {
code: 'RUNTIME_LAUNCH_EVIDENCE_MISMATCH', details: { reservationId: reservation.id },
});
}
if (record.outcome !== outcome || record.terminal !== terminal) {
throw new TorchError('Runtime executor evidence has the wrong lifecycle outcome', {
code: 'RUNTIME_LAUNCH_EVIDENCE_MISMATCH', details: { reservationId: reservation.id },
});
}
for (const field of ['status', 'signal', 'errorCode']) requiredEvidenceField(record, field);
const status = boundedEvidenceValue(record.status, 'evidence.status');
const signal = boundedEvidenceValue(record.signal, 'evidence.signal');
const errorCode = boundedEvidenceValue(record.errorCode, 'evidence.errorCode');
const reason = terminal === false ? boundedEvidenceText(record.reason, 'evidence.reason') : undefined;
const verified = (() => {
try {
return this.verifyRuntimeLaunchEvidence({ kind, evidence: record, reservation });
} catch {
return false;
}
})();
if (verified !== true) {
throw new TorchError('Runtime launch evidence was not issued by the executor boundary', {
code: 'RUNTIME_LAUNCH_EVIDENCE_UNVERIFIED', details: { reservationId: reservation.id, kind },
});
}
return {
source, attemptId: evidenceAttemptId, reservationId: evidenceReservationId, observedAt,
outcome, terminal, status, signal, errorCode, ...(reason ? { reason } : {}),
};
}
runtimeLaunchStoppedProof({ proof, reservation }) {
const record = requiredRecord(proof, 'proof');
const attemptId = boundedEvidenceText(record.attemptId, 'proof.attemptId');
const reservationId = boundedEvidenceText(record.reservationId, 'proof.reservationId');
const observedAt = boundedEvidenceText(record.observedAt, 'proof.observedAt');
const processId = boundedEvidenceText(record.processId, 'proof.processId');
const processStart = boundedEvidenceText(record.processStart, 'proof.processStart');
const verifier = boundedEvidenceText(record.verifier, 'proof.verifier');
if (record.kind !== 'process-proof'
|| attemptId !== reservation.attempt_id
|| reservationId !== reservation.id) {
throw new TorchError('Stopped-process proof is not attributable to this reservation', {
code: 'RUNTIME_LAUNCH_RECONCILIATION_BLOCKED', details: { reservationId: reservation.id },
});
}
const verified = (() => {
try {
return this.verifyRuntimeLaunchEvidence({ kind: 'process-proof', evidence: record, reservation });
} catch {
return false;
}
})();
if (verified !== true) {
throw new TorchError('Stopped-process proof was not issued by the executor boundary', {
code: 'RUNTIME_LAUNCH_RECONCILIATION_BLOCKED', details: { reservationId: reservation.id },
});
}
return { kind: 'process-proof', attemptId, reservationId, observedAt, processId, processStart, verifier };
}
reserveRuntimeLaunch({ areaId, runtime, attemptId, mode, runtimeSessionId } = {}) {
const id = this.assertIdentity(areaId);
const normalizedRuntime = requiredText(runtime, 'runtime');
const normalizedAttempt = requiredText(attemptId, 'attemptId');
const normalizedMode = requiredText(mode, 'mode');
if (!['fresh', 'resume'].includes(normalizedMode)) {
throw new TorchError('Runtime launch mode must be fresh or resume', { code: 'INVALID_RUNTIME_LAUNCH_MODE' });
}
const requestedRuntimeSessionId = optionalText(runtimeSessionId, 'runtimeSessionId');
if (normalizedMode === 'resume' && !requestedRuntimeSessionId) {
throw new TorchError('Resumed runtime launches require their expected session ID', {
code: 'RUNTIME_LAUNCH_SESSION_REQUIRED', details: { areaId: id, mode: normalizedMode },
});
}
const createdAt = this.clock().toISOString();
const reservation = {
id: this.idFactory(), projectId: this.projectId, areaId: id, runtime: normalizedRuntime,
attemptId: normalizedAttempt, mode: normalizedMode, expectedRuntimeSessionId: null,
capturedRuntimeSessionId: null, state: 'starting', evidence: null, createdAt, settledAt: null,
};
this.database.exec('BEGIN IMMEDIATE');
try {
const identity = this.database.prepare('SELECT * FROM identities WHERE area_id = ?').get(id);
const registeredRuntimeSessionId = identity?.runtime_session_id ?? null;
if (normalizedMode === 'resume' && registeredRuntimeSessionId !== requestedRuntimeSessionId) {
throw new TorchError('Resumed runtime launch does not match the registered provider session ID', {
code: 'RUNTIME_LAUNCH_SESSION_MISMATCH', details: { areaId: id, expected: registeredRuntimeSessionId },
});
}
if (normalizedMode === 'fresh' && requestedRuntimeSessionId
&& registeredRuntimeSessionId && requestedRuntimeSessionId !== registeredRuntimeSessionId) {
throw new TorchError('Fresh runtime launch cannot replace a registered provider session ID', {
code: 'RUNTIME_LAUNCH_SESSION_MISMATCH', details: { areaId: id, expected: registeredRuntimeSessionId },
});
}
const replay = this.database.prepare(`
SELECT id FROM runtime_launch_reservations
WHERE project_id = ? AND area_id = ? AND attempt_id = ?
`).get(this.projectId, id, normalizedAttempt);
if (replay) {
throw new TorchError('A durable runtime launch attempt cannot be replayed', {
code: 'RUNTIME_LAUNCH_ATTEMPT_REPLAYED', details: { reservationId: replay.id, areaId: id },
});
}
const active = this.database.prepare(`
SELECT * FROM runtime_launch_reservations
WHERE project_id = ? AND area_id = ? AND state IN ('starting', 'held')
ORDER BY created_at ASC, id ASC LIMIT 1
`).get(this.projectId, id);
if (active) {
throw new TorchError('A runtime launch reservation already blocks this identity', {
code: 'RUNTIME_LAUNCH_ALREADY_RESERVED',
details: { reservationId: active.id, areaId: id, state: active.state },
});
}
reservation.expectedRuntimeSessionId = registeredRuntimeSessionId;
this.database.prepare(`
INSERT INTO runtime_launch_reservations (
id, project_id, area_id, runtime, attempt_id, mode, expected_runtime_session_id,
captured_runtime_session_id, state, evidence, created_at, settled_at
) VALUES (?, ?, ?, ?, ?, ?, ?, NULL, 'starting', NULL, ?, NULL)
`).run(
reservation.id, reservation.projectId, reservation.areaId, reservation.runtime,
reservation.attemptId, reservation.mode, reservation.expectedRuntimeSessionId, reservation.createdAt,
);
this.database.prepare(`
UPDATE identities SET runtime = ?, runtime_session_id = ?, state = 'starting',
summary = ?, heartbeat_at = ?, updated_at = ? WHERE area_id = ?
`).run(normalizedRuntime, registeredRuntimeSessionId, 'Runtime launch reserved before provider dispatch.', createdAt, createdAt, id);
this.audit({
actorId: id, operation: 'runtime.launch.reserve', entityType: 'runtime-launch-reservation', entityId: reservation.id,
details: { attemptId: normalizedAttempt, mode: normalizedMode, expectedRuntimeSessionId: registeredRuntimeSessionId },
});
this.database.exec('COMMIT');
return reservation;
} catch (error) {
this.database.exec('ROLLBACK');
throw error;
}
}
settleRuntimeLaunch({ reservationId, attemptId, outcome, runtimeSessionId, evidence } = {}) {
const id = requiredText(reservationId, 'reservationId');
const normalizedAttempt = requiredText(attemptId, 'attemptId');
const normalizedOutcome = requiredText(outcome, 'outcome');
if (!['succeeded', 'failed'].includes(normalizedOutcome)) {
throw new TorchError('Runtime launch settlement must be succeeded or failed', { code: 'INVALID_RUNTIME_LAUNCH_OUTCOME' });
}
const capturedRuntimeSessionId = optionalText(runtimeSessionId, 'runtimeSessionId');
this.database.exec('BEGIN IMMEDIATE');
try {
const reservation = this.database.prepare('SELECT * FROM runtime_launch_reservations WHERE id = ? AND project_id = ?').get(id, this.projectId);
if (!reservation) throw new TorchError('Unknown runtime launch reservation', { code: 'RUNTIME_LAUNCH_RESERVATION_NOT_FOUND' });
if (!['starting', 'held'].includes(reservation.state)) throw new TorchError('Runtime launch reservation is no longer active', { code: 'RUNTIME_LAUNCH_NOT_ACTIVE' });
if (reservation.attempt_id !== normalizedAttempt) throw new TorchError('Runtime launch attempt does not match its reservation', { code: 'RUNTIME_LAUNCH_ATTEMPT_MISMATCH' });
const terminalEvidence = this.runtimeLaunchEvidence({
evidence, reservation, outcome: normalizedOutcome, terminal: true, kind: 'terminal',
});
if (reservation.mode === 'resume' && capturedRuntimeSessionId
&& capturedRuntimeSessionId !== reservation.expected_runtime_session_id) {
throw new TorchError('Resumed runtime launch captured an unexpected provider session ID', { code: 'RUNTIME_LAUNCH_SESSION_MISMATCH' });
}
const settledAt = this.clock().toISOString();
const knownSessionId = reservation.captured_runtime_session_id ?? reservation.expected_runtime_session_id ?? null;
if (capturedRuntimeSessionId && knownSessionId && capturedRuntimeSessionId !== knownSessionId) {
throw new TorchError('Runtime launch captured an unexpected provider session ID', { code: 'RUNTIME_LAUNCH_SESSION_MISMATCH' });
}
const boundSessionId = capturedRuntimeSessionId ?? knownSessionId;
this.database.prepare(`
UPDATE runtime_launch_reservations SET state = ?, captured_runtime_session_id = ?, evidence = ?, settled_at = ? WHERE id = ?
`).run(normalizedOutcome, boundSessionId, JSON.stringify(terminalEvidence), settledAt, id);
this.database.prepare(`
UPDATE identities SET runtime_session_id = ?, state = ?, summary = ?, heartbeat_at = ?, updated_at = ? WHERE area_id = ?
`).run(boundSessionId, normalizedOutcome === 'succeeded' ? 'idle' : 'offline',
normalizedOutcome === 'succeeded' ? 'Runtime launch completed.' : 'Runtime launch failed with terminal evidence.', settledAt, settledAt, reservation.area_id);
this.audit({ actorId: reservation.area_id, operation: 'runtime.launch.settle', entityType: 'runtime-launch-reservation', entityId: id,
details: { attemptId: normalizedAttempt, outcome: normalizedOutcome, capturedRuntimeSessionId: boundSessionId } });
this.database.exec('COMMIT');
return this.runtimeLaunchReservations({ areaId: reservation.area_id }).find((entry) => entry.id === id);
} catch (error) {
this.database.exec('ROLLBACK');
throw error;
}
}
holdRuntimeLaunch({ reservationId, attemptId, outcome, runtimeSessionId, capturedRuntimeSessionId, evidence } = {}) {
const id = requiredText(reservationId, 'reservationId');
const normalizedAttempt = requiredText(attemptId, 'attemptId');
if (requiredText(outcome, 'outcome') !== 'unknown') {
throw new TorchError('Only unknown runtime interruptions may be held', { code: 'INVALID_RUNTIME_LAUNCH_HOLD' });
}
const runtimeSession = optionalText(runtimeSessionId, 'runtimeSessionId');
const capturedSession = optionalText(capturedRuntimeSessionId, 'capturedRuntimeSessionId');
if (runtimeSession && capturedSession && runtimeSession !== capturedSession) {
throw new TorchError('Runtime launch hold received conflicting captured provider IDs', { code: 'RUNTIME_LAUNCH_SESSION_MISMATCH' });
}
const observedRuntimeSessionId = capturedSession ?? runtimeSession;
this.database.exec('BEGIN IMMEDIATE');
try {
const reservation = this.database.prepare('SELECT * FROM runtime_launch_reservations WHERE id = ? AND project_id = ?').get(id, this.projectId);
if (!reservation) throw new TorchError('Unknown runtime launch reservation', { code: 'RUNTIME_LAUNCH_RESERVATION_NOT_FOUND' });
if (reservation.state !== 'starting') throw new TorchError('Only starting runtime launches may be held', { code: 'RUNTIME_LAUNCH_NOT_ACTIVE' });
if (reservation.attempt_id !== normalizedAttempt) throw new TorchError('Runtime launch attempt does not match its reservation', { code: 'RUNTIME_LAUNCH_ATTEMPT_MISMATCH' });
const unknownEvidence = this.runtimeLaunchEvidence({
evidence, reservation, outcome: 'unknown', terminal: false, kind: 'unknown',
});
const knownSessionId = reservation.captured_runtime_session_id ?? reservation.expected_runtime_session_id ?? null;
if (observedRuntimeSessionId && knownSessionId && observedRuntimeSessionId !== knownSessionId) {
throw new TorchError('Runtime launch captured an unexpected provider session ID', { code: 'RUNTIME_LAUNCH_SESSION_MISMATCH' });
}
const boundSessionId = observedRuntimeSessionId ?? knownSessionId;
const updatedAt = this.clock().toISOString();
this.database.prepare('UPDATE runtime_launch_reservations SET state = ?, captured_runtime_session_id = ?, evidence = ? WHERE id = ?')
.run('held', boundSessionId, JSON.stringify(unknownEvidence), id);
this.database.prepare(`UPDATE identities SET runtime_session_id = ?, state = 'working', summary = ?, heartbeat_at = ?, updated_at = ? WHERE area_id = ?`)
.run(boundSessionId, 'Runtime executor completion is unknown; duplicate launch is blocked pending reconciliation.', updatedAt, updatedAt, reservation.area_id);
this.audit({ actorId: reservation.area_id, operation: 'runtime.launch.hold', entityType: 'runtime-launch-reservation', entityId: id,
details: { attemptId: normalizedAttempt } });
this.database.exec('COMMIT');
return this.runtimeLaunchReservations({ areaId: reservation.area_id }).find((entry) => entry.id === id);
} catch (error) {
this.database.exec('ROLLBACK');
throw error;
}
}
reconcileRuntimeLaunch({ reservationId, proof } = {}) {
const id = requiredText(reservationId, 'reservationId');
this.database.exec('BEGIN IMMEDIATE');
try {
const reservation = this.database.prepare('SELECT * FROM runtime_launch_reservations WHERE id = ? AND project_id = ?').get(id, this.projectId);
if (!reservation) throw new TorchError('Unknown runtime launch reservation', { code: 'RUNTIME_LAUNCH_RESERVATION_NOT_FOUND' });
if (reservation.state !== 'held') throw new TorchError('Only held runtime launches require reconciliation', { code: 'RUNTIME_LAUNCH_RECONCILIATION_NOT_ALLOWED' });
const stoppedProof = this.runtimeLaunchStoppedProof({ proof, reservation });
const settledAt = this.clock().toISOString();
this.database.prepare('UPDATE runtime_launch_reservations SET state = ?, evidence = ?, settled_at = ? WHERE id = ?')
.run('reconciled-stopped', JSON.stringify(stoppedProof), settledAt, id);
this.database.prepare(`UPDATE identities SET state = 'offline', summary = ?, heartbeat_at = ?, updated_at = ? WHERE area_id = ?`)
.run('Runtime stopped by explicit reconciliation evidence.', settledAt, settledAt, reservation.area_id);
this.audit({ actorId: reservation.area_id, operation: 'runtime.launch.reconcile', entityType: 'runtime-launch-reservation', entityId: id,
details: { attemptId: reservation.attempt_id, proof: 'process-proof' } });
this.database.exec('COMMIT');
return this.runtimeLaunchReservations({ areaId: reservation.area_id }).find((entry) => entry.id === id);
} catch (error) {
this.database.exec('ROLLBACK');
throw error;
}
}
writeReport({ areaId, kind, summary, task, evidence } = {}) {
const id = this.assertIdentity(areaId);
const createdAt = this.clock().toISOString();
+224 -1
View File
@@ -1,8 +1,9 @@
import assert from 'node:assert/strict';
import { execFileSync } from 'node:child_process';
import { execFileSync, fork } from 'node:child_process';
import { mkdirSync, mkdtempSync, readFileSync, writeFileSync, existsSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { DatabaseSync } from 'node:sqlite';
import test from 'node:test';
import { analyzeRepository } from '../../src/kernel/analyze.mjs';
import { diagnoseProject } from '../../src/kernel/doctor.mjs';
@@ -89,6 +90,228 @@ test('SCN-install-doctor-purge: a fresh install is healthy and exactly reversibl
assert.equal(readFileSync(join(root, 'index.js'), 'utf8'), 'export const ready = true;\n');
});
test('SCN-runtime-launch-reservation: durable attempts, provider identities, and unknown launches fail closed', () => {
const { root, env } = fixture();
const repository = inspectRepository(root);
const proposal = approvedProposal(repository);
installProject({ repository, proposal, env, projectId: 'reservation-project' });
const areaId = proposal.domains[0].id;
const trustedExecutorRecords = new WeakSet();
const control = openControlPlane({
repositoryRoot: root,
env,
verifyRuntimeLaunchEvidence: ({ evidence, reservation }) => trustedExecutorRecords.has(evidence)
&& evidence.reservationId === reservation.id && evidence.attemptId === reservation.attempt_id,
});
const executorEvidence = (reservation, outcome, extra = {}) => {
const evidence = {
source: 'runtime-executor', attemptId: reservation.attemptId, reservationId: reservation.id,
observedAt: '2026-10-01T00:00:00.000Z', outcome, terminal: outcome !== 'unknown',
status: null, signal: null, errorCode: null, ...extra,
};
trustedExecutorRecords.add(evidence);
return evidence;
};
const stoppedProof = (reservation) => {
const proof = {
kind: 'process-proof', attemptId: reservation.attemptId, reservationId: reservation.id,
observedAt: '2026-10-01T00:01:00.000Z', processId: 'fixture-process-1',
processStart: '2026-10-01T00:00:00.000Z', verifier: 'fixture-executor',
};
trustedExecutorRecords.add(proof);
return proof;
};
try {
const fresh = control.reserveRuntimeLaunch({
areaId, runtime: 'codex', attemptId: 'fresh-attempt', mode: 'fresh',
});
assert.equal(fresh.expectedRuntimeSessionId, null, 'fresh launches reserve before a provider ID exists');
assert.equal(control.identity(areaId).state, 'starting');
assert.equal(control.identity(areaId).runtimeSessionId, null);
assert.throws(() => control.reserveRuntimeLaunch({
areaId, runtime: 'codex', attemptId: 'competing-attempt', mode: 'fresh',
}), (error) => error.code === 'RUNTIME_LAUNCH_ALREADY_RESERVED');
const held = control.holdRuntimeLaunch({
reservationId: fresh.id, attemptId: fresh.attemptId, outcome: 'unknown',
evidence: executorEvidence(fresh, 'unknown', { reason: 'executor-interrupted' }),
});
assert.equal(held.state, 'held');
assert.equal(held.capturedRuntimeSessionId, null, 'an unknown launch without an ID remains guarded without inventing one');
assert.equal(control.identity(areaId).state, 'working', 'unknown fresh launch remains guarded rather than offline');
assert.throws(() => control.reconcileRuntimeLaunch({
reservationId: fresh.id,
proof: {
kind: 'process-proof', attemptId: fresh.attemptId, reservationId: fresh.id,
observedAt: '2026-10-01T00:01:00.000Z', processId: 'forged-process',
processStart: '2026-10-01T00:00:00.000Z', verifier: 'untrusted-caller',
},
}), (error) => error.code === 'RUNTIME_LAUNCH_RECONCILIATION_BLOCKED');
assert.equal(control.reconcileRuntimeLaunch({
reservationId: fresh.id, proof: stoppedProof(fresh),
}).state, 'reconciled-stopped');
assert.throws(() => control.reserveRuntimeLaunch({
areaId, runtime: 'codex', attemptId: fresh.attemptId, mode: 'fresh',
}), (error) => error.code === 'RUNTIME_LAUNCH_ATTEMPT_REPLAYED');
const captured = control.reserveRuntimeLaunch({
areaId, runtime: 'codex', attemptId: 'fresh-captured', mode: 'fresh',
});
assert.equal(control.holdRuntimeLaunch({
reservationId: captured.id, attemptId: captured.attemptId, outcome: 'unknown',
capturedRuntimeSessionId: 'captured-after-dispatch',
evidence: executorEvidence(captured, 'unknown', { reason: 'executor-disconnected' }),
}).capturedRuntimeSessionId, 'captured-after-dispatch');
assert.equal(control.identity(areaId).runtimeSessionId, 'captured-after-dispatch');
assert.equal(control.reconcileRuntimeLaunch({ reservationId: captured.id, proof: stoppedProof(captured) }).state, 'reconciled-stopped');
control.reportStatus({
areaId, state: 'idle', runtime: 'codex', runtimeSessionId: 'registered-provider-id', summary: 'Registered provider persists.',
});
const preservingFresh = control.reserveRuntimeLaunch({
areaId, runtime: 'codex', attemptId: 'preserving-fresh', mode: 'fresh',
});
assert.equal(preservingFresh.expectedRuntimeSessionId, 'registered-provider-id');
assert.equal(control.identity(areaId).runtimeSessionId, 'registered-provider-id');
assert.equal(control.settleRuntimeLaunch({
reservationId: preservingFresh.id, attemptId: preservingFresh.attemptId, outcome: 'succeeded',
evidence: executorEvidence(preservingFresh, 'succeeded'),
}).capturedRuntimeSessionId, 'registered-provider-id');
assert.throws(() => control.reserveRuntimeLaunch({
areaId, runtime: 'codex', attemptId: preservingFresh.attemptId, mode: 'fresh',
}), (error) => error.code === 'RUNTIME_LAUNCH_ATTEMPT_REPLAYED');
assert.throws(() => control.reserveRuntimeLaunch({
areaId, runtime: 'codex', attemptId: 'wrong-resume', mode: 'resume', runtimeSessionId: 'wrong-provider-id',
}), (error) => error.code === 'RUNTIME_LAUNCH_SESSION_MISMATCH');
assert.equal(control.identity(areaId).runtimeSessionId, 'registered-provider-id', 'a rejected resume cannot rebind identity state');
assert.equal(control.identity(areaId).state, 'idle', 'a rejected resume cannot mutate presence state');
const resumed = control.reserveRuntimeLaunch({
areaId, runtime: 'codex', attemptId: 'resume-attempt', mode: 'resume', runtimeSessionId: 'registered-provider-id',
});
assert.throws(() => control.settleRuntimeLaunch({
reservationId: resumed.id, attemptId: resumed.attemptId, outcome: 'succeeded',
runtimeSessionId: 'different-provider-id', evidence: executorEvidence(resumed, 'succeeded'),
}), (error) => error.code === 'RUNTIME_LAUNCH_SESSION_MISMATCH');
assert.throws(() => control.settleRuntimeLaunch({
reservationId: resumed.id, attemptId: 'wrong-attempt', outcome: 'succeeded',
runtimeSessionId: 'registered-provider-id', evidence: executorEvidence(resumed, 'succeeded'),
}), (error) => error.code === 'RUNTIME_LAUNCH_ATTEMPT_MISMATCH');
assert.equal(control.settleRuntimeLaunch({
reservationId: resumed.id, attemptId: resumed.attemptId, outcome: 'failed',
runtimeSessionId: 'registered-provider-id', evidence: executorEvidence(resumed, 'failed'),
}).state, 'failed');
assert.equal(control.identity(areaId).state, 'offline');
} finally {
control.close();
}
});
test('SCN-runtime-launch-contention: independent contender processes reserve one durable launch token', async () => {
const { root, env } = fixture();
const repository = inspectRepository(root);
const proposal = approvedProposal(repository);
installProject({ repository, proposal, env, projectId: 'reservation-contention-project' });
const areaId = 'session-manager';
const initializer = openControlPlane({ repositoryRoot: root, env });
initializer.close();
const workerPath = join(root, 'reservation-contender.mjs');
const serviceUrl = new URL('../../src/control-plane/service.mjs', import.meta.url).href;
writeFileSync(workerPath, `
import { openControlPlane } from ${JSON.stringify(serviceUrl)};
process.on('message', (input) => {
const control = openControlPlane({ repositoryRoot: input.root, env: input.env });
let result;
try {
result = { ok: true, reservation: control.reserveRuntimeLaunch({
areaId: input.areaId, runtime: 'codex', attemptId: input.attemptId, mode: 'fresh',
}).id };
} catch (error) {
result = { ok: false, code: error.code };
} finally {
control.close();
}
process.send(result, () => process.disconnect());
});
process.send({ ready: true });
`);
const contender = (attemptId) => {
const child = fork(workerPath, [], { stdio: ['ignore', 'ignore', 'ignore', 'ipc'] });
let resolveReady;
let resolveResult;
const ready = new Promise((resolve) => { resolveReady = resolve; });
const result = new Promise((resolve, reject) => {
resolveResult = resolve;
child.once('error', reject);
child.on('exit', (code) => {
if (code !== 0) reject(new Error(`reservation contender exited ${code}`));
});
});
child.on('message', (message) => {
if (message.ready) resolveReady();
else resolveResult(message);
});
return { child, ready, result, input: { root, env, areaId, attemptId } };
};
const first = contender('manual-dispatch-attempt');
const second = contender('scheduled-dispatch-attempt');
await Promise.all([first.ready, second.ready]);
first.child.send(first.input);
second.child.send(second.input);
const outcomes = await Promise.all([first.result, second.result]);
assert.equal(outcomes.filter((outcome) => outcome.ok).length, 1, 'exactly one process may reserve the identity');
assert.deepEqual(outcomes.filter((outcome) => !outcome.ok).map((outcome) => outcome.code), ['RUNTIME_LAUNCH_ALREADY_RESERVED']);
const control = openControlPlane({ repositoryRoot: root, env });
try {
const reservations = control.runtimeLaunchReservations({ areaId });
assert.equal(reservations.length, 1);
assert.equal(reservations[0].state, 'starting');
} finally {
control.close();
}
});
test('SCN-runtime-launch-schema-compatibility: reservation state migrates forward and rejects engine downgrades', () => {
const { root, env } = fixture();
const repository = inspectRepository(root);
const installed = installProject({ repository, proposal: approvedProposal(repository), env, projectId: 'reservation-schema-project' });
const databasePath = join(installed.stateRoot, 'state.db');
const initial = openControlPlane({ repositoryRoot: root, env });
initial.close();
const database = new DatabaseSync(databasePath);
try {
assert.equal(database.prepare('PRAGMA user_version').get().user_version, 3);
database.exec('DROP TABLE runtime_launch_reservations; PRAGMA user_version = 2;');
} finally {
database.close();
}
const migrated = openControlPlane({ repositoryRoot: root, env });
migrated.close();
const downgraded = new DatabaseSync(databasePath);
try {
downgraded.exec('PRAGMA user_version = 2;');
} finally {
downgraded.close();
}
assert.throws(
() => openControlPlane({ repositoryRoot: root, env }),
(error) => error.code === 'CONTROL_PLANE_SCHEMA_DOWNGRADE_DETECTED',
);
const newer = new DatabaseSync(databasePath);
try {
newer.exec('PRAGMA user_version = 4;');
} finally {
newer.close();
}
assert.throws(
() => openControlPlane({ repositoryRoot: root, env }),
(error) => error.code === 'CONTROL_PLANE_SCHEMA_NEWER_THAN_ENGINE',
);
});
test('SCN-existing-repository-preservation: install and purge preserve history, branches, config, and untracked files', () => {
const { root, env } = fixture();
const externalEnv = { ...env, XDG_DATA_HOME: mkdtempSync(join(tmpdir(), 'torch-existing-repo-state-')) };