mirror of
https://github.com/PR0M3TH3AN/TORCH.git
synced 2026-10-28 16:31:56 +00:00
TASK-125dd886-6859-4e3c-a0c7-abbfde9eac24: enforce atomic runtime reservations
This commit is contained in:
1 parent
28c941d1ec
commit
ff285ffd29
4 files changed
+667
-3
No files matched your search
@@ -451,6 +451,19 @@ failure. Planning alone never invokes a provider. Legacy
|
||||
`session_manager.start_last: true` remains accepted in existing configs but is
|
||||
ignored in favor of the active organization graph.
|
||||
|
||||
## ADR-024: Runtime reservation schema compatibility
|
||||
|
||||
**Status:** accepted.
|
||||
|
||||
Runtime reservation leases require control-plane schema version 3 or later.
|
||||
An engine may migrate state that predates reservations, but it must refuse a
|
||||
reservation-bearing database relabelled by an older engine and must refuse a
|
||||
database with a newer schema. Runtime and Release must keep the reservation-
|
||||
capable engine active for any project that has reservation state; rollback to
|
||||
an older launcher is unsupported until an explicit, recovery-tested migration
|
||||
exists. This is a compatibility boundary, not permission to install, activate,
|
||||
or start a runtime.
|
||||
|
||||
## Consequences
|
||||
|
||||
These decisions keep the first usable release local-first and provider-
|
||||
|
||||
@@ -1,5 +1,43 @@
|
||||
# Test Integrity Notes
|
||||
|
||||
## 2026-10-01 — Atomic runtime identity reservations
|
||||
|
||||
```yaml
|
||||
test_integrity_note:
|
||||
change_type: new_tests
|
||||
scenarios:
|
||||
- id: SCN-runtime-launch-reservation
|
||||
given: A fresh or resumed identity with a durable launch-attempt token and executor-issued lifecycle evidence
|
||||
when: A launch is reserved, held as unknown, settled, or reconciled
|
||||
then: Attempts cannot replay, known provider IDs persist, only reservation-bound evidence changes state, and unknown launches remain held
|
||||
- id: SCN-runtime-launch-contention
|
||||
given: Independent manual and scheduled-dispatch contender processes connected to the same SQLite project state
|
||||
when: Both request a fresh reservation for one identity at the same release boundary
|
||||
then: Exactly one durable reservation exists and the other contender observes the active-reservation refusal
|
||||
- id: SCN-runtime-launch-schema-compatibility
|
||||
given: Fresh, migratable legacy, downgraded, and newer local control-plane databases
|
||||
when: The reservation-capable engine opens their persistent state
|
||||
then: It migrates only safe older state and refuses both an old-engine relabel with retained leases and a newer unsupported schema
|
||||
observable_outcomes:
|
||||
- Durable reservation token, state, expected and captured provider identity, and stable refusal codes
|
||||
- Persisted identity state after rejected resume, unknown hold, terminal settlement, and trusted stopped-process reconciliation
|
||||
- Exactly one persisted reservation across independent process connections
|
||||
- Recorded schema version and stable refusal codes for downgrade or newer-engine incompatibility
|
||||
determinism_controls:
|
||||
- Disposable Git and SQLite fixture, fixed evidence timestamps, and injected executor provenance boundary
|
||||
- Parent IPC releases two local child processes only after both report readiness; no provider calls, network, retries, sleeps, or timeout assertions
|
||||
anti_cheat_rationale:
|
||||
prevents:
|
||||
- Reusing a settled lifecycle token to dispatch a second provider
|
||||
- Clearing or replacing a known provider ID during a fresh reservation or rejected resume
|
||||
- Releasing an unknown launch from a caller-supplied boolean, free-form text, or forged process proof
|
||||
- Passing a same-process-only lock test while separate dispatch connections can double-reserve
|
||||
- Relabelling reservation-bearing state after an older engine has ignored its lease guard
|
||||
relaxation:
|
||||
did_relax_any_assertion: false
|
||||
if_true_explain_spec_basis: ""
|
||||
```
|
||||
|
||||
## 2026-09-30 — Bounded routine coordination
|
||||
|
||||
```yaml
|
||||
@@ -2531,3 +2569,30 @@ test_integrity_note:
|
||||
did_relax_any_assertion: false
|
||||
if_true_explain_spec_basis: ""
|
||||
```
|
||||
|
||||
# Runtime launch reservations (2026-10-01)
|
||||
|
||||
```yaml
|
||||
test_integrity_note:
|
||||
change_type: new_tests
|
||||
scenarios:
|
||||
- id: SCN-runtime-launch-reservation
|
||||
given: "An installed deterministic project with fresh and resumed runtime launch attempts"
|
||||
when: "A launch reserves, is held without terminal evidence, settles with attributable evidence, or presents a mismatched resumed ID or attempt"
|
||||
then: "Exactly one per-project identity reservation blocks contenders; fresh launches may bind only a post-dispatch captured ID; unknown launches remain guarded; mismatches fail closed; only explicit reconciliation or terminal evidence releases presence"
|
||||
observable_outcomes:
|
||||
- "Durable reservation rows, refusal codes, audit events and externally observable identity presence"
|
||||
- "Fresh null provider IDs, captured ID binding, guarded unknown state and offline terminal failure"
|
||||
determinism_controls:
|
||||
- "Disposable Git and XDG fixtures with direct local control-plane calls"
|
||||
- "Fixed attempt and provider IDs; no timers, retries, sleeps, real providers or wake execution"
|
||||
anti_cheat_rationale:
|
||||
prevents:
|
||||
- "Creating duplicate starts while a fresh provider ID is not yet capturable"
|
||||
- "Treating unknown completion or elapsed time as permission to restart"
|
||||
- "Binding a captured provider ID to a different resumed attempt"
|
||||
- "Hiding provider or wake invocation behind reservation acquisition"
|
||||
relaxation:
|
||||
did_relax_any_assertion: false
|
||||
if_true_explain_spec_basis: ""
|
||||
```
|
||||
@@ -17,6 +17,8 @@ const MESSAGE_KINDS = new Set([
|
||||
'approval-request', 'approval-decision',
|
||||
]);
|
||||
|
||||
const CONTROL_PLANE_SCHEMA_VERSION = 3;
|
||||
|
||||
function readJson(path, code) {
|
||||
try {
|
||||
return JSON.parse(readFileSync(path, 'utf8'));
|
||||
@@ -39,6 +41,40 @@ function optionalText(value, name) {
|
||||
return requiredText(value, name);
|
||||
}
|
||||
|
||||
function requiredRecord(value, name) {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
||||
throw new TorchError(`${name} must be a structured record`, {
|
||||
code: 'INVALID_RUNTIME_LAUNCH_EVIDENCE', details: { field: name },
|
||||
});
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requiredEvidenceField(record, field) {
|
||||
if (!Object.hasOwn(record, field)) {
|
||||
throw new TorchError(`Runtime launch evidence requires ${field}`, {
|
||||
code: 'INVALID_RUNTIME_LAUNCH_EVIDENCE', details: { field },
|
||||
});
|
||||
}
|
||||
return record[field];
|
||||
}
|
||||
|
||||
function boundedEvidenceText(value, name) {
|
||||
const text = requiredText(value, name);
|
||||
if (text.length > 240) {
|
||||
throw new TorchError(`${name} exceeds the bounded executor-evidence limit`, {
|
||||
code: 'INVALID_RUNTIME_LAUNCH_EVIDENCE', details: { field: name, maximum: 240 },
|
||||
});
|
||||
}
|
||||
return text;
|
||||
}
|
||||
|
||||
function boundedEvidenceValue(value, name) {
|
||||
if (value === null) return null;
|
||||
if (typeof value === 'number' && Number.isFinite(value)) return value;
|
||||
return boundedEvidenceText(value, name);
|
||||
}
|
||||
|
||||
function normalizeReferences(references = {}) {
|
||||
return {
|
||||
task: optionalText(references.task, 'references.task'),
|
||||
@@ -132,6 +168,28 @@ function rowToMessage(row) {
|
||||
}
|
||||
|
||||
function initializeSchema(database) {
|
||||
const currentVersion = database.prepare('PRAGMA user_version').get().user_version;
|
||||
const reservationTable = database.prepare(`
|
||||
SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'runtime_launch_reservations'
|
||||
`).get();
|
||||
if (currentVersion > CONTROL_PLANE_SCHEMA_VERSION) {
|
||||
throw new TorchError('Local control-plane state requires a newer engine', {
|
||||
code: 'CONTROL_PLANE_SCHEMA_NEWER_THAN_ENGINE',
|
||||
details: { currentVersion, supportedVersion: CONTROL_PLANE_SCHEMA_VERSION },
|
||||
});
|
||||
}
|
||||
if (currentVersion < CONTROL_PLANE_SCHEMA_VERSION && reservationTable) {
|
||||
throw new TorchError('Local runtime reservation state was opened by an older engine and cannot be relabelled', {
|
||||
code: 'CONTROL_PLANE_SCHEMA_DOWNGRADE_DETECTED',
|
||||
details: { currentVersion, requiredVersion: CONTROL_PLANE_SCHEMA_VERSION },
|
||||
});
|
||||
}
|
||||
if (currentVersion === CONTROL_PLANE_SCHEMA_VERSION && !reservationTable) {
|
||||
throw new TorchError('Local control-plane schema is incomplete for its recorded version', {
|
||||
code: 'CONTROL_PLANE_SCHEMA_CORRUPT',
|
||||
details: { currentVersion, missingTable: 'runtime_launch_reservations' },
|
||||
});
|
||||
}
|
||||
database.exec(`
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA foreign_keys = ON;
|
||||
@@ -213,21 +271,46 @@ function initializeSchema(database) {
|
||||
decided_by TEXT,
|
||||
decision_note TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS runtime_launch_reservations (
|
||||
id TEXT PRIMARY KEY,
|
||||
project_id TEXT NOT NULL,
|
||||
area_id TEXT NOT NULL,
|
||||
runtime TEXT NOT NULL,
|
||||
attempt_id TEXT NOT NULL,
|
||||
mode TEXT NOT NULL CHECK (mode IN ('fresh', 'resume')),
|
||||
expected_runtime_session_id TEXT,
|
||||
captured_runtime_session_id TEXT,
|
||||
state TEXT NOT NULL CHECK (state IN ('starting', 'held', 'succeeded', 'failed', 'reconciled-stopped')),
|
||||
evidence TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
settled_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS approval_requests_pending_approver
|
||||
ON approval_requests(project_id, approver_id, status, created_at, id);
|
||||
CREATE INDEX IF NOT EXISTS approval_requests_pending_requester
|
||||
ON approval_requests(project_id, requester_id, status, created_at, id);
|
||||
CREATE INDEX IF NOT EXISTS messages_recipient_created
|
||||
ON messages(recipient_id, created_at, id);
|
||||
PRAGMA user_version = 2;
|
||||
CREATE INDEX IF NOT EXISTS runtime_launch_reservations_active_identity
|
||||
ON runtime_launch_reservations(project_id, area_id, state, created_at, id);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS runtime_launch_reservations_unique_attempt
|
||||
ON runtime_launch_reservations(project_id, area_id, attempt_id);
|
||||
PRAGMA user_version = ${CONTROL_PLANE_SCHEMA_VERSION};
|
||||
`);
|
||||
}
|
||||
|
||||
export class ControlPlane {
|
||||
constructor({ repositoryRoot, env = process.env, clock = () => new Date(), idFactory = randomUUID } = {}) {
|
||||
constructor({
|
||||
repositoryRoot,
|
||||
env = process.env,
|
||||
clock = () => new Date(),
|
||||
idFactory = randomUUID,
|
||||
verifyRuntimeLaunchEvidence = () => false,
|
||||
} = {}) {
|
||||
this.repositoryRoot = resolve(repositoryRoot ?? process.cwd());
|
||||
this.clock = clock;
|
||||
this.idFactory = idFactory;
|
||||
this.verifyRuntimeLaunchEvidence = verifyRuntimeLaunchEvidence;
|
||||
this.manifest = readInstallManifest(this.repositoryRoot);
|
||||
this.config = loadProjectConfig(this.repositoryRoot);
|
||||
this.roster = readJson(join(this.repositoryRoot, '.torch', 'roster.yaml'), 'INVALID_TORCH_ROSTER');
|
||||
@@ -485,6 +568,286 @@ export class ControlPlane {
|
||||
return this.identity(id);
|
||||
}
|
||||
|
||||
runtimeLaunchReservations({ areaId } = {}) {
|
||||
const clauses = ['project_id = ?'];
|
||||
const values = [this.projectId];
|
||||
if (areaId !== undefined) {
|
||||
clauses.push('area_id = ?');
|
||||
values.push(this.assertIdentity(areaId));
|
||||
}
|
||||
return this.database.prepare(`
|
||||
SELECT * FROM runtime_launch_reservations WHERE ${clauses.join(' AND ')}
|
||||
ORDER BY created_at ASC, id ASC
|
||||
`).all(...values).map((row) => ({
|
||||
id: row.id, projectId: row.project_id, areaId: row.area_id, runtime: row.runtime,
|
||||
attemptId: row.attempt_id, mode: row.mode,
|
||||
expectedRuntimeSessionId: row.expected_runtime_session_id ?? null,
|
||||
capturedRuntimeSessionId: row.captured_runtime_session_id ?? null,
|
||||
state: row.state, evidence: row.evidence ?? null, createdAt: row.created_at,
|
||||
settledAt: row.settled_at ?? null,
|
||||
}));
|
||||
}
|
||||
|
||||
runtimeLaunchEvidence({ evidence, reservation, outcome, terminal, kind }) {
|
||||
const record = requiredRecord(evidence, 'evidence');
|
||||
const source = boundedEvidenceText(record.source, 'evidence.source');
|
||||
const evidenceAttemptId = boundedEvidenceText(record.attemptId, 'evidence.attemptId');
|
||||
const evidenceReservationId = boundedEvidenceText(record.reservationId, 'evidence.reservationId');
|
||||
const observedAt = boundedEvidenceText(record.observedAt, 'evidence.observedAt');
|
||||
if (source !== 'runtime-executor'
|
||||
|| evidenceAttemptId !== reservation.attempt_id
|
||||
|| evidenceReservationId !== reservation.id) {
|
||||
throw new TorchError('Runtime executor evidence is not attributable to this reservation', {
|
||||
code: 'RUNTIME_LAUNCH_EVIDENCE_MISMATCH', details: { reservationId: reservation.id },
|
||||
});
|
||||
}
|
||||
if (record.outcome !== outcome || record.terminal !== terminal) {
|
||||
throw new TorchError('Runtime executor evidence has the wrong lifecycle outcome', {
|
||||
code: 'RUNTIME_LAUNCH_EVIDENCE_MISMATCH', details: { reservationId: reservation.id },
|
||||
});
|
||||
}
|
||||
for (const field of ['status', 'signal', 'errorCode']) requiredEvidenceField(record, field);
|
||||
const status = boundedEvidenceValue(record.status, 'evidence.status');
|
||||
const signal = boundedEvidenceValue(record.signal, 'evidence.signal');
|
||||
const errorCode = boundedEvidenceValue(record.errorCode, 'evidence.errorCode');
|
||||
const reason = terminal === false ? boundedEvidenceText(record.reason, 'evidence.reason') : undefined;
|
||||
const verified = (() => {
|
||||
try {
|
||||
return this.verifyRuntimeLaunchEvidence({ kind, evidence: record, reservation });
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})();
|
||||
if (verified !== true) {
|
||||
throw new TorchError('Runtime launch evidence was not issued by the executor boundary', {
|
||||
code: 'RUNTIME_LAUNCH_EVIDENCE_UNVERIFIED', details: { reservationId: reservation.id, kind },
|
||||
});
|
||||
}
|
||||
return {
|
||||
source, attemptId: evidenceAttemptId, reservationId: evidenceReservationId, observedAt,
|
||||
outcome, terminal, status, signal, errorCode, ...(reason ? { reason } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
runtimeLaunchStoppedProof({ proof, reservation }) {
|
||||
const record = requiredRecord(proof, 'proof');
|
||||
const attemptId = boundedEvidenceText(record.attemptId, 'proof.attemptId');
|
||||
const reservationId = boundedEvidenceText(record.reservationId, 'proof.reservationId');
|
||||
const observedAt = boundedEvidenceText(record.observedAt, 'proof.observedAt');
|
||||
const processId = boundedEvidenceText(record.processId, 'proof.processId');
|
||||
const processStart = boundedEvidenceText(record.processStart, 'proof.processStart');
|
||||
const verifier = boundedEvidenceText(record.verifier, 'proof.verifier');
|
||||
if (record.kind !== 'process-proof'
|
||||
|| attemptId !== reservation.attempt_id
|
||||
|| reservationId !== reservation.id) {
|
||||
throw new TorchError('Stopped-process proof is not attributable to this reservation', {
|
||||
code: 'RUNTIME_LAUNCH_RECONCILIATION_BLOCKED', details: { reservationId: reservation.id },
|
||||
});
|
||||
}
|
||||
const verified = (() => {
|
||||
try {
|
||||
return this.verifyRuntimeLaunchEvidence({ kind: 'process-proof', evidence: record, reservation });
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})();
|
||||
if (verified !== true) {
|
||||
throw new TorchError('Stopped-process proof was not issued by the executor boundary', {
|
||||
code: 'RUNTIME_LAUNCH_RECONCILIATION_BLOCKED', details: { reservationId: reservation.id },
|
||||
});
|
||||
}
|
||||
return { kind: 'process-proof', attemptId, reservationId, observedAt, processId, processStart, verifier };
|
||||
}
|
||||
|
||||
reserveRuntimeLaunch({ areaId, runtime, attemptId, mode, runtimeSessionId } = {}) {
|
||||
const id = this.assertIdentity(areaId);
|
||||
const normalizedRuntime = requiredText(runtime, 'runtime');
|
||||
const normalizedAttempt = requiredText(attemptId, 'attemptId');
|
||||
const normalizedMode = requiredText(mode, 'mode');
|
||||
if (!['fresh', 'resume'].includes(normalizedMode)) {
|
||||
throw new TorchError('Runtime launch mode must be fresh or resume', { code: 'INVALID_RUNTIME_LAUNCH_MODE' });
|
||||
}
|
||||
const requestedRuntimeSessionId = optionalText(runtimeSessionId, 'runtimeSessionId');
|
||||
if (normalizedMode === 'resume' && !requestedRuntimeSessionId) {
|
||||
throw new TorchError('Resumed runtime launches require their expected session ID', {
|
||||
code: 'RUNTIME_LAUNCH_SESSION_REQUIRED', details: { areaId: id, mode: normalizedMode },
|
||||
});
|
||||
}
|
||||
const createdAt = this.clock().toISOString();
|
||||
const reservation = {
|
||||
id: this.idFactory(), projectId: this.projectId, areaId: id, runtime: normalizedRuntime,
|
||||
attemptId: normalizedAttempt, mode: normalizedMode, expectedRuntimeSessionId: null,
|
||||
capturedRuntimeSessionId: null, state: 'starting', evidence: null, createdAt, settledAt: null,
|
||||
};
|
||||
this.database.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
const identity = this.database.prepare('SELECT * FROM identities WHERE area_id = ?').get(id);
|
||||
const registeredRuntimeSessionId = identity?.runtime_session_id ?? null;
|
||||
if (normalizedMode === 'resume' && registeredRuntimeSessionId !== requestedRuntimeSessionId) {
|
||||
throw new TorchError('Resumed runtime launch does not match the registered provider session ID', {
|
||||
code: 'RUNTIME_LAUNCH_SESSION_MISMATCH', details: { areaId: id, expected: registeredRuntimeSessionId },
|
||||
});
|
||||
}
|
||||
if (normalizedMode === 'fresh' && requestedRuntimeSessionId
|
||||
&& registeredRuntimeSessionId && requestedRuntimeSessionId !== registeredRuntimeSessionId) {
|
||||
throw new TorchError('Fresh runtime launch cannot replace a registered provider session ID', {
|
||||
code: 'RUNTIME_LAUNCH_SESSION_MISMATCH', details: { areaId: id, expected: registeredRuntimeSessionId },
|
||||
});
|
||||
}
|
||||
const replay = this.database.prepare(`
|
||||
SELECT id FROM runtime_launch_reservations
|
||||
WHERE project_id = ? AND area_id = ? AND attempt_id = ?
|
||||
`).get(this.projectId, id, normalizedAttempt);
|
||||
if (replay) {
|
||||
throw new TorchError('A durable runtime launch attempt cannot be replayed', {
|
||||
code: 'RUNTIME_LAUNCH_ATTEMPT_REPLAYED', details: { reservationId: replay.id, areaId: id },
|
||||
});
|
||||
}
|
||||
const active = this.database.prepare(`
|
||||
SELECT * FROM runtime_launch_reservations
|
||||
WHERE project_id = ? AND area_id = ? AND state IN ('starting', 'held')
|
||||
ORDER BY created_at ASC, id ASC LIMIT 1
|
||||
`).get(this.projectId, id);
|
||||
if (active) {
|
||||
throw new TorchError('A runtime launch reservation already blocks this identity', {
|
||||
code: 'RUNTIME_LAUNCH_ALREADY_RESERVED',
|
||||
details: { reservationId: active.id, areaId: id, state: active.state },
|
||||
});
|
||||
}
|
||||
reservation.expectedRuntimeSessionId = registeredRuntimeSessionId;
|
||||
this.database.prepare(`
|
||||
INSERT INTO runtime_launch_reservations (
|
||||
id, project_id, area_id, runtime, attempt_id, mode, expected_runtime_session_id,
|
||||
captured_runtime_session_id, state, evidence, created_at, settled_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, ?, NULL, 'starting', NULL, ?, NULL)
|
||||
`).run(
|
||||
reservation.id, reservation.projectId, reservation.areaId, reservation.runtime,
|
||||
reservation.attemptId, reservation.mode, reservation.expectedRuntimeSessionId, reservation.createdAt,
|
||||
);
|
||||
this.database.prepare(`
|
||||
UPDATE identities SET runtime = ?, runtime_session_id = ?, state = 'starting',
|
||||
summary = ?, heartbeat_at = ?, updated_at = ? WHERE area_id = ?
|
||||
`).run(normalizedRuntime, registeredRuntimeSessionId, 'Runtime launch reserved before provider dispatch.', createdAt, createdAt, id);
|
||||
this.audit({
|
||||
actorId: id, operation: 'runtime.launch.reserve', entityType: 'runtime-launch-reservation', entityId: reservation.id,
|
||||
details: { attemptId: normalizedAttempt, mode: normalizedMode, expectedRuntimeSessionId: registeredRuntimeSessionId },
|
||||
});
|
||||
this.database.exec('COMMIT');
|
||||
return reservation;
|
||||
} catch (error) {
|
||||
this.database.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
settleRuntimeLaunch({ reservationId, attemptId, outcome, runtimeSessionId, evidence } = {}) {
|
||||
const id = requiredText(reservationId, 'reservationId');
|
||||
const normalizedAttempt = requiredText(attemptId, 'attemptId');
|
||||
const normalizedOutcome = requiredText(outcome, 'outcome');
|
||||
if (!['succeeded', 'failed'].includes(normalizedOutcome)) {
|
||||
throw new TorchError('Runtime launch settlement must be succeeded or failed', { code: 'INVALID_RUNTIME_LAUNCH_OUTCOME' });
|
||||
}
|
||||
const capturedRuntimeSessionId = optionalText(runtimeSessionId, 'runtimeSessionId');
|
||||
this.database.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
const reservation = this.database.prepare('SELECT * FROM runtime_launch_reservations WHERE id = ? AND project_id = ?').get(id, this.projectId);
|
||||
if (!reservation) throw new TorchError('Unknown runtime launch reservation', { code: 'RUNTIME_LAUNCH_RESERVATION_NOT_FOUND' });
|
||||
if (!['starting', 'held'].includes(reservation.state)) throw new TorchError('Runtime launch reservation is no longer active', { code: 'RUNTIME_LAUNCH_NOT_ACTIVE' });
|
||||
if (reservation.attempt_id !== normalizedAttempt) throw new TorchError('Runtime launch attempt does not match its reservation', { code: 'RUNTIME_LAUNCH_ATTEMPT_MISMATCH' });
|
||||
const terminalEvidence = this.runtimeLaunchEvidence({
|
||||
evidence, reservation, outcome: normalizedOutcome, terminal: true, kind: 'terminal',
|
||||
});
|
||||
if (reservation.mode === 'resume' && capturedRuntimeSessionId
|
||||
&& capturedRuntimeSessionId !== reservation.expected_runtime_session_id) {
|
||||
throw new TorchError('Resumed runtime launch captured an unexpected provider session ID', { code: 'RUNTIME_LAUNCH_SESSION_MISMATCH' });
|
||||
}
|
||||
const settledAt = this.clock().toISOString();
|
||||
const knownSessionId = reservation.captured_runtime_session_id ?? reservation.expected_runtime_session_id ?? null;
|
||||
if (capturedRuntimeSessionId && knownSessionId && capturedRuntimeSessionId !== knownSessionId) {
|
||||
throw new TorchError('Runtime launch captured an unexpected provider session ID', { code: 'RUNTIME_LAUNCH_SESSION_MISMATCH' });
|
||||
}
|
||||
const boundSessionId = capturedRuntimeSessionId ?? knownSessionId;
|
||||
this.database.prepare(`
|
||||
UPDATE runtime_launch_reservations SET state = ?, captured_runtime_session_id = ?, evidence = ?, settled_at = ? WHERE id = ?
|
||||
`).run(normalizedOutcome, boundSessionId, JSON.stringify(terminalEvidence), settledAt, id);
|
||||
this.database.prepare(`
|
||||
UPDATE identities SET runtime_session_id = ?, state = ?, summary = ?, heartbeat_at = ?, updated_at = ? WHERE area_id = ?
|
||||
`).run(boundSessionId, normalizedOutcome === 'succeeded' ? 'idle' : 'offline',
|
||||
normalizedOutcome === 'succeeded' ? 'Runtime launch completed.' : 'Runtime launch failed with terminal evidence.', settledAt, settledAt, reservation.area_id);
|
||||
this.audit({ actorId: reservation.area_id, operation: 'runtime.launch.settle', entityType: 'runtime-launch-reservation', entityId: id,
|
||||
details: { attemptId: normalizedAttempt, outcome: normalizedOutcome, capturedRuntimeSessionId: boundSessionId } });
|
||||
this.database.exec('COMMIT');
|
||||
return this.runtimeLaunchReservations({ areaId: reservation.area_id }).find((entry) => entry.id === id);
|
||||
} catch (error) {
|
||||
this.database.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
holdRuntimeLaunch({ reservationId, attemptId, outcome, runtimeSessionId, capturedRuntimeSessionId, evidence } = {}) {
|
||||
const id = requiredText(reservationId, 'reservationId');
|
||||
const normalizedAttempt = requiredText(attemptId, 'attemptId');
|
||||
if (requiredText(outcome, 'outcome') !== 'unknown') {
|
||||
throw new TorchError('Only unknown runtime interruptions may be held', { code: 'INVALID_RUNTIME_LAUNCH_HOLD' });
|
||||
}
|
||||
const runtimeSession = optionalText(runtimeSessionId, 'runtimeSessionId');
|
||||
const capturedSession = optionalText(capturedRuntimeSessionId, 'capturedRuntimeSessionId');
|
||||
if (runtimeSession && capturedSession && runtimeSession !== capturedSession) {
|
||||
throw new TorchError('Runtime launch hold received conflicting captured provider IDs', { code: 'RUNTIME_LAUNCH_SESSION_MISMATCH' });
|
||||
}
|
||||
const observedRuntimeSessionId = capturedSession ?? runtimeSession;
|
||||
this.database.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
const reservation = this.database.prepare('SELECT * FROM runtime_launch_reservations WHERE id = ? AND project_id = ?').get(id, this.projectId);
|
||||
if (!reservation) throw new TorchError('Unknown runtime launch reservation', { code: 'RUNTIME_LAUNCH_RESERVATION_NOT_FOUND' });
|
||||
if (reservation.state !== 'starting') throw new TorchError('Only starting runtime launches may be held', { code: 'RUNTIME_LAUNCH_NOT_ACTIVE' });
|
||||
if (reservation.attempt_id !== normalizedAttempt) throw new TorchError('Runtime launch attempt does not match its reservation', { code: 'RUNTIME_LAUNCH_ATTEMPT_MISMATCH' });
|
||||
const unknownEvidence = this.runtimeLaunchEvidence({
|
||||
evidence, reservation, outcome: 'unknown', terminal: false, kind: 'unknown',
|
||||
});
|
||||
const knownSessionId = reservation.captured_runtime_session_id ?? reservation.expected_runtime_session_id ?? null;
|
||||
if (observedRuntimeSessionId && knownSessionId && observedRuntimeSessionId !== knownSessionId) {
|
||||
throw new TorchError('Runtime launch captured an unexpected provider session ID', { code: 'RUNTIME_LAUNCH_SESSION_MISMATCH' });
|
||||
}
|
||||
const boundSessionId = observedRuntimeSessionId ?? knownSessionId;
|
||||
const updatedAt = this.clock().toISOString();
|
||||
this.database.prepare('UPDATE runtime_launch_reservations SET state = ?, captured_runtime_session_id = ?, evidence = ? WHERE id = ?')
|
||||
.run('held', boundSessionId, JSON.stringify(unknownEvidence), id);
|
||||
this.database.prepare(`UPDATE identities SET runtime_session_id = ?, state = 'working', summary = ?, heartbeat_at = ?, updated_at = ? WHERE area_id = ?`)
|
||||
.run(boundSessionId, 'Runtime executor completion is unknown; duplicate launch is blocked pending reconciliation.', updatedAt, updatedAt, reservation.area_id);
|
||||
this.audit({ actorId: reservation.area_id, operation: 'runtime.launch.hold', entityType: 'runtime-launch-reservation', entityId: id,
|
||||
details: { attemptId: normalizedAttempt } });
|
||||
this.database.exec('COMMIT');
|
||||
return this.runtimeLaunchReservations({ areaId: reservation.area_id }).find((entry) => entry.id === id);
|
||||
} catch (error) {
|
||||
this.database.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
reconcileRuntimeLaunch({ reservationId, proof } = {}) {
|
||||
const id = requiredText(reservationId, 'reservationId');
|
||||
this.database.exec('BEGIN IMMEDIATE');
|
||||
try {
|
||||
const reservation = this.database.prepare('SELECT * FROM runtime_launch_reservations WHERE id = ? AND project_id = ?').get(id, this.projectId);
|
||||
if (!reservation) throw new TorchError('Unknown runtime launch reservation', { code: 'RUNTIME_LAUNCH_RESERVATION_NOT_FOUND' });
|
||||
if (reservation.state !== 'held') throw new TorchError('Only held runtime launches require reconciliation', { code: 'RUNTIME_LAUNCH_RECONCILIATION_NOT_ALLOWED' });
|
||||
const stoppedProof = this.runtimeLaunchStoppedProof({ proof, reservation });
|
||||
const settledAt = this.clock().toISOString();
|
||||
this.database.prepare('UPDATE runtime_launch_reservations SET state = ?, evidence = ?, settled_at = ? WHERE id = ?')
|
||||
.run('reconciled-stopped', JSON.stringify(stoppedProof), settledAt, id);
|
||||
this.database.prepare(`UPDATE identities SET state = 'offline', summary = ?, heartbeat_at = ?, updated_at = ? WHERE area_id = ?`)
|
||||
.run('Runtime stopped by explicit reconciliation evidence.', settledAt, settledAt, reservation.area_id);
|
||||
this.audit({ actorId: reservation.area_id, operation: 'runtime.launch.reconcile', entityType: 'runtime-launch-reservation', entityId: id,
|
||||
details: { attemptId: reservation.attempt_id, proof: 'process-proof' } });
|
||||
this.database.exec('COMMIT');
|
||||
return this.runtimeLaunchReservations({ areaId: reservation.area_id }).find((entry) => entry.id === id);
|
||||
} catch (error) {
|
||||
this.database.exec('ROLLBACK');
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
writeReport({ areaId, kind, summary, task, evidence } = {}) {
|
||||
const id = this.assertIdentity(areaId);
|
||||
const createdAt = this.clock().toISOString();
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { execFileSync, fork } from 'node:child_process';
|
||||
import { mkdirSync, mkdtempSync, readFileSync, writeFileSync, existsSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { DatabaseSync } from 'node:sqlite';
|
||||
import test from 'node:test';
|
||||
import { analyzeRepository } from '../../src/kernel/analyze.mjs';
|
||||
import { diagnoseProject } from '../../src/kernel/doctor.mjs';
|
||||
@@ -89,6 +90,228 @@ test('SCN-install-doctor-purge: a fresh install is healthy and exactly reversibl
|
||||
assert.equal(readFileSync(join(root, 'index.js'), 'utf8'), 'export const ready = true;\n');
|
||||
});
|
||||
|
||||
test('SCN-runtime-launch-reservation: durable attempts, provider identities, and unknown launches fail closed', () => {
|
||||
const { root, env } = fixture();
|
||||
const repository = inspectRepository(root);
|
||||
const proposal = approvedProposal(repository);
|
||||
installProject({ repository, proposal, env, projectId: 'reservation-project' });
|
||||
const areaId = proposal.domains[0].id;
|
||||
const trustedExecutorRecords = new WeakSet();
|
||||
const control = openControlPlane({
|
||||
repositoryRoot: root,
|
||||
env,
|
||||
verifyRuntimeLaunchEvidence: ({ evidence, reservation }) => trustedExecutorRecords.has(evidence)
|
||||
&& evidence.reservationId === reservation.id && evidence.attemptId === reservation.attempt_id,
|
||||
});
|
||||
const executorEvidence = (reservation, outcome, extra = {}) => {
|
||||
const evidence = {
|
||||
source: 'runtime-executor', attemptId: reservation.attemptId, reservationId: reservation.id,
|
||||
observedAt: '2026-10-01T00:00:00.000Z', outcome, terminal: outcome !== 'unknown',
|
||||
status: null, signal: null, errorCode: null, ...extra,
|
||||
};
|
||||
trustedExecutorRecords.add(evidence);
|
||||
return evidence;
|
||||
};
|
||||
const stoppedProof = (reservation) => {
|
||||
const proof = {
|
||||
kind: 'process-proof', attemptId: reservation.attemptId, reservationId: reservation.id,
|
||||
observedAt: '2026-10-01T00:01:00.000Z', processId: 'fixture-process-1',
|
||||
processStart: '2026-10-01T00:00:00.000Z', verifier: 'fixture-executor',
|
||||
};
|
||||
trustedExecutorRecords.add(proof);
|
||||
return proof;
|
||||
};
|
||||
try {
|
||||
const fresh = control.reserveRuntimeLaunch({
|
||||
areaId, runtime: 'codex', attemptId: 'fresh-attempt', mode: 'fresh',
|
||||
});
|
||||
assert.equal(fresh.expectedRuntimeSessionId, null, 'fresh launches reserve before a provider ID exists');
|
||||
assert.equal(control.identity(areaId).state, 'starting');
|
||||
assert.equal(control.identity(areaId).runtimeSessionId, null);
|
||||
assert.throws(() => control.reserveRuntimeLaunch({
|
||||
areaId, runtime: 'codex', attemptId: 'competing-attempt', mode: 'fresh',
|
||||
}), (error) => error.code === 'RUNTIME_LAUNCH_ALREADY_RESERVED');
|
||||
|
||||
const held = control.holdRuntimeLaunch({
|
||||
reservationId: fresh.id, attemptId: fresh.attemptId, outcome: 'unknown',
|
||||
evidence: executorEvidence(fresh, 'unknown', { reason: 'executor-interrupted' }),
|
||||
});
|
||||
assert.equal(held.state, 'held');
|
||||
assert.equal(held.capturedRuntimeSessionId, null, 'an unknown launch without an ID remains guarded without inventing one');
|
||||
assert.equal(control.identity(areaId).state, 'working', 'unknown fresh launch remains guarded rather than offline');
|
||||
assert.throws(() => control.reconcileRuntimeLaunch({
|
||||
reservationId: fresh.id,
|
||||
proof: {
|
||||
kind: 'process-proof', attemptId: fresh.attemptId, reservationId: fresh.id,
|
||||
observedAt: '2026-10-01T00:01:00.000Z', processId: 'forged-process',
|
||||
processStart: '2026-10-01T00:00:00.000Z', verifier: 'untrusted-caller',
|
||||
},
|
||||
}), (error) => error.code === 'RUNTIME_LAUNCH_RECONCILIATION_BLOCKED');
|
||||
assert.equal(control.reconcileRuntimeLaunch({
|
||||
reservationId: fresh.id, proof: stoppedProof(fresh),
|
||||
}).state, 'reconciled-stopped');
|
||||
assert.throws(() => control.reserveRuntimeLaunch({
|
||||
areaId, runtime: 'codex', attemptId: fresh.attemptId, mode: 'fresh',
|
||||
}), (error) => error.code === 'RUNTIME_LAUNCH_ATTEMPT_REPLAYED');
|
||||
|
||||
const captured = control.reserveRuntimeLaunch({
|
||||
areaId, runtime: 'codex', attemptId: 'fresh-captured', mode: 'fresh',
|
||||
});
|
||||
assert.equal(control.holdRuntimeLaunch({
|
||||
reservationId: captured.id, attemptId: captured.attemptId, outcome: 'unknown',
|
||||
capturedRuntimeSessionId: 'captured-after-dispatch',
|
||||
evidence: executorEvidence(captured, 'unknown', { reason: 'executor-disconnected' }),
|
||||
}).capturedRuntimeSessionId, 'captured-after-dispatch');
|
||||
assert.equal(control.identity(areaId).runtimeSessionId, 'captured-after-dispatch');
|
||||
assert.equal(control.reconcileRuntimeLaunch({ reservationId: captured.id, proof: stoppedProof(captured) }).state, 'reconciled-stopped');
|
||||
|
||||
control.reportStatus({
|
||||
areaId, state: 'idle', runtime: 'codex', runtimeSessionId: 'registered-provider-id', summary: 'Registered provider persists.',
|
||||
});
|
||||
const preservingFresh = control.reserveRuntimeLaunch({
|
||||
areaId, runtime: 'codex', attemptId: 'preserving-fresh', mode: 'fresh',
|
||||
});
|
||||
assert.equal(preservingFresh.expectedRuntimeSessionId, 'registered-provider-id');
|
||||
assert.equal(control.identity(areaId).runtimeSessionId, 'registered-provider-id');
|
||||
assert.equal(control.settleRuntimeLaunch({
|
||||
reservationId: preservingFresh.id, attemptId: preservingFresh.attemptId, outcome: 'succeeded',
|
||||
evidence: executorEvidence(preservingFresh, 'succeeded'),
|
||||
}).capturedRuntimeSessionId, 'registered-provider-id');
|
||||
assert.throws(() => control.reserveRuntimeLaunch({
|
||||
areaId, runtime: 'codex', attemptId: preservingFresh.attemptId, mode: 'fresh',
|
||||
}), (error) => error.code === 'RUNTIME_LAUNCH_ATTEMPT_REPLAYED');
|
||||
|
||||
assert.throws(() => control.reserveRuntimeLaunch({
|
||||
areaId, runtime: 'codex', attemptId: 'wrong-resume', mode: 'resume', runtimeSessionId: 'wrong-provider-id',
|
||||
}), (error) => error.code === 'RUNTIME_LAUNCH_SESSION_MISMATCH');
|
||||
assert.equal(control.identity(areaId).runtimeSessionId, 'registered-provider-id', 'a rejected resume cannot rebind identity state');
|
||||
assert.equal(control.identity(areaId).state, 'idle', 'a rejected resume cannot mutate presence state');
|
||||
|
||||
const resumed = control.reserveRuntimeLaunch({
|
||||
areaId, runtime: 'codex', attemptId: 'resume-attempt', mode: 'resume', runtimeSessionId: 'registered-provider-id',
|
||||
});
|
||||
assert.throws(() => control.settleRuntimeLaunch({
|
||||
reservationId: resumed.id, attemptId: resumed.attemptId, outcome: 'succeeded',
|
||||
runtimeSessionId: 'different-provider-id', evidence: executorEvidence(resumed, 'succeeded'),
|
||||
}), (error) => error.code === 'RUNTIME_LAUNCH_SESSION_MISMATCH');
|
||||
assert.throws(() => control.settleRuntimeLaunch({
|
||||
reservationId: resumed.id, attemptId: 'wrong-attempt', outcome: 'succeeded',
|
||||
runtimeSessionId: 'registered-provider-id', evidence: executorEvidence(resumed, 'succeeded'),
|
||||
}), (error) => error.code === 'RUNTIME_LAUNCH_ATTEMPT_MISMATCH');
|
||||
assert.equal(control.settleRuntimeLaunch({
|
||||
reservationId: resumed.id, attemptId: resumed.attemptId, outcome: 'failed',
|
||||
runtimeSessionId: 'registered-provider-id', evidence: executorEvidence(resumed, 'failed'),
|
||||
}).state, 'failed');
|
||||
assert.equal(control.identity(areaId).state, 'offline');
|
||||
} finally {
|
||||
control.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('SCN-runtime-launch-contention: independent contender processes reserve one durable launch token', async () => {
|
||||
const { root, env } = fixture();
|
||||
const repository = inspectRepository(root);
|
||||
const proposal = approvedProposal(repository);
|
||||
installProject({ repository, proposal, env, projectId: 'reservation-contention-project' });
|
||||
const areaId = 'session-manager';
|
||||
const initializer = openControlPlane({ repositoryRoot: root, env });
|
||||
initializer.close();
|
||||
const workerPath = join(root, 'reservation-contender.mjs');
|
||||
const serviceUrl = new URL('../../src/control-plane/service.mjs', import.meta.url).href;
|
||||
writeFileSync(workerPath, `
|
||||
import { openControlPlane } from ${JSON.stringify(serviceUrl)};
|
||||
process.on('message', (input) => {
|
||||
const control = openControlPlane({ repositoryRoot: input.root, env: input.env });
|
||||
let result;
|
||||
try {
|
||||
result = { ok: true, reservation: control.reserveRuntimeLaunch({
|
||||
areaId: input.areaId, runtime: 'codex', attemptId: input.attemptId, mode: 'fresh',
|
||||
}).id };
|
||||
} catch (error) {
|
||||
result = { ok: false, code: error.code };
|
||||
} finally {
|
||||
control.close();
|
||||
}
|
||||
process.send(result, () => process.disconnect());
|
||||
});
|
||||
process.send({ ready: true });
|
||||
`);
|
||||
|
||||
const contender = (attemptId) => {
|
||||
const child = fork(workerPath, [], { stdio: ['ignore', 'ignore', 'ignore', 'ipc'] });
|
||||
let resolveReady;
|
||||
let resolveResult;
|
||||
const ready = new Promise((resolve) => { resolveReady = resolve; });
|
||||
const result = new Promise((resolve, reject) => {
|
||||
resolveResult = resolve;
|
||||
child.once('error', reject);
|
||||
child.on('exit', (code) => {
|
||||
if (code !== 0) reject(new Error(`reservation contender exited ${code}`));
|
||||
});
|
||||
});
|
||||
child.on('message', (message) => {
|
||||
if (message.ready) resolveReady();
|
||||
else resolveResult(message);
|
||||
});
|
||||
return { child, ready, result, input: { root, env, areaId, attemptId } };
|
||||
};
|
||||
|
||||
const first = contender('manual-dispatch-attempt');
|
||||
const second = contender('scheduled-dispatch-attempt');
|
||||
await Promise.all([first.ready, second.ready]);
|
||||
first.child.send(first.input);
|
||||
second.child.send(second.input);
|
||||
const outcomes = await Promise.all([first.result, second.result]);
|
||||
assert.equal(outcomes.filter((outcome) => outcome.ok).length, 1, 'exactly one process may reserve the identity');
|
||||
assert.deepEqual(outcomes.filter((outcome) => !outcome.ok).map((outcome) => outcome.code), ['RUNTIME_LAUNCH_ALREADY_RESERVED']);
|
||||
const control = openControlPlane({ repositoryRoot: root, env });
|
||||
try {
|
||||
const reservations = control.runtimeLaunchReservations({ areaId });
|
||||
assert.equal(reservations.length, 1);
|
||||
assert.equal(reservations[0].state, 'starting');
|
||||
} finally {
|
||||
control.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('SCN-runtime-launch-schema-compatibility: reservation state migrates forward and rejects engine downgrades', () => {
|
||||
const { root, env } = fixture();
|
||||
const repository = inspectRepository(root);
|
||||
const installed = installProject({ repository, proposal: approvedProposal(repository), env, projectId: 'reservation-schema-project' });
|
||||
const databasePath = join(installed.stateRoot, 'state.db');
|
||||
const initial = openControlPlane({ repositoryRoot: root, env });
|
||||
initial.close();
|
||||
const database = new DatabaseSync(databasePath);
|
||||
try {
|
||||
assert.equal(database.prepare('PRAGMA user_version').get().user_version, 3);
|
||||
database.exec('DROP TABLE runtime_launch_reservations; PRAGMA user_version = 2;');
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
const migrated = openControlPlane({ repositoryRoot: root, env });
|
||||
migrated.close();
|
||||
const downgraded = new DatabaseSync(databasePath);
|
||||
try {
|
||||
downgraded.exec('PRAGMA user_version = 2;');
|
||||
} finally {
|
||||
downgraded.close();
|
||||
}
|
||||
assert.throws(
|
||||
() => openControlPlane({ repositoryRoot: root, env }),
|
||||
(error) => error.code === 'CONTROL_PLANE_SCHEMA_DOWNGRADE_DETECTED',
|
||||
);
|
||||
const newer = new DatabaseSync(databasePath);
|
||||
try {
|
||||
newer.exec('PRAGMA user_version = 4;');
|
||||
} finally {
|
||||
newer.close();
|
||||
}
|
||||
assert.throws(
|
||||
() => openControlPlane({ repositoryRoot: root, env }),
|
||||
(error) => error.code === 'CONTROL_PLANE_SCHEMA_NEWER_THAN_ENGINE',
|
||||
);
|
||||
});
|
||||
|
||||
test('SCN-existing-repository-preservation: install and purge preserve history, branches, config, and untracked files', () => {
|
||||
const { root, env } = fixture();
|
||||
const externalEnv = { ...env, XDG_DATA_HOME: mkdtempSync(join(tmpdir(), 'torch-existing-repo-state-')) };
|
||||
|
||||
Reference in new issue
Block a user