mirror of
https://github.com/PR0M3TH3AN/TORCH.git
synced 2026-10-27 07:51:52 +00:00
TASK-company-runtime-recovery: bind continuation controller locks
This commit is contained in:
1 parent
0bb5ca4a08
commit
cfa0e87506
3 files changed
+183
-9
No files matched your search
@@ -3114,3 +3114,29 @@ test_integrity_note:
|
||||
did_relax_any_assertion: false
|
||||
if_true_explain_spec_basis: ""
|
||||
```
|
||||
|
||||
```yaml
|
||||
test_integrity_note:
|
||||
change_type: new_tests
|
||||
scenarios:
|
||||
- id: SCN-runtime-continuation-provenance-lock
|
||||
given: "An owner-authorized continuation with an isolated fixed process observation"
|
||||
when: "The controller creates its lock before reserving an eligible launch"
|
||||
then: "A 0600, versioned owner record binds project, PID, UID and start ticks; task/session stay unchanged and the lock is exact-live only"
|
||||
- id: SCN-runtime-continuation-provenance-refusal
|
||||
given: "A bare, malformed, foreign-project, PID-replaced, live-quiet, or observer-unavailable retained controller lock"
|
||||
when: "Continuation runs across fixed UTC clocks"
|
||||
then: "The lock remains held, zero launches occur, and only matching UID, PID and start ticks is observed as active; no stopped inference or cleanup occurs"
|
||||
observable_outcomes:
|
||||
- "Owner-record permissions and provenance fields, identity task/session, launch count, attempt ledger, retained lock path, and controller observation state"
|
||||
determinism_controls:
|
||||
- "Hermetic fixture repositories, fixed UTC clocks, and an injected process-observation boundary only; no live provider, timer, termination, retry, or nonce inspection"
|
||||
anti_cheat_rationale:
|
||||
prevents:
|
||||
- "Treating quietness, lock age, malformed data, or observer failure as stopped-process proof"
|
||||
- "Deleting retained locks or launching around an unresolved controller"
|
||||
- "Forging active controller status without matching UID, PID, and start ticks"
|
||||
relaxation:
|
||||
did_relax_any_assertion: false
|
||||
if_true_explain_spec_basis: ""
|
||||
```
|
||||
@@ -1,9 +1,74 @@
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmdirSync, writeFileSync } from 'node:fs';
|
||||
import { existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmdirSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { TorchError } from '../kernel/errors.mjs';
|
||||
|
||||
function statePath(control) { return join(control.stateRoot, 'continuation.json'); }
|
||||
function controllerLockPath(stateRoot) { return join(stateRoot, 'continuation.lock'); }
|
||||
function controllerOwnerPath(stateRoot) { return join(controllerLockPath(stateRoot), 'owner.json'); }
|
||||
|
||||
function nativeControllerProcess(pid) {
|
||||
if (process.platform !== 'linux') throw new Error('Controller process observation unavailable');
|
||||
const directory = `/proc/${pid}`;
|
||||
const stat = readFileSync(join(directory, 'stat'), 'utf8');
|
||||
const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
|
||||
return { uid: statSync(directory).uid, state: fields[0], startTicks: fields[19] };
|
||||
}
|
||||
|
||||
// Observational only. A retained controller lock is never a recovery authority:
|
||||
// only an exact live process match is active, while every other state remains
|
||||
// held and requires an owner-controlled reconciliation path outside this module.
|
||||
export function observeContinuationController({ stateRoot, projectId,
|
||||
inspectProcess = nativeControllerProcess, uid = process.getuid?.(), now = () => new Date() } = {}) {
|
||||
let record;
|
||||
try { record = JSON.parse(readFileSync(controllerOwnerPath(stateRoot), 'utf8')); }
|
||||
catch (error) {
|
||||
return { state: 'unknown', reason: error.code === 'ENOENT' ? 'controller-lock-unprovenanced' : 'controller-lock-unavailable' };
|
||||
}
|
||||
if (record.version !== 1 || record.projectId !== projectId || !Number.isSafeInteger(record.pid) || record.pid < 1
|
||||
|| !Number.isSafeInteger(record.uid) || record.uid < 0 || typeof record.startTicks !== 'string' || !record.startTicks
|
||||
|| typeof record.nonce !== 'string' || !record.nonce) {
|
||||
return { state: 'unknown', reason: 'controller-lock-invalid' };
|
||||
}
|
||||
if (record.uid !== uid) return { state: 'unknown', reason: 'controller-lock-foreign-user' };
|
||||
try {
|
||||
const live = inspectProcess(record.pid);
|
||||
if (live.uid !== record.uid || ['Z', 'X'].includes(live.state)) return { state: 'unknown', reason: 'controller-not-live' };
|
||||
if (live.startTicks !== record.startTicks) return { state: 'unknown', reason: 'controller-replaced' };
|
||||
return { state: 'active', source: 'live-process-and-controller-lock', observedAt: now().toISOString() };
|
||||
} catch { return { state: 'unknown', reason: 'controller-unobservable' }; }
|
||||
}
|
||||
|
||||
function createContinuationControllerLock(control, { inspectProcess = nativeControllerProcess, uid = process.getuid?.() } = {}) {
|
||||
let live;
|
||||
try { live = inspectProcess(process.pid); } catch { return null; }
|
||||
if (live.uid !== uid || ['Z', 'X'].includes(live.state) || typeof live.startTicks !== 'string' || !live.startTicks) return null;
|
||||
const lock = controllerLockPath(control.stateRoot);
|
||||
try { mkdirSync(lock); } catch (error) {
|
||||
if (error.code === 'EEXIST') return { lock, record: null };
|
||||
throw error;
|
||||
}
|
||||
const record = { version: 1, projectId: control.projectId, pid: process.pid, uid: live.uid,
|
||||
startTicks: live.startTicks, nonce: randomUUID() };
|
||||
try { writeFileSync(controllerOwnerPath(control.stateRoot), JSON.stringify(record), { flag: 'wx', mode: 0o600 }); }
|
||||
catch (error) {
|
||||
try { rmdirSync(lock); } catch { /* A changed artifact must remain fail-closed. */ }
|
||||
throw error;
|
||||
}
|
||||
return { lock, record };
|
||||
}
|
||||
|
||||
function releaseContinuationControllerLock(control, record) {
|
||||
if (!record) return;
|
||||
const path = controllerOwnerPath(control.stateRoot);
|
||||
try {
|
||||
const current = JSON.parse(readFileSync(path, 'utf8'));
|
||||
if (current.version !== record.version || current.projectId !== record.projectId || current.pid !== record.pid
|
||||
|| current.uid !== record.uid || current.startTicks !== record.startTicks || current.nonce !== record.nonce) return;
|
||||
unlinkSync(path);
|
||||
rmdirSync(controllerLockPath(control.stateRoot));
|
||||
} catch { /* Preserve malformed, foreign, or replaced lock artifacts. */ }
|
||||
}
|
||||
function occupiedAreas(control) {
|
||||
const path = join(control.stateRoot, 'sessions', 'turn-guards');
|
||||
return existsSync(path) ? readdirSync(path, { withFileTypes: true }).filter(d => d.isDirectory()).map(d => d.name) : [];
|
||||
@@ -122,14 +187,18 @@ export function planContinuation(control, backlog, { at = new Date(), limit } =
|
||||
candidates: candidates.slice(0, Math.min(capacity, cap - attempts)), mutationPerformed: false };
|
||||
}
|
||||
|
||||
export async function runContinuation(control, backlog, { actorId, launch, at, clock = () => at ?? new Date(), limit } = {}) {
|
||||
export async function runContinuation(control, backlog, { actorId, launch, at, clock = () => at ?? new Date(), limit,
|
||||
inspectProcess = nativeControllerProcess, uid = process.getuid?.() } = {}) {
|
||||
control.assertOwnerActor(actorId);
|
||||
if (typeof launch !== 'function') throw new TorchError('Continuation needs an authorized launcher.', { code: 'RUNTIME_EXECUTION_NOT_AUTHORIZED' });
|
||||
const lock = join(control.stateRoot, 'continuation.lock');
|
||||
try { mkdirSync(lock); } catch (error) {
|
||||
if (error.code === 'EEXIST') return { reason: 'controller-active-or-unreconciled', launched: [] };
|
||||
throw error;
|
||||
if (existsSync(controllerLockPath(control.stateRoot))) {
|
||||
return { reason: 'controller-active-or-unreconciled',
|
||||
controller: observeContinuationController({ stateRoot: control.stateRoot, projectId: control.projectId, inspectProcess, uid }), launched: [] };
|
||||
}
|
||||
const controller = createContinuationControllerLock(control, { inspectProcess, uid });
|
||||
if (controller === null) return { reason: 'controller-observation-unavailable', launched: [] };
|
||||
if (controller.record === null) return { reason: 'controller-active-or-unreconciled',
|
||||
controller: observeContinuationController({ stateRoot: control.stateRoot, projectId: control.projectId, inspectProcess, uid }), launched: [] };
|
||||
try {
|
||||
const pending = new Map();
|
||||
const launched = [];
|
||||
@@ -174,5 +243,5 @@ export async function runContinuation(control, backlog, { actorId, launch, at, c
|
||||
}
|
||||
}
|
||||
return { launched, reason };
|
||||
} finally { rmdirSync(lock); }
|
||||
} finally { releaseContinuationControllerLock(control, controller.record); }
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmdirSync, unlinkSync, writeFileSync } from 'node:fs';
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmdirSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import test from 'node:test';
|
||||
@@ -12,7 +12,7 @@ import { openControlPlane } from '../../src/control-plane/service.mjs';
|
||||
import { issueStoppedExecutorEvidence } from '../../src/runtime/stopped-executor-evidence.mjs';
|
||||
import { observeRuntimeTurn, withRuntimeTurnGuard } from '../../src/runtime/turn-guard.mjs';
|
||||
import { observeProject } from '../../src/observability/snapshot.mjs';
|
||||
import { configureContinuation, observeContinuation, planContinuation, runContinuation } from '../../src/runtime/continuation.mjs';
|
||||
import { configureContinuation, observeContinuation, observeContinuationController, planContinuation, runContinuation } from '../../src/runtime/continuation.mjs';
|
||||
|
||||
function fixture({ recoveryWorktree = true, extraWorker = false, workerCount = 0 } = {}) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'torch-legacy-recovery-'));
|
||||
@@ -487,3 +487,82 @@ test('SCN-legacy-runtime-recovery-project: evidence cannot be applied to a diffe
|
||||
assert.equal(second.control.getAgent(second.areaId).state, 'working');
|
||||
} finally { first.control.close(); second.control.close(); }
|
||||
});
|
||||
|
||||
test('SCN-runtime-continuation-provenance-lock: a controller lock binds exact live process provenance without changing identity state', async () => {
|
||||
const f = fixture();
|
||||
try {
|
||||
const uid = process.getuid?.();
|
||||
const processRecord = { uid, state: 'S', startTicks: 'fixed-controller-start' };
|
||||
const inspectProcess = () => processRecord;
|
||||
f.control.reportStatus({ areaId: f.areaId, state: 'idle', task: 'TASK-preserved', runtimeSessionId: 'fixture-session',
|
||||
summary: 'Ready for an authorized continuation' });
|
||||
const before = f.control.getAgent(f.areaId);
|
||||
f.control.sendOwnerRequest({ actorId: 'owner', recipient: f.areaId, body: 'Eligible handoff' });
|
||||
configureContinuation(f.control, { actorId: 'owner', enabled: true, maxTurnsPerDay: 2 });
|
||||
let inspected = false;
|
||||
let inspectionFailure;
|
||||
const result = await runContinuation(f.control, { list: () => [] }, { actorId: 'owner', inspectProcess,
|
||||
launch: () => {
|
||||
try {
|
||||
const path = join(f.control.stateRoot, 'continuation.lock', 'owner.json');
|
||||
const record = JSON.parse(readFileSync(path, 'utf8'));
|
||||
assert.equal(record.version, 1);
|
||||
assert.equal(record.projectId, f.control.projectId);
|
||||
assert.equal(record.pid, process.pid);
|
||||
assert.equal(record.startTicks, 'fixed-controller-start');
|
||||
assert.equal(statSync(path).mode & 0o777, 0o600);
|
||||
assert.equal(f.control.getAgent(f.areaId).currentTask, before.currentTask);
|
||||
assert.equal(f.control.getAgent(f.areaId).runtimeSessionId, before.runtimeSessionId);
|
||||
assert.equal(observeContinuation({ stateRoot: f.control.stateRoot }).attempts, 1,
|
||||
'Only the subsequent launch reservation charges the attempt ledger');
|
||||
assert.equal(observeContinuationController({ stateRoot: f.control.stateRoot, projectId: f.control.projectId,
|
||||
inspectProcess }).state, 'active');
|
||||
} catch (error) { inspectionFailure = error; }
|
||||
inspected = true;
|
||||
return 0;
|
||||
} });
|
||||
assert.equal(inspected, true, JSON.stringify(result));
|
||||
assert.ifError(inspectionFailure);
|
||||
assert.equal(result.launched.length, 1);
|
||||
assert.equal(existsSync(join(f.control.stateRoot, 'continuation.lock')), false);
|
||||
} finally { f.control.close(); }
|
||||
});
|
||||
|
||||
test('SCN-runtime-continuation-provenance-refusal: retained controller locks never infer stopped or launch', async () => {
|
||||
const cases = [
|
||||
{ id: 'bare', write: () => {}, inspect: () => { throw new Error('must not inspect bare lock'); }, state: 'unknown' },
|
||||
{ id: 'malformed', write: path => writeFileSync(path, 'not-json'), inspect: () => { throw new Error('unavailable'); }, state: 'unknown' },
|
||||
{ id: 'foreign-project', write: (path, record) => writeFileSync(path, JSON.stringify({ ...record, projectId: 'foreign' })), inspect: () => ({ uid: process.getuid?.(), state: 'S', startTicks: 'fixed' }), state: 'unknown' },
|
||||
{ id: 'pid-replaced', write: (path, record) => writeFileSync(path, JSON.stringify(record)), inspect: () => ({ uid: process.getuid?.(), state: 'S', startTicks: 'reused' }), state: 'unknown' },
|
||||
{ id: 'live-quiet', write: (path, record) => writeFileSync(path, JSON.stringify(record)), inspect: () => ({ uid: process.getuid?.(), state: 'S', startTicks: 'fixed' }), state: 'active' },
|
||||
{ id: 'observer-unavailable', write: (path, record) => writeFileSync(path, JSON.stringify(record)), inspect: () => { throw new Error('process boundary unavailable'); }, state: 'unknown' },
|
||||
];
|
||||
for (const scenario of cases) {
|
||||
const f = fixture();
|
||||
const at = new Date('2026-10-03T00:00:00Z');
|
||||
const lock = join(f.control.stateRoot, 'continuation.lock');
|
||||
const owner = join(lock, 'owner.json');
|
||||
try {
|
||||
f.control.reportStatus({ areaId: f.areaId, state: 'idle', summary: 'Quiet status is not stopped-process proof' });
|
||||
f.control.sendOwnerRequest({ actorId: 'owner', recipient: f.areaId, body: 'Eligible handoff' });
|
||||
configureContinuation(f.control, { actorId: 'owner', enabled: true, maxTurnsPerDay: 2 });
|
||||
mkdirSync(lock);
|
||||
const record = { version: 1, projectId: f.control.projectId, pid: 999, uid: process.getuid?.(), startTicks: 'fixed', nonce: 'opaque' };
|
||||
scenario.write(owner, record);
|
||||
let launches = 0;
|
||||
for (const clock of [at, new Date('2026-10-04T00:00:00Z')]) {
|
||||
const result = await runContinuation(f.control, { list: () => [] }, { actorId: 'owner', at: clock,
|
||||
inspectProcess: scenario.inspect, launch: () => { launches++; return 0; } });
|
||||
assert.equal(result.reason, 'controller-active-or-unreconciled', scenario.id);
|
||||
assert.equal(result.controller.state, scenario.state, scenario.id);
|
||||
assert.equal(existsSync(lock), true, scenario.id);
|
||||
}
|
||||
assert.equal(launches, 0, scenario.id);
|
||||
assert.equal(planContinuation(f.control, { list: () => [] }, { at }).attempts, 0, scenario.id);
|
||||
} finally {
|
||||
if (existsSync(owner)) unlinkSync(owner);
|
||||
if (existsSync(lock)) rmdirSync(lock);
|
||||
f.control.close();
|
||||
}
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user