diff --git a/docs/TEST_INTEGRITY.md b/docs/TEST_INTEGRITY.md index aacb180..48c6ef6 100644 --- a/docs/TEST_INTEGRITY.md +++ b/docs/TEST_INTEGRITY.md @@ -1,5 +1,38 @@ # Test Integrity Notes +## 2026-10-03 — Process-verified runtime presence + +```yaml +test_integrity_note: + change_type: new_tests + scenarios: + - id: SCN-runtime-live-presence + given: "A Linux executor holds a real identity guard while durable state is idle" + when: "The observation API reads starting, working, approval waiting and terminal phases" + then: "Live phases display accurately, semantic waits persist, terminal guards vanish and durable reports are not fabricated" + - id: SCN-runtime-live-presence-refusal + given: "A guarded identity with injected OS observations" + when: "PID is replaced, dead, foreign-user, unavailable or project binding is wrong" + then: "It never reports active execution" + - id: SCN-runtime-live-presence-legacy + given: "A pre-observation guard without a process-start token" + when: "The exact TORCH up identity and canonical cwd match or disagree" + then: "Only a verified matching live executor displays working" + observable_outcomes: + - "Observation API state, reportedState, executor source and unchanged durable identity" + determinism_controls: + - "Hermetic installed repository and injected PID observations for refusal cases" + - "Supplemental Linux /proc integration observes the test process itself, with no provider or timing sleeps" + anti_cheat_rationale: + prevents: + - "Painting all idle badges working without process evidence" + - "Treating a crashed guard or reused PID as an active agent" + - "Hiding approval waits or fabricating heartbeat/task progress" + relaxation: + did_relax_any_assertion: false + if_true_explain_spec_basis: "" +``` + ## 2026-09-30 — Bounded routine coordination ```yaml diff --git a/docs/spec_changes/2026-10-03-live-runtime-presence.md b/docs/spec_changes/2026-10-03-live-runtime-presence.md new file mode 100644 index 0000000..4dab7d2 --- /dev/null +++ b/docs/spec_changes/2026-10-03-live-runtime-presence.md @@ -0,0 +1,13 @@ +# Live runtime presence instead of stale idle badges + +The owner requires active executor turns to be distinguishable from old self-reported idle/offline state. Display is observational, not a new task-state machine or a substitute for authorization. + +Each new physical turn guard records a starting phase and Linux process-start token, then enters working immediately before invoking the authorized executor. The read-only observation API validates the exact project/identity guard against a live same-user process with the matching start token. Existing legacy guards additionally require the exact TORCH up identity and canonical working directory. Dead, foreign-user, replaced or unobservable processes are not called active; retained unverified guards display unknown instead of false idle/offline. + +An active verified guard overrides idle/offline display with starting/working. Original `reportedState`, `reportedSummary`, task and heartbeat are retained. Waiting/blocked reports remain visible even during a live turn. Guard removal naturally restores the last reported state after completion. No report, task, authority, launch policy, budget or receipt is changed by observation. + +On unsupported OS process-observation surfaces, guarded execution remains unknown, not optimistically working. Other platform qualification remains open. Native harnesses with no resident TORCH executor may continue to rely on durable reports; this change does not claim to discover every external runtime process. + +The live console pilot must load the source candidate to expose this observation. Updating its owned server does not upgrade the installed distribution or restart running agent turns. Broader startup/launch reservation work remains tracked by `TASK-pilot-starting-presence`. + +Test integrity scenarios are recorded in docs/TEST_INTEGRITY.md. No existing expectations or test timeouts are relaxed. diff --git a/site/console.js b/site/console.js index 2cd75cb..c82d85d 100644 --- a/site/console.js +++ b/site/console.js @@ -1087,7 +1087,7 @@ function render(snapshot) { return `
${escapeHtml(agent.title ?? agent.areaId)}${escapeHtml(agent.summary ?? profile)}${escapeHtml(agent.areaId)} · ${escapeHtml(profile)}
-
${escapeHtml(agent.state ?? 'unknown')}${escapeHtml(ageLabel(agent.heartbeatAt))}
+
${escapeHtml(agent.state ?? 'unknown')}${escapeHtml(agent.executor?.state === 'active' ? 'executor verified live' : ageLabel(agent.heartbeatAt))}
`; }).join('') : empty('Install an approved Fleet to see persistent identities.')); diff --git a/src/observability/snapshot.mjs b/src/observability/snapshot.mjs index 12ab6d0..a8d9879 100644 --- a/src/observability/snapshot.mjs +++ b/src/observability/snapshot.mjs @@ -16,6 +16,7 @@ import { assessBacklogHealth, backlogObservedCommitDistance } from '../backlog/s import { observeTaskActivity } from '../backlog/activity.mjs'; import { ScheduleLauncherService } from '../schedules/launcher.mjs'; import { hierarchyOrder } from '../runtime/hierarchy-order.mjs'; +import { observeRuntimeTurn } from '../runtime/turn-guard.mjs'; function json(path) { return JSON.parse(readFileSync(path, 'utf8')); @@ -417,11 +418,21 @@ export function observeProject({ repositoryRoot, env = process.env, now = () => `).all().map((row) => [row.area_id, row])); result.agents = result.agents.map((agent) => { const row = live.get(agent.areaId); - return row ? { + const reported = row ? { ...agent, state: row.state, runtime: row.runtime ?? agent.runtime, runtimeSessionId: row.runtime_session_id, summary: row.summary, task: row.task, heartbeatAt: row.heartbeat_at, } : agent; + const executor = observeRuntimeTurn({ stateRoot, repositoryRoot: repository.root, + projectId: manifest.projectId, areaId: agent.areaId, now }); + const staleIdle = ['idle', 'offline'].includes(reported.state); + return { ...reported, reportedState: reported.state, reportedSummary: reported.summary, executor, + ...(executor.state === 'active' && staleIdle ? { + state: executor.phase, + summary: `Live runtime executor is ${executor.phase} (last reported state: ${reported.state}).`, + } : executor.state === 'unknown' && staleIdle ? { + state: 'unknown', summary: 'Runtime turn guard exists, but its executor could not be verified.', + } : {}) }; }); } if (hasTable(database, 'schedule_wake_reservations')) { diff --git a/src/runtime/lifecycle.mjs b/src/runtime/lifecycle.mjs index 4bbdede..0683d2c 100644 --- a/src/runtime/lifecycle.mjs +++ b/src/runtime/lifecycle.mjs @@ -571,8 +571,9 @@ export function startFleet({ plan, controlPlane, executor, adapters } = {}) { const runtimes = adapterMap({ adapters }); for (const action of plan.actions) { controlPlane.assertIdentity(action.areaId); - withRuntimeTurnGuard(controlPlane, action.areaId, () => { + withRuntimeTurnGuard(controlPlane, action.areaId, setPhase => { writeCombinedPrompt(action); + setPhase('working'); const result = executor({ ...action.launch, areaId: action.areaId, runtime: action.runtime, runtimeSessionId: action.runtimeSessionId, diff --git a/src/runtime/turn-guard.mjs b/src/runtime/turn-guard.mjs index 55bcca9..ea7df4a 100644 --- a/src/runtime/turn-guard.mjs +++ b/src/runtime/turn-guard.mjs @@ -1,8 +1,48 @@ import { randomUUID } from 'node:crypto'; -import { mkdirSync, readFileSync, rmdirSync, unlinkSync, writeFileSync } from 'node:fs'; -import { join } from 'node:path'; +import { mkdirSync, readFileSync, readlinkSync, renameSync, rmdirSync, statSync, unlinkSync, writeFileSync } from 'node:fs'; +import { basename, join, resolve } from 'node:path'; import { TorchError } from '../kernel/errors.mjs'; +function nativeProcess(pid) { + if (process.platform !== 'linux') throw new Error('Process observation unavailable'); + const directory = `/proc/${pid}`; + const stat = readFileSync(join(directory, 'stat'), 'utf8'); + const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' '); + return { uid: statSync(directory).uid, state: fields[0], startTicks: fields[19], + cwd: readlinkSync(join(directory, 'cwd')), + argv: readFileSync(join(directory, 'cmdline'), 'utf8').split('\0').filter(Boolean) }; +} + +// Observational only: never changes assignments, heartbeat, authority or launch eligibility. +export function observeRuntimeTurn({ stateRoot, repositoryRoot, projectId, areaId, + inspectProcess = nativeProcess, uid = process.getuid?.(), now = () => new Date() } = {}) { + if (!/^[a-z0-9][a-z0-9-]*$/.test(areaId)) return { state: 'unknown', reason: 'invalid-identity' }; + let record; + try { record = JSON.parse(readFileSync(join(stateRoot, 'sessions', 'turn-guards', areaId, 'owner.json'), 'utf8')); } + catch (error) { return { state: error.code === 'ENOENT' ? 'inactive' : 'unknown', reason: 'guard-unavailable' }; } + if (record.projectId !== projectId || record.areaId !== areaId || !Number.isSafeInteger(record.pid) + || record.pid < 1 || typeof record.nonce !== 'string' || !record.nonce) { + return { state: 'unknown', reason: 'guard-invalid' }; + } + try { + const live = inspectProcess(record.pid); + if (live.uid !== uid || ['Z', 'X'].includes(live.state)) return { state: 'unknown', reason: 'executor-not-live' }; + if (record.startTicks) { + if (record.startTicks !== live.startTicks) return { state: 'unknown', reason: 'executor-replaced' }; + } else { + // Existing pre-observation guards: require an exact identity-bound TORCH up process. + const only = live.argv.indexOf('--only'); + if (resolve(live.cwd) !== resolve(repositoryRoot) || !live.argv.includes('up') + || only < 0 || live.argv[only + 1] !== areaId + || !live.argv.some(arg => ['torch.mjs', 'torch'].includes(basename(arg)))) { + return { state: 'unknown', reason: 'legacy-executor-unverified' }; + } + } + return { state: 'active', phase: record.phase === 'starting' ? 'starting' : 'working', + observedAt: now().toISOString(), source: 'live-process-and-turn-guard' }; + } catch { return { state: 'unknown', reason: 'executor-unobservable' }; } +} + // One physical executor per identity, including the interval after an agent // reports idle but before its harness actually exits. Crashed guards never expire. export function withRuntimeTurnGuard(controlPlane, areaId, run) { @@ -19,8 +59,19 @@ export function withRuntimeTurnGuard(controlPlane, areaId, run) { } const path = join(directory, 'owner.json'); const nonce = randomUUID(); - writeFileSync(path, JSON.stringify({ nonce, pid: process.pid, areaId, projectId: controlPlane.projectId }), { flag: 'wx', mode: 0o600 }); - try { return run(); } finally { + let startTicks; + try { startTicks = nativeProcess(process.pid).startTicks; } catch { /* Unavailable hosts remain observationally unknown. */ } + const record = { nonce, pid: process.pid, areaId, projectId: controlPlane.projectId, startTicks, phase: 'starting' }; + writeFileSync(path, JSON.stringify(record), { flag: 'wx', mode: 0o600 }); + const setPhase = phase => { + if (!['starting', 'working'].includes(phase)) throw new TorchError('Invalid runtime phase.', { code: 'INVALID_RUNTIME_INPUT' }); + const current = JSON.parse(readFileSync(path, 'utf8')); + if (current.nonce !== nonce || current.pid !== process.pid) throw new TorchError('Runtime guard changed.', { code: 'RUNTIME_TURN_ACTIVE' }); + const temporary = `${path}.${nonce}.tmp`; + writeFileSync(temporary, JSON.stringify({ ...record, phase }), { flag: 'wx', mode: 0o600 }); + renameSync(temporary, path); + }; + try { return run(setPhase); } finally { // Delete only the exact guard created by this invocation; preserve foreign/changed guards. const record = JSON.parse(readFileSync(path, 'utf8')); if (record.nonce === nonce && record.pid === process.pid) { diff --git a/test/fleet/legacy-runtime-recovery.test.mjs b/test/fleet/legacy-runtime-recovery.test.mjs index cf9cf7e..a5718f1 100644 --- a/test/fleet/legacy-runtime-recovery.test.mjs +++ b/test/fleet/legacy-runtime-recovery.test.mjs @@ -10,10 +10,11 @@ import { proposeDomains } from '../../src/kernel/domains.mjs'; import { installProject } from '../../src/kernel/install.mjs'; import { openControlPlane } from '../../src/control-plane/service.mjs'; import { issueStoppedExecutorEvidence } from '../../src/runtime/stopped-executor-evidence.mjs'; -import { withRuntimeTurnGuard } from '../../src/runtime/turn-guard.mjs'; +import { observeRuntimeTurn, withRuntimeTurnGuard } from '../../src/runtime/turn-guard.mjs'; +import { observeProject } from '../../src/observability/snapshot.mjs'; import { configureContinuation, planContinuation, runContinuation } from '../../src/runtime/continuation.mjs'; -function fixture() { +function fixture({ recoveryWorktree = true } = {}) { const root = mkdtempSync(join(tmpdir(), 'torch-legacy-recovery-')); for (const args of [['init', '-b', 'main'], ['config', 'user.email', 'test@example.invalid'], ['config', 'user.name', 'Recovery Fixture']]) execFileSync('git', args, { cwd: root, stdio: 'pipe' }); @@ -28,7 +29,7 @@ function fixture() { const actualPath = join(root, '.torch', 'install-manifest.json'); const manifest = JSON.parse(readFileSync(actualPath, 'utf8')); const areaId = proposal.domains[0].id; - manifest.external.push({ type: 'worktree', area: areaId, path: join(root, 'worker') }); + if (recoveryWorktree) manifest.external.push({ type: 'worktree', area: areaId, path: join(root, 'worker') }); writeFileSync(actualPath, JSON.stringify(manifest)); let time = Date.parse('2026-10-03T00:00:00Z'); const control = openControlPlane({ repositoryRoot: root, env, clock: () => new Date(time) }); @@ -44,7 +45,7 @@ function fixture() { invocationId: 'a'.repeat(32), readUnit: () => ({ ...properties }), readProcesses: () => processes, now: () => time }); const apply = issued => control.recoverOwnerStoppedUnknownIdentity({ actorId: 'owner', areaId, expectedIdentity: issued.plan.expectedIdentity, evidence: issued.evidence }); - return { control, identity, areaId, properties, issue, apply, + return { root, env, control, identity, areaId, properties, issue, apply, advance: ms => { time += ms; }, processes: value => { processes = value; } }; } @@ -122,6 +123,75 @@ test('SCN-runtime-turn-guard: overlapping executors refuse even if an agent repo } finally { f.control.close(); } }); +test('SCN-runtime-live-presence: live guard phases override stale idle display without changing durable reports or waits', { skip: process.platform !== 'linux' }, () => { + const f = fixture({ recoveryWorktree: false }); + try { + f.control.reportStatus({ areaId: f.areaId, state: 'idle', summary: 'Previous turn completed' }); + const observe = () => observeProject({ repositoryRoot: f.root, env: f.env }).agents.find(a => a.areaId === f.areaId); + withRuntimeTurnGuard(f.control, f.areaId, phase => { + let agent = observe(); + assert.equal(agent.state, 'starting'); + assert.equal(agent.reportedState, 'idle'); + assert.equal(agent.executor.source, 'live-process-and-turn-guard'); + phase('working'); + agent = observe(); + assert.equal(agent.state, 'working'); + assert.equal(f.control.getAgent(f.areaId).state, 'idle', 'Observation is not an identity mutation'); + f.control.reportStatus({ areaId: f.areaId, state: 'waiting', summary: 'Waiting for named approval' }); + assert.equal(observe().state, 'waiting', 'Live execution must not conceal semantic approval waits'); + }); + assert.equal(observe().executor.state, 'inactive'); + assert.equal(observe().state, 'waiting'); + f.control.reportStatus({ areaId: f.areaId, state: 'idle', summary: 'Turn completed' }); + assert.equal(observe().state, 'idle'); + } finally { f.control.close(); } +}); + +test('SCN-runtime-live-presence-refusal: dead, replaced, foreign-user and malformed guards never claim active execution', () => { + const f = fixture(); + try { + withRuntimeTurnGuard(f.control, f.areaId, () => { + const path = join(f.control.stateRoot, 'sessions', 'turn-guards', f.areaId, 'owner.json'); + const original = JSON.parse(readFileSync(path, 'utf8')); + writeFileSync(path, JSON.stringify({ ...original, startTicks: 'fixture-start' })); + const read = live => observeRuntimeTurn({ stateRoot: f.control.stateRoot, repositoryRoot: f.root, + projectId: f.control.projectId, areaId: f.areaId, uid: 1000, inspectProcess: () => live }); + const live = { uid: 1000, startTicks: 'fixture-start', state: 'S' }; + assert.equal(read(live).state, 'active'); + for (const mismatch of [{ ...live, uid: 2000 }, { ...live, startTicks: 'reused-pid' }, { ...live, state: 'Z' }]) { + assert.equal(read(mismatch).state, 'unknown'); + } + const absent = observeRuntimeTurn({ stateRoot: f.control.stateRoot, repositoryRoot: f.root, + projectId: f.control.projectId, areaId: f.areaId, inspectProcess: () => { throw new Error('Process gone'); } }); + assert.equal(absent.state, 'unknown'); + writeFileSync(path, JSON.stringify({ ...original, projectId: 'foreign' })); + assert.equal(read(live).reason, 'guard-invalid'); + writeFileSync(path, JSON.stringify(original)); + }); + } finally { f.control.close(); } +}); + +test('SCN-runtime-live-presence-legacy: pre-observation guards require exact TORCH identity and project cwd', () => { + const f = fixture(); + try { + withRuntimeTurnGuard(f.control, f.areaId, () => { + const path = join(f.control.stateRoot, 'sessions', 'turn-guards', f.areaId, 'owner.json'); + const original = JSON.parse(readFileSync(path, 'utf8')); + const legacy = { ...original }; + delete legacy.startTicks; + delete legacy.phase; + writeFileSync(path, JSON.stringify(legacy)); + const inspect = live => observeRuntimeTurn({ stateRoot: f.control.stateRoot, repositoryRoot: f.root, + projectId: f.control.projectId, areaId: f.areaId, uid: 1000, inspectProcess: () => live }); + const live = { uid: 1000, state: 'S', cwd: f.root, argv: ['node', '/runtime/bin/torch.mjs', 'up', '--only', f.areaId] }; + assert.equal(inspect(live).phase, 'working'); + assert.equal(inspect({ ...live, cwd: '/foreign' }).state, 'unknown'); + assert.equal(inspect({ ...live, argv: ['node', 'torch.mjs', 'up', '--only', 'someone-else'] }).state, 'unknown'); + writeFileSync(path, JSON.stringify(original)); + }); + } finally { f.control.close(); } +}); + test('SCN-runtime-continuation: explicit enable, event deduplication, daily cap and pause govern authorized turns', async () => { const f = fixture(); const backlog = { list: () => [] };