From e26a0fb50608a0cd62db6ee9a6341cb11098d70c Mon Sep 17 00:00:00 2001 From: thePR0M3TH3AN Date: Wed, 30 Sep 2026 21:00:06 -0400 Subject: [PATCH] TASK-pilot-executor-outcomes qualify isolated CLI outcomes --- docs/TEST_INTEGRITY.md | 35 +++++ src/cli.mjs | 6 +- test/fleet/executor-outcomes.test.mjs | 183 +++++++++++++++++++++++++- 3 files changed, 215 insertions(+), 9 deletions(-) diff --git a/docs/TEST_INTEGRITY.md b/docs/TEST_INTEGRITY.md index 203e6ac..31bac41 100644 --- a/docs/TEST_INTEGRITY.md +++ b/docs/TEST_INTEGRITY.md @@ -105,6 +105,41 @@ test_integrity_note: if_true_explain_spec_basis: "" ``` +## 2026-09-30 — Early identity and safe no-code diagnostics + +```yaml +test_integrity_note: + change_type: new_tests + scenarios: + - id: SCN-codex-interrupted-identity + given: A real local helper whose child emits thread.started then remains alive + when: The parent observes the reduced bounded identity and sends SIGTERM to the helper + then: The helper closes by SIGTERM without a terminal success, while lifecycle retains the identity as working and uncertain + - id: SCN-codex-code-less-diagnostics + given: Code-less structured model, authentication, or quota errors containing prompt, reasoning, and image adversarial fields + when: The bounded reducer and adapter classify startup failure + then: Only the allowlisted category crosses the boundary; no free-form diagnostic or adversarial field remains + - id: SCN-cli-codex-real-executable + given: An installed disposable project and an isolated executable named codex + when: The actual runCli up command starts the selected domain + then: The reducer carries identity and terminal completion through the CLI boundary and durable identity becomes idle + observable_outcomes: + - Real helper close signal, exact reduced protocol bytes, and working retained runtime identity + - Safe diagnostic category with no prompt, reasoning, image, or free-form message + - Actual CLI JSON response and persisted identity state from a fake executable on an isolated PATH + determinism_controls: + - Local child process signal handshake driven by receipt of thread.started; a bounded watchdog only terminates and awaits the owned fixture on failed emission, never retries or asserts product timing + - Disposable Git/XDG fixtures and a hermetic executable; no provider, network, or account access + anti_cheat_rationale: + prevents: + - Delaying identity output until child close and losing it on interruption + - Treating code-less model/auth/quota errors as successful or retaining their raw text + - Testing only an injected launch helper instead of the CLI execution boundary + relaxation: + did_relax_any_assertion: false + if_true_explain_spec_basis: "" +``` + ## 2026-09-30 — Exact observed-file ownership evidence ```yaml diff --git a/src/cli.mjs b/src/cli.mjs index 27f84c4..5b01283 100644 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -1122,7 +1122,7 @@ export async function runCli(argv = process.argv.slice(2), { print(startFleet({ plan, controlPlane: control, adapters, executor: (launch) => executeRuntimeLaunch(spawn, launch, { - cwd: launch.cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], + cwd: launch.cwd, env, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: MAX_RUNTIME_EXECUTOR_OUTPUT_BYTES, }), }), { json }); @@ -1456,7 +1456,7 @@ export async function runCli(argv = process.argv.slice(2), { print(startFleet({ plan, controlPlane: control, adapters, executor: (launch) => executeRuntimeLaunch(spawn, launch, { - cwd: launch.cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], + cwd: launch.cwd, env, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: MAX_RUNTIME_EXECUTOR_OUTPUT_BYTES, }), }), { json }); @@ -1631,7 +1631,7 @@ export async function runCli(argv = process.argv.slice(2), { return startFleet({ plan, controlPlane: control, adapters, executor: (launch) => executeRuntimeLaunch(spawn, launch, { - cwd: launch.cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], + cwd: launch.cwd, env, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout: 300_000, maxBuffer: MAX_RUNTIME_EXECUTOR_OUTPUT_BYTES, killSignal: 'SIGTERM', }), }); diff --git a/test/fleet/executor-outcomes.test.mjs b/test/fleet/executor-outcomes.test.mjs index ff71df8..a3f13e2 100644 --- a/test/fleet/executor-outcomes.test.mjs +++ b/test/fleet/executor-outcomes.test.mjs @@ -1,11 +1,11 @@ import assert from 'node:assert/strict'; -import { execFileSync, spawnSync } from 'node:child_process'; -import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { execFileSync, spawn, spawnSync } from 'node:child_process'; +import { chmodSync, existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import test from 'node:test'; import { createCodexAdapter } from '../../src/adapters/codex.mjs'; -import { executeRuntimeLaunch } from '../../src/cli.mjs'; +import { executeRuntimeLaunch, runCli } from '../../src/cli.mjs'; import { openControlPlane } from '../../src/control-plane/service.mjs'; import { analyzeRepository } from '../../src/kernel/analyze.mjs'; import { proposeDomains } from '../../src/kernel/domains.mjs'; @@ -37,7 +37,9 @@ function fixture() { }]; proposal.review = { status: 'approved', reviewedAt: '2026-09-30T00:00:00Z', reviewedBy: 'fixture-owner', notes: [] }; const env = { ...process.env, XDG_DATA_HOME: mkdtempSync(join(tmpdir(), 'torch-executor-outcomes-state-')) }; - installProject({ repository, proposal, env, projectId: 'executor-outcomes-fixture' }); + installProject({ + repository, proposal, env, projectId: 'executor-outcomes-fixture', runtimes: ['codex'], defaultRuntime: 'codex', + }); return { root, env, worker }; } @@ -56,7 +58,7 @@ function planFor(context, { }; } -function expectInterruptedStart({ context, result, expectedReason, plan = planFor(context) }) { +function expectInterruptedStart({ context, result, expectedReason, plan = planFor(context), expectedRuntimeSessionId }) { const control = openControlPlane({ repositoryRoot: context.root, env: context.env }); try { let failure; @@ -70,11 +72,82 @@ function expectInterruptedStart({ context, result, expectedReason, plan = planFo assert.equal(failure.details.executorOutcome.reason, expectedReason); assert.equal(control.identity(context.worker).state, 'working'); assert.equal(control.identity(context.worker).runtimeSessionId, - plan.actions[0].runtimeSessionId ?? 'overflow-thread'); + expectedRuntimeSessionId ?? plan.actions[0].runtimeSessionId ?? 'overflow-thread'); return failure; } finally { control.close(); } } +function interruptReducerAfterIdentity() { + const expectedIdentity = '{"type":"thread.started","thread_id":"interrupted-thread"}\n'; + const childProgram = [ + 'process.stdout.write(JSON.stringify({type:"thread.started",thread_id:"interrupted-thread"}) + "\\n");', + 'process.on("SIGTERM", () => process.exit(0));', + 'setInterval(() => {}, 1_000);', + ].join(''); + const helper = spawn(process.execPath, [CODEX_PROTOCOL_REDUCER, process.execPath, '-e', childProgram], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + helper.stdout.setEncoding('utf8'); + helper.stderr.setEncoding('utf8'); + return new Promise((resolve, reject) => { + let stdout = ''; + let stderr = ''; + let interrupted = false; + let failure = null; + const terminate = () => { + if (helper.exitCode === null && helper.signalCode === null) helper.kill('SIGTERM'); + }; + const watchdog = setTimeout(() => { + failure ??= new Error('helper did not emit its bounded identity; terminating owned fixture process'); + terminate(); + }, 1_000); + helper.stdout.on('data', (chunk) => { + stdout += chunk; + try { + if (!interrupted && stdout === expectedIdentity) { + interrupted = true; + assert.equal(helper.exitCode, null, 'helper must still be live when the reduced identity arrives'); + assert.equal(helper.signalCode, null, 'identity must not depend on a close event'); + assert.equal(helper.kill('SIGTERM'), true, 'parent must forward interruption through the real helper process'); + } else if (!expectedIdentity.startsWith(stdout)) { + failure ??= new Error(`helper emitted unexpected reduced protocol bytes: ${JSON.stringify(stdout)}`); + terminate(); + } + } catch (error) { + failure ??= error; + terminate(); + } + }); + helper.stderr.on('data', (chunk) => { stderr += chunk; }); + helper.once('error', (error) => { + failure ??= error; + terminate(); + }); + helper.once('close', (status, signal) => { + clearTimeout(watchdog); + try { + assert.throws(() => process.kill(helper.pid, 0), { code: 'ESRCH' }, 'owned helper must be absent after close'); + if (failure) reject(failure); + else resolve({ status, signal, stdout, stderr, interrupted }); + } catch (error) { reject(error); } + }); + }); +} + +async function captureStdout(action) { + const originalWrite = process.stdout.write; + let output = ''; + process.stdout.write = function capture(chunk, ...args) { + output += typeof chunk === 'string' ? chunk : chunk.toString('utf8'); + return originalWrite.call(this, '', ...args); + }; + try { + return { status: await action(), output }; + } finally { + process.stdout.write = originalWrite; + } +} + test('SCN-executor-interrupted-outcomes: a graceful ready timeout, signal, or ENOBUFS cannot create an idle turn or disclose output', () => { const readyDirectory = mkdtempSync(join(tmpdir(), 'torch-executor-ready-')); const readyPath = join(readyDirectory, 'ready'); @@ -173,6 +246,45 @@ test('SCN-codex-final-terminal-state: a later failed terminal overrides an earli assert.equal(unavailable.stdout, '{"type":"error","code":"ENOENT"}\n'); }); +test('SCN-codex-interrupted-identity: a real helper emits a bounded identity before deterministic parent interruption and remains uncertain', async () => { + const result = await interruptReducerAfterIdentity(); + assert.equal(result.interrupted, true); + assert.equal(result.status, null); + assert.equal(result.signal, 'SIGTERM'); + assert.equal(result.stdout, '{"type":"thread.started","thread_id":"interrupted-thread"}\n'); + assert.equal(result.stderr, ''); + const context = fixture(); + expectInterruptedStart({ + context, + plan: planFor(context, { runtimeSessionId: null, requiresRuntimeIdCapture: true }), + result, + expectedReason: 'executor-signal', + expectedRuntimeSessionId: 'interrupted-thread', + }); +}); + +test('SCN-codex-code-less-diagnostics: no-code model, authentication, and quota errors retain only safe categories', () => { + const cases = [ + ['The requested model is not supported; prompt=private-prompt', 'model-rejection'], + ['Authentication rejected this credential; private reasoning=secret-analysis', 'authentication'], + ['Quota exceeded for this account; image=data:image/png;base64,fixture-image', 'quota'], + ]; + for (const [message, category] of cases) { + const childProgram = `process.stdout.write(${JSON.stringify(`${JSON.stringify({ type: 'error', message, prompt: 'private-prompt', reasoning: 'secret-analysis', image_url: 'data:image/png;base64,fixture-image' })}\n`)});`; + const result = spawnSync(process.execPath, [CODEX_PROTOCOL_REDUCER, process.execPath, '-e', childProgram], { + encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], + }); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, `{"type":"turn.failed","category":"${category}"}\n`); + const diagnostic = createCodexAdapter().diagnoseStartupFailure(result); + assert.deepEqual(diagnostic, { + schema: 'torch.dev/runtime-startup-diagnostic/v1alpha1', outcome: 'known', category, stderr: null, + }); + assert.doesNotMatch(`${result.stdout}\n${result.stderr}\n${JSON.stringify(diagnostic)}`, + /private-prompt|secret-analysis|fixture-image|image\/png/); + } +}); + test('SCN-cli-codex-protocol-routing: the CLI routes Codex launch output through the bounded protocol reducer', () => { const result = executeRuntimeLaunch(spawnSync, { runtime: 'codex', command: process.execPath, @@ -182,6 +294,65 @@ test('SCN-cli-codex-protocol-routing: the CLI routes Codex launch output through assert.equal(result.stdout, '{"type":"thread.started","thread_id":"cli-thread"}\n{"type":"turn.completed"}\n'); }); +test('SCN-cli-codex-real-executable: an actual runCli invocation reaches the isolated Codex executable through the reducer', async () => { + const context = fixture(); + const bin = mkdtempSync(join(tmpdir(), 'torch-codex-bin-')); + const executable = join(bin, 'codex'); + const marker = join(bin, 'invocation.json'); + const markerEnv = 'TORCH_FAKE_CODEX_INVOCATION_MARKER'; + const inheritedMarker = process.env[markerEnv]; + writeFileSync(executable, [ + '#!/usr/bin/env node', + 'const fs = require("node:fs");', + `const marker = process.env.${markerEnv};`, + 'if (!marker) { process.stderr.write("missing isolated invocation marker\\n"); process.exit(86); }', + 'fs.writeFileSync(marker, JSON.stringify({ cwd: process.cwd(), args: process.argv.slice(2) }));', + 'process.stdout.write(JSON.stringify({type:"thread.started",thread_id:"cli-boundary-thread"}) + "\\n");', + 'process.stdout.write(JSON.stringify({type:"turn.completed"}) + "\\n");', + ].join('\n')); + chmodSync(executable, 0o755); + const cliOptions = { + cwd: context.root, + env: { ...context.env, PATH: `${bin}:${process.env.PATH}`, [markerEnv]: marker }, + }; + const worktreeParent = mkdtempSync(join(tmpdir(), 'torch-cli-worktrees-')); + execFileSync('git', ['-C', context.root, 'add', '.torch']); + execFileSync('git', ['-C', context.root, 'commit', '-m', 'commit fixture installation state']); + const worktrees = await captureStdout(() => runCli([ + 'worktrees', '--parent', worktreeParent, '--yes', '--json', + ], cliOptions)); + assert.equal(worktrees.status, 0, worktrees.output); + const missingMarker = await captureStdout(() => runCli([ + 'up', '--fresh', '--only', context.worker, '--yes', '--json', + ], { + cwd: context.root, + env: { ...context.env, PATH: `${bin}:${process.env.PATH}` }, + })); + assert.equal(missingMarker.status, 2, missingMarker.output); + assert.equal(JSON.parse(missingMarker.output).error, 'FLEET_START_FAILED'); + assert.equal(existsSync(marker), false, 'fake executable must fail before it can claim invocation without its marker'); + assert.ok(Buffer.byteLength(missingMarker.output) < 4 * 1024, 'failed CLI response must remain bounded'); + const captured = await captureStdout(() => runCli([ + 'up', '--fresh', '--only', context.worker, '--yes', '--json', + ], cliOptions)); + assert.equal(captured.status, 0, captured.output); + assert.equal(process.env[markerEnv], inheritedMarker, 'runCli must not mutate the process environment'); + const invocation = JSON.parse(readFileSync(marker, 'utf8')); + assert.equal(invocation.args.includes('exec'), true, 'the isolated executable must receive the Codex exec invocation'); + assert.equal(invocation.args.includes('--json'), true); + assert.match(invocation.cwd, /\/core$/); + assert.ok(Buffer.byteLength(captured.output) < 4 * 1024, 'CLI response must remain bounded'); + const response = JSON.parse(captured.output); + assert.deepEqual(response.started, [{ + areaId: context.worker, runtimeSessionId: 'cli-boundary-thread', mode: 'create', + }]); + const control = openControlPlane({ repositoryRoot: context.root, env: context.env }); + try { + assert.equal(control.identity(context.worker).state, 'idle'); + assert.equal(control.identity(context.worker).runtimeSessionId, 'cli-boundary-thread'); + } finally { control.close(); } +}); + test('SCN-scheduled-manager-wake-terminal-failure: the real manager-check-in boundary fails rather than invoking a successful wake receipt', () => { const context = fixture(); const configPath = join(context.root, '.torch', 'torch.yaml');