fleet update

tools/fleet/bin/core/ops.mjs

@@ -5,6 +5,7 @@ import path from 'node:path';
 import { spawnSync } from 'node:child_process';
 import { area as getArea, die, pidAlive, tryGit, git, mainRef, writeAtomic, today, landerIsUp, matchesAny, startLander, runTimedSync } from '../lib.mjs';
 import { send as mailboxSendFile, read as mailboxReadDir, archive as mailboxArchiveDir } from '../../adapters/mailbox.mjs';
+import { appendWakeLog, priorWakes } from './wake-log.mjs';
 
 const STATUSES = ['open', 'in-progress', 'blocked', 'done', 'deferred'];
 const SPECIAL = ['owner', 'scheduler', 'interpreter'];
@@ -119,13 +120,19 @@ function knownParty(c, id) {
   getArea(c, id);
 }
 
-export function sendMailbox(c, { to, from, body, subject = '' } = {}) {
+export function sendMailbox(c, { to, from, body, subject = '', trigger = 'send' } = {}) {
   if (!from) die('--from <your id> required');
   knownParty(c, from);
   if (!body || !String(body).trim()) die('empty message');
   const targets = to === 'all' ? c.areas.map((a) => a.id).filter((id) => id !== from) : [to];
-  const files = targets.map((t) => { knownParty(c, t); return mailboxSendFile(c.mailbox.dir, t, from, body, subject ?? ''); });
-  const wakes = targets.filter((t) => t !== from).map((t) => ({ to: t, ...nudgeMail(c, t, from) }));
+  const files = [];
+  const wakes = [];
+  for (const t of targets) {
+    knownParty(c, t);
+    const file = mailboxSendFile(c.mailbox.dir, t, from, body, subject ?? '');
+    files.push(file);
+    if (t !== from) wakes.push({ to: t, ...nudgeMail(c, t, from, { trigger, file: path.basename(file) }) });
+  }
   return Object.assign(files, { wakes });
 }
 
@@ -193,7 +200,7 @@ export function setupCommit(repo, { message = 'fleet init' } = {}) {
 export function tellDispatch(c, { body, from = 'interpreter', subject = '' } = {}) {
   const hub = c.areas.find((a) => a.role === 'dispatcher');
   if (!hub) die('no session manager (role dispatcher)');
-  const files = sendMailbox(c, { to: hub.id, from, body, subject });
+  const files = sendMailbox(c, { to: hub.id, from, body, subject, trigger: 'tell' });
   const wake = (files.wakes || []).find((w) => w.to === hub.id) || { submitted: false, reason: 'no-window' };
   const status = fleetStatus(c);
   return { files: [...files], to: hub.id, from, nudged: Boolean(wake.submitted), wake, recovered: status.recovered };
@@ -243,6 +250,25 @@ function stillInInput(target, line) {
   return cursorRow(target, capturePane(target)).includes(line);
 }
 
+function inspectPane(target) {
+  if (tmuxText(['list-panes', '-t', target]).status !== 0) return { found: false, dead: false, interrupt: false, idleSec: null, pane: '' };
+  const info = tmuxText(['display-message', '-p', '-t', target, '#{pane_dead} #{window_activity}']);
+  const match = String(info.stdout || '').trim().match(/^([01])\s+(\d+)$/);
+  const pane = capturePane(target);
+  let idleSec = null;
+  let dead = false;
+  if (match) {
+    dead = match[1] === '1';
+    idleSec = Math.max(0, Math.round(Date.now() / 1000 - Number(match[2])));
+  }
+  return { found: true, dead, interrupt: BUSY_RE.test(pane), idleSec, pane };
+}
+
+function paneTail(pane) {
+  const rows = String(pane).split('\n').map((s) => s.trim()).filter(Boolean).slice(-4);
+  return rows.map((s) => s.slice(0, 120)).join(' | ').slice(0, 400);
+}
+
 /**
  * Wake an open tmux window with a short line, not the mail body.
  * Codex and Claude treat a fast burst as a paste and swallow Enter, so the
@@ -250,21 +276,38 @@ function stillInInput(target, line) {
  * If the cursor row still holds the line, Enter is sent once more.
  * A pane that shows an interrupt hint is left alone; a later status check can nudge it.
  */
-export function nudgeMail(c, areaId, from) {
+export function nudgeMail(c, areaId, from, meta = {}) {
   const target = tmuxTarget(c, areaId);
-  if (tmuxText(['list-panes', '-t', target]).status !== 0) return { submitted: false, reason: 'no-window' };
+  const before = inspectPane(target);
+  const record = {
+    t: new Date().toISOString(),
+    area: areaId,
+    window: target,
+    file: meta.file || null,
+    trigger: meta.trigger || 'send',
+    before: { found: before.found, dead: before.dead, interrupt: before.interrupt, idleSec: before.idleSec },
+  };
+  const finish = (fields) => {
+    const { line, reason, ...logged } = fields;
+    const pane = logged.action === 'no-window' ? '' : capturePane(target);
+    appendWakeLog(c, { ...record, ...logged, tail: paneTail(pane) });
+    return { submitted: logged.outcome === 'submitted', reason, ...(line ? { line } : {}) };
+  };
+  if (!before.found) return finish({ action: 'no-window', enterAttempts: 0, outcome: 'no-window', reason: 'no-window' });
   const line = nudgeLine(from);
-  if (BUSY_RE.test(capturePane(target))) return { submitted: false, reason: 'working', line };
-  if (tmuxText(['send-keys', '-t', target, '-l', line]).status !== 0) return { submitted: false, reason: 'send-failed', line };
+  if (before.interrupt) return finish({ action: 'skipped-busy', enterAttempts: 0, outcome: 'skipped-busy', reason: 'working', line });
+  if (tmuxText(['send-keys', '-t', target, '-l', line]).status !== 0) return finish({ action: 'send-failed', enterAttempts: 0, outcome: 'send-failed', reason: 'send-failed', line });
+  let enterAttempts = 0;
+  const hitEnter = () => { tmuxText(['send-keys', '-t', target, 'Enter']); enterAttempts += 1; };
   sleepMs(NUDGE_PAUSE_MS);
-  tmuxText(['send-keys', '-t', target, 'Enter']);
+  hitEnter();
   sleepMs(NUDGE_SETTLE_MS);
-  if (!stillInInput(target, line)) return { submitted: true, reason: 'submitted', line };
+  if (!stillInInput(target, line)) return finish({ action: 'typed', enterAttempts, outcome: 'submitted', reason: 'submitted', line });
   sleepMs(NUDGE_RETRY_MS);
-  tmuxText(['send-keys', '-t', target, 'Enter']);
+  hitEnter();
   sleepMs(NUDGE_SETTLE_MS);
   const submitted = !stillInInput(target, line);
-  return { submitted, reason: submitted ? 'submitted' : 'not-submitted', line };
+  return finish({ action: 'typed', enterAttempts, outcome: submitted ? 'submitted' : 'swallowed', reason: submitted ? 'submitted' : 'not-submitted', line });
 }
 
 function unreadFiles(c, id) {
@@ -325,7 +368,12 @@ export function fleetStatus(c, { recoverAfterMs = RECOVER_AFTER_MS, idleAfterMs
     const age = oldest ? Date.now() - oldest.mtimeMs : 0;
     const already = (state[a.id] || []).includes(oldest?.file);
     if (view.window === 'alive' && view.activity === 'idle' && oldest && age >= recoverAfterMs && !already) {
-      const wake = nudgeMail(c, a.id, senderOf(oldest.path));
+      const prior = priorWakes(c, a.id, oldest.file).slice(-5).map((p) => ({ t: p.t, trigger: p.trigger, action: p.action, outcome: p.outcome }));
+      appendWakeLog(c, {
+        t: new Date().toISOString(), kind: 'stale', area: a.id, window: tmuxTarget(c, a.id),
+        file: oldest.file, ageSec: Math.round(age / 1000), prior,
+      });
+      const wake = nudgeMail(c, a.id, senderOf(oldest.path), { trigger: 'recovery', file: oldest.file });
       state[a.id] = [...(state[a.id] || []), oldest.file];
       stateDirty = true;
       row.recovered = true;

tools/fleet/bin/core/wake-log.mjs

@@ -0,0 +1,66 @@
+// One JSON line per wake attempt, in the fleet state dir (not the checkout).
+// The line records what the pane looked like and what the nudge did. It does
+// not record mail bodies. The file is capped so it cannot grow without bound.
+import fs from 'node:fs';
+import path from 'node:path';
+import { writeAtomic } from '../lib.mjs';
+
+export const WAKE_LOG_MAX = 400;
+
+export function wakeLogPath(c) {
+  return path.join(c.stateDir, 'wake-log.jsonl');
+}
+
+export function appendWakeLog(c, entry) {
+  const file = wakeLogPath(c);
+  fs.mkdirSync(path.dirname(file), { recursive: true });
+  let lines = [];
+  try { lines = fs.readFileSync(file, 'utf8').split('\n').filter(Boolean); } catch { /* new log */ }
+  lines.push(JSON.stringify(entry));
+  if (lines.length > WAKE_LOG_MAX) lines = lines.slice(lines.length - WAKE_LOG_MAX);
+  writeAtomic(file, `${lines.join('\n')}\n`);
+}
+
+export function priorWakes(c, area, file) {
+  return readRaw(c).filter((e) => e && e.kind !== 'stale' && e.area === area && e.file === file);
+}
+
+function readRaw(c) {
+  try {
+    return fs.readFileSync(wakeLogPath(c), 'utf8').split('\n').filter(Boolean).map((line) => {
+      try { return JSON.parse(line); } catch { return null; }
+    }).filter(Boolean);
+  } catch { return []; }
+}
+
+/** Plain words for the failure a later reader should tell the user. */
+export function wakeFlag(entry) {
+  if (!entry) return 'unknown';
+  if (entry.kind === 'stale') {
+    const last = Array.isArray(entry.prior) ? entry.prior[entry.prior.length - 1] : null;
+    const age = entry.ageSec ?? '?';
+    if (!last) return `still unread after ${age}s; no earlier wake`;
+    if (last.outcome === 'swallowed') return `woke at ${last.t} (${last.trigger}), wake line still in the composer, still unread after ${age}s`;
+    if (last.action === 'skipped-busy') return `woke at ${last.t} (${last.trigger}), skipped because an interrupt hint was showing, still unread after ${age}s`;
+    if (last.action === 'no-window' || last.outcome === 'no-window') return `woke at ${last.t} (${last.trigger}), pane not found, still unread after ${age}s`;
+    if (last.outcome === 'submitted') return `woke at ${last.t} (${last.trigger}), line left the composer, still unread after ${age}s`;
+    return `woke at ${last.t} (${last.trigger}), outcome ${last.outcome}, still unread after ${age}s`;
+  }
+  if (entry.action === 'no-window' || entry.outcome === 'no-window') return 'pane not found';
+  if (entry.action === 'skipped-busy') return 'skipped: interrupt hint was showing, so the pane looked mid-turn';
+  if (entry.action === 'send-failed') return 'send-keys failed before Enter';
+  if (entry.outcome === 'swallowed') return `wake line still on the cursor row after ${entry.enterAttempts} Enter`;
+  if (entry.outcome === 'submitted') {
+    if (entry.before && entry.before.idleSec != null && entry.before.idleSec < 3 && !entry.before.interrupt) {
+      return 'typed and the line left the composer while the pane was recently active';
+    }
+    return 'typed and the line left the composer';
+  }
+  return entry.outcome || entry.action || 'unknown';
+}
+
+export function readWakeLog(c, { limit = 40 } = {}) {
+  const all = readRaw(c);
+  const entries = all.slice(-Math.max(1, Number(limit) || 40)).map((entry) => ({ ...entry, flag: wakeFlag(entry) }));
+  return { path: wakeLogPath(c), entries };
+}

tools/fleet/bin/fleet.mjs

@@ -37,6 +37,7 @@
 //   fleet doctor | landq | backlog | mailbox
 //                                     the same tools as tools/fleet/bin/<name>.mjs
 //   fleet status                      areas, run state (running / exited / failed / timeout / stopped), lander up or down
+//   fleet wakes                       recent wake attempts, and plain words for a likely failure
 //   fleet prompt <id>                 print the composed system prompt for an area
 //   fleet crontab                     PRINT the crontab lines for scheduled jobs (never installs)
 //   fleet run-job <name>              one headless run of docs/agents/schedules/<name>.md (sched.sh calls this)
@@ -51,6 +52,7 @@ import path from 'node:path';
 import { spawnSync } from 'node:child_process';
 import { KIT, loadConfig, loadAdapter, area as getArea, dispatcher, sessionName, flag, die, writeAtomic, tryGit, mainRef, today, runProcessGroup, runForeground, finishHeadless, withHandoff, applyHeadlessSession, assignLaunchSession, areaBudgetMin, readRun, writeRun, outcomeOfRun, pidAlive, killProcessGroup, landerIsUp, landerPid, startLander, writableRoots } from './lib.mjs';
 import { commitWork, setupCommit, tellDispatch, hearInterpreter, fleetStatus } from './core/ops.mjs';
+import { readWakeLog } from './core/wake-log.mjs';
 import { fleetUpdate, recordInstalled, isTemplatePath } from './core/update.mjs';
 
 const argv = process.argv.slice(2);
@@ -308,6 +310,19 @@ async function status() {
   for (const id of report.recovered) console.log(`nudged ${id} again`);
 }
 
+function wakesCmd() {
+  const report = readWakeLog(c);
+  if (!report.entries.length) {
+    console.log(`no wake attempts in ${report.path}`);
+    return;
+  }
+  console.log(report.path);
+  for (const e of report.entries) {
+    if (e.kind === 'stale') console.log(`${e.t}  stale ${e.area} ${e.file}  ${e.flag}`);
+    else console.log(`${e.t}  ${e.trigger} ${e.area} ${e.window} ${e.file || '-'}  ${e.action} enter=${e.enterAttempts} ${e.outcome}  ${e.flag}`);
+  }
+}
+
 function ensureWorktree(a) {
   if (fs.existsSync(a.worktree)) return true;
   const dirty = uncommittedFleetFiles();
@@ -618,7 +633,7 @@ function tool(name) {
 const run = { up, down, status, supervise: () => supervise(argv[1]), worktrees, crontab,
   prompt: async () => console.log(composePrompt(getArea(c, argv[1]), await adapterFor(getArea(c, argv[1]).harness))),
   'run-job': () => runJob(argv[1]), 'sched-env': () => schedEnv(argv[1]), lander: () => lander(argv[1] ?? 'status'),
-  commit: commitCmd, 'setup-commit': setupCommitCmd, update: updateCmd, tell: tellCmd, hear: hearCmd,
+  commit: commitCmd, 'setup-commit': setupCommitCmd, update: updateCmd, tell: tellCmd, hear: hearCmd, wakes: wakesCmd,
   doctor: () => tool('doctor'), landq: () => tool('landq'), backlog: () => tool('backlog'),
   mailbox: () => tool('mailbox') }[cmd];
 if (!run) die(`unknown command "${cmd}" (fleet help)`);

tools/fleet/bin/mcp.mjs

@@ -10,6 +10,7 @@ import {
   setupCommit, tellDispatch, hearInterpreter, fleetStatus,
 } from './core/ops.mjs';
 import { fleetUpdate } from './core/update.mjs';
+import { readWakeLog } from './core/wake-log.mjs';
 
 export const SERVER = { name: 'agent-fleet', version: '0.2.0' };
 const INSTRUCTIONS = [
@@ -18,7 +19,7 @@ const INSTRUCTIONS = [
   'mailbox_read at the start of a turn. Mail is always delivered; the prompt says when to act on it. Sandboxed agents commit with the commit tool (message plus paths, or all), not git commit. submit on that tool also queues the tip. Then go back to work. At your next natural checkpoint, land_status can report that submission. Waiting is optional. Never infer the lander or the land result from silence. landq_submit queues the committed tip (a dirty worktree warns and still queues HEAD) and starts a drain. Do not push main.',
   'A scarce machine is a prompt convention: say so in .task and wait if a peer already has it. There is no slot lock.',
   'Sessions, worktrees, and cron stay on the fleet CLI. This server does not start them.',
-  'setup_commit commits only the fleet setup files on the main checkout. A sandboxed agent uses it instead of git commit. update refreshes vendored runtime and the registered skill to this kit, leaves an edited prompt in place (the new template is a .kit file beside it), and commits only the paths it changed. It does not change fleet.config.json, product files, mail, or the landing queue. If that commit cannot write the index, it falls back to setup_commit. Mail to an open tmux window wakes it with a short line, not the body. mailbox_read and hear take those messages out of unread. fleet_status reports each window alive or dead, the unread count, and a working or idle guess. An idle area with mail that has been waiting is nudged once by that check. mailbox_read, hear, and tell run that same check. A second look does not nudge again. Do not scrape the tmux pane.',
+  'setup_commit commits only the fleet setup files on the main checkout. A sandboxed agent uses it instead of git commit. update refreshes vendored runtime and the registered skill to this kit, leaves an edited prompt in place (the new template is a .kit file beside it), and commits only the paths it changed. It does not change fleet.config.json, product files, mail, or the landing queue. If that commit cannot write the index, it falls back to setup_commit. Mail to an open tmux window wakes it with a short line, not the body. mailbox_read and hear take those messages out of unread. fleet_status reports each window alive or dead, the unread count, and a working or idle guess. An idle area with mail that has been waiting is nudged once by that check. mailbox_read, hear, and tell run that same check. A second look does not nudge again. wake_log says whether a wake was swallowed, skipped, missing, or submitted while the mail is still unread. Tell the user that case. Do not ask them to open the log. Do not scrape the tmux pane.',
 ].join(' ');
 
 const cwdProp = { type: 'string', description: 'Directory to resolve the fleet from. Defaults to this server\'s cwd.' };
@@ -39,6 +40,7 @@ export const TOOLS = [
   { name: 'tell', description: 'Mail the session manager from the interpreter. Wakes its tmux window with a short line when that window is open. The body stays in the mail. Replies are read with hear.', inputSchema: { type: 'object', properties: { body: { type: 'string' }, subject: { type: 'string' }, cwd: cwdProp }, required: ['body'] } },
   { name: 'hear', description: 'Read unread mailbox messages for the interpreter and move them out of unread.', inputSchema: { type: 'object', properties: { cwd: cwdProp } } },
   { name: 'fleet_status', description: 'Each area: tmux window alive or dead, unread mail count, and a working or idle guess. Idle with unread mail that has been waiting is nudged once.', inputSchema: { type: 'object', properties: { cwd: cwdProp } } },
+  { name: 'wake_log', description: 'Recent wake attempts from the fleet state dir. Each line says the pane state, what was typed or skipped, and a plain-words flag when a wake likely failed. No mail bodies.', inputSchema: { type: 'object', properties: { limit: { type: 'number' }, cwd: cwdProp } } },
 ];
 
 function configFor(args) {
@@ -66,6 +68,7 @@ export function dispatchTool(name, args = {}) {
     case 'tell': return tellDispatch(c, { body: args.body, subject: args.subject ?? '' });
     case 'hear': return hearInterpreter(c);
     case 'fleet_status': return fleetStatus(c);
+    case 'wake_log': return readWakeLog(c, { limit: args.limit });
     default: throw new FleetError(`unknown tool ${name}`);
   }
 }