feat: truncate logs and inject memory into prompts to reduce token burn

This change implements several optimizations to reduce token consumption and improve agent context:

1.  **Log Truncation**: `scripts/agent/run-scheduler-cycle.mjs` now truncates the echoed output of spawned processes to 20KB per stream (stdout/stderr). This prevents the scheduler agent's context from being flooded with excessive logs from sub-agents.
2.  **Memory Injection**: `torch-config.json`'s `retrieveCommand` has been updated to write retrieved memories to `.scheduler-memory/latest/<cadence>/memories.md`. `scripts/agent/run-selected-prompt.mjs` now checks for this file and injects its content into the agent's prompt, providing immediate context from the memory system.
3.  **CLI Optimization**: The `torch-lock list-memories` command now truncates the `content` field of memories to 200 characters by default. A new `--full` flag has been added to `src/cli-parser.mjs` and `src/lib.mjs` to view the full content when needed.

These changes aim to make the agent process more efficient ("error on the side of caution") and leverage the memory system more effectively.

Co-authored-by: PR0M3TH3AN <[email protected]>

scripts/agent/run-scheduler-cycle.mjs

@@ -86,16 +86,48 @@ async function runCommand(command, args = [], options = {}) {
     let stdout = '';
     let stderr = '';
 
+    const OUTPUT_LIMIT = 20000; // 20KB
+    let stdoutWritten = 0;
+    let stderrWritten = 0;
+    let stdoutTruncated = false;
+    let stderrTruncated = false;
+
     child.stdout.on('data', (chunk) => {
       const text = chunk.toString();
       stdout += text;
-      process.stdout.write(text);
+
+      if (stdoutWritten < OUTPUT_LIMIT) {
+        const remaining = OUTPUT_LIMIT - stdoutWritten;
+        if (text.length <= remaining) {
+          process.stdout.write(text);
+          stdoutWritten += text.length;
+        } else {
+          process.stdout.write(text.slice(0, remaining));
+          stdoutWritten += remaining;
+        }
+      } else if (!stdoutTruncated) {
+        process.stdout.write('\n...[stdout truncated]...\n');
+        stdoutTruncated = true;
+      }
     });
 
     child.stderr.on('data', (chunk) => {
       const text = chunk.toString();
       stderr += text;
-      process.stderr.write(text);
+
+      if (stderrWritten < OUTPUT_LIMIT) {
+        const remaining = OUTPUT_LIMIT - stderrWritten;
+        if (text.length <= remaining) {
+          process.stderr.write(text);
+          stderrWritten += text.length;
+        } else {
+          process.stderr.write(text.slice(0, remaining));
+          stderrWritten += remaining;
+        }
+      } else if (!stderrTruncated) {
+        process.stderr.write('\n...[stderr truncated]...\n');
+        stderrTruncated = true;
+      }
     });
 
     child.on('close', (code) => {

scripts/agent/run-selected-prompt.mjs

@@ -52,6 +52,20 @@ async function main() {
     fail(`Scheduler prompt at ${promptPath} is empty.`);
   }
 
+  const cadence = process.env.SCHEDULER_CADENCE;
+  const memoriesPath = path.resolve(process.cwd(), `.scheduler-memory/latest/${cadence}/memories.md`);
+  try {
+    const memories = await fs.readFile(memoriesPath, 'utf8');
+    if (memories && memories.trim()) {
+      promptMarkdown += `\n\n# Retrieved Memories\n\n${memories.trim()}\n`;
+      console.error(`[scheduler] Injected memories from ${memoriesPath}`);
+    }
+  } catch (error) {
+    if (error.code !== 'ENOENT') {
+      console.error(`[scheduler] Warning: Failed to read memories from ${memoriesPath}: ${error.message}`);
+    }
+  }
+
   const runner = resolveRunner(promptMarkdown);
   const child = spawn(runner.command, runner.args, {
     cwd: process.cwd(),

src/cli-parser.mjs

@@ -31,6 +31,7 @@ export function parseArgs(argv) {
     allRelaysDownMinutes: null,
     minSuccessRate: null,
     windowMinutes: null,
+    full: false,
   };
   let i = 0;
 
@@ -117,6 +118,8 @@ export function parseArgs(argv) {
       args.windowMinutes = parseInt(argv[++i], 10);
     } else if (arg.startsWith('--window-minutes=')) {
       args.windowMinutes = parseInt(arg.split('=')[1], 10);
+    } else if (arg === '--full') {
+      args.full = true;
     }
   }
 

src/lib.mjs

@@ -541,7 +541,7 @@ Commands:
   init      [--force]                              Initialize torch/ directory in current project
   update    [--force]                              Update torch/ configuration (backups, merges)
 
-  list-memories           [--agent <id>] [--type <type>] [--tags <a,b>] [--pinned <true|false>]
+  list-memories           [--agent <id>] [--type <type>] [--tags <a,b>] [--pinned <true|false>] [--full]
   inspect-memory          --id <memoryId>
   pin-memory              --id <memoryId>
   unpin-memory            --id <memoryId>
@@ -684,6 +684,15 @@ export async function main(argv) {
           limit: args.limit,
           offset: args.offset,
         });
+
+        if (!args.full && Array.isArray(result)) {
+          for (const memory of result) {
+            if (typeof memory.content === 'string' && memory.content.length > 200) {
+              memory.content = memory.content.slice(0, 200) + '... (truncated, use --full to see all)';
+            }
+          }
+        }
+
         console.log(JSON.stringify(result, null, 2));
         break;
       }

torch-config.json

@@ -25,7 +25,7 @@
     "memoryPolicyByCadence": {
       "daily": {
         "mode": "required",
-        "retrieveCommand": "set -euo pipefail; node --input-type=module -e \"import { ingestEvents, getRelevantMemories } from './src/services/memory/index.js'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; const fallbackCadence='daily'; const cadence=process.env.SCHEDULER_CADENCE || fallbackCadence; const agentId=process.env.SCHEDULER_AGENT || ('scheduler-memory-' + cadence); const promptPath=process.env.SCHEDULER_PROMPT_PATH || ''; const runId=(process.env.SCHEDULER_RUN_ID || process.env.SCHEDULER_SESSION_ID || process.env.RUN_ID || ('session-' + Date.now().toString(36))); const promptRaw=promptPath ? readFileSync(promptPath, 'utf8') : ''; const promptLines=promptRaw.split(/\\r?\\n/).map((line)=>line.trim()).filter(Boolean); const promptIntent=promptLines.find((line)=>line.startsWith('#') || line.toLowerCase().startsWith('goal')) || promptLines[0] || 'scheduler memory retrieval'; const query=['scheduler', cadence, 'agent', agentId, 'intent', promptIntent.replace(/^#+\\s*/, '')].join(' | '); const timestamp=Date.now(); const events=[{ agent_id: agentId, content: 'Memory retrieval seed for ' + cadence + ' :: ' + promptIntent, timestamp, tags:['scheduler', cadence, 'retrieve'], metadata:{ session_id:runId, source:'scheduler-retrieve', importance:0.4, prompt_path:promptPath } }]; const ingested=await ingestEvents(events,{ agent_id: agentId }); const retrieved=await getRelevantMemories({ agent_id: agentId, query, k: 5 }); const sessionDir='.scheduler-memory/' + runId; const latestDir='.scheduler-memory/latest/' + cadence; mkdirSync(sessionDir,{ recursive:true }); mkdirSync(latestDir,{ recursive:true }); const artifact={ cadence, operation:'retrieve', runId, servicePath:'src/services/memory/index.js#getRelevantMemories', inputs:{ agentId, promptPath, promptIntent, query, events:events.length }, outputs:{ ingestedCount:ingested.length, retrievedCount:retrieved.length }, status:'ok' }; writeFileSync(sessionDir + '/retrieve.json', JSON.stringify(artifact, null, 2)); writeFileSync(sessionDir + '/retrieve.ok', 'MEMORY_RETRIEVED\\n'); writeFileSync(latestDir + '/retrieve.ok', 'MEMORY_RETRIEVED\\n');\"; echo MEMORY_RETRIEVED",
+        "retrieveCommand": "set -euo pipefail; node --input-type=module -e \"import { ingestEvents, getRelevantMemories } from './src/services/memory/index.js'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; const fallbackCadence='daily'; const cadence=process.env.SCHEDULER_CADENCE || fallbackCadence; const agentId=process.env.SCHEDULER_AGENT || ('scheduler-memory-' + cadence); const promptPath=process.env.SCHEDULER_PROMPT_PATH || ''; const runId=(process.env.SCHEDULER_RUN_ID || process.env.SCHEDULER_SESSION_ID || process.env.RUN_ID || ('session-' + Date.now().toString(36))); const promptRaw=promptPath ? readFileSync(promptPath, 'utf8') : ''; const promptLines=promptRaw.split(/\\r?\\n/).map((line)=>line.trim()).filter(Boolean); const promptIntent=promptLines.find((line)=>line.startsWith('#') || line.toLowerCase().startsWith('goal')) || promptLines[0] || 'scheduler memory retrieval'; const query=['scheduler', cadence, 'agent', agentId, 'intent', promptIntent.replace(/^#+\\s*/, '')].join(' | '); const timestamp=Date.now(); const events=[{ agent_id: agentId, content: 'Memory retrieval seed for ' + cadence + ' :: ' + promptIntent, timestamp, tags:['scheduler', cadence, 'retrieve'], metadata:{ session_id:runId, source:'scheduler-retrieve', importance:0.4, prompt_path:promptPath } }]; const ingested=await ingestEvents(events,{ agent_id: agentId }); const retrieved=await getRelevantMemories({ agent_id: agentId, query, k: 5 }); const sessionDir='.scheduler-memory/' + runId; const latestDir='.scheduler-memory/latest/' + cadence; mkdirSync(sessionDir,{ recursive:true }); mkdirSync(latestDir,{ recursive:true }); const artifact={ cadence, operation:'retrieve', runId, servicePath:'src/services/memory/index.js#getRelevantMemories', inputs:{ agentId, promptPath, promptIntent, query, events:events.length }, outputs:{ ingestedCount:ingested.length, retrievedCount:retrieved.length }, status:'ok' }; writeFileSync(sessionDir + '/retrieve.json', JSON.stringify(artifact, null, 2)); writeFileSync(sessionDir + '/retrieve.ok', 'MEMORY_RETRIEVED\\n'); writeFileSync(latestDir + '/retrieve.ok', 'MEMORY_RETRIEVED\\n'); writeFileSync(latestDir + '/memories.md', retrieved.map(m => '- ' + m.content).join('\\n'));\"; echo MEMORY_RETRIEVED",
         "storeCommand": "set -euo pipefail; node --input-type=module -e \"import { ingestEvents } from './src/services/memory/index.js'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; const fallbackCadence='daily'; const cadence=process.env.SCHEDULER_CADENCE || fallbackCadence; const agentId=process.env.SCHEDULER_AGENT || ('scheduler-memory-' + cadence); const promptPath=process.env.SCHEDULER_PROMPT_PATH || ''; const runId=(process.env.SCHEDULER_RUN_ID || process.env.SCHEDULER_SESSION_ID || process.env.RUN_ID || ('session-' + Date.now().toString(36))); const promptRaw=promptPath ? readFileSync(promptPath, 'utf8') : ''; const promptLines=promptRaw.split(/\\r?\\n/).map((line)=>line.trim()).filter(Boolean); const promptIntent=promptLines.find((line)=>line.startsWith('#') || line.toLowerCase().startsWith('goal')) || promptLines[0] || 'scheduler memory store'; const baseTs=Date.now(); const events=[{ agent_id: agentId, content: 'Store memory event A for ' + cadence + ' :: ' + promptIntent, timestamp: baseTs, tags:['scheduler', cadence, 'store'], metadata:{ session_id:runId, source:'scheduler-store', importance:0.55, prompt_path:promptPath } },{ agent_id: agentId, content: 'Store memory event B for ' + cadence + ' :: ' + promptIntent, timestamp: baseTs + 1, tags:['scheduler', cadence, 'store'], metadata:{ session_id:runId, source:'scheduler-store', importance:0.55, prompt_path:promptPath } }]; const stored=await ingestEvents(events,{ agent_id: agentId }); const sessionDir='.scheduler-memory/' + runId; const latestDir='.scheduler-memory/latest/' + cadence; mkdirSync(sessionDir,{ recursive:true }); mkdirSync(latestDir,{ recursive:true }); const artifact={ cadence, operation:'store', runId, servicePath:'src/services/memory/index.js#ingestEvents', inputs:{ agentId, promptPath, promptIntent, events:events.length }, outputs:{ storedCount:stored.length, summaries:stored.map((m)=>m.summary) }, status:'ok' }; writeFileSync(sessionDir + '/store.json', JSON.stringify(artifact, null, 2)); writeFileSync(sessionDir + '/store.ok', 'MEMORY_STORED\\n'); writeFileSync(latestDir + '/store.ok', 'MEMORY_STORED\\n');\"; echo MEMORY_STORED",
         "retrieveSuccessMarkers": [
           "MEMORY_RETRIEVED"
@@ -42,7 +42,7 @@
       },
       "weekly": {
         "mode": "required",
-        "retrieveCommand": "set -euo pipefail; node --input-type=module -e \"import { ingestEvents, getRelevantMemories } from './src/services/memory/index.js'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; const fallbackCadence='weekly'; const cadence=process.env.SCHEDULER_CADENCE || fallbackCadence; const agentId=process.env.SCHEDULER_AGENT || ('scheduler-memory-' + cadence); const promptPath=process.env.SCHEDULER_PROMPT_PATH || ''; const runId=(process.env.SCHEDULER_RUN_ID || process.env.SCHEDULER_SESSION_ID || process.env.RUN_ID || ('session-' + Date.now().toString(36))); const promptRaw=promptPath ? readFileSync(promptPath, 'utf8') : ''; const promptLines=promptRaw.split(/\\r?\\n/).map((line)=>line.trim()).filter(Boolean); const promptIntent=promptLines.find((line)=>line.startsWith('#') || line.toLowerCase().startsWith('goal')) || promptLines[0] || 'scheduler memory retrieval'; const query=['scheduler', cadence, 'agent', agentId, 'intent', promptIntent.replace(/^#+\\s*/, '')].join(' | '); const timestamp=Date.now(); const events=[{ agent_id: agentId, content: 'Memory retrieval seed for ' + cadence + ' :: ' + promptIntent, timestamp, tags:['scheduler', cadence, 'retrieve'], metadata:{ session_id:runId, source:'scheduler-retrieve', importance:0.4, prompt_path:promptPath } }]; const ingested=await ingestEvents(events,{ agent_id: agentId }); const retrieved=await getRelevantMemories({ agent_id: agentId, query, k: 5 }); const sessionDir='.scheduler-memory/' + runId; const latestDir='.scheduler-memory/latest/' + cadence; mkdirSync(sessionDir,{ recursive:true }); mkdirSync(latestDir,{ recursive:true }); const artifact={ cadence, operation:'retrieve', runId, servicePath:'src/services/memory/index.js#getRelevantMemories', inputs:{ agentId, promptPath, promptIntent, query, events:events.length }, outputs:{ ingestedCount:ingested.length, retrievedCount:retrieved.length }, status:'ok' }; writeFileSync(sessionDir + '/retrieve.json', JSON.stringify(artifact, null, 2)); writeFileSync(sessionDir + '/retrieve.ok', 'MEMORY_RETRIEVED\\n'); writeFileSync(latestDir + '/retrieve.ok', 'MEMORY_RETRIEVED\\n');\"; echo MEMORY_RETRIEVED",
+        "retrieveCommand": "set -euo pipefail; node --input-type=module -e \"import { ingestEvents, getRelevantMemories } from './src/services/memory/index.js'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; const fallbackCadence='weekly'; const cadence=process.env.SCHEDULER_CADENCE || fallbackCadence; const agentId=process.env.SCHEDULER_AGENT || ('scheduler-memory-' + cadence); const promptPath=process.env.SCHEDULER_PROMPT_PATH || ''; const runId=(process.env.SCHEDULER_RUN_ID || process.env.SCHEDULER_SESSION_ID || process.env.RUN_ID || ('session-' + Date.now().toString(36))); const promptRaw=promptPath ? readFileSync(promptPath, 'utf8') : ''; const promptLines=promptRaw.split(/\\r?\\n/).map((line)=>line.trim()).filter(Boolean); const promptIntent=promptLines.find((line)=>line.startsWith('#') || line.toLowerCase().startsWith('goal')) || promptLines[0] || 'scheduler memory retrieval'; const query=['scheduler', cadence, 'agent', agentId, 'intent', promptIntent.replace(/^#+\\s*/, '')].join(' | '); const timestamp=Date.now(); const events=[{ agent_id: agentId, content: 'Memory retrieval seed for ' + cadence + ' :: ' + promptIntent, timestamp, tags:['scheduler', cadence, 'retrieve'], metadata:{ session_id:runId, source:'scheduler-retrieve', importance:0.4, prompt_path:promptPath } }]; const ingested=await ingestEvents(events,{ agent_id: agentId }); const retrieved=await getRelevantMemories({ agent_id: agentId, query, k: 5 }); const sessionDir='.scheduler-memory/' + runId; const latestDir='.scheduler-memory/latest/' + cadence; mkdirSync(sessionDir,{ recursive:true }); mkdirSync(latestDir,{ recursive:true }); const artifact={ cadence, operation:'retrieve', runId, servicePath:'src/services/memory/index.js#getRelevantMemories', inputs:{ agentId, promptPath, promptIntent, query, events:events.length }, outputs:{ ingestedCount:ingested.length, retrievedCount:retrieved.length }, status:'ok' }; writeFileSync(sessionDir + '/retrieve.json', JSON.stringify(artifact, null, 2)); writeFileSync(sessionDir + '/retrieve.ok', 'MEMORY_RETRIEVED\\n'); writeFileSync(latestDir + '/retrieve.ok', 'MEMORY_RETRIEVED\\n'); writeFileSync(latestDir + '/memories.md', retrieved.map(m => '- ' + m.content).join('\\n'));\"; echo MEMORY_RETRIEVED",
         "storeCommand": "set -euo pipefail; node --input-type=module -e \"import { ingestEvents } from './src/services/memory/index.js'; import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'; const fallbackCadence='weekly'; const cadence=process.env.SCHEDULER_CADENCE || fallbackCadence; const agentId=process.env.SCHEDULER_AGENT || ('scheduler-memory-' + cadence); const promptPath=process.env.SCHEDULER_PROMPT_PATH || ''; const runId=(process.env.SCHEDULER_RUN_ID || process.env.SCHEDULER_SESSION_ID || process.env.RUN_ID || ('session-' + Date.now().toString(36))); const promptRaw=promptPath ? readFileSync(promptPath, 'utf8') : ''; const promptLines=promptRaw.split(/\\r?\\n/).map((line)=>line.trim()).filter(Boolean); const promptIntent=promptLines.find((line)=>line.startsWith('#') || line.toLowerCase().startsWith('goal')) || promptLines[0] || 'scheduler memory store'; const baseTs=Date.now(); const events=[{ agent_id: agentId, content: 'Store memory event A for ' + cadence + ' :: ' + promptIntent, timestamp: baseTs, tags:['scheduler', cadence, 'store'], metadata:{ session_id:runId, source:'scheduler-store', importance:0.55, prompt_path:promptPath } },{ agent_id: agentId, content: 'Store memory event B for ' + cadence + ' :: ' + promptIntent, timestamp: baseTs + 1, tags:['scheduler', cadence, 'store'], metadata:{ session_id:runId, source:'scheduler-store', importance:0.55, prompt_path:promptPath } }]; const stored=await ingestEvents(events,{ agent_id: agentId }); const sessionDir='.scheduler-memory/' + runId; const latestDir='.scheduler-memory/latest/' + cadence; mkdirSync(sessionDir,{ recursive:true }); mkdirSync(latestDir,{ recursive:true }); const artifact={ cadence, operation:'store', runId, servicePath:'src/services/memory/index.js#ingestEvents', inputs:{ agentId, promptPath, promptIntent, events:events.length }, outputs:{ storedCount:stored.length, summaries:stored.map((m)=>m.summary) }, status:'ok' }; writeFileSync(sessionDir + '/store.json', JSON.stringify(artifact, null, 2)); writeFileSync(sessionDir + '/store.ok', 'MEMORY_STORED\\n'); writeFileSync(latestDir + '/store.ok', 'MEMORY_STORED\\n');\"; echo MEMORY_STORED",
         "retrieveSuccessMarkers": [
           "MEMORY_RETRIEVED"
@@ -59,4 +59,4 @@
       }
     }
   }
-}
+}