Merge branch 'beta' into main (safe conflict resolution)

torch/src/services/governance/index.js

@@ -0,0 +1,288 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { execSync } from 'node:child_process';
+import { createHash } from 'node:crypto';
+
+const PROPOSALS_DIR = path.resolve(process.cwd(), 'src/proposals');
+const HISTORY_DIR = path.resolve(process.cwd(), '.torch/prompt-history');
+const ALLOWED_TARGET_DIRS = [
+  path.resolve(process.cwd(), 'src/prompts/daily'),
+  path.resolve(process.cwd(), 'src/prompts/weekly'),
+];
+
+async function ensureDirs() {
+  await fs.mkdir(PROPOSALS_DIR, { recursive: true });
+  await fs.mkdir(HISTORY_DIR, { recursive: true });
+}
+
+export async function listProposals() {
+  await ensureDirs();
+  const entries = await fs.readdir(PROPOSALS_DIR, { withFileTypes: true });
+  const proposals = [];
+
+  for (const entry of entries) {
+    if (entry.isDirectory()) {
+      const metaPath = path.join(PROPOSALS_DIR, entry.name, 'meta.json');
+      try {
+        const metaContent = await fs.readFile(metaPath, 'utf8');
+        const meta = JSON.parse(metaContent);
+        proposals.push({
+          id: entry.name,
+          ...meta
+        });
+      } catch (e) {
+        // Ignore invalid proposals
+      }
+    }
+  }
+  return proposals;
+}
+
+export async function getProposal(id) {
+  const dir = path.join(PROPOSALS_DIR, id);
+  const metaPath = path.join(dir, 'meta.json');
+  const newPath = path.join(dir, 'new.md');
+  const diffPath = path.join(dir, 'change.diff');
+
+  try {
+    const meta = JSON.parse(await fs.readFile(metaPath, 'utf8'));
+    const newContent = await fs.readFile(newPath, 'utf8');
+    let diff = '';
+    try {
+      diff = await fs.readFile(diffPath, 'utf8');
+    } catch {
+      // Ignore missing diff
+    }
+
+    return {
+      id,
+      meta,
+      newContent,
+      diff,
+      dir
+    };
+  } catch (e) {
+    throw new Error(`Proposal ${id} not found or invalid: ${e.message}`, { cause: e });
+  }
+}
+
+export async function createProposal({ agent, target, newContent, reason }) {
+  await ensureDirs();
+
+  // Validate target path is allowed
+  const absoluteTarget = path.resolve(process.cwd(), target);
+  const isAllowed = ALLOWED_TARGET_DIRS.some(allowed => absoluteTarget.startsWith(allowed));
+
+  if (!isAllowed) {
+    throw new Error(`Target ${target} is not in an allowed directory (src/prompts/daily or src/prompts/weekly).`);
+  }
+
+  // Generate ID
+  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+  const id = `${timestamp}_${agent}`;
+  const dir = path.join(PROPOSALS_DIR, id);
+
+  await fs.mkdir(dir, { recursive: true });
+
+  // Write content
+  await fs.writeFile(path.join(dir, 'new.md'), newContent);
+
+  // Generate Diff if target exists
+  let diff = '';
+  try {
+    // Check if target exists
+    await fs.access(absoluteTarget);
+
+    // Create a temporary file for the diff logic to work cleanly
+    // But direct path is fine.
+    const newFile = path.join(dir, 'new.md');
+    const diffCmd = `git diff --no-index --color=never "${absoluteTarget}" "${newFile}"`;
+    try {
+      execSync(diffCmd, { stdio: 'pipe' }); // Will throw if exit code 1 (diff found)
+    } catch (e) {
+      if (e.status === 1 && e.stdout) {
+        diff = e.stdout.toString();
+      } else if (e.status !== 1) {
+          // Some other error
+          console.error('Diff generation failed:', e.message);
+      }
+    }
+  } catch (err) {
+    diff = '(New File)';
+  }
+
+  await fs.writeFile(path.join(dir, 'change.diff'), diff);
+
+  const meta = {
+    id,
+    author: agent,
+    target,
+    reason,
+    timestamp: new Date().toISOString(),
+    status: 'pending'
+  };
+
+  await fs.writeFile(path.join(dir, 'meta.json'), JSON.stringify(meta, null, 2));
+
+  return { id, diff };
+}
+
+export async function validateProposal(id) {
+  const { meta, newContent } = await getProposal(id);
+
+  // 1. Allowlist check (redundant but safe)
+  const absoluteTarget = path.resolve(process.cwd(), meta.target);
+  const isAllowed = ALLOWED_TARGET_DIRS.some(allowed => absoluteTarget.startsWith(allowed));
+  if (!isAllowed) {
+    return { valid: false, reason: 'Target not in allowed directories.' };
+  }
+
+  // 2. Invariant checks
+  // Check for required headers based on common prompt structure
+  const requiredPatterns = [
+    /Shared contract \(required\):/i,
+    /Required startup \+ artifacts \+ memory \+ issue capture/i,
+  ];
+
+  for (const pattern of requiredPatterns) {
+    if (!pattern.test(newContent)) {
+       return { valid: false, reason: `Missing required header pattern: ${pattern}` };
+    }
+  }
+
+  return { valid: true };
+}
+
+export async function applyProposal(id) {
+  const proposal = await getProposal(id);
+  const { meta, newContent, dir } = proposal;
+
+  if (meta.status !== 'pending') {
+    throw new Error(`Proposal is ${meta.status}, cannot apply.`);
+  }
+
+  const validation = await validateProposal(id);
+  if (!validation.valid) {
+    // Mark as rejected if validation fails?
+    // Or just throw? The governance agent should decide.
+    // We throw here to prevent accidental application.
+    throw new Error(`Validation failed: ${validation.reason}`);
+  }
+
+  const absoluteTarget = path.resolve(process.cwd(), meta.target);
+
+  // Archive old
+  try {
+    const oldContent = await fs.readFile(absoluteTarget, 'utf8');
+    const hash = createHash('sha256').update(oldContent).digest('hex');
+
+    // We store archives in .torch/prompt-history/<target_path>/<hash>.md
+    // Note: meta.target is relative path like 'src/prompts/daily/agent.md'
+    // We want to keep that structure.
+    const archiveDir = path.join(HISTORY_DIR, path.dirname(meta.target));
+    await fs.mkdir(archiveDir, { recursive: true });
+
+    // Add timestamp to filename for easier sorting later?
+    // Or just rely on hash and store metadata elsewhere?
+    // User requested <path>/<hash>.md.
+    const archivePath = path.join(archiveDir, `${path.basename(meta.target, '.md')}_${hash}.md`);
+    await fs.writeFile(archivePath, oldContent);
+  } catch (e) {
+    // If file didn't exist, nothing to archive
+  }
+
+  // Apply new
+  await fs.mkdir(path.dirname(absoluteTarget), { recursive: true });
+  await fs.writeFile(absoluteTarget, newContent);
+
+  // Update meta
+  meta.status = 'applied';
+  meta.appliedAt = new Date().toISOString();
+  await fs.writeFile(path.join(dir, 'meta.json'), JSON.stringify(meta, null, 2));
+
+  // Git Commit (optional)
+  try {
+    // We use execSync for git operations
+    // Ensure we are in repo root? cwd is repo root.
+    execSync(`git add "${meta.target}"`);
+    const commitMsg = `feat(prompts): apply proposal ${id} by ${meta.author}`;
+    execSync(`git commit -m "${commitMsg}"`);
+    meta.gitCommit = execSync('git rev-parse HEAD').toString().trim();
+    await fs.writeFile(path.join(dir, 'meta.json'), JSON.stringify(meta, null, 2));
+  } catch (gitErr) {
+    console.warn('Git commit failed (ignoring):', gitErr.message);
+  }
+
+  return { success: true };
+}
+
+export async function rejectProposal(id, reason) {
+  const proposal = await getProposal(id);
+  const { meta, dir } = proposal;
+
+  meta.status = 'rejected';
+  meta.rejectionReason = reason;
+  meta.rejectedAt = new Date().toISOString();
+
+  await fs.writeFile(path.join(dir, 'meta.json'), JSON.stringify(meta, null, 2));
+  return { success: true };
+}
+
+export async function rollbackPrompt(target, hashOrStrategy = 'latest') {
+  // target is relative path e.g. src/prompts/daily/agent.md
+  const archiveDir = path.join(HISTORY_DIR, path.dirname(target));
+  const absoluteTarget = path.resolve(process.cwd(), target);
+
+  let sourceContent = null;
+  let sourceName = null;
+
+  try {
+    // Try Local Archive first
+    if (hashOrStrategy === 'latest') {
+       // Find latest file in archiveDir
+       // Our naming is <agent>_<hash>.md.
+       // We can rely on file mtime.
+       const files = await fs.readdir(archiveDir);
+       const targetBase = path.basename(target, '.md');
+       const prefix = targetBase + '_';
+       const candidates = files.filter(f => f.startsWith(prefix));
+
+       if (candidates.length > 0) {
+           const stats = await Promise.all(candidates.map(async f => ({
+               name: f,
+               ...await fs.stat(path.join(archiveDir, f))
+           })));
+           const latest = stats.sort((a, b) => b.mtimeMs - a.mtimeMs)[0];
+           sourceName = latest.name;
+           sourceContent = await fs.readFile(path.join(archiveDir, sourceName), 'utf8');
+       }
+    } else {
+        // Try to find file containing hash
+        const files = await fs.readdir(archiveDir);
+        const match = files.find(f => f.includes(hashOrStrategy));
+        if (match) {
+            sourceName = match;
+            sourceContent = await fs.readFile(path.join(archiveDir, match), 'utf8');
+        }
+    }
+  } catch (e) {
+      // Archive lookup failed
+  }
+
+  if (sourceContent) {
+      await fs.writeFile(absoluteTarget, sourceContent);
+      return { success: true, source: 'archive', restored: sourceName };
+  }
+
+  // Fallback to Git
+  try {
+      const commit = hashOrStrategy === 'latest' ? 'HEAD' : hashOrStrategy;
+      // Use git show instead of checkout to avoid detaching head if we just want content
+      // But user said "git checkout" implying revert state.
+      // Actually `git checkout <commit> -- <path>` overwrites the file in working tree.
+      execSync(`git checkout "${commit}" -- "${target}"`);
+      return { success: true, source: 'git', restored: commit };
+  } catch (e) {
+      throw new Error(`Rollback failed: Local archive not found and git failed (${e.message})`, { cause: e });
+  }
+}

torch/src/services/memory/index.js

@@ -6,8 +6,11 @@ import { startScheduler } from './scheduler.js';
 import { getMemoryPruneMode, isMemoryIngestEnabled } from './feature-flags.js';
 import fs from 'node:fs';
 import path from 'node:path';
+import util from 'node:util';
+import { ensureDir } from '../../utils.mjs';
 
 const MEMORY_FILE_PATH = path.join(process.cwd(), '.scheduler-memory', 'memory-store.json');
+const debug = util.debuglog('torch-memory');
 
 function loadMemoryStore() {
   try {
@@ -24,26 +27,52 @@ function loadMemoryStore() {
   return new Map();
 }
 
-function saveMemoryStore(store) {
+let currentSavePromise = null;
+let pendingSavePromise = null;
+let pendingSaveResolve = null;
+
+async function performSave(store) {
   try {
     const dir = path.dirname(MEMORY_FILE_PATH);
-    if (!fs.existsSync(dir)) {
-      fs.mkdirSync(dir, { recursive: true });
-    }
+    await fs.promises.mkdir(dir, { recursive: true });
     const entries = [...store.entries()];
-    fs.writeFileSync(MEMORY_FILE_PATH, JSON.stringify(entries, null, 2), 'utf8');
+    await fs.promises.writeFile(MEMORY_FILE_PATH, JSON.stringify(entries, null, 2), 'utf8');
   } catch (err) {
     console.error('Failed to save memory store:', err);
+  } finally {
+    currentSavePromise = null;
+    if (pendingSavePromise) {
+      const resolve = pendingSaveResolve;
+      pendingSavePromise = null;
+      pendingSaveResolve = null;
+      currentSavePromise = performSave(store).then(() => resolve());
+    }
   }
 }
 
+async function saveMemoryStore(store) {
+  if (pendingSavePromise) {
+    return pendingSavePromise;
+  }
+
+  if (currentSavePromise) {
+    pendingSavePromise = new Promise((resolve) => {
+      pendingSaveResolve = resolve;
+    });
+    return pendingSavePromise;
+  }
+
+  currentSavePromise = performSave(store);
+  return currentSavePromise;
+}
+
 const memoryStore = loadMemoryStore();
 const cache = createMemoryCache();
 
 const memoryRepository = {
   async insertMemory(memory) {
     memoryStore.set(memory.id, memory);
-    saveMemoryStore(memoryStore);
+    await saveMemoryStore(memoryStore);
     return memory;
   },
   async updateMemoryUsage(id, lastSeen = Date.now()) {
@@ -51,7 +80,7 @@ const memoryRepository = {
     if (!existing) return null;
     const updated = { ...existing, last_seen: lastSeen };
     memoryStore.set(id, updated);
-    saveMemoryStore(memoryStore);
+    await saveMemoryStore(memoryStore);
     return updated;
   },
   async listPruneCandidates({ cutoff }) {
@@ -61,7 +90,7 @@ const memoryRepository = {
     const existing = memoryStore.get(id);
     if (!existing) return false;
     memoryStore.set(id, { ...existing, merged_into: mergedInto, last_seen: Date.now() });
-    saveMemoryStore(memoryStore);
+    await saveMemoryStore(memoryStore);
     return true;
   },
   async setPinned(id, pinned) {
@@ -69,7 +98,7 @@ const memoryRepository = {
     if (!existing) return null;
     const updated = { ...existing, pinned, last_seen: Date.now() };
     memoryStore.set(id, updated);
-    saveMemoryStore(memoryStore);
+    await saveMemoryStore(memoryStore);
     return updated;
   },
   async getMemoryById(id) {
@@ -113,7 +142,7 @@ function buildTelemetryEmitter(options = {}) {
   }
 
   return (event, payload) => {
-    console.log('memory_telemetry', {
+    debug('memory_telemetry', {
       event,
       payload: toSafeTelemetryPayload(payload),
       ts: Date.now(),
@@ -132,7 +161,7 @@ function emitMetric(options = {}, metric, payload) {
     return;
   }
 
-  console.log('memory_metric', { metric, payload, ts: Date.now() });
+  debug('memory_metric', { metric, payload, ts: Date.now() });
 }
 
 function applyMemoryFilters(memories, filters = {}) {
@@ -196,10 +225,15 @@ export async function ingestEvents(events, options = {}) {
  * @returns {Promise<import('./schema.js').MemoryRecord[]>} Sorted list of relevant memories.
  */
 export async function getRelevantMemories(params) {
-  const { repository = memoryRepository, ...queryParams } = params;
+  const {
+    repository = memoryRepository,
+    ranker = filterAndRankMemories,
+    cache: cacheProvider = cache,
+    ...queryParams
+  } = params;
   const telemetry = buildTelemetryEmitter(params);
   const cacheKey = JSON.stringify(queryParams);
-  const cached = cache.get(cacheKey);
+  const cached = cacheProvider.get(cacheKey);
   if (cached) {
     telemetry('memory:retrieved', {
       agent_id: queryParams.agent_id,
@@ -214,9 +248,13 @@ export async function getRelevantMemories(params) {
     return cached;
   }
 
-  const ranked = await filterAndRankMemories([...memoryStore.values()], queryParams);
+  const source = typeof repository.listMemories === 'function'
+    ? await repository.listMemories(queryParams)
+    : [...memoryStore.values()];
+
+  const ranked = await ranker(source, queryParams);
   await updateMemoryUsage(repository, ranked.map((memory) => memory.id));
-  cache.set(cacheKey, ranked);
+  cacheProvider.set(cacheKey, ranked);
 
   telemetry('memory:retrieved', {
     agent_id: queryParams.agent_id,
@@ -275,7 +313,7 @@ export async function runPruneCycle(options = {}) {
   }
 
   if (prunable.length > 0) {
-    saveMemoryStore(memoryStore);
+    await saveMemoryStore(memoryStore);
   }
 
   memoryStatsState.deleted += prunable.length;
@@ -330,8 +368,9 @@ export async function unpinMemory(id, options = {}) {
  * @param {string} id
  * @param {string} mergedInto
  */
-export async function markMemoryMerged(id, mergedInto) {
-  const merged = await memoryRepository.markMerged(id, mergedInto);
+export async function markMemoryMerged(id, mergedInto, options = {}) {
+  const repository = options.repository ?? memoryRepository;
+  const merged = await repository.markMerged(id, mergedInto);
   cache.clear();
   return merged;
 }

torch/src/services/memory/pruner.js

@@ -67,22 +67,29 @@ function evaluateCandidate(memory, policy) {
  * @param {{ getEmbedding?: (memory: import('./schema.js').MemoryRecord) => number[] | null | undefined, similarityThreshold: number, duplicateWindowMs: number }} options
  */
 function groupNearDuplicates(memories, options) {
+  // Sort by created_at to enable early exit in the inner loop
+  const sortedMemories = [...memories].sort((a, b) => a.created_at - b.created_at);
+
   const groups = [];
   const consumed = new Set();
 
-  for (let i = 0; i < memories.length; i += 1) {
-    const base = memories[i];
+  for (let i = 0; i < sortedMemories.length; i += 1) {
+    const base = sortedMemories[i];
     if (consumed.has(base.id) || base.merged_into) continue;
 
     const baseEmbedding = options.getEmbedding?.(base);
     const group = [base];
 
-    for (let j = i + 1; j < memories.length; j += 1) {
-      const probe = memories[j];
+    for (let j = i + 1; j < sortedMemories.length; j += 1) {
+      const probe = sortedMemories[j];
+
+      // Optimization: Since sorted by created_at, if the time difference exceeds the window,
+      // all subsequent items will also be outside the window.
+      if (probe.created_at - base.created_at > options.duplicateWindowMs) break;
+
       if (consumed.has(probe.id) || probe.merged_into || probe.id === base.id) continue;
 
-      const withinWindow = Math.abs(base.created_at - probe.created_at) <= options.duplicateWindowMs;
-      if (!withinWindow || !hasTagOverlap(base.tags, probe.tags)) continue;
+      if (!hasTagOverlap(base.tags, probe.tags)) continue;
 
       const similarity = cosineSimilarity(baseEmbedding, options.getEmbedding?.(probe));
       if (similarity >= options.similarityThreshold) {
@@ -189,12 +196,13 @@ export async function createLifecyclePlan(memories, options) {
   });
 
   const templates = loadMemoryPromptTemplates({ templateDir: options.templateDir });
-  const condensedGroups = [];
+  const condensedGroups = await Promise.all(
+    groups.map((group) => condenseGroup(group, options.generateSummary, templates.condense))
+  );
+
   const mergedIntoMap = new Map();
 
-  for (const group of groups) {
-    const condensed = await condenseGroup(group, options.generateSummary, templates.condense);
-    condensedGroups.push(condensed);
+  for (const condensed of condensedGroups) {
     for (const mergedId of condensed.mergedIds) {
       mergedIntoMap.set(mergedId, condensed.merged.id);
     }
@@ -265,9 +273,7 @@ export async function createLifecyclePlan(memories, options) {
  * @param {Awaited<ReturnType<typeof createLifecyclePlan>>} plan
  */
 export async function applyLifecycleActions(repository, plan) {
-  const results = [];
-
-  for (const decision of plan.actions) {
+  const results = await Promise.all(plan.actions.map(async (decision) => {
     if (decision.merged_into && typeof repository.markMerged === 'function') {
       await repository.markMerged(decision.id, decision.merged_into);
     }
@@ -280,8 +286,8 @@ export async function applyLifecycleActions(repository, plan) {
       await repository.keepMemory?.(decision.id, decision.reason);
     }
 
-    results.push(decision);
-  }
+    return decision;
+  }));
 
   return {
     applied: results,

torch/src/services/memory/retriever.js

@@ -31,17 +31,20 @@ function computeRecencyDecay(timestamp, now, halfLifeMs) {
   return Math.exp(-(age / halfLifeMs));
 }
 
-function textMatchSimilarity(query, memory) {
-  const normalizedQuery = String(query ?? '').trim().toLowerCase();
-  if (!normalizedQuery) return 0;
+function escapeRegExp(string) {
+  return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
 
-  const haystack = `${memory.summary} ${memory.content}`.toLowerCase();
-  if (haystack.includes(normalizedQuery)) return 1;
+function textMatchSimilarity(queryTerms, queryRegexes, memory) {
+  if (!queryTerms || queryTerms.length === 0) return 0;
 
-  const queryTerms = normalizedQuery.split(/\s+/).filter(Boolean);
-  if (queryTerms.length === 0) return 0;
+  let matchedTerms = 0;
+  for (const regex of queryRegexes) {
+    if (regex.test(memory.summary) || regex.test(memory.content)) {
+      matchedTerms++;
+    }
+  }
 
-  const matchedTerms = queryTerms.filter((term) => haystack.includes(term)).length;
   return matchedTerms / queryTerms.length;
 }
 
@@ -102,8 +105,11 @@ async function buildSimilarityIndex(memories, params) {
     }
   }
 
+  const queryTerms = normalizedQuery.toLowerCase().split(/\s+/).filter(Boolean);
+  const queryRegexes = queryTerms.map((t) => new RegExp(escapeRegExp(t), 'i'));
+
   for (const memory of memories) {
-    const lexicalScore = textMatchSimilarity(normalizedQuery, memory);
+    const lexicalScore = textMatchSimilarity(queryTerms, queryRegexes, memory);
     if (lexicalScore > 0) {
       similarityById.set(memory.id, lexicalScore);
     }