From 811f45b3affe3c7d1b1d5775b034f1f47417e74b Mon Sep 17 00:00:00 2001 From: thePR0M3TH3AN <53631862+PR0M3TH3AN@users.noreply.github.com> Date: Thu, 26 Feb 2026 08:18:47 -0500 Subject: [PATCH] Harden memory workflow to reduce merge conflicts --- .gitattributes | 5 ++ .../memory-architecture-report-2026-02-26.md | 66 +++++++++++++++++++ scripts/agent/prompt-safety-audit.mjs | 9 ++- scripts/agent/run-selected-prompt.mjs | 2 +- scripts/memory/store.mjs | 6 +- src/cmd-doctor.mjs | 2 +- src/ops.mjs | 2 +- src/prompts/scheduler-flow.md | 2 +- 8 files changed, 84 insertions(+), 10 deletions(-) create mode 100644 reports/protocol/memory-architecture-report-2026-02-26.md diff --git a/.gitattributes b/.gitattributes index dfe0770..ca6b180 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,7 @@ # Auto detect text files and perform LF normalization * text=auto + +# Reduce merge conflicts for append-style memory artifacts. +memory-updates/*.md merge=union +.scheduler-memory/latest/daily/memories.md merge=union +.scheduler-memory/latest/weekly/memories.md merge=union diff --git a/reports/protocol/memory-architecture-report-2026-02-26.md b/reports/protocol/memory-architecture-report-2026-02-26.md new file mode 100644 index 0000000..49f50c9 --- /dev/null +++ b/reports/protocol/memory-architecture-report-2026-02-26.md @@ -0,0 +1,66 @@ +# Memory Architecture Report - 2026-02-26 + +## Problem +Recurring merge conflicts come from shared write-hotspot files (`memory-update.md`, aggregate `memories.md`) being edited by many branches/runs. + +## Goals +- Preserve reliable short-term and long-term memory for agents. +- Keep memory artifacts traceable in git. +- Minimize manual merge work and conflict risk. + +## Core Learnings +1. Shared mutable files are conflict magnets. +2. Per-run immutable files are merge-friendly and auditable. +3. Aggregate views should be generated artifacts, not canonical sources of truth. +4. Reliable memory quality comes from deterministic ingestion and stable schema, not from one giant markdown file. + +## Recommended Structure +- Source of truth (tracked): `memory-updates/__.md` +- Optional index/metadata (tracked): machine-readable pointers (run id, agent, cadence, tags, embedding/version). +- Generated read model (tracked or regenerated): `.scheduler-memory/latest//memories.md` + +Canonical rule: +- Write path: only per-run file (`$SCHEDULER_MEMORY_FILE`). +- Read path: generated latest aggregate and/or retrieval service. +- Never write directly to shared aggregate files from agents. + +## Merge Policy +- Keep `memory-updates/*.md` append-style and use `merge=union` to reduce non-overlapping merge collisions. +- For generated aggregates (`.scheduler-memory/latest/*/memories.md`): + - Prefer keeping one side during merge, then regenerate from source files. + - Treat as a cache/read model, not primary truth. + +## Determinism and Reliability Controls +- Stable timestamp format (`YYYY-MM-DDTHH-MM-SSZ`) and agent-id naming. +- Ingestion writes structured metadata (agent, cadence, run id, tags, source path). +- Retrieval/storage must emit verifiable markers/artifacts (`MEMORY_RETRIEVED`, `MEMORY_STORED`, JSON evidence). +- Avoid retries/sleeps as correctness fixes; remove nondeterminism at source. + +## Short-Term Memory Design +- Scope by cadence and recent window in retrieval queries. +- Prioritize high-signal recent memories by recency + importance + semantic match. +- Keep run-local context in artifacts (`context/`, `todo/`, `decisions/`, `test_logs/`) and link memory entries to those artifacts. + +## Long-Term Memory Design +- Periodic consolidation jobs should summarize repeated findings into durable patterns. +- Promote stable insights to a curated long-term layer (policy/decision docs or pinned memory records). +- Keep provenance: every promoted insight references original run files. + +## Suggested Evolution Path +1. Enforce write contract: only `$SCHEDULER_MEMORY_FILE` in `memory-updates/`. +2. Regenerate aggregates from `memory-updates/` in scheduler cycle. +3. Add lightweight memory manifest for faster deterministic rebuilds. +4. Add pruning/compaction rules that preserve high-value long-term insights. +5. Add integrity checks: fail if agents write to deprecated shared files. + +## Operational Playbook for Conflicts +When conflicts involve memory files: +1. Resolve source files (`memory-updates/*`) by keeping both sides when possible. +2. Resolve aggregate files (`.scheduler-memory/latest/*/memories.md`) by choosing one side. +3. Regenerate aggregates from source files. +4. Run memory policy tests before merge completion. + +## Success Criteria +- Significant drop in memory-related merge conflicts. +- No loss in retrieval quality for recent and historical tasks. +- Rebuild of aggregate memory is deterministic from tracked source files. diff --git a/scripts/agent/prompt-safety-audit.mjs b/scripts/agent/prompt-safety-audit.mjs index fd1d956..1f2b656 100644 --- a/scripts/agent/prompt-safety-audit.mjs +++ b/scripts/agent/prompt-safety-audit.mjs @@ -8,10 +8,14 @@ const ROOT_DIR = path.resolve(__dirname, '../../'); const DAILY_DIR = path.join(ROOT_DIR, 'src/prompts/daily'); const WEEKLY_DIR = path.join(ROOT_DIR, 'src/prompts/weekly'); const REPORT_DIR = path.join(ROOT_DIR, 'reports/prompt-safety'); +const MEMORY_DIR = path.join(ROOT_DIR, 'memory-updates'); if (!fs.existsSync(REPORT_DIR)) { fs.mkdirSync(REPORT_DIR, { recursive: true }); } +if (!fs.existsSync(MEMORY_DIR)) { + fs.mkdirSync(MEMORY_DIR, { recursive: true }); +} const SAFETY_KEYWORDS = [ 'FAILURE MODES', @@ -96,8 +100,9 @@ function runAudit() { memoryContent += `Flagged prompts: ${results.filter(r => r.status === 'NEEDS_REVIEW').map(r => r.filename).join(', ')}\n`; } - fs.writeFileSync(path.join(ROOT_DIR, 'memory-update.md'), memoryContent); - console.log('Memory update written to memory-update.md'); + const memoryFile = path.join(MEMORY_DIR, `${timestamp}__prompt-safety-audit-agent.md`); + fs.writeFileSync(memoryFile, memoryContent); + console.log(`Memory update written to ${memoryFile}`); } runAudit(); diff --git a/scripts/agent/run-selected-prompt.mjs b/scripts/agent/run-selected-prompt.mjs index a622aef..58e74ba 100644 --- a/scripts/agent/run-selected-prompt.mjs +++ b/scripts/agent/run-selected-prompt.mjs @@ -69,7 +69,7 @@ async function main() { const runner = resolveRunner(promptMarkdown); // Compute a stable, per-run memory output file path so agents know where to write learnings. - // store.mjs reads this env var; if absent it falls back to the root memory-update.md placeholder. + // store.mjs reads this env var and ingests this file. const agentName = process.env.SCHEDULER_AGENT || 'unknown-agent'; const runTs = new Date().toISOString().replace(/:/g, '-').replace(/\.\d+Z$/, 'Z'); const memoryFile = `memory-updates/${runTs}__${agentName}.md`; diff --git a/scripts/memory/store.mjs b/scripts/memory/store.mjs index cf0e170..b0bd81d 100644 --- a/scripts/memory/store.mjs +++ b/scripts/memory/store.mjs @@ -35,9 +35,7 @@ const cliArgs = process.argv.slice(2); const fileArgIndex = cliArgs.indexOf('--file'); const explicitFile = fileArgIndex !== -1 ? cliArgs[fileArgIndex + 1] : null; const envFile = process.env.SCHEDULER_MEMORY_FILE; -const defaultFile = 'memory-update.md'; - -const targetFile = explicitFile || (envFile && existsSync(envFile) ? envFile : (existsSync(defaultFile) ? defaultFile : null)); +const targetFile = explicitFile || (envFile && existsSync(envFile) ? envFile : null); if (targetFile) { try { @@ -71,7 +69,7 @@ if (memoryContent) { } else { // No memory file found — skip ingest rather than storing meaningless placeholder events. // Placeholder events pollute the store and degrade retrieval quality on future runs. - console.warn(`No memory input found (checked --file, env SCHEDULER_MEMORY_FILE=${envFile}, and default ${defaultFile}). Skipping ingest.`); + console.warn(`No memory input found (checked --file and env SCHEDULER_MEMORY_FILE=${envFile}). Skipping ingest.`); } try { diff --git a/src/cmd-doctor.mjs b/src/cmd-doctor.mjs index 27320f2..5506b5e 100644 --- a/src/cmd-doctor.mjs +++ b/src/cmd-doctor.mjs @@ -4,7 +4,7 @@ import { detectTorchInstallDir } from './cmd-remove.mjs'; const MEMORY_HEADING = '## TORCH Memory Integration'; const MEMORY_READ_LINE = '`.scheduler-memory/latest/${cadence}/memories.md`'; -const MEMORY_WRITE_LINE = '`memory-update.md`'; +const MEMORY_WRITE_LINE = '`$SCHEDULER_MEMORY_FILE`'; function parseMajorNodeVersion(version) { const major = Number.parseInt(String(version).split('.')[0], 10); diff --git a/src/ops.mjs b/src/ops.mjs index 567e2a9..b573ad5 100644 --- a/src/ops.mjs +++ b/src/ops.mjs @@ -59,7 +59,7 @@ const MEMORY_INTEGRATION_HEADING = '## TORCH Memory Integration'; const MEMORY_INTEGRATION_BLOCK = `${MEMORY_INTEGRATION_HEADING} You have access to the TORCH memory system. 1. READ: Check \`.scheduler-memory/latest/\${cadence}/memories.md\` for past learnings. -2. WRITE: Before exiting, save new insights to \`memory-update.md\` so future runs can learn from this session.`; +2. WRITE: Before exiting, save new insights to \`$SCHEDULER_MEMORY_FILE\` (for example \`memory-updates/__.md\`) so future runs can learn from this session.`; /** * Resolves the absolute paths for the installation. diff --git a/src/prompts/scheduler-flow.md b/src/prompts/scheduler-flow.md index e79d804..c3fda6e 100644 --- a/src/prompts/scheduler-flow.md +++ b/src/prompts/scheduler-flow.md @@ -57,7 +57,7 @@ Every agent prompt invoked by the schedulers (daily/weekly) MUST enforce this co - Write only facts that will help the **same agent** on its **next run**. Skip meta-commentary about the task structure. - Run configured memory storage workflow after prompt execution (for example via `scheduler.memoryPolicyByCadence..storeCommand`) - Storage command MUST call real memory services (`src/services/memory/index.js#ingestEvents`, which uses ingestor/summarizer pipeline) and MUST emit deterministic marker `MEMORY_STORED`. - - Storage command MUST ingest content from the file at `$SCHEDULER_MEMORY_FILE` (or fallback to `memory-update.md` if present). + - Storage command MUST ingest content from the file at `$SCHEDULER_MEMORY_FILE`. - Storage command MUST write cadence-scoped evidence artifacts: - `.scheduler-memory/store-.ok` - `.scheduler-memory/store-.json` containing operation inputs/outputs (`agentId`, input event count, stored count, generated summaries).