feat: Weekly Prompt Safety Audit

Co-authored-by: PR0M3TH3AN <[email protected]>

scripts/agent/prompt-safety-audit.mjs

@@ -0,0 +1,103 @@
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT_DIR = path.resolve(__dirname, '../../');
+const DAILY_DIR = path.join(ROOT_DIR, 'src/prompts/daily');
+const WEEKLY_DIR = path.join(ROOT_DIR, 'src/prompts/weekly');
+const REPORT_DIR = path.join(ROOT_DIR, 'reports/prompt-safety');
+
+if (!fs.existsSync(REPORT_DIR)) {
+  fs.mkdirSync(REPORT_DIR, { recursive: true });
+}
+
+const SAFETY_KEYWORDS = [
+  'FAILURE MODES',
+  'EXIT CRITERIA',
+  'SKIP',
+  'NO-OP',
+  'STOP IF',
+  'DO NOTHING',
+  'NO ACTION'
+];
+
+const FORCEFUL_KEYWORDS = [
+  'MUST',
+  'ALWAYS',
+  'FORCE',
+  'REQUIRED'
+];
+
+function analyzeFile(filepath) {
+  const content = fs.readFileSync(filepath, 'utf8');
+  const upperContent = content.toUpperCase();
+  const filename = path.basename(filepath);
+
+  const hasSafetySection = SAFETY_KEYWORDS.some(k => upperContent.includes(k));
+
+  // Simple heuristic for forceful language without mitigation
+  // This is a naive check and might have false positives
+  let forcefulCount = 0;
+  FORCEFUL_KEYWORDS.forEach(k => {
+    const regex = new RegExp(`\\b${k}\\b`, 'g');
+    const matches = (upperContent.match(regex) || []).length;
+    forcefulCount += matches;
+  });
+
+  return {
+    filename,
+    hasSafetySection,
+    forcefulCount,
+    status: hasSafetySection ? 'SAFE' : 'NEEDS_REVIEW'
+  };
+}
+
+function runAudit() {
+  const files = [
+    ...fs.readdirSync(DAILY_DIR).map(f => path.join(DAILY_DIR, f)),
+    ...fs.readdirSync(WEEKLY_DIR).map(f => path.join(WEEKLY_DIR, f))
+  ].filter(f => f.endsWith('.md'));
+
+  const results = files.map(analyzeFile);
+
+  const safeCount = results.filter(r => r.status === 'SAFE').length;
+  const needsReviewCount = results.filter(r => r.status === 'NEEDS_REVIEW').length;
+  const totalCount = results.length;
+
+  const timestamp = new Date().toISOString().replace(/:/g, '-').split('.')[0];
+  const reportFile = path.join(REPORT_DIR, `audit-${timestamp}.md`);
+
+  let reportContent = `# Prompt Safety Audit Report - ${timestamp}\n\n`;
+  reportContent += `**Total Prompts:** ${totalCount}\n`;
+  reportContent += `**Safe:** ${safeCount}\n`;
+  reportContent += `**Needs Review:** ${needsReviewCount}\n\n`;
+
+  reportContent += `## Findings\n\n`;
+  reportContent += `| Filename | Status | Has Safety Section | Forceful Keywords (Approx) |\n`;
+  reportContent += `|---|---|---|---|\n`;
+
+  results.forEach(r => {
+    reportContent += `| ${r.filename} | ${r.status} | ${r.hasSafetySection ? 'Yes' : 'No'} | ${r.forcefulCount} |\n`;
+  });
+
+  reportContent += `\n## Recommendations\n`;
+  reportContent += `- Review prompts marked 'NEEDS_REVIEW'.\n`;
+  reportContent += `- Ensure all prompts have a 'FAILURE MODES' or 'EXIT CRITERIA' section.\n`;
+
+  fs.writeFileSync(reportFile, reportContent);
+  console.log(`Report written to ${reportFile}`);
+
+  // Memory Update
+  let memoryContent = `Goal: Audit prompt safety.\n`;
+  memoryContent += `Result: Audited ${totalCount} prompts. ${safeCount} safe, ${needsReviewCount} need review.\n`;
+  if (needsReviewCount > 0) {
+      memoryContent += `Flagged prompts: ${results.filter(r => r.status === 'NEEDS_REVIEW').map(r => r.filename).join(', ')}\n`;
+  }
+
+  fs.writeFileSync(path.join(ROOT_DIR, 'memory-update.md'), memoryContent);
+  console.log('Memory update written to memory-update.md');
+}
+
+runAudit();