TORCH/scripts/agent/prompt-safety-audit.mjs

import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const ROOT_DIR = path.resolve(__dirname, '../../');
const DAILY_DIR = path.join(ROOT_DIR, 'src/prompts/daily');
const WEEKLY_DIR = path.join(ROOT_DIR, 'src/prompts/weekly');
const REPORT_DIR = path.join(ROOT_DIR, 'reports/prompt-safety');
const MEMORY_DIR = path.join(ROOT_DIR, 'memory-updates');

if (!fs.existsSync(REPORT_DIR)) {
  fs.mkdirSync(REPORT_DIR, { recursive: true });
}
if (!fs.existsSync(MEMORY_DIR)) {
  fs.mkdirSync(MEMORY_DIR, { recursive: true });
}

const SAFETY_KEYWORDS = [
  'FAILURE MODES',
  'EXIT CRITERIA',
  'SKIP',
  'NO-OP',
  'STOP IF',
  'DO NOTHING',
  'NO ACTION'
];

const FORCEFUL_KEYWORDS = [
  'MUST',
  'ALWAYS',
  'FORCE',
  'REQUIRED'
];

function analyzeFile(filepath) {
  const content = fs.readFileSync(filepath, 'utf8');
  const upperContent = content.toUpperCase();
  const filename = path.basename(filepath);

  const hasSafetySection = SAFETY_KEYWORDS.some(k => upperContent.includes(k));

  // Simple heuristic for forceful language without mitigation
  // This is a naive check and might have false positives
  let forcefulCount = 0;
  FORCEFUL_KEYWORDS.forEach(k => {
    const regex = new RegExp(`\\b${k}\\b`, 'g');
    const matches = (upperContent.match(regex) || []).length;
    forcefulCount += matches;
  });

  return {
    filename,
    hasSafetySection,
    forcefulCount,
    status: hasSafetySection ? 'SAFE' : 'NEEDS_REVIEW'
  };
}

function runAudit() {
  const files = [
    ...fs.readdirSync(DAILY_DIR).map(f => path.join(DAILY_DIR, f)),
    ...fs.readdirSync(WEEKLY_DIR).map(f => path.join(WEEKLY_DIR, f))
  ].filter(f => f.endsWith('.md'));

  const results = files.map(analyzeFile);

  const safeCount = results.filter(r => r.status === 'SAFE').length;
  const needsReviewCount = results.filter(r => r.status === 'NEEDS_REVIEW').length;
  const totalCount = results.length;

  const timestamp = new Date().toISOString().replace(/:/g, '-').split('.')[0];
  const reportFile = path.join(REPORT_DIR, `audit-${timestamp}.md`);

  let reportContent = `# Prompt Safety Audit Report - ${timestamp}\n\n`;
  reportContent += `**Total Prompts:** ${totalCount}\n`;
  reportContent += `**Safe:** ${safeCount}\n`;
  reportContent += `**Needs Review:** ${needsReviewCount}\n\n`;

  reportContent += `## Findings\n\n`;
  reportContent += `| Filename | Status | Has Safety Section | Forceful Keywords (Approx) |\n`;
  reportContent += `|---|---|---|---|\n`;

  results.forEach(r => {
    reportContent += `| ${r.filename} | ${r.status} | ${r.hasSafetySection ? 'Yes' : 'No'} | ${r.forcefulCount} |\n`;
  });

  reportContent += `\n## Recommendations\n`;
  reportContent += `- Review prompts marked 'NEEDS_REVIEW'.\n`;
  reportContent += `- Ensure all prompts have a 'FAILURE MODES' or 'EXIT CRITERIA' section.\n`;

  fs.writeFileSync(reportFile, reportContent);
  console.log(`Report written to ${reportFile}`);

  // Memory Update
  let memoryContent = `Goal: Audit prompt safety.\n`;
  memoryContent += `Result: Audited ${totalCount} prompts. ${safeCount} safe, ${needsReviewCount} need review.\n`;
  if (needsReviewCount > 0) {
      memoryContent += `Flagged prompts: ${results.filter(r => r.status === 'NEEDS_REVIEW').map(r => r.filename).join(', ')}\n`;
  }

  const memoryFile = path.join(MEMORY_DIR, `${timestamp}__prompt-safety-audit-agent.md`);
  fs.writeFileSync(memoryFile, memoryContent);
  console.log(`Memory update written to ${memoryFile}`);
}

runAudit();