From 909a7f9541ece5ccf103fbffe57028141d622050 Mon Sep 17 00:00:00 2001 From: thePR0M3TH3AN <53631862+PR0M3TH3AN@users.noreply.github.com> Date: Sat, 14 Feb 2026 18:05:54 -0500 Subject: [PATCH] Strengthen run-artifact verification metadata and issue linkage --- scripts/agent/run-scheduler-cycle.mjs | 8 ++ scripts/agent/verify-run-artifacts.mjs | 187 +++++++++++++++++++++---- test/verify-run-artifacts.test.mjs | 104 ++++++++++++++ 3 files changed, 275 insertions(+), 24 deletions(-) create mode 100644 test/verify-run-artifacts.test.mjs diff --git a/scripts/agent/run-scheduler-cycle.mjs b/scripts/agent/run-scheduler-cycle.mjs index d5e9f47..06016e3 100644 --- a/scripts/agent/run-scheduler-cycle.mjs +++ b/scripts/agent/run-scheduler-cycle.mjs @@ -398,6 +398,14 @@ async function main() { 'scripts/agent/verify-run-artifacts.mjs', '--since', runArtifactSince, + '--agent', + selectedAgent, + '--cadence', + cadence, + '--prompt-path', + promptPath, + '--run-start', + runArtifactSince, '--check-failure-notes', ]); if (artifactCheck.code !== 0) { diff --git a/scripts/agent/verify-run-artifacts.mjs b/scripts/agent/verify-run-artifacts.mjs index 1040d8c..4d2588c 100644 --- a/scripts/agent/verify-run-artifacts.mjs +++ b/scripts/agent/verify-run-artifacts.mjs @@ -37,13 +37,18 @@ const REQUIRED_ARTIFACTS = [ ]; const FAILURE_KEYWORD_RE = /\b(fail|failed|failure|error)\b/i; -const UNRESOLVED_REFERENCE_RE = /\bunresolved\s+finding(s)?\b|\bunresolved\b/i; +const STATUS_RESOLVED_RE = /\b(resolved|closed|fixed)\b/i; +const IDENTIFIER_INLINE_PATTERN = String.raw`\[([A-Za-z0-9._-]{3,})\]`; function parseArgs(argv) { const args = { since: null, session: null, checkFailureNotes: false, + agent: null, + cadence: null, + promptPath: null, + runStart: null, }; for (let i = 0; i < argv.length; i += 1) { @@ -60,6 +65,26 @@ function parseArgs(argv) { } if (value === '--check-failure-notes') { args.checkFailureNotes = true; + continue; + } + if (value === '--agent') { + args.agent = argv[i + 1] || null; + i += 1; + continue; + } + if (value === '--cadence') { + args.cadence = argv[i + 1] || null; + i += 1; + continue; + } + if (value === '--prompt-path') { + args.promptPath = argv[i + 1] || null; + i += 1; + continue; + } + if (value === '--run-start') { + args.runStart = argv[i + 1] || null; + i += 1; } } @@ -118,60 +143,166 @@ async function hasValidArtifactStructure(files, artifact) { return false; } +function readMetadataValue(content, keys) { + for (const key of keys) { + const re = new RegExp(`^\\s*(?:[-*]\\s*)?\\**${key}\\**\\s*:\\s*(.+)$`, 'im'); + const match = content.match(re); + if (match && match[1]) { + return match[1].trim(); + } + } + return null; +} + +function extractIdentifiers(content) { + const ids = new Set(); + const lines = content.split(/\r?\n/); + for (const line of lines) { + if (/(issue|incident|failure)[-_\s]*id/i.test(line)) { + const match = line.match(/([A-Za-z0-9._-]{3,})\s*$/); + if (match) ids.add(match[1]); + } + for (const inline of line.matchAll(new RegExp(IDENTIFIER_INLINE_PATTERN, 'g'))) { + ids.add(inline[1]); + } + } + return ids; +} + +function collectUnresolvedKnownIssueIds(content) { + const ids = new Set(); + const sections = content.split(/^###\s+/m).slice(1); + for (const sectionRaw of sections) { + const section = sectionRaw.trim(); + const statusMatch = section.match(/\*\*Status:\*\*\s*(.+)$/im); + const status = (statusMatch?.[1] || '').trim(); + if (status && STATUS_RESOLVED_RE.test(status)) { + continue; + } + for (const id of extractIdentifiers(section)) { + ids.add(id); + } + } + return ids; +} + +async function validateArtifactMetadata(files, { agent, cadence, promptPath, runStart }) { + const failures = []; + const promptPathResolved = promptPath ? path.resolve(process.cwd(), promptPath) : null; + const promptRelative = promptPathResolved ? path.relative(process.cwd(), promptPathResolved) : null; + const promptBase = promptPathResolved ? path.basename(promptPathResolved) : null; + + for (const file of files) { + const content = await fs.readFile(file.filePath, 'utf8').catch(() => ''); + const fileErrors = []; + + const agentValue = readMetadataValue(content, ['agent']); + const cadenceValue = readMetadataValue(content, ['cadence']); + const sessionValue = readMetadataValue(content, ['session']); + const runStartValue = readMetadataValue(content, ['run-start', 'run_start', 'run start']); + + if (!agentValue) fileErrors.push('missing `agent:` metadata'); + if (!cadenceValue) fileErrors.push('missing `cadence:` metadata'); + if (!sessionValue && !runStartValue) fileErrors.push('missing `session:` or `run-start:` metadata'); + + if (agent && agentValue && agentValue !== agent) { + fileErrors.push(`agent metadata "${agentValue}" does not match active SCHEDULER_AGENT "${agent}"`); + } + if (cadence && cadenceValue && cadenceValue !== cadence) { + fileErrors.push(`cadence metadata "${cadenceValue}" does not match active cadence "${cadence}"`); + } + if (runStart && runStartValue && runStartValue !== runStart) { + fileErrors.push(`run-start metadata "${runStartValue}" does not match active run-start "${runStart}"`); + } + + if (agent && !content.includes(agent)) { + fileErrors.push(`content does not reference active SCHEDULER_AGENT "${agent}"`); + } + + if (promptPathResolved) { + const referencesPrompt = [promptPathResolved, promptRelative, promptBase].filter(Boolean) + .some((candidate) => content.includes(candidate)); + if (!referencesPrompt) { + fileErrors.push(`content does not reference active prompt file (${promptRelative || promptPathResolved})`); + } + } + + if (fileErrors.length) { + failures.push(`${path.relative(process.cwd(), file.filePath)}: ${fileErrors.join('; ')}`); + } + } + + return failures; +} + async function checkFailureTracking({ sinceMs }) { const recentTestLogs = await listRecentFiles(path.resolve(process.cwd(), 'src/test_logs'), sinceMs); - let sawFailure = false; + const failureIds = new Set(); + const logsWithFailureNoIds = []; for (const file of recentTestLogs) { const content = await fs.readFile(file.filePath, 'utf8').catch(() => ''); if (FAILURE_KEYWORD_RE.test(content)) { - sawFailure = true; - break; + const ids = extractIdentifiers(content); + if (!ids.size) { + logsWithFailureNoIds.push(path.relative(process.cwd(), file.filePath)); + } + for (const id of ids) failureIds.add(id); } } - if (!sawFailure) { + if (!failureIds.size && !logsWithFailureNoIds.length) { return { ok: true, message: null }; } - const knownIssuesPath = path.resolve(process.cwd(), 'KNOWN_ISSUES.md'); - const knownIssuesStat = await fs.stat(knownIssuesPath).catch(() => null); - const knownIssuesContent = await fs.readFile(knownIssuesPath, 'utf8').catch(() => ''); - const incidents = await listRecentFiles(path.resolve(process.cwd(), 'docs/agent-handoffs/incidents'), sinceMs); - - const touchedKnownIssues = Boolean(knownIssuesStat && (sinceMs == null || knownIssuesStat.mtimeMs >= sinceMs)); - const knownIssuesHasExplicitReference = touchedKnownIssues - && UNRESOLVED_REFERENCE_RE.test(knownIssuesContent) - && FAILURE_KEYWORD_RE.test(knownIssuesContent); - - let incidentHasExplicitReference = false; - for (const incident of incidents) { - const incidentContent = await fs.readFile(incident.filePath, 'utf8').catch(() => ''); - if (UNRESOLVED_REFERENCE_RE.test(incidentContent) && FAILURE_KEYWORD_RE.test(incidentContent)) { - incidentHasExplicitReference = true; - break; - } + if (logsWithFailureNoIds.length) { + return { + ok: false, + message: `Failure-related output detected in test logs without identifiers. Add Issue/Incident identifiers to: ${logsWithFailureNoIds.join(', ')}`, + }; } - if (knownIssuesHasExplicitReference || incidentHasExplicitReference) { + const knownIssuesPath = path.resolve(process.cwd(), 'KNOWN_ISSUES.md'); + const knownIssuesContent = await fs.readFile(knownIssuesPath, 'utf8').catch(() => ''); + const unresolvedKnownIssueIds = collectUnresolvedKnownIssueIds(knownIssuesContent); + const incidents = await listRecentFiles(path.resolve(process.cwd(), 'docs/agent-handoffs/incidents'), sinceMs); + + const incidentIds = new Set(); + for (const incident of incidents) { + const incidentContent = await fs.readFile(incident.filePath, 'utf8').catch(() => ''); + for (const id of extractIdentifiers(incidentContent)) incidentIds.add(id); + } + + const unmatched = [...failureIds].filter((id) => !unresolvedKnownIssueIds.has(id) && !incidentIds.has(id)); + + if (!unmatched.length) { return { ok: true, message: null }; } return { ok: false, - message: 'Failure-related output detected in src/test_logs; add an explicit unresolved-finding reference (with failure context) in updated KNOWN_ISSUES.md or a new incident note.', + message: `Failure identifiers found in src/test_logs are not cross-linked to unresolved KNOWN_ISSUES.md entries or new incident notes: ${unmatched.join(', ')}`, }; } async function main() { const args = parseArgs(process.argv.slice(2)); const sinceMs = toMs(args.since); + const activeAgent = args.agent || process.env.SCHEDULER_AGENT || null; + const activeCadence = args.cadence || process.env.SCHEDULER_CADENCE || null; + const activePromptPath = args.promptPath || process.env.SCHEDULER_PROMPT_PATH || null; + const activeRunStart = args.runStart || null; if (args.since && sinceMs == null) { console.error(`Invalid --since value: ${args.since}`); process.exit(2); } + if (!activeAgent || !activePromptPath) { + console.error('Missing scheduler context: provide --agent and --prompt-path (or set SCHEDULER_AGENT/SCHEDULER_PROMPT_PATH).'); + process.exit(2); + } + const missing = []; for (const artifact of REQUIRED_ARTIFACTS) { @@ -197,6 +328,14 @@ async function main() { if (!hasValidStructure) { missing.push(`${artifact.dir}: ${artifact.structureHint}`); } + + const metadataFailures = await validateArtifactMetadata(matchingArtifacts, { + agent: activeAgent, + cadence: activeCadence, + promptPath: activePromptPath, + runStart: activeRunStart, + }); + missing.push(...metadataFailures); } if (args.checkFailureNotes) { diff --git a/test/verify-run-artifacts.test.mjs b/test/verify-run-artifacts.test.mjs new file mode 100644 index 0000000..c280cb0 --- /dev/null +++ b/test/verify-run-artifacts.test.mjs @@ -0,0 +1,104 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; + +const SCRIPT = path.resolve('scripts/agent/verify-run-artifacts.mjs'); + +async function runVerify(cwd, args = []) { + return new Promise((resolve) => { + const child = spawn(process.execPath, [SCRIPT, ...args], { + cwd, + stdio: ['ignore', 'pipe', 'pipe'], + env: { ...process.env }, + }); + + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { stdout += chunk.toString(); }); + child.stderr.on('data', (chunk) => { stderr += chunk.toString(); }); + child.on('close', (code) => resolve({ code: code ?? 1, stdout, stderr })); + }); +} + +async function setupFixture() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), 'verify-run-artifacts-')); + await fs.mkdir(path.join(root, 'src', 'context'), { recursive: true }); + await fs.mkdir(path.join(root, 'src', 'todo'), { recursive: true }); + await fs.mkdir(path.join(root, 'src', 'decisions'), { recursive: true }); + await fs.mkdir(path.join(root, 'src', 'test_logs'), { recursive: true }); + await fs.mkdir(path.join(root, 'src', 'prompts', 'daily'), { recursive: true }); + await fs.mkdir(path.join(root, 'docs', 'agent-handoffs', 'incidents'), { recursive: true }); + + await fs.writeFile(path.join(root, 'src', 'prompts', 'daily', 'docs-agent.md'), '# docs-agent\n', 'utf8'); + + return root; +} + +async function writeArtifacts(root, { includeMetadata = true, failureId = 'ISSUE-123' } = {}) { + const metadata = includeMetadata + ? ['agent: docs-agent', 'cadence: daily', 'session: session-1', 'run-start: 2026-02-14T00:00:00.000Z', 'prompt-file: src/prompts/daily/docs-agent.md'].join('\n') + : ''; + + await fs.writeFile(path.join(root, 'src', 'context', 'CONTEXT_1.md'), `${metadata}\n\nGoal: update docs\nScope: scripts\nConstraints: minimal\n`, 'utf8'); + await fs.writeFile(path.join(root, 'src', 'todo', 'TODO_1.md'), `${metadata}\n\nPending tasks:\n- finish checks\n`, 'utf8'); + await fs.writeFile(path.join(root, 'src', 'decisions', 'DECISIONS_1.md'), `${metadata}\n\nDecision: keep scope tight\nRationale: minimal risk\n`, 'utf8'); + await fs.writeFile(path.join(root, 'src', 'test_logs', 'TEST_LOG_1.md'), `${metadata}\n\nCommand: npm test\nResult: failed\nFailure-ID: ${failureId}\n`, 'utf8'); +} + +test('passes when artifacts include metadata and failure IDs map to unresolved known issue entries', async () => { + const root = await setupFixture(); + await writeArtifacts(root); + await fs.writeFile( + path.join(root, 'KNOWN_ISSUES.md'), + `### [Issue Tracking]\n- **Status:** Open\n- **Issue-ID:** ISSUE-123\n- **Symptom:** reproducible failure\n`, + 'utf8', + ); + + const result = await runVerify(root, [ + '--agent', 'docs-agent', + '--cadence', 'daily', + '--prompt-path', 'src/prompts/daily/docs-agent.md', + '--run-start', '2026-02-14T00:00:00.000Z', + '--check-failure-notes', + ]); + + assert.equal(result.code, 0, result.stderr || result.stdout); +}); + +test('fails when required metadata is missing from artifacts', async () => { + const root = await setupFixture(); + await writeArtifacts(root, { includeMetadata: false }); + await fs.writeFile(path.join(root, 'KNOWN_ISSUES.md'), '_No active issues currently documented._\n', 'utf8'); + + const result = await runVerify(root, [ + '--agent', 'docs-agent', + '--cadence', 'daily', + '--prompt-path', 'src/prompts/daily/docs-agent.md', + ]); + + assert.equal(result.code, 1); + assert.match(result.stderr, /missing `agent:` metadata/i); +}); + +test('fails when failure identifiers are not cross-linked to known issues or incidents', async () => { + const root = await setupFixture(); + await writeArtifacts(root, { includeMetadata: true, failureId: 'ISSUE-999' }); + await fs.writeFile( + path.join(root, 'KNOWN_ISSUES.md'), + `### [Issue Tracking]\n- **Status:** Open\n- **Issue-ID:** ISSUE-123\n`, + 'utf8', + ); + + const result = await runVerify(root, [ + '--agent', 'docs-agent', + '--cadence', 'daily', + '--prompt-path', 'src/prompts/daily/docs-agent.md', + '--check-failure-notes', + ]); + + assert.equal(result.code, 1); + assert.match(result.stderr, /not cross-linked/i); +});