From 9fef4c4f46ef19c1e314c7533220d60c0a535046 Mon Sep 17 00:00:00 2001 From: thePR0M3TH3AN <53631862+PR0M3TH3AN@users.noreply.github.com> Date: Sat, 21 Feb 2026 15:16:45 -0500 Subject: [PATCH] update --- KNOWN_ISSUES.md | 13 +- TORCH.md | 3 + ...ex-test-failures-platform-and-telemetry.md | 27 + .../2026-02-21T19-48-22Z__test-audit-agent.md | 13 + ...02-21T20-08-13Z__scheduler-update-agent.md | 13 + reports/test-audit/flakiness-matrix.json | 622 +----------------- reports/test-audit/flakiness-runs.json | 32 + reports/test-audit/flaky-run-01.tap | 34 + reports/test-audit/flaky-run-02.tap | 34 + reports/test-audit/flaky-run-03.tap | 34 + reports/test-audit/flaky-run-04.tap | 34 + reports/test-audit/flaky-run-05.tap | 34 + .../scheduler-preflight-direct.exitcode | 1 + .../test-audit/scheduler-preflight.exitcode | 1 + reports/test-audit/suspicious-tests.json | 18 +- .../test-audit-report-2026-02-21.md | 89 +++ .../test-audit-run-flaky-check.exitcode | 1 + scripts/agent/run-scheduler-cycle.mjs | 241 ++++++- scripts/memory/retrieve.mjs | 1 + scripts/memory/store.mjs | 1 + scripts/test-audit/run-flaky-check.mjs | 48 +- src/context/CONTEXT_2026-02-21T19-48-18Z.md | 17 + .../DECISIONS_2026-02-21T19-48-18Z.md | 20 + src/lib.mjs | 8 +- src/lock-utils.mjs | 2 +- src/ops.mjs | 26 +- .../TEST_LOG_2026-02-21T19-48-18Z.md | 26 + src/todo/TODO_2026-02-21T19-48-18Z.md | 17 + ...26-02-21T19-46-36Z__style-agent__failed.md | 20 + ...T19-55-27Z__test-audit-agent__completed.md | 18 + .../flaky-check/fail.fixture.test.mjs | 6 + .../flaky-check/pass.fixture.test.mjs | 6 + test/lib.test.mjs | 18 + ...run-scheduler-cycle-memory-policy.test.mjs | 116 +++- test/test-audit-run-flaky-check.test.mjs | 45 ++ torch-config.example.json | 36 +- torch-config.json | 14 +- 37 files changed, 1019 insertions(+), 670 deletions(-) create mode 100644 memory-updates/2026-02-21T19-48-22Z__test-audit-agent.md create mode 100644 memory-updates/2026-02-21T20-08-13Z__scheduler-update-agent.md create mode 100644 reports/test-audit/flakiness-runs.json create mode 100644 reports/test-audit/flaky-run-01.tap create mode 100644 reports/test-audit/flaky-run-02.tap create mode 100644 reports/test-audit/flaky-run-03.tap create mode 100644 reports/test-audit/flaky-run-04.tap create mode 100644 reports/test-audit/flaky-run-05.tap create mode 100644 reports/test-audit/scheduler-preflight-direct.exitcode create mode 100644 reports/test-audit/scheduler-preflight.exitcode create mode 100644 reports/test-audit/test-audit-report-2026-02-21.md create mode 100644 reports/test-audit/test-audit-run-flaky-check.exitcode create mode 100644 src/context/CONTEXT_2026-02-21T19-48-18Z.md create mode 100644 src/decisions/DECISIONS_2026-02-21T19-48-18Z.md create mode 100644 src/test_logs/TEST_LOG_2026-02-21T19-48-18Z.md create mode 100644 src/todo/TODO_2026-02-21T19-48-18Z.md create mode 100644 task-logs/daily/2026-02-21T19-46-36Z__style-agent__failed.md create mode 100644 task-logs/daily/2026-02-21T19-55-27Z__test-audit-agent__completed.md create mode 100644 test/fixtures/flaky-check/fail.fixture.test.mjs create mode 100644 test/fixtures/flaky-check/pass.fixture.test.mjs create mode 100644 test/test-audit-run-flaky-check.test.mjs diff --git a/KNOWN_ISSUES.md b/KNOWN_ISSUES.md index d71fcec..72644d5 100644 --- a/KNOWN_ISSUES.md +++ b/KNOWN_ISSUES.md @@ -17,14 +17,15 @@ Track only **active, reproducible, unresolved** issues. ## Active issues ### `test/scheduler-preflight-lock.e2e.test.mjs` platform mismatch (`unknown` vs `codex`) -- **Status:** Resolved +- **Status:** Active +- **Issue-ID:** KNOWN-ISSUE-scheduler-preflight-platform - **Area:** Test -- **Symptom:** Historical issue where test expected `platform: unknown` while scheduler logs used `platform: codex`. -- **Trigger/Conditions:** Previously reproed in Codex. -- **Workaround:** None needed after assertion behavior update. -- **Impact:** No longer blocking. +- **Symptom:** The first e2e scenario fails because expected frontmatter platform resolves to `unknown` in test context while scheduler output persists `platform: codex`. +- **Trigger/Conditions:** Running `node test/scheduler-preflight-lock.e2e.test.mjs` or `node --test test/scheduler-preflight-lock.e2e.test.mjs` in this environment. +- **Workaround:** Exclude `test/scheduler-preflight-lock.e2e.test.mjs` from local sandbox triage runs until the expectation source is aligned with scheduler platform semantics. +- **Impact:** Medium — scheduler e2e bundle reports a deterministic failure unrelated to most feature changes. - **Related notes:** `docs/agent-handoffs/incidents/2026-02-20-codex-test-failures-platform-and-telemetry.md` -- **Last verified:** 2026-02-20 +- **Last verified:** 2026-02-21 ### `test/memory-telemetry.test.mjs` expects child-process stdout/stderr that are empty in this environment - **Status:** Active diff --git a/TORCH.md b/TORCH.md index 72b1357..7247e71 100644 --- a/TORCH.md +++ b/TORCH.md @@ -330,6 +330,9 @@ Common settings: - `dashboard.hashtag` — custom hashtag for lock events (defaults to `-agent-lock`). - `scheduler.firstPromptByCadence.daily` / `.weekly` — first-run scheduler starting agent. - `scheduler.handoffCommandByCadence.daily` / `.weekly` — shell command run after lock acquisition; command must use `SCHEDULER_AGENT`, `SCHEDULER_CADENCE`, and `SCHEDULER_PROMPT_PATH` provided by `scripts/agent/run-scheduler-cycle.mjs`. +- `scheduler.handoffPolicyByCadence.daily` / `.weekly` — optional retry/fallback policy for handoff execution (`maxAttempts`, `retryBackoffMs`, `retryOnPatterns`, `fallbackPlatform`). +- `scheduler.runnerHealthCommandByCadence.daily` / `.weekly` — optional command run before lock acquisition to verify runner availability; non-zero exits fail the run before prompt execution. +- `scheduler.memoryPolicyByCadence..retrieveArtifacts` / `.storeArtifacts` — evidence file list; recommended canonical files include `.scheduler-memory/latest//retrieve.ok`, `.scheduler-memory/latest//retrieve.json`, `.scheduler-memory/latest//store.ok`, `.scheduler-memory/latest//store.json`. - `scheduler.paused.daily` / `.weekly` — array of agent names to exclude from scheduler rotation. - `scheduler.strict_lock` — lock backend policy switch (default: `true`); when `false`, scheduler defers backend-unavailable lock failures before converting the run to failed. - `scheduler.degraded_lock_retry_window` — non-strict deferral window in milliseconds; backend lock failures outside this window immediately consume failure budget and mark run failed. diff --git a/docs/agent-handoffs/incidents/2026-02-20-codex-test-failures-platform-and-telemetry.md b/docs/agent-handoffs/incidents/2026-02-20-codex-test-failures-platform-and-telemetry.md index cfafaee..b95ce41 100644 --- a/docs/agent-handoffs/incidents/2026-02-20-codex-test-failures-platform-and-telemetry.md +++ b/docs/agent-handoffs/incidents/2026-02-20-codex-test-failures-platform-and-telemetry.md @@ -28,3 +28,30 @@ - Treat these as environment-sensitive test failures until assertions are aligned with Codex platform semantics and child-process output behavior. - If updating tests, follow spec-correction protocol and preserve strict behavioral intent. - Keep `KNOWN_ISSUES.md` dates/status current after remediation. + +## Update 2026-02-21 + +### Context +- Agent: `test-audit-agent` +- Scope: scenario-first audit run using `scripts/test-audit/run-flaky-check.mjs` and targeted scheduler e2e verification. + +### Observation +- The scheduler platform mismatch is still reproducible: + - Expected in test assertion: `platform: unknown` (derived from test-side `detectPlatform() || 'unknown'`) + - Actual log frontmatter: `platform: codex` +- Reproduced with: + - `node test/scheduler-preflight-lock.e2e.test.mjs` + - `node --test test/scheduler-preflight-lock.e2e.test.mjs` + +### Action taken +- Reclassified the issue in `KNOWN_ISSUES.md` from Resolved back to Active (`KNOWN-ISSUE-scheduler-preflight-platform`) with updated workaround and verification date. +- Did not weaken/modify scenario expectations in the e2e test without explicit spec correction. + +### Validation performed +- `node test/scheduler-preflight-lock.e2e.test.mjs` (fails with platform mismatch) +- `node --test test/scheduler-preflight-lock.e2e.test.mjs` (fails) + +### Recommendation for next agents +- Resolve via spec correction only: + 1. Decide canonical platform source for scheduler logs in fixtures (`AGENT_PLATFORM` fallback vs runtime detector). + 2. Update assertion to match documented scheduler behavior, not incidental environment state. diff --git a/memory-updates/2026-02-21T19-48-22Z__test-audit-agent.md b/memory-updates/2026-02-21T19-48-22Z__test-audit-agent.md new file mode 100644 index 0000000..c1ab2ff --- /dev/null +++ b/memory-updates/2026-02-21T19-48-22Z__test-audit-agent.md @@ -0,0 +1,13 @@ +# Memory Update - test-audit-agent - 2026-02-21 + +## Key findings +- `scripts/test-audit/run-flaky-check.mjs` can produce false-green empty output when relying on piped child stdout in this environment. +- Writing node:test TAP output to explicit reporter destination files is stable across this sandbox and keeps flakiness evidence auditable. + +## Patterns / reusable knowledge +- For audit tooling that consumes child test output, prefer file-based reporter destinations over stdio capture. +- Add deterministic pass/fail fixtures to test audit scripts so regressions in signal extraction fail immediately. + +## Warnings / gotchas +- `test/scheduler-preflight-lock.e2e.test.mjs` remains reproducibly red on platform mismatch (`unknown` vs `codex`) and should be treated as active until spec correction. +- Running node:test from inside another node:test process may require clearing `NODE_TEST_CONTEXT` in child env for stable behavior. diff --git a/memory-updates/2026-02-21T20-08-13Z__scheduler-update-agent.md b/memory-updates/2026-02-21T20-08-13Z__scheduler-update-agent.md new file mode 100644 index 0000000..d2cc9bc --- /dev/null +++ b/memory-updates/2026-02-21T20-08-13Z__scheduler-update-agent.md @@ -0,0 +1,13 @@ +# Memory Update - scheduler-update-agent - 2026-02-21 + +## Key findings +- Scheduler reliability improves when handoff failures are classified and retried only for retryable network signatures. +- `lock:check --json --quiet` still needs defensive parsing; consume structured JSON events and choose the payload with exclusion keys. + +## Patterns / reusable knowledge +- For spawned-agent handoff, use bounded retries with backoff plus optional fallback platform as a last resort. +- For runner availability, run a preflight command before lock acquisition to avoid claiming work that cannot execute. + +## Warnings / gotchas +- `cmdCheck` should suppress query health/error loggers in quiet JSON mode to keep automation parsing deterministic. +- Memory evidence is easier to validate when latest cadence artifacts include both marker and JSON (`retrieve/store.ok` + `retrieve/store.json`). diff --git a/reports/test-audit/flakiness-matrix.json b/reports/test-audit/flakiness-matrix.json index 5368ac2..828887a 100644 --- a/reports/test-audit/flakiness-matrix.json +++ b/reports/test-audit/flakiness-matrix.json @@ -1,618 +1,14 @@ { - "Build artifacts verification": { + "test/lock-ops-publish-retry.test.mjs": { + "pass": 5, + "fail": 0 + }, + "test/scheduler-lock-failure-schema.contract.test.mjs": { + "pass": 5, + "fail": 0 + }, + "test/scheduler-preflight-lock.e2e.test.mjs": { "pass": 0, "fail": 5 - }, - "parseArgs should parse basic command": { - "pass": 5, - "fail": 0 - }, - "parseArgs should parse agent and cadence": { - "pass": 5, - "fail": 0 - }, - "parseArgs should parse equals sign syntax": { - "pass": 5, - "fail": 0 - }, - "parseArgs should parse dry-run flag": { - "pass": 5, - "fail": 0 - }, - "parseArgs should parse port with default": { - "pass": 5, - "fail": 0 - }, - "parseArgs should parse custom port": { - "pass": 5, - "fail": 0 - }, - "parseArgs should parse ignore-logs flag": { - "pass": 5, - "fail": 0 - }, - "parseArgs should parse memory flags": { - "pass": 5, - "fail": 0 - }, - "parseArgs should parse memory flags with equals": { - "pass": 5, - "fail": 0 - }, - "cmdList": { - "pass": 5, - "fail": 0 - }, - "Dashboard Authentication": { - "pass": 5, - "fail": 0 - }, - "Dashboard Configuration Exposure Security": { - "pass": 5, - "fail": 0 - }, - "Dashboard Configuration Security": { - "pass": 5, - "fail": 0 - }, - "Dashboard Path Traversal Security Fix": { - "pass": 5, - "fail": 0 - }, - "Dashboard File Access Security": { - "pass": 5, - "fail": 0 - }, - "cmdInit should validate install directory name": { - "pass": 5, - "fail": 0 - }, - "src/lib.mjs": { - "pass": 5, - "fail": 0 - }, - "ci-resilience lock acquisition under unstable relays": { - "pass": 5, - "fail": 0 - }, - "publishLock retry behavior": { - "pass": 5, - "fail": 0 - }, - "lock-ops": { - "pass": 5, - "fail": 0 - }, - "lock-utils": { - "pass": 5, - "fail": 0 - }, - "listMemories, inspectMemory, pinMemory, and unpinMemory support admin hooks": { - "pass": 5, - "fail": 0 - }, - "triggerPruneDryRun returns candidate metadata without deleting records": { - "pass": 5, - "fail": 0 - }, - "memoryStats reports counts/rates and retrieval telemetry remains redaction-safe": { - "pass": 5, - "fail": 0 - }, - "runPruneCycle respects prune feature-flag modes": { - "pass": 5, - "fail": 0 - }, - "markMemoryMerged delegates to repository and clears cache": { - "pass": 5, - "fail": 0 - }, - "stores runtime events in signer/session namespace and clears scope": { - "pass": 5, - "fail": 0 - }, - "expires runtime events and records expiration metrics": { - "pass": 5, - "fail": 0 - }, - "enforces LRU bounds and tracks eviction metrics": { - "pass": 5, - "fail": 0 - }, - "blocks decrypted/session-sensitive data from durable promotion without sanitizer pass": { - "pass": 5, - "fail": 0 - }, - "createEmbedderAdapter selects in-memory backend and supports vector CRUD": { - "pass": 5, - "fail": 0 - }, - "createEmbedderAdapter reads backend from environment": { - "pass": 5, - "fail": 0 - }, - "memory feature flags parse defaults and canary allow-lists": { - "pass": 5, - "fail": 0 - }, - "memory prune flag supports off, dry-run, and active modes": { - "pass": 5, - "fail": 0 - }, - "estimateTokenCount returns heuristic count": { - "pass": 5, - "fail": 0 - }, - "formatMemoryBlock converts memory record to display block": { - "pass": 5, - "fail": 0 - }, - "formatMemoryBlock handles missing optional fields and default clamping": { - "pass": 5, - "fail": 0 - }, - "formatMemoryBlock respects custom clamping options": { - "pass": 5, - "fail": 0 - }, - "formatMemoryBlock handles edge cases for clamping": { - "pass": 5, - "fail": 0 - }, - "formatMemoriesForPrompt enforces token budget and k limit": { - "pass": 5, - "fail": 0 - }, - "formatMemoriesForPrompt handles aggressive budget fitting": { - "pass": 5, - "fail": 0 - }, - "renderPromptWithMemoryContext injects context when enabled": { - "pass": 5, - "fail": 0 - }, - "renderPromptWithMemoryContext returns base prompt when disabled": { - "pass": 5, - "fail": 0 - }, - "renderPromptWithMemoryContext handles missing query or service": { - "pass": 5, - "fail": 0 - }, - "renderPromptWithMemoryContext handles circular agentContext gracefully": { - "pass": 5, - "fail": 0 - }, - "renderPromptWithMemoryContext handles agentContext as array of strings": { - "pass": 5, - "fail": 0 - }, - "renderPromptWithMemoryContext handles agentContext as plain object": { - "pass": 5, - "fail": 0 - }, - "renderPromptWithMemoryContext handles agentContext as primitives": { - "pass": 5, - "fail": 0 - }, - "renderPromptWithMemoryContext calls updateMemoryUsage for retrieved memories": { - "pass": 5, - "fail": 0 - }, - "renderPromptWithMemoryContext handles empty memories gracefully": { - "pass": 5, - "fail": 0 - }, - "renderPromptWithMemoryContext handles null/undefined memories gracefully": { - "pass": 5, - "fail": 0 - }, - "renderPromptWithMemoryContext respects TORCH_MEMORY_ENABLED=false": { - "pass": 5, - "fail": 0 - }, - "renderPromptWithMemoryContext respects retrieval allowlist": { - "pass": 5, - "fail": 0 - }, - "renderPromptWithMemoryContext passes k and tokenBudget": { - "pass": 5, - "fail": 0 - }, - "getRelevantMemories uses cached results when available": { - "pass": 5, - "fail": 0 - }, - "getRelevantMemories calls ranker and updates cache on miss": { - "pass": 5, - "fail": 0 - }, - "getRelevantMemories passes correct parameters to ranker": { - "pass": 5, - "fail": 0 - }, - "getRelevantMemories triggers updateMemoryUsage on repository": { - "pass": 5, - "fail": 0 - }, - "ingestEvents": { - "pass": 5, - "fail": 0 - }, - "ingestMemoryWindow gathers sources, redacts pii, chunks, embeds, links, and emits telemetry": { - "pass": 5, - "fail": 0 - }, - "ingestMemoryWindow dedupes overlapping windows by hashed content + agent + timestamp bucket": { - "pass": 5, - "fail": 0 - }, - "pinMemory pins the memory and clears the cache": { - "pass": 5, - "fail": 0 - }, - "pinMemory returns null for non-existent ID but still clears cache": { - "pass": 5, - "fail": 0 - }, - "unpinMemory unpins the memory and clears the cache": { - "pass": 5, - "fail": 0 - }, - "unpinMemory returns null for non-existent ID but still clears cache": { - "pass": 5, - "fail": 0 - }, - "createLifecyclePlan respects delete safety constraints and logs reasons": { - "pass": 5, - "fail": 0 - }, - "createLifecyclePlan condenses near-duplicates and archives merged source records": { - "pass": 5, - "fail": 0 - }, - "applyLifecycleActions executes keep/archive/delete and merge markers": { - "pass": 5, - "fail": 0 - }, - "filterAndRankMemories applies metadata filters and composite score": { - "pass": 5, - "fail": 0 - }, - "formatMemoriesForPrompt returns prompt blocks and enforces token budget trimming": { - "pass": 5, - "fail": 0 - }, - "isMemoryRetrievalEnabled supports global toggle and agent canary list": { - "pass": 5, - "fail": 0 - }, - "renderPromptWithMemoryContext injects bounded memory block before base prompt": { - "pass": 5, - "fail": 0 - }, - "runPruneCycle respects off mode": { - "pass": 5, - "fail": 0 - }, - "runPruneCycle respects dry-run mode": { - "pass": 5, - "fail": 0 - }, - "runPruneCycle removes stale unpinned memories in active mode": { - "pass": 5, - "fail": 0 - }, - "runPruneCycle starts scheduler if requested": { - "pass": 5, - "fail": 0 - }, - "createDbAdvisoryLockProvider acquires and releases postgres advisory locks": { - "pass": 5, - "fail": 0 - }, - "startMemoryMaintenanceScheduler runs all jobs and emits metrics including retries": { - "pass": 5, - "fail": 0 - }, - "startMemoryMaintenanceScheduler skips jobs when feature flags disable memory": { - "pass": 5, - "fail": 0 - }, - "normalizeMemoryItem applies durable schema defaults and coercions": { - "pass": 5, - "fail": 0 - }, - "validateMemoryItem reports field-level errors": { - "pass": 5, - "fail": 0 - }, - "ingestEvents rejects invalid writes and logs structured validation details": { - "pass": 5, - "fail": 0 - }, - "memoryStats with empty repository returns zero counts and rates": { - "pass": 5, - "fail": 0 - }, - "memoryStats with populated repository returns correct counts and totals": { - "pass": 5, - "fail": 0 - }, - "memoryStats calculates ingest throughput based on window": { - "pass": 5, - "fail": 0 - }, - "memoryStats reflects deleted count from prune cycle": { - "pass": 5, - "fail": 0 - }, - "memoryStats handles windowMs parameter correctly": { - "pass": 5, - "fail": 0 - }, - "loadMemoryPromptTemplates loads all memory prompt templates from disk": { - "pass": 5, - "fail": 0 - }, - "summarizeEvents retries with repair prompt when first response is malformed JSON": { - "pass": 5, - "fail": 0 - }, - "summarizeEvents falls back to deterministic minimal summary with conservative importance when parsing fails twice": { - "pass": 5, - "fail": 0 - }, - "telemetry should not log to stdout or stderr by default": { - "pass": 5, - "fail": 0 - }, - "telemetry should log to stderr when NODE_DEBUG=torch-memory is set": { - "pass": 5, - "fail": 0 - }, - "schema validation covers required fields, coercion, and rejection paths": { - "pass": 5, - "fail": 0 - }, - "summarizer parser accepts valid json and recovers after malformed json response": { - "pass": 5, - "fail": 0 - }, - "ingest integration persists memory records and links embeddings from event batches": { - "pass": 5, - "fail": 0 - }, - "retrieval ranking blends semantic, importance, and recency signals": { - "pass": 5, - "fail": 0 - }, - "pruner policy protects pins, merges duplicates, and picks archive/delete outcomes": { - "pass": 5, - "fail": 0 - }, - "scheduler smoke test handles mocked clock and lock contention": { - "pass": 5, - "fail": 0 - }, - "CLI Smoke Test": { - "pass": 5, - "fail": 0 - }, - "cmdInit creates directory structure and files": { - "pass": 5, - "fail": 0 - }, - "cmdInit fails if directory exists without force": { - "pass": 5, - "fail": 0 - }, - "cmdUpdate preserves modified prompt": { - "pass": 5, - "fail": 0 - }, - "cmdUpdate overwrites prompt with force": { - "pass": 5, - "fail": 0 - }, - "cmdUpdate creates backup": { - "pass": 5, - "fail": 0 - }, - "torch-lock check respects local roster": { - "pass": 5, - "fail": 0 - }, - "cmdInit creates torch-config.json with random namespace": { - "pass": 5, - "fail": 0 - }, - "relay fanout quorum integration": { - "pass": 5, - "fail": 0 - }, - "RelayHealthManager": { - "pass": 5, - "fail": 0 - }, - "evaluateAlertThresholds": { - "pass": 5, - "fail": 0 - }, - "summarizeHistory - basic statistics calculation": { - "pass": 5, - "fail": 0 - }, - "summarizeHistory - handles empty history": { - "pass": 5, - "fail": 0 - }, - "summarizeHistory - respects time window for stats": { - "pass": 5, - "fail": 0 - }, - "summarizeHistory - allDownDurationMinutes considers outside window entries": { - "pass": 5, - "fail": 0 - }, - "summarizeHistory - allDownDurationMinutes is null if never healthy": { - "pass": 5, - "fail": 0 - }, - "summarizeHistory - allDownDurationMinutes is 0 if currently healthy": { - "pass": 5, - "fail": 0 - }, - "summarizeHistory - handles malformed entries gracefully": { - "pass": 5, - "fail": 0 - }, - "summarizeHistory - handles valid timestamp but missing summary": { - "pass": 5, - "fail": 0 - }, - "summarizeHistory - healthy entry with invalid timestamp results in NaN duration": { - "pass": 5, - "fail": 0 - }, - "Roster (Unit Tests with Dependency Injection)": { - "pass": 5, - "fail": 0 - }, - "fails required memory policy when retrieval/storage evidence is missing": { - "pass": 5, - "fail": 0 - }, - "accepts required memory policy when markers or artifacts are produced": { - "pass": 5, - "fail": 0 - }, - "records backend failure metadata when lock command exits with code 2": { - "pass": 5, - "fail": 0 - }, - "defers backend failures in non-strict mode and reuses idempotency key on successful retry": { - "pass": 5, - "fail": 0 - }, - "fails after exceeding non-strict deferral budget": { - "pass": 5, - "fail": 0 - }, - "strict lock mode fails immediately without deferral": { - "pass": 5, - "fail": 0 - }, - "skips lock health preflight by default when not enabled": { - "pass": 5, - "fail": 0 - }, - "fails scheduler with preflight metadata when lock health preflight is enabled": { - "pass": 5, - "fail": 0 - }, - "categorizes unreadable prompt files as prompt_parse_error": { - "pass": 5, - "fail": 0 - }, - "categorizes invalid prompt schema as prompt_schema_error": { - "pass": 5, - "fail": 0 - }, - "categorizes handoff runtime failures as execution_error": { - "pass": 5, - "fail": 0 - }, - "retries lock acquisition for backend error and succeeds on a later attempt": { - "pass": 5, - "fail": 0 - }, - "stops retrying after configured lock backend retries are exhausted": { - "pass": 5, - "fail": 0 - }, - "does not use backend retry flow for exit code 3 lock conflicts": { - "pass": 5, - "fail": 0 - }, - "runs selected scheduler prompt through configured runner command": { - "pass": 5, - "fail": 0 - }, - "propagates non-zero runner exit code": { - "pass": 5, - "fail": 0 - }, - "scheduler lock backend failure artifact matches required frontmatter schema": { - "pass": 5, - "fail": 0 - }, - "schema contract catches missing required lock-backend field": { - "pass": 5, - "fail": 0 - }, - "schema contract catches misnamed lock-backend key": { - "pass": 5, - "fail": 0 - }, - "lock preflight e2e: successful lock writes completed status snapshot": { - "pass": 5, - "fail": 0 - }, - "lock preflight e2e: exit code 2 quorum failure persists failed backend status and prompt-not-started marker": { - "pass": 5, - "fail": 0 - }, - "lock preflight e2e: non-lock failure exits failed without prompt parse/schema classification": { - "pass": 5, - "fail": 0 - }, - "Scheduler Ratchet Logic (Log Checking)": { - "pass": 5, - "fail": 0 - }, - "Scheduler cycle ordering guarantees": { - "pass": 5, - "fail": 0 - }, - "secureRandom": { - "pass": 5, - "fail": 0 - }, - "summarizes recent lock reliability by platform/cadence/backend/relay": { - "pass": 5, - "fail": 0 - }, - "loadTorchConfig (Unit Tests with Mocked FS)": { - "pass": 5, - "fail": 0 - }, - "torch-config": { - "pass": 5, - "fail": 0 - }, - "Date Utilities": { - "pass": 5, - "fail": 0 - }, - "Platform Utilities": { - "pass": 5, - "fail": 0 - }, - "File Utilities": { - "pass": 5, - "fail": 0 - }, - "passes when artifacts include metadata and failure IDs map to unresolved known issue entries": { - "pass": 5, - "fail": 0 - }, - "fails when required metadata is missing from artifacts": { - "pass": 5, - "fail": 0 - }, - "fails when failure identifiers are not cross-linked to known issues or incidents": { - "pass": 5, - "fail": 0 } } \ No newline at end of file diff --git a/reports/test-audit/flakiness-runs.json b/reports/test-audit/flakiness-runs.json new file mode 100644 index 0000000..3cde1b2 --- /dev/null +++ b/reports/test-audit/flakiness-runs.json @@ -0,0 +1,32 @@ +[ + { + "run": 1, + "code": 1, + "tapOutputPath": "reports/test-audit/flaky-run-01.tap", + "observed": 3 + }, + { + "run": 2, + "code": 1, + "tapOutputPath": "reports/test-audit/flaky-run-02.tap", + "observed": 3 + }, + { + "run": 3, + "code": 1, + "tapOutputPath": "reports/test-audit/flaky-run-03.tap", + "observed": 3 + }, + { + "run": 4, + "code": 1, + "tapOutputPath": "reports/test-audit/flaky-run-04.tap", + "observed": 3 + }, + { + "run": 5, + "code": 1, + "tapOutputPath": "reports/test-audit/flaky-run-05.tap", + "observed": 3 + } +] \ No newline at end of file diff --git a/reports/test-audit/flaky-run-01.tap b/reports/test-audit/flaky-run-01.tap new file mode 100644 index 0000000..06c390d --- /dev/null +++ b/reports/test-audit/flaky-run-01.tap @@ -0,0 +1,34 @@ +TAP version 13 +# Subtest: test/lock-ops-publish-retry.test.mjs +ok 1 - test/lock-ops-publish-retry.test.mjs + --- + duration_ms: 102.263713 + type: 'test' + ... +# Subtest: test/scheduler-lock-failure-schema.contract.test.mjs +ok 2 - test/scheduler-lock-failure-schema.contract.test.mjs + --- + duration_ms: 111.249474 + type: 'test' + ... +# Subtest: test/scheduler-preflight-lock.e2e.test.mjs +not ok 3 - test/scheduler-preflight-lock.e2e.test.mjs + --- + duration_ms: 286.217684 + type: 'test' + location: '/home/user/Documents/GitHub/TORCH/test/scheduler-preflight-lock.e2e.test.mjs:1:1' + failureType: 'testCodeFailure' + exitCode: 1 + signal: ~ + error: 'test failed' + code: 'ERR_TEST_FAILURE' + ... +1..3 +# tests 3 +# suites 0 +# pass 2 +# fail 1 +# cancelled 0 +# skipped 0 +# todo 0 +# duration_ms 300.601253 diff --git a/reports/test-audit/flaky-run-02.tap b/reports/test-audit/flaky-run-02.tap new file mode 100644 index 0000000..38c5c10 --- /dev/null +++ b/reports/test-audit/flaky-run-02.tap @@ -0,0 +1,34 @@ +TAP version 13 +# Subtest: test/lock-ops-publish-retry.test.mjs +ok 1 - test/lock-ops-publish-retry.test.mjs + --- + duration_ms: 136.571153 + type: 'test' + ... +# Subtest: test/scheduler-lock-failure-schema.contract.test.mjs +ok 2 - test/scheduler-lock-failure-schema.contract.test.mjs + --- + duration_ms: 150.466335 + type: 'test' + ... +# Subtest: test/scheduler-preflight-lock.e2e.test.mjs +not ok 3 - test/scheduler-preflight-lock.e2e.test.mjs + --- + duration_ms: 315.710253 + type: 'test' + location: '/home/user/Documents/GitHub/TORCH/test/scheduler-preflight-lock.e2e.test.mjs:1:1' + failureType: 'testCodeFailure' + exitCode: 1 + signal: ~ + error: 'test failed' + code: 'ERR_TEST_FAILURE' + ... +1..3 +# tests 3 +# suites 0 +# pass 2 +# fail 1 +# cancelled 0 +# skipped 0 +# todo 0 +# duration_ms 333.778214 diff --git a/reports/test-audit/flaky-run-03.tap b/reports/test-audit/flaky-run-03.tap new file mode 100644 index 0000000..698c285 --- /dev/null +++ b/reports/test-audit/flaky-run-03.tap @@ -0,0 +1,34 @@ +TAP version 13 +# Subtest: test/lock-ops-publish-retry.test.mjs +ok 1 - test/lock-ops-publish-retry.test.mjs + --- + duration_ms: 134.112659 + type: 'test' + ... +# Subtest: test/scheduler-lock-failure-schema.contract.test.mjs +ok 2 - test/scheduler-lock-failure-schema.contract.test.mjs + --- + duration_ms: 135.921496 + type: 'test' + ... +# Subtest: test/scheduler-preflight-lock.e2e.test.mjs +not ok 3 - test/scheduler-preflight-lock.e2e.test.mjs + --- + duration_ms: 316.604483 + type: 'test' + location: '/home/user/Documents/GitHub/TORCH/test/scheduler-preflight-lock.e2e.test.mjs:1:1' + failureType: 'testCodeFailure' + exitCode: 1 + signal: ~ + error: 'test failed' + code: 'ERR_TEST_FAILURE' + ... +1..3 +# tests 3 +# suites 0 +# pass 2 +# fail 1 +# cancelled 0 +# skipped 0 +# todo 0 +# duration_ms 334.498823 diff --git a/reports/test-audit/flaky-run-04.tap b/reports/test-audit/flaky-run-04.tap new file mode 100644 index 0000000..20d36d6 --- /dev/null +++ b/reports/test-audit/flaky-run-04.tap @@ -0,0 +1,34 @@ +TAP version 13 +# Subtest: test/lock-ops-publish-retry.test.mjs +ok 1 - test/lock-ops-publish-retry.test.mjs + --- + duration_ms: 125.402872 + type: 'test' + ... +# Subtest: test/scheduler-lock-failure-schema.contract.test.mjs +ok 2 - test/scheduler-lock-failure-schema.contract.test.mjs + --- + duration_ms: 132.678397 + type: 'test' + ... +# Subtest: test/scheduler-preflight-lock.e2e.test.mjs +not ok 3 - test/scheduler-preflight-lock.e2e.test.mjs + --- + duration_ms: 442.786008 + type: 'test' + location: '/home/user/Documents/GitHub/TORCH/test/scheduler-preflight-lock.e2e.test.mjs:1:1' + failureType: 'testCodeFailure' + exitCode: 1 + signal: ~ + error: 'test failed' + code: 'ERR_TEST_FAILURE' + ... +1..3 +# tests 3 +# suites 0 +# pass 2 +# fail 1 +# cancelled 0 +# skipped 0 +# todo 0 +# duration_ms 460.996364 diff --git a/reports/test-audit/flaky-run-05.tap b/reports/test-audit/flaky-run-05.tap new file mode 100644 index 0000000..51077d3 --- /dev/null +++ b/reports/test-audit/flaky-run-05.tap @@ -0,0 +1,34 @@ +TAP version 13 +# Subtest: test/lock-ops-publish-retry.test.mjs +ok 1 - test/lock-ops-publish-retry.test.mjs + --- + duration_ms: 118.053138 + type: 'test' + ... +# Subtest: test/scheduler-lock-failure-schema.contract.test.mjs +ok 2 - test/scheduler-lock-failure-schema.contract.test.mjs + --- + duration_ms: 127.486819 + type: 'test' + ... +# Subtest: test/scheduler-preflight-lock.e2e.test.mjs +not ok 3 - test/scheduler-preflight-lock.e2e.test.mjs + --- + duration_ms: 340.626807 + type: 'test' + location: '/home/user/Documents/GitHub/TORCH/test/scheduler-preflight-lock.e2e.test.mjs:1:1' + failureType: 'testCodeFailure' + exitCode: 1 + signal: ~ + error: 'test failed' + code: 'ERR_TEST_FAILURE' + ... +1..3 +# tests 3 +# suites 0 +# pass 2 +# fail 1 +# cancelled 0 +# skipped 0 +# todo 0 +# duration_ms 357.903984 diff --git a/reports/test-audit/scheduler-preflight-direct.exitcode b/reports/test-audit/scheduler-preflight-direct.exitcode new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/reports/test-audit/scheduler-preflight-direct.exitcode @@ -0,0 +1 @@ +1 diff --git a/reports/test-audit/scheduler-preflight.exitcode b/reports/test-audit/scheduler-preflight.exitcode new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/reports/test-audit/scheduler-preflight.exitcode @@ -0,0 +1 @@ +1 diff --git a/reports/test-audit/suspicious-tests.json b/reports/test-audit/suspicious-tests.json index 0294568..1cf3e8a 100644 --- a/reports/test-audit/suspicious-tests.json +++ b/reports/test-audit/suspicious-tests.json @@ -5,22 +5,16 @@ "No obvious assertions" ] }, - { - "file": "test/memory-cache.test.mjs", - "issues": [ - "Found setTimeout()" - ] - }, - { - "file": "test/memory.test.js", - "issues": [ - "Found setTimeout()" - ] - }, { "file": "test/relay-fanout-quorum.integration.test.mjs", "issues": [ "Found setTimeout()" ] + }, + { + "file": "test/utils-async.test.mjs", + "issues": [ + "Found setTimeout()" + ] } ] \ No newline at end of file diff --git a/reports/test-audit/test-audit-report-2026-02-21.md b/reports/test-audit/test-audit-report-2026-02-21.md new file mode 100644 index 0000000..f047c1b --- /dev/null +++ b/reports/test-audit/test-audit-report-2026-02-21.md @@ -0,0 +1,89 @@ +# Test Audit Report - 2026-02-21 + +## Scope +- Agent: `test-audit-agent` +- Cadence: `daily` +- Prompt: `src/prompts/daily/test-audit-agent.md` +- Focus: scenario fidelity, determinism, and cheat-vector resistance in audit tooling/tests. + +## Discovery +- Test runner entry points: + - `npm test` -> `npm run validate:scheduler && npm run test:integration:e2e && node --test --test-timeout=30000 test/*.test.mjs test/*.test.js` + - `npm run test:integration:e2e` -> `node --test test/scheduler-preflight-lock.e2e.test.mjs` +- CI entry points: + - `.github/workflows/ci.yml` runs `npm run lint` and `npm test`. +- Audit tools used: + - `scripts/test-audit/run-static-analysis.mjs` + - `scripts/test-audit/run-flaky-check.mjs` + +## Findings +1. **High - flakiness signal could be silently false-green** + - `scripts/test-audit/run-flaky-check.mjs` parsed only piped child stdout and produced `{}` in this environment, even when tests emitted pass/fail TAP lines. + - Risk: audit report can claim "no flakes" with zero observed outcomes. +2. **Medium - unresolved scheduler e2e platform mismatch** + - `test/scheduler-preflight-lock.e2e.test.mjs` still fails on `platform` expectation (`unknown` expected vs `codex` actual). + - Treated as unresolved spec/contract mismatch; test expectation was not weakened. +3. **Low - static-analysis noise** + - `reports/test-audit/suspicious-tests.json` flags `test/fixtures/memory-fixtures.js` for assertions (fixture-only file), plus intentional `setTimeout` usages in timing-related tests. + +## Changes Applied +- Hardened `scripts/test-audit/run-flaky-check.mjs` to: + - force TAP reporter output to per-run files (`flaky-run-*.tap`), + - parse TAP files deterministically, + - emit run diagnostics (`flakiness-runs.json`), + - fail closed when zero outcomes are observed. +- Added integration regression coverage: + - `test/test-audit-run-flaky-check.test.mjs` + - fixtures: `test/fixtures/flaky-check/pass.fixture.test.mjs`, `test/fixtures/flaky-check/fail.fixture.test.mjs` + +## Scenario List (Given/When/Then) +- `SCN-flaky-check-captures-results` + - Given deterministic pass and fail fixture tests, + - When `run-flaky-check.mjs` executes 5 runs, + - Then `flakiness-matrix.json` records 5 pass counts for the pass fixture and 5 fail counts for the fail fixture. +- `SCN-flaky-check-fail-closed-on-no-signal` + - Given no observed TAP outcomes across all runs, + - When `run-flaky-check.mjs` completes, + - Then the command exits non-zero to prevent false-green flakiness reports. + +## Validation +- `node scripts/test-audit/run-static-analysis.mjs --output-dir reports/test-audit` +- `node scripts/test-audit/run-flaky-check.mjs --output-dir reports/test-audit test/scheduler-preflight-lock.e2e.test.mjs test/scheduler-lock-failure-schema.contract.test.mjs test/lock-ops-publish-retry.test.mjs` +- `node --test test/test-audit-run-flaky-check.test.mjs` +- `node test/scheduler-preflight-lock.e2e.test.mjs` (repro unresolved issue) + +## Needs Spec Clarification +- Scheduler e2e platform assertion source of truth is unclear in this environment. +- Candidate scenarios: + - Given scheduler completion log frontmatter, when runtime platform is detected/declared, then platform value should come from one canonical source (`AGENT_PLATFORM` or detector), consistently in code and tests. + +## Test Integrity Note +```yaml +test_integrity_note: + change_type: ["new_tests", "refactor_tests", "flake_fix"] + scenarios: + - id: SCN-flaky-check-captures-results + given: "A deterministic pass fixture and deterministic fail fixture" + when: "run-flaky-check executes five node:test runs" + then: "flakiness matrix records pass/fail counts per fixture across all runs" + - id: SCN-flaky-check-fail-closed-on-no-signal + given: "No TAP outcomes are captured for all runs" + when: "run-flaky-check finishes aggregation" + then: "script exits non-zero instead of emitting a false-green empty matrix" + observable_outcomes: + - "reports/test-audit/flakiness-matrix.json includes concrete pass/fail counters" + - "reports/test-audit/flakiness-runs.json includes per-run observed counts and exit codes" + - "process exit code is non-zero when observed outcome count is zero" + determinism_controls: + - "Fixture tests are deterministic and local" + - "TAP reporter output is written to explicit files instead of environment-sensitive piped stdio" + anti_cheat_rationale: + prevents: + - "hard-coded empty flakiness output" + - "false-green flake audit when no outcomes are parsed" + - "environment-dependent stdout capture masking failures" + - "retry/sleep-based flake masking" + relaxation: + did_relax_any_assertion: false + if_true_explain_spec_basis: "" +``` diff --git a/reports/test-audit/test-audit-run-flaky-check.exitcode b/reports/test-audit/test-audit-run-flaky-check.exitcode new file mode 100644 index 0000000..573541a --- /dev/null +++ b/reports/test-audit/test-audit-run-flaky-check.exitcode @@ -0,0 +1 @@ +0 diff --git a/scripts/agent/run-scheduler-cycle.mjs b/scripts/agent/run-scheduler-cycle.mjs index 5c7f088..7dcdf3b 100644 --- a/scripts/agent/run-scheduler-cycle.mjs +++ b/scripts/agent/run-scheduler-cycle.mjs @@ -47,6 +47,7 @@ import { randomUUID } from 'node:crypto'; import { runCommand, parseJsonFromOutput, + parseJsonEventsFromOutput, readJson, normalizeStringList, parseNonNegativeInt, @@ -61,6 +62,7 @@ import { parseTimestampFromFilename, isStrictSchedulerLogFilename, parseAgentFromFilename, + sleep, } from './scheduler-utils.mjs'; import { @@ -82,6 +84,12 @@ const FAILURE_CATEGORY = { EXECUTION: 'execution_error', }; +const HANDOFF_FAILURE = { + NETWORK: 'network_error', + RUNNER_UNAVAILABLE: 'runner_unavailable', + UNKNOWN: 'unknown_error', +}; + /** * Parses CLI arguments into a structured options object. * Supports both positional cadence (`daily`|`weekly`) and named flags. @@ -141,6 +149,23 @@ function categorizeFailureMetadata(failureCategory, extraMetadata = {}) { }; } +function getDefaultRunnerHealthCommand({ platform, handoffCommand }) { + if (!handoffCommand || !handoffCommand.includes('run-selected-prompt.mjs')) { + return null; + } + const normalizedPlatform = String(platform || '').trim().toLowerCase(); + if (normalizedPlatform === 'codex') { + return 'command -v codex >/dev/null 2>&1'; + } + if (normalizedPlatform === 'claude') { + return 'command -v claude >/dev/null 2>&1'; + } + if (normalizedPlatform === 'linux') { + return 'command -v echo >/dev/null 2>&1'; + } + return null; +} + /** * Validates that a prompt file is readable and contains a valid markdown heading * or blockquote as its first non-empty line. @@ -225,6 +250,55 @@ async function verifyMemoryStep({ name, markers, artifacts, outputText, sinceMs }; } +function classifyHandoffFailure(text) { + const normalized = String(text || '').toLowerCase(); + if (!normalized) return HANDOFF_FAILURE.UNKNOWN; + + if ( + normalized.includes('stream disconnected') + || normalized.includes('econnreset') + || normalized.includes('enotfound') + || normalized.includes('eai_again') + || normalized.includes('timed out') + || normalized.includes('timeout') + || normalized.includes('temporarily unavailable') + || normalized.includes('connection reset') + || normalized.includes('502') + || normalized.includes('503') + || normalized.includes('504') + ) { + return HANDOFF_FAILURE.NETWORK; + } + + if ( + normalized.includes('runner command not found') + || normalized.includes('command not found') + || normalized.includes('enoent') + || normalized.includes('unsupported agent_platform') + ) { + return HANDOFF_FAILURE.RUNNER_UNAVAILABLE; + } + + return HANDOFF_FAILURE.UNKNOWN; +} + +function isRetryableHandoffFailure({ failureCategory, combinedOutput, retryOnPatterns }) { + if (failureCategory === HANDOFF_FAILURE.NETWORK) { + return true; + } + const text = String(combinedOutput || ''); + for (const pattern of retryOnPatterns || []) { + try { + if (new RegExp(pattern, 'i').test(text)) { + return true; + } + } catch { + // Invalid regex in config; ignore and continue. + } + } + return false; +} + /** * Finds the most recent valid scheduler task log file in `logDir`. @@ -333,6 +407,7 @@ async function getSchedulerConfig(cadence, { isInteractive }) { const handoffCommand = scheduler.handoffCommandByCadence?.[cadence] || null; const missingHandoffCommandForMode = !isInteractive && !handoffCommand; const memoryPolicyRaw = scheduler.memoryPolicyByCadence?.[cadence] || {}; + const handoffPolicyRaw = scheduler.handoffPolicyByCadence?.[cadence] || {}; const mode = memoryPolicyRaw.mode === 'required' ? 'required' : 'optional'; const lockRetryRaw = scheduler.lockRetry || {}; const maxRetries = parseNonNegativeInt( @@ -365,11 +440,42 @@ async function getSchedulerConfig(cadence, { isInteractive }) { process.env.SCHEDULER_MAX_DEFERRALS, parseNonNegativeInt(scheduler.max_deferrals, 3), ); + const handoffMaxAttempts = Math.max(1, parseNonNegativeInt( + process.env.SCHEDULER_HANDOFF_MAX_ATTEMPTS, + parseNonNegativeInt(handoffPolicyRaw.maxAttempts, 1), + )); + const handoffRetryBackoffMs = parseNonNegativeInt( + process.env.SCHEDULER_HANDOFF_RETRY_BACKOFF_MS, + parseNonNegativeInt(handoffPolicyRaw.retryBackoffMs, 1000), + ); + const runnerHealthCommand = (process.env.SCHEDULER_RUNNER_HEALTH_COMMAND + || scheduler.runnerHealthCommandByCadence?.[cadence] + || null); + const fallbackPlatform = (process.env.SCHEDULER_HANDOFF_FALLBACK_PLATFORM + || handoffPolicyRaw.fallbackPlatform + || null); + const retryOnPatterns = normalizeStringList(handoffPolicyRaw.retryOnPatterns, [ + 'stream disconnected', + 'timeout', + 'temporarily unavailable', + 'connection reset', + ]); return { firstPrompt: scheduler.firstPromptByCadence?.[cadence] || null, handoffCommand, + runnerHealthCommand: typeof runnerHealthCommand === 'string' && runnerHealthCommand.trim() + ? runnerHealthCommand.trim() + : null, missingHandoffCommandForMode, + handoffPolicy: { + maxAttempts: handoffMaxAttempts, + retryBackoffMs: handoffRetryBackoffMs, + retryOnPatterns, + fallbackPlatform: typeof fallbackPlatform === 'string' && fallbackPlatform.trim() + ? fallbackPlatform.trim() + : null, + }, validationCommands: Array.isArray(scheduler.validationCommandsByCadence?.[cadence]) ? scheduler.validationCommandsByCadence[cadence].filter((cmd) => typeof cmd === 'string' && cmd.trim()) : ['npm run lint'], @@ -582,6 +688,8 @@ async function main() { const isInteractive = Boolean(process.stdin.isTTY && process.stdout.isTTY); const schedulerConfig = await getSchedulerConfig(cadence, { isInteractive }); + const runnerHealthPreflightCommand = schedulerConfig.runnerHealthCommand + || getDefaultRunnerHealthCommand({ platform, handoffCommand: schedulerConfig.handoffCommand }); const logDir = path.resolve(process.cwd(), 'task-logs', cadence); const agentsPath = path.resolve(process.cwd(), 'AGENTS.md'); @@ -636,7 +744,14 @@ async function main() { // Step 6: Build exclusion set from relay state. Prefer the top-level `excluded` // field (computed by cmdCheck), falling back to the union of locked/paused/completed. const checkResult = await runCommand('npm', ['run', `lock:check:${cadence}`, '--', '--json', '--quiet']); - const checkPayload = parseJsonFromOutput(`${checkResult.stdout}\n${checkResult.stderr}`) || {}; + const checkCombinedOutput = `${checkResult.stdout}\n${checkResult.stderr}`; + const checkEvents = parseJsonEventsFromOutput(checkCombinedOutput); + const checkPayload = checkEvents.findLast((event) => ( + Array.isArray(event.excluded) + || Array.isArray(event.locked) + || Array.isArray(event.paused) + || Array.isArray(event.completed) + )) || parseJsonFromOutput(checkCombinedOutput) || {}; const excluded = Array.isArray(checkPayload.excluded) ? checkPayload.excluded : [...new Set([ @@ -711,6 +826,43 @@ async function main() { }); } + if (runnerHealthPreflightCommand) { + const runnerPreflightPromptPath = path.resolve(process.cwd(), 'src/prompts', cadence, `${selectedAgent}.md`); + const runnerPreflight = await runCommand('bash', ['-lc', runnerHealthPreflightCommand], { + env: { + AGENT_PLATFORM: platform, + ...(model ? { AGENT_MODEL: model } : {}), + SCHEDULER_AGENT: selectedAgent, + SCHEDULER_CADENCE: cadence, + SCHEDULER_PROMPT_PATH: runnerPreflightPromptPath, + }, + }); + if (runnerPreflight.code !== 0) { + await writeLog({ + cadence, + agent: selectedAgent, + status: 'failed', + platform, + reason: 'Runner health preflight failed', + detail: `Prompt not executed. Command failed: ${runnerHealthPreflightCommand}`, + metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { + failure_class: 'runner_unavailable', + runner_health_command: runnerHealthPreflightCommand, + runner_health_stdout_excerpt: excerptText(runnerPreflight.stdout) || '(empty)', + runner_health_stderr_excerpt: excerptText(runnerPreflight.stderr) || '(empty)', + }), + }); + await exitWithSummary(runnerPreflight.code, { + status: 'failed', + agent: selectedAgent, + promptPath: runnerPreflightPromptPath, + reason: 'Runner health preflight failed', + detail: runnerHealthPreflightCommand, + platform, + }); + } + } + const deferralForAgent = schedulerRunState.lock_deferral?.selected_agent === selectedAgent ? schedulerRunState.lock_deferral : null; @@ -910,26 +1062,95 @@ async function main() { } if (schedulerConfig.handoffCommand) { - const handoff = await runCommand('bash', ['-lc', schedulerConfig.handoffCommand], { - env: schedulerEnv, - }); - outputChunks.push(handoff.stdout, handoff.stderr); - if (handoff.code !== 0) { + const handoffAttempts = []; + const maxAttempts = schedulerConfig.handoffPolicy.maxAttempts; + let handoffSucceeded = false; + let fallbackUsed = false; + let attempt = 0; + let finalHandoff = { code: 1, stdout: '', stderr: '' }; + + while (attempt < maxAttempts) { + attempt += 1; + const handoff = await runCommand('bash', ['-lc', schedulerConfig.handoffCommand], { + env: schedulerEnv, + }); + handoffAttempts.push(handoff); + outputChunks.push(handoff.stdout, handoff.stderr); + finalHandoff = handoff; + if (handoff.code === 0) { + handoffSucceeded = true; + break; + } + const combinedOutput = `${handoff.stdout}\n${handoff.stderr}`; + const failureCategory = classifyHandoffFailure(combinedOutput); + const retryable = isRetryableHandoffFailure({ + failureCategory, + combinedOutput, + retryOnPatterns: schedulerConfig.handoffPolicy.retryOnPatterns, + }); + if (!retryable || attempt >= maxAttempts) { + break; + } + if (schedulerConfig.handoffPolicy.retryBackoffMs > 0) { + await sleep(schedulerConfig.handoffPolicy.retryBackoffMs * attempt); + } + } + + if ( + !handoffSucceeded + && schedulerConfig.handoffPolicy.fallbackPlatform + && schedulerConfig.handoffPolicy.fallbackPlatform !== platform + ) { + const fallbackPlatform = schedulerConfig.handoffPolicy.fallbackPlatform; + const fallbackResult = await runCommand('bash', ['-lc', schedulerConfig.handoffCommand], { + env: { + ...schedulerEnv, + AGENT_PLATFORM: fallbackPlatform, + }, + }); + handoffAttempts.push(fallbackResult); + outputChunks.push(fallbackResult.stdout, fallbackResult.stderr); + finalHandoff = fallbackResult; + fallbackUsed = true; + if (fallbackResult.code === 0) { + handoffSucceeded = true; + } + } + + if (!handoffSucceeded) { + const combinedOutput = `${finalHandoff.stdout}\n${finalHandoff.stderr}`; + const failureCategory = classifyHandoffFailure(combinedOutput); + const retryable = isRetryableHandoffFailure({ + failureCategory, + combinedOutput, + retryOnPatterns: schedulerConfig.handoffPolicy.retryOnPatterns, + }); + const detail = `Handoff callback failed after ${handoffAttempts.length} attempt(s)${fallbackUsed ? ` including fallback platform ${schedulerConfig.handoffPolicy.fallbackPlatform}` : ''}.`; await writeLog({ cadence, agent: selectedAgent, status: 'failed', platform, reason: 'Prompt/handoff execution failed', - detail: 'Handoff callback failed.', - metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { failure_class: 'prompt_validation_error' }), + detail, + metadata: categorizeFailureMetadata(FAILURE_CATEGORY.EXECUTION, { + failure_class: 'prompt_validation_error', + handoff_failure_category: failureCategory, + handoff_retryable: retryable, + handoff_attempts: handoffAttempts.length, + handoff_fallback_used: fallbackUsed, + handoff_fallback_platform: fallbackUsed ? schedulerConfig.handoffPolicy.fallbackPlatform : null, + handoff_last_exit_code: finalHandoff.code, + handoff_stdout_excerpt: excerptText(finalHandoff.stdout) || '(empty)', + handoff_stderr_excerpt: excerptText(finalHandoff.stderr) || '(empty)', + }), }); - await exitWithSummary(handoff.code, { + await exitWithSummary(finalHandoff.code || 1, { status: 'failed', agent: selectedAgent, promptPath, reason: 'Prompt/handoff execution failed', - detail: 'Handoff callback failed', + detail, memoryFile, platform, }); diff --git a/scripts/memory/retrieve.mjs b/scripts/memory/retrieve.mjs index 4bdda00..4cf1a75 100644 --- a/scripts/memory/retrieve.mjs +++ b/scripts/memory/retrieve.mjs @@ -84,6 +84,7 @@ try { writeFileSync(path.join(sessionDir, 'retrieve.json'), JSON.stringify(artifact, null, 2)); writeFileSync(path.join(sessionDir, 'retrieve.ok'), 'MEMORY_RETRIEVED\n'); writeFileSync(path.join(latestDir, 'retrieve.ok'), 'MEMORY_RETRIEVED\n'); + writeFileSync(path.join(latestDir, 'retrieve.json'), JSON.stringify(artifact, null, 2)); // Format retrieved memories as readable markdown sections. // Prefer m.summary (compact excerpt) over m.content (full blob) for prompt injection. const memoriesMarkdown = retrieved.length > 0 diff --git a/scripts/memory/store.mjs b/scripts/memory/store.mjs index cf0e170..e365e46 100644 --- a/scripts/memory/store.mjs +++ b/scripts/memory/store.mjs @@ -108,6 +108,7 @@ try { writeFileSync(path.join(sessionDir, 'store.json'), JSON.stringify(artifact, null, 2)); writeFileSync(path.join(sessionDir, 'store.ok'), 'MEMORY_STORED\n'); writeFileSync(path.join(latestDir, 'store.ok'), 'MEMORY_STORED\n'); + writeFileSync(path.join(latestDir, 'store.json'), JSON.stringify(artifact, null, 2)); // 4. Output success marker to stdout (for scheduler verification) console.log('MEMORY_STORED'); diff --git a/scripts/test-audit/run-flaky-check.mjs b/scripts/test-audit/run-flaky-check.mjs index 575e518..e693d65 100644 --- a/scripts/test-audit/run-flaky-check.mjs +++ b/scripts/test-audit/run-flaky-check.mjs @@ -1,9 +1,10 @@ import { spawn } from 'node:child_process'; -import { writeFileSync, mkdirSync } from 'node:fs'; +import { writeFileSync, mkdirSync, readFileSync, existsSync } from 'node:fs'; import { join } from 'node:path'; const RUNS = 5; const results = {}; +const runDiagnostics = []; // Parse arguments to separate our flags from test runner args const args = process.argv.slice(2); @@ -22,44 +23,65 @@ for (let i = 0; i < args.length; i++) { async function runTests(i) { console.log(`Run ${i + 1}/${RUNS}...`); return new Promise((resolve) => { - // Pass strictly the test args to the child process - const child = spawn('node', ['--test', ...testArgs], { - stdio: ['ignore', 'pipe', 'pipe'] + const tapOutputPath = join(outputDir, `flaky-run-${String(i + 1).padStart(2, '0')}.tap`); + const child = spawn(process.execPath, [ + '--test', + '--test-reporter=tap', + `--test-reporter-destination=${tapOutputPath}`, + ...testArgs + ], { + stdio: ['ignore', 'ignore', 'ignore'] }); - let stdout = ''; - - child.stdout.on('data', (d) => stdout += d.toString()); - - child.on('close', (_code) => { - const lines = stdout.split('\n'); + child.on('close', (code) => { + let observed = 0; + const tapOutput = existsSync(tapOutputPath) ? readFileSync(tapOutputPath, 'utf8') : ''; + const lines = tapOutput.split('\n'); for (const line of lines) { if (line.startsWith('not ok ') && !line.includes('# skip') && !line.includes('# todo')) { const name = line.substring(line.indexOf('-') + 1).trim(); if (!results[name]) results[name] = { pass: 0, fail: 0 }; results[name].fail++; + observed++; } else if (line.startsWith('ok ') && !line.includes('# skip') && !line.includes('# todo')) { const name = line.substring(line.indexOf('-') + 1).trim(); if (!results[name]) results[name] = { pass: 0, fail: 0 }; results[name].pass++; + observed++; } } + runDiagnostics.push({ + run: i + 1, + code: code ?? 1, + tapOutputPath, + observed + }); resolve(); }); }); } async function main() { + // Ensure output directory exists before creating per-run TAP files + mkdirSync(outputDir, { recursive: true }); + for (let i = 0; i < RUNS; i++) { await runTests(i); } - // Ensure output directory exists - mkdirSync(outputDir, { recursive: true }); - const outputPath = join(outputDir, 'flakiness-matrix.json'); writeFileSync(outputPath, JSON.stringify(results, null, 2)); console.log(`Flakiness matrix written to ${outputPath}.`); + + const diagnosticsPath = join(outputDir, 'flakiness-runs.json'); + writeFileSync(diagnosticsPath, JSON.stringify(runDiagnostics, null, 2)); + + // Fail closed: an empty matrix means the signal is missing, not "no flakiness". + const totalObserved = runDiagnostics.reduce((sum, run) => sum + run.observed, 0); + if (totalObserved === 0) { + console.error('No TAP test outcomes were observed across runs; flakiness report is not trustworthy.'); + process.exitCode = 1; + } } main(); diff --git a/src/context/CONTEXT_2026-02-21T19-48-18Z.md b/src/context/CONTEXT_2026-02-21T19-48-18Z.md new file mode 100644 index 0000000..b592053 --- /dev/null +++ b/src/context/CONTEXT_2026-02-21T19-48-18Z.md @@ -0,0 +1,17 @@ +agent: test-audit-agent +cadence: daily +session: daily-20260221T194818Z-test-audit-agent +run-start: 2026-02-21T19:48:18Z +prompt-file: src/prompts/daily/test-audit-agent.md + +Goal +- Audit test integrity and strengthen scenario-level anti-cheat coverage. + +Scope +- Inspect test/CI entry points and run audit tools. +- Fix audit signal integrity defects. +- Add regression tests for audit tooling behavior. + +Constraints +- No test weakening. +- No scheduler completion/log publication from spawned agent. diff --git a/src/decisions/DECISIONS_2026-02-21T19-48-18Z.md b/src/decisions/DECISIONS_2026-02-21T19-48-18Z.md new file mode 100644 index 0000000..2d5e2f6 --- /dev/null +++ b/src/decisions/DECISIONS_2026-02-21T19-48-18Z.md @@ -0,0 +1,20 @@ +agent: test-audit-agent +cadence: daily +session: daily-20260221T194818Z-test-audit-agent +run-start: 2026-02-21T19:48:18Z +prompt-file: src/prompts/daily/test-audit-agent.md + +Decision +- Refactor `scripts/test-audit/run-flaky-check.mjs` to parse TAP reporter files per run and fail closed when zero outcomes are observed. + +Rationale +- Piped child-process output is environment-sensitive and produced empty matrices in this run. +- A truthful flakiness audit must not report success when no evidence was captured. + +Alternatives considered +- Keep stdout parsing and parse stderr too: still vulnerable to no-output child-process environments. +- Loosen checks and allow empty matrices: rejected as anti-cheat regression. + +Consequence +- Flaky-check now exposes missing signal as a failing audit outcome. +- Added dedicated regression test with deterministic pass/fail fixtures. diff --git a/src/lib.mjs b/src/lib.mjs index ab7b9c3..ddb87e1 100644 --- a/src/lib.mjs +++ b/src/lib.mjs @@ -102,7 +102,13 @@ export async function cmdCheck(cadence, deps = {}) { } } - const locks = await queryLocks(relays, cadence, dateStr, namespace); + const locks = await queryLocks( + relays, + cadence, + dateStr, + namespace, + quiet ? { errorLogger: () => {}, healthLogger: () => {} } : {}, + ); const lockedAgents = [...new Set(locks.map((l) => l.agent).filter(Boolean))]; const roster = await getRoster(cadence); const rosterSet = new Set(roster); diff --git a/src/lock-utils.mjs b/src/lock-utils.mjs index 7abcad2..8e68dd2 100644 --- a/src/lock-utils.mjs +++ b/src/lock-utils.mjs @@ -3,7 +3,7 @@ import path from 'node:path'; import { randomInt } from 'node:crypto'; import { todayDateStr, getIsoWeekStr } from './utils.mjs'; -export { relayListLabel } from './utils.mjs'; +export { relayListLabel, mergeRelayList } from './utils.mjs'; /** * Scans the local log directory to identify agents that have already completed their task diff --git a/src/ops.mjs b/src/ops.mjs index 5c831ad..d471aac 100644 --- a/src/ops.mjs +++ b/src/ops.mjs @@ -289,8 +289,14 @@ function configureTorch(cwd, paths, installDir, namespace, relays, hashtag) { mode: "required", retrieveSuccessMarkers: ["MEMORY_RETRIEVED"], storeSuccessMarkers: ["MEMORY_STORED"], - retrieveArtifacts: [`.scheduler-memory/latest/${cadence}/retrieve.ok`], - storeArtifacts: [`.scheduler-memory/latest/${cadence}/store.ok`] + retrieveArtifacts: [ + `.scheduler-memory/latest/${cadence}/retrieve.ok`, + `.scheduler-memory/latest/${cadence}/retrieve.json`, + ], + storeArtifacts: [ + `.scheduler-memory/latest/${cadence}/store.ok`, + `.scheduler-memory/latest/${cadence}/store.json`, + ], }; } @@ -298,6 +304,22 @@ function configureTorch(cwd, paths, installDir, namespace, relays, hashtag) { const policy = configData.scheduler.memoryPolicyByCadence[cadence]; policy.retrieveCommand = `node ${scriptPrefix}scripts/memory/retrieve.mjs`; policy.storeCommand = `node ${scriptPrefix}scripts/memory/store.mjs`; + const retrieveArtifacts = Array.isArray(policy.retrieveArtifacts) ? policy.retrieveArtifacts : []; + const storeArtifacts = Array.isArray(policy.storeArtifacts) ? policy.storeArtifacts : []; + if (!retrieveArtifacts.includes(`.scheduler-memory/latest/${cadence}/retrieve.ok`)) { + retrieveArtifacts.push(`.scheduler-memory/latest/${cadence}/retrieve.ok`); + } + if (!retrieveArtifacts.includes(`.scheduler-memory/latest/${cadence}/retrieve.json`)) { + retrieveArtifacts.push(`.scheduler-memory/latest/${cadence}/retrieve.json`); + } + if (!storeArtifacts.includes(`.scheduler-memory/latest/${cadence}/store.ok`)) { + storeArtifacts.push(`.scheduler-memory/latest/${cadence}/store.ok`); + } + if (!storeArtifacts.includes(`.scheduler-memory/latest/${cadence}/store.json`)) { + storeArtifacts.push(`.scheduler-memory/latest/${cadence}/store.json`); + } + policy.retrieveArtifacts = retrieveArtifacts; + policy.storeArtifacts = storeArtifacts; }); fs.writeFileSync(configPath, JSON.stringify(configData, null, 2), 'utf8'); diff --git a/src/test_logs/TEST_LOG_2026-02-21T19-48-18Z.md b/src/test_logs/TEST_LOG_2026-02-21T19-48-18Z.md new file mode 100644 index 0000000..f0eaab9 --- /dev/null +++ b/src/test_logs/TEST_LOG_2026-02-21T19-48-18Z.md @@ -0,0 +1,26 @@ +agent: test-audit-agent +cadence: daily +session: daily-20260221T194818Z-test-audit-agent +run-start: 2026-02-21T19:48:18Z +prompt-file: src/prompts/daily/test-audit-agent.md + +Command: node scripts/memory/retrieve.mjs +Result: pass +Notes: emitted MEMORY_RETRIEVED; artifact marker at .scheduler-memory/latest/daily/retrieve.ok + +Command: node scripts/test-audit/run-static-analysis.mjs --output-dir reports/test-audit +Result: pass +Notes: wrote reports/test-audit/suspicious-tests.json + +Command: node scripts/test-audit/run-flaky-check.mjs --output-dir reports/test-audit test/scheduler-preflight-lock.e2e.test.mjs test/scheduler-lock-failure-schema.contract.test.mjs test/lock-ops-publish-retry.test.mjs +Result: pass +Notes: wrote flakiness-matrix.json + flakiness-runs.json + flaky-run-*.tap + +Command: node --test test/test-audit-run-flaky-check.test.mjs +Result: pass +Notes: regression test for deterministic flakiness signal capture + +Command: node test/scheduler-preflight-lock.e2e.test.mjs +Result: fail +Failure-ID: KNOWN-ISSUE-scheduler-preflight-platform +Notes: expected platform unknown vs actual codex; issue tracked in KNOWN_ISSUES.md and incidents note. diff --git a/src/todo/TODO_2026-02-21T19-48-18Z.md b/src/todo/TODO_2026-02-21T19-48-18Z.md new file mode 100644 index 0000000..acbb80c --- /dev/null +++ b/src/todo/TODO_2026-02-21T19-48-18Z.md @@ -0,0 +1,17 @@ +agent: test-audit-agent +cadence: daily +session: daily-20260221T194818Z-test-audit-agent +run-start: 2026-02-21T19:48:18Z +prompt-file: src/prompts/daily/test-audit-agent.md + +Completed +- Read required baseline files and scheduler shared contract. +- Executed memory retrieval command and verified retrieval marker/artifacts. +- Ran static-analysis and flaky-check audit tools. +- Fixed flaky-check deterministic signal capture. +- Added regression test and deterministic fixtures. +- Reproduced unresolved scheduler platform mismatch and updated issue/incidents tracking. + +Remaining +- Execute memory store workflow. +- Final summary handoff to scheduler. diff --git a/task-logs/daily/2026-02-21T19-46-36Z__style-agent__failed.md b/task-logs/daily/2026-02-21T19-46-36Z__style-agent__failed.md new file mode 100644 index 0000000..a74cd46 --- /dev/null +++ b/task-logs/daily/2026-02-21T19-46-36Z__style-agent__failed.md @@ -0,0 +1,20 @@ +--- +cadence: daily +agent: style-agent +status: failed +reason: 'Prompt/handoff execution failed' +detail: 'Command failed: AGENT_PLATFORM=codex node scripts/agent/run-selected-prompt.mjs (stream disconnected before completion)' +created_at: 2026-02-21T19:46:36Z +timestamp: 2026-02-21T19:46:36Z +platform: 'codex' +failure_category: 'execution_error' +failure_class: 'prompt_validation_error' +prompt_path: 'src/prompts/daily/style-agent.md' +--- +# Scheduler failed +- reason: Prompt/handoff execution failed +- detail: Command failed: AGENT_PLATFORM=codex node scripts/agent/run-selected-prompt.mjs (stream disconnected before completion) +- platform: codex +- failure_category: execution_error +- failure_class: prompt_validation_error +- prompt: src/prompts/daily/style-agent.md diff --git a/task-logs/daily/2026-02-21T19-55-27Z__test-audit-agent__completed.md b/task-logs/daily/2026-02-21T19-55-27Z__test-audit-agent__completed.md new file mode 100644 index 0000000..7b82ccf --- /dev/null +++ b/task-logs/daily/2026-02-21T19-55-27Z__test-audit-agent__completed.md @@ -0,0 +1,18 @@ +--- +cadence: daily +agent: test-audit-agent +status: completed +reason: 'Completed successfully' +detail: 'Prompt executed, memory retrieval/storage evidence validated, lint passed, and completion published.' +created_at: 2026-02-21T19:55:27Z +timestamp: 2026-02-21T19:55:27Z +platform: 'codex' +prompt_path: 'src/prompts/daily/test-audit-agent.md' +memory_file: 'memory-updates/2026-02-21T19-48-22Z__test-audit-agent.md' +--- +# Scheduler completed +- reason: Completed successfully +- detail: Prompt executed, memory retrieval/storage evidence validated, lint passed, and completion published. +- platform: codex +- prompt: src/prompts/daily/test-audit-agent.md +- memory_file: memory-updates/2026-02-21T19-48-22Z__test-audit-agent.md diff --git a/test/fixtures/flaky-check/fail.fixture.test.mjs b/test/fixtures/flaky-check/fail.fixture.test.mjs new file mode 100644 index 0000000..1ec5b57 --- /dev/null +++ b/test/fixtures/flaky-check/fail.fixture.test.mjs @@ -0,0 +1,6 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +test('fail fixture stays red', () => { + assert.equal(1, 2); +}); diff --git a/test/fixtures/flaky-check/pass.fixture.test.mjs b/test/fixtures/flaky-check/pass.fixture.test.mjs new file mode 100644 index 0000000..4c96b53 --- /dev/null +++ b/test/fixtures/flaky-check/pass.fixture.test.mjs @@ -0,0 +1,6 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; + +test('pass fixture stays green', () => { + assert.equal(2 + 2, 4); +}); diff --git a/test/lib.test.mjs b/test/lib.test.mjs index 050308e..04e10b8 100644 --- a/test/lib.test.mjs +++ b/test/lib.test.mjs @@ -96,6 +96,24 @@ describe('src/lib.mjs', () => { // unknown agents are not in roster, so available should be full roster (since no roster agent is locked) assert.deepStrictEqual(result.available, mockRoster.sort()); }); + + it('suppresses relay query loggers when quiet mode is enabled', async () => { + let receivedQueryDeps = null; + const deps = { + ...baseDeps, + json: true, + quiet: true, + queryLocks: async (_relays, _cadence, _dateStr, _namespace, queryDeps) => { + receivedQueryDeps = queryDeps; + return []; + }, + }; + + await cmdCheck('daily', deps); + + assert.equal(typeof receivedQueryDeps.errorLogger, 'function'); + assert.equal(typeof receivedQueryDeps.healthLogger, 'function'); + }); }); describe('cmdLock', () => { diff --git a/test/run-scheduler-cycle-memory-policy.test.mjs b/test/run-scheduler-cycle-memory-policy.test.mjs index b738e61..6af066b 100644 --- a/test/run-scheduler-cycle-memory-policy.test.mjs +++ b/test/run-scheduler-cycle-memory-policy.test.mjs @@ -59,6 +59,9 @@ async function setupFixture({ roster = ['agent-a'], lockHealthPreflight = undefined, schedulerPolicy = {}, + handoffCommand = 'echo HANDOFF_OK', + validationCommands = ['true'], + lockCheckOutput = `echo '{"excluded":[]}'`, preflightScript = '#!/usr/bin/env node\nconsole.log(JSON.stringify({ ok: true, relays: ["wss://relay.test"] }));\nprocess.exit(0);\n', }) { const root = await fs.mkdtemp(path.join(os.tmpdir(), 'scheduler-memory-policy-')); @@ -103,8 +106,8 @@ async function setupFixture({ JSON.stringify({ scheduler: { firstPromptByCadence: { daily: 'agent-a' }, - handoffCommandByCadence: { daily: 'echo HANDOFF_OK' }, - validationCommandsByCadence: { daily: ['true'] }, + handoffCommandByCadence: { daily: handoffCommand }, + validationCommandsByCadence: { daily: validationCommands }, memoryPolicyByCadence: { daily: memoryPolicy }, ...schedulerPolicy, ...(lockHealthPreflight === undefined ? {} : { lockHealthPreflight }), @@ -122,7 +125,7 @@ if [[ "$1" != "run" ]]; then fi case "$2" in lock:check:daily) - echo '{"excluded":[]}' +${lockCheckOutput} ;; lock:health) shift 2 @@ -611,3 +614,110 @@ test('does not use backend retry flow for exit code 3 lock conflicts', { concurr assert.equal(result.code, 0, `stdout: ${result.stdout}\nstderr: ${result.stderr}`); assert.doesNotMatch(result.stdout, /"event":"scheduler.lock.retry"/); }); + +test('retries handoff command for retryable network failures and succeeds', { concurrency: false }, async () => { + const fixture = await setupFixture({ + memoryPolicy: { mode: 'optional' }, + handoffCommand: `count_file="$PWD/.handoff-count" +count=0 +if [[ -f "$count_file" ]]; then + count="$(cat "$count_file")" +fi +count=$((count + 1)) +echo "$count" > "$count_file" +if [[ "$count" -lt 2 ]]; then + echo "stream disconnected before completion" >&2 + exit 12 +fi +echo HANDOFF_OK`, + schedulerPolicy: { + handoffPolicyByCadence: { + daily: { + maxAttempts: 3, + retryBackoffMs: 0, + }, + }, + }, + }); + + const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], { + cwd: fixture.root, + env: fixture.env, + }); + + assert.equal(result.code, 0, `stdout: ${result.stdout}\nstderr: ${result.stderr}`); + const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily')); + assert.ok(logs.some((name) => name.endsWith('__completed.md'))); +}); + +test('uses fallback platform for handoff when primary platform fails', { concurrency: false }, async () => { + const fixture = await setupFixture({ + memoryPolicy: { mode: 'optional' }, + handoffCommand: `if [[ "$AGENT_PLATFORM" == "linux" ]]; then + echo HANDOFF_FALLBACK_OK + exit 0 +fi +echo "stream disconnected before completion" >&2 +exit 13`, + schedulerPolicy: { + handoffPolicyByCadence: { + daily: { + maxAttempts: 1, + fallbackPlatform: 'linux', + }, + }, + }, + }); + + const result = await runNode(fixture.scriptPath, ['--cadence', 'daily', '--platform', 'codex'], { + cwd: fixture.root, + env: fixture.env, + }); + + assert.equal(result.code, 0, `stdout: ${result.stdout}\nstderr: ${result.stderr}`); + const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily')); + assert.ok(logs.some((name) => name.endsWith('__completed.md'))); +}); + +test('fails before lock acquisition when runner health preflight command fails', { concurrency: false }, async () => { + const fixture = await setupFixture({ + memoryPolicy: { mode: 'optional' }, + schedulerPolicy: { + runnerHealthCommandByCadence: { + daily: 'echo RUNNER_UNAVAILABLE >&2; exit 9', + }, + }, + }); + + const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], { + cwd: fixture.root, + env: fixture.env, + }); + + assert.equal(result.code, 9, `stdout: ${result.stdout}\nstderr: ${result.stderr}`); + const logs = await fs.readdir(path.join(fixture.root, 'task-logs', 'daily')); + const failedLog = logs.find((name) => name.endsWith('__failed.md')); + assert.ok(failedLog); + const failedBody = await fs.readFile(path.join(fixture.root, 'task-logs', 'daily', failedLog), 'utf8'); + assert.match(failedBody, /reason: Runner health preflight failed/); + assert.match(failedBody, /failure_class: 'runner_unavailable'/); + assert.match(failedBody, /failure_category: 'execution_error'/); + assert.match(failedBody, /Prompt not executed\./); +}); + +test('uses exclusion payload from mixed JSON lock:check output', { concurrency: false }, async () => { + const fixture = await setupFixture({ + memoryPolicy: { mode: 'optional' }, + roster: ['agent-a', 'agent-b'], + lockCheckOutput: `echo '{"event":"relay_health_snapshot"}' +echo '{"excluded":["agent-a"]}'`, + }); + + const result = await runNode(fixture.scriptPath, ['--cadence', 'daily'], { + cwd: fixture.root, + env: fixture.env, + }); + + assert.equal(result.code, 0, `stdout: ${result.stdout}\nstderr: ${result.stderr}`); + assert.match(result.stdout, /Agent:\s+agent-b/); +}); diff --git a/test/test-audit-run-flaky-check.test.mjs b/test/test-audit-run-flaky-check.test.mjs new file mode 100644 index 0000000..73fceaf --- /dev/null +++ b/test/test-audit-run-flaky-check.test.mjs @@ -0,0 +1,45 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { spawn } from 'node:child_process'; + +const SCRIPT = path.resolve('scripts/test-audit/run-flaky-check.mjs'); + +function runFlakyCheck(args) { + return new Promise((resolve) => { + const childEnv = { ...process.env }; + delete childEnv.NODE_TEST_CONTEXT; + delete childEnv.NODE_UNIQUE_ID; + + const child = spawn(process.execPath, [SCRIPT, ...args], { + cwd: process.cwd(), + stdio: ['ignore', 'pipe', 'pipe'], + env: childEnv, + }); + + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (chunk) => { stdout += chunk.toString(); }); + child.stderr.on('data', (chunk) => { stderr += chunk.toString(); }); + child.on('close', (code) => resolve({ code: code ?? 1, stdout, stderr })); + }); +} + +test('run-flaky-check reports pass/fail counts from TAP output files', async () => { + const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flaky-check-report-')); + const passFixture = 'test/fixtures/flaky-check/pass.fixture.test.mjs'; + const failFixture = 'test/fixtures/flaky-check/fail.fixture.test.mjs'; + + const result = await runFlakyCheck(['--output-dir', outputDir, passFixture, failFixture]); + assert.equal(result.code, 0, result.stderr || result.stdout); + + const matrix = JSON.parse(await fs.readFile(path.join(outputDir, 'flakiness-matrix.json'), 'utf8')); + assert.deepEqual(matrix[passFixture], { pass: 5, fail: 0 }); + assert.deepEqual(matrix[failFixture], { pass: 0, fail: 5 }); + + const diagnostics = JSON.parse(await fs.readFile(path.join(outputDir, 'flakiness-runs.json'), 'utf8')); + assert.equal(diagnostics.length, 5); + assert.ok(diagnostics.every((run) => run.observed >= 2)); +}); diff --git a/torch-config.example.json b/torch-config.example.json index aafd0bf..dafd23d 100644 --- a/torch-config.example.json +++ b/torch-config.example.json @@ -26,6 +26,30 @@ "daily": "scheduler-update-agent", "weekly": "bug-reproducer-agent" }, + "handoffCommandByCadence": { + "daily": "node torch/scripts/agent/run-selected-prompt.mjs", + "weekly": "node torch/scripts/agent/run-selected-prompt.mjs" + }, + "handoffPolicyByCadence": { + "daily": { + "maxAttempts": 2, + "retryBackoffMs": 1000, + "retryOnPatterns": [ + "stream disconnected", + "timeout", + "temporarily unavailable" + ], + "fallbackPlatform": "linux" + }, + "weekly": { + "maxAttempts": 2, + "retryBackoffMs": 1000 + } + }, + "runnerHealthCommandByCadence": { + "daily": "command -v codex >/dev/null 2>&1", + "weekly": "command -v codex >/dev/null 2>&1" + }, "memoryPolicyByCadence": { "daily": { "mode": "required", @@ -38,10 +62,12 @@ "MEMORY_STORED" ], "retrieveArtifacts": [ - ".scheduler-memory/latest/daily/retrieve.ok" + ".scheduler-memory/latest/daily/retrieve.ok", + ".scheduler-memory/latest/daily/retrieve.json" ], "storeArtifacts": [ - ".scheduler-memory/latest/daily/store.ok" + ".scheduler-memory/latest/daily/store.ok", + ".scheduler-memory/latest/daily/store.json" ] }, "weekly": { @@ -55,10 +81,12 @@ "MEMORY_STORED" ], "retrieveArtifacts": [ - ".scheduler-memory/latest/weekly/retrieve.ok" + ".scheduler-memory/latest/weekly/retrieve.ok", + ".scheduler-memory/latest/weekly/retrieve.json" ], "storeArtifacts": [ - ".scheduler-memory/latest/weekly/store.ok" + ".scheduler-memory/latest/weekly/store.ok", + ".scheduler-memory/latest/weekly/store.json" ] } }, diff --git a/torch-config.json b/torch-config.json index 60cda0f..96fe9b9 100644 --- a/torch-config.json +++ b/torch-config.json @@ -34,10 +34,12 @@ "MEMORY_STORED" ], "retrieveArtifacts": [ - ".scheduler-memory/latest/daily/retrieve.ok" + ".scheduler-memory/latest/daily/retrieve.ok", + ".scheduler-memory/latest/daily/retrieve.json" ], "storeArtifacts": [ - ".scheduler-memory/latest/daily/store.ok" + ".scheduler-memory/latest/daily/store.ok", + ".scheduler-memory/latest/daily/store.json" ] }, "weekly": { @@ -51,12 +53,14 @@ "MEMORY_STORED" ], "retrieveArtifacts": [ - ".scheduler-memory/latest/weekly/retrieve.ok" + ".scheduler-memory/latest/weekly/retrieve.ok", + ".scheduler-memory/latest/weekly/retrieve.json" ], "storeArtifacts": [ - ".scheduler-memory/latest/weekly/store.ok" + ".scheduler-memory/latest/weekly/store.ok", + ".scheduler-memory/latest/weekly/store.json" ] } } } -} \ No newline at end of file +}