mirror of
https://github.com/PR0M3TH3AN/TORCH.git
synced 2026-11-03 03:11:57 +00:00
Run daily test-audit-agent
Co-authored-by: PR0M3TH3AN <[email protected]>
This commit is contained in:
google-labs-jules[bot]andPR0M3TH3AN committed
1 parent
7dbb56745c
commit
b1eb7f07f0
16 files changed
+3727
No files matched your search
@@ -0,0 +1,18 @@
|
|||||||
|
{
|
||||||
|
"cadence": "daily",
|
||||||
|
"operation": "retrieve",
|
||||||
|
"runId": "session-mlny4il2",
|
||||||
|
"servicePath": "src/services/memory/index.js#getRelevantMemories",
|
||||||
|
"inputs": {
|
||||||
|
"agentId": "test-audit-agent",
|
||||||
|
"promptPath": "src/prompts/daily/test-audit-agent.md",
|
||||||
|
"promptIntent": "## Required startup + artifacts + memory + issue capture",
|
||||||
|
"query": "scheduler | daily | agent | test-audit-agent | intent | Required startup + artifacts + memory + issue capture",
|
||||||
|
"events": 1
|
||||||
|
},
|
||||||
|
"outputs": {
|
||||||
|
"ingestedCount": 1,
|
||||||
|
"retrievedCount": 1
|
||||||
|
},
|
||||||
|
"status": "ok"
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
MEMORY_RETRIEVED
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
{
|
||||||
|
"cadence": "daily",
|
||||||
|
"operation": "store",
|
||||||
|
"runId": "session-mlnyc1lp",
|
||||||
|
"servicePath": "src/services/memory/index.js#ingestEvents",
|
||||||
|
"inputs": {
|
||||||
|
"agentId": "test-audit-agent",
|
||||||
|
"promptPath": "src/prompts/daily/test-audit-agent.md",
|
||||||
|
"promptIntent": "## Required startup + artifacts + memory + issue capture",
|
||||||
|
"events": 2
|
||||||
|
},
|
||||||
|
"outputs": {
|
||||||
|
"storedCount": 1,
|
||||||
|
"summaries": [
|
||||||
|
"Store memory event A for daily :: ## Required startup + artifacts + memory + issue capture Store memory event B for daily :: ## Required startup + artifacts + memory + issue capture"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"status": "ok"
|
||||||
|
}
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
MEMORY_STORED
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
---
|
||||||
|
agent: test-audit-agent
|
||||||
|
cadence: daily
|
||||||
|
run-start: 2026-02-15T16-15-05Z
|
||||||
|
prompt: src/prompts/daily/test-audit-agent.md
|
||||||
|
---
|
||||||
|
|
||||||
|
# Goal
|
||||||
|
Run daily test audit to verify test quality, coverage, and flakiness.
|
||||||
|
|
||||||
|
# Scope
|
||||||
|
- Run unit tests with coverage.
|
||||||
|
- Detect flaky tests.
|
||||||
|
- Static analysis for bad patterns.
|
||||||
|
- Map coverage to critical files.
|
||||||
|
|
||||||
|
# Constraints
|
||||||
|
- Do not modify production code without strict justification.
|
||||||
|
- Keep CI green.
|
||||||
|
- Prefer test-only fixes.
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
---
|
||||||
|
agent: test-audit-agent
|
||||||
|
cadence: daily
|
||||||
|
run-start: 2026-02-15T16-15-05Z
|
||||||
|
prompt: src/prompts/daily/test-audit-agent.md
|
||||||
|
---
|
||||||
|
|
||||||
|
# Decision
|
||||||
|
- Using manual execution of agent steps as 'run-selected-prompt.mjs' is incompatible with sandbox environment.
|
||||||
|
|
||||||
|
# Rationale
|
||||||
|
- Sandbox lacks 'codex'/'claude' CLI tools.
|
||||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,15 @@
|
|||||||
|
---
|
||||||
|
agent: test-audit-agent
|
||||||
|
cadence: daily
|
||||||
|
run-start: 2026-02-15T16-15-05Z
|
||||||
|
prompt: src/prompts/daily/test-audit-agent.md
|
||||||
|
---
|
||||||
|
|
||||||
|
# Pending tasks
|
||||||
|
- [ ] Run tests with coverage
|
||||||
|
- [ ] Detect flakiness
|
||||||
|
- [ ] Run static analysis
|
||||||
|
- [ ] Generate report
|
||||||
|
|
||||||
|
# Completed tasks
|
||||||
|
- [x] Initialize artifacts
|
||||||
@@ -0,0 +1,10 @@
|
|||||||
|
---
|
||||||
|
agent: test-audit-agent
|
||||||
|
cadence: daily
|
||||||
|
platform: sandbox
|
||||||
|
---
|
||||||
|
|
||||||
|
# Completed
|
||||||
|
Task completed successfully.
|
||||||
|
Artifacts created in src/context, src/todo, src/decisions, src/test_logs.
|
||||||
|
Report: test-audit-report-2026-02-15.md
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
# Test Audit Report 2026-02-15
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
- **Test Runner**: Node.js --test
|
||||||
|
- **Coverage**: ~64% Line Coverage
|
||||||
|
- **Flakiness**: 0 flaky tests found in 5 runs.
|
||||||
|
- **Static Analysis**: 5 files flagged for timeouts.
|
||||||
|
|
||||||
|
## Failing Tests
|
||||||
|
None found during flakiness check.
|
||||||
|
|
||||||
|
## Flakiness
|
||||||
|
See test-audit/flakiness-matrix.json. All tests passed 5/5 runs.
|
||||||
|
|
||||||
|
## Suspicious Tests
|
||||||
|
The following files contain `setTimeout` or lack assertions:
|
||||||
|
- test/memory-cache.test.mjs
|
||||||
|
- test/memory-scheduler.test.mjs
|
||||||
|
- test/relay-fanout-quorum.integration.test.mjs
|
||||||
|
- tests/fixtures/memory-fixtures.js (Fixture, expected)
|
||||||
|
- tests/memory.test.js
|
||||||
|
|
||||||
|
## Coverage Gaps
|
||||||
|
Based on coverage run:
|
||||||
|
- Overall Line Coverage: 64.37%
|
||||||
|
- Overall Branch Coverage: 58.20%
|
||||||
|
- Overall Function Coverage: 75.46%
|
||||||
|
|
||||||
|
Critical files with potential gaps:
|
||||||
|
- src/run-scheduler-cycle.mjs (53% line coverage)
|
||||||
|
- src/scripts/agent/check-relay-health.mjs (100%)
|
||||||
|
|
||||||
|
## Recommendations
|
||||||
|
1. Replace `setTimeout` with fake timers or deterministic waits where possible.
|
||||||
|
2. Increase coverage for `run-scheduler-cycle.mjs`.
|
||||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,151 @@
|
|||||||
|
TAP version 13
|
||||||
|
# Subtest: parseArgs should parse basic command
|
||||||
|
ok 1 - parseArgs should parse basic command
|
||||||
|
---
|
||||||
|
duration_ms: 2.074523
|
||||||
|
type: 'test'
|
||||||
|
...
|
||||||
|
# Subtest: parseArgs should parse agent and cadence
|
||||||
|
ok 2 - parseArgs should parse agent and cadence
|
||||||
|
---
|
||||||
|
duration_ms: 0.373647
|
||||||
|
type: 'test'
|
||||||
|
...
|
||||||
|
# Subtest: parseArgs should parse equals sign syntax
|
||||||
|
ok 3 - parseArgs should parse equals sign syntax
|
||||||
|
---
|
||||||
|
duration_ms: 0.204255
|
||||||
|
type: 'test'
|
||||||
|
...
|
||||||
|
# Subtest: parseArgs should parse dry-run flag
|
||||||
|
ok 4 - parseArgs should parse dry-run flag
|
||||||
|
---
|
||||||
|
duration_ms: 0.16969
|
||||||
|
type: 'test'
|
||||||
|
...
|
||||||
|
# Subtest: parseArgs should parse port with default
|
||||||
|
ok 5 - parseArgs should parse port with default
|
||||||
|
---
|
||||||
|
duration_ms: 0.170406
|
||||||
|
type: 'test'
|
||||||
|
...
|
||||||
|
# Subtest: parseArgs should parse custom port
|
||||||
|
ok 6 - parseArgs should parse custom port
|
||||||
|
---
|
||||||
|
duration_ms: 0.211104
|
||||||
|
type: 'test'
|
||||||
|
...
|
||||||
|
# Subtest: parseArgs should parse ignore-logs flag
|
||||||
|
ok 7 - parseArgs should parse ignore-logs flag
|
||||||
|
---
|
||||||
|
duration_ms: 0.273681
|
||||||
|
type: 'test'
|
||||||
|
...
|
||||||
|
# Subtest: parseArgs should parse memory flags
|
||||||
|
ok 8 - parseArgs should parse memory flags
|
||||||
|
---
|
||||||
|
duration_ms: 5.033281
|
||||||
|
type: 'test'
|
||||||
|
...
|
||||||
|
# Subtest: parseArgs should parse memory flags with equals
|
||||||
|
ok 9 - parseArgs should parse memory flags with equals
|
||||||
|
---
|
||||||
|
duration_ms: 0.269666
|
||||||
|
type: 'test'
|
||||||
|
...
|
||||||
|
1..9
|
||||||
|
# tests 9
|
||||||
|
# suites 0
|
||||||
|
# pass 9
|
||||||
|
# fail 0
|
||||||
|
# cancelled 0
|
||||||
|
# skipped 0
|
||||||
|
# todo 0
|
||||||
|
# duration_ms 32.484347
|
||||||
|
# start of coverage report
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
# file | line % | branch % | funcs % | uncovered lines
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
# src | | | |
|
||||||
|
# cli-parser.mjs | 76.99 | 85.71 | 100.00 | 52 56 86-109
|
||||||
|
# constants.mjs | 100.00 | 100.00 | 100.00 |
|
||||||
|
# test | | | |
|
||||||
|
# cli-parser.test.mjs | 100.00 | 100.00 | 100.00 |
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
# all files | 87.50 | 88.33 | 100.00 |
|
||||||
|
# ---------------------------------------------------------------------
|
||||||
|
# end of coverage report
|
||||||
|
TAP version 13
|
||||||
|
# node:internal/modules/cjs/loader:1386
|
||||||
|
# throw err;
|
||||||
|
# ^
|
||||||
|
# Error: Cannot find module '/app/test'
|
||||||
|
# at Function._resolveFilename (node:internal/modules/cjs/loader:1383:15)
|
||||||
|
# at defaultResolveImpl (node:internal/modules/cjs/loader:1025:19)
|
||||||
|
# at resolveForCJSWithHooks (node:internal/modules/cjs/loader:1030:22)
|
||||||
|
# at Function._load (node:internal/modules/cjs/loader:1192:37)
|
||||||
|
# at TracingChannel.traceSync (node:diagnostics_channel:328:14)
|
||||||
|
# at wrapModuleLoad (node:internal/modules/cjs/loader:237:24)
|
||||||
|
# at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:171:5)
|
||||||
|
# at node:internal/main/run_main_module:36:49 {
|
||||||
|
# code: 'MODULE_NOT_FOUND',
|
||||||
|
# requireStack: []
|
||||||
|
# }
|
||||||
|
# Node.js v22.22.0
|
||||||
|
# Subtest: test
|
||||||
|
not ok 1 - test
|
||||||
|
---
|
||||||
|
duration_ms: 51.535733
|
||||||
|
type: 'test'
|
||||||
|
location: '/app/test:1:1'
|
||||||
|
failureType: 'testCodeFailure'
|
||||||
|
exitCode: 1
|
||||||
|
signal: ~
|
||||||
|
error: 'test failed'
|
||||||
|
code: 'ERR_TEST_FAILURE'
|
||||||
|
...
|
||||||
|
# node:internal/modules/cjs/loader:1386
|
||||||
|
# throw err;
|
||||||
|
# ^
|
||||||
|
# Error: Cannot find module '/app/tests'
|
||||||
|
# at Function._resolveFilename (node:internal/modules/cjs/loader:1383:15)
|
||||||
|
# at defaultResolveImpl (node:internal/modules/cjs/loader:1025:19)
|
||||||
|
# at resolveForCJSWithHooks (node:internal/modules/cjs/loader:1030:22)
|
||||||
|
# at Function._load (node:internal/modules/cjs/loader:1192:37)
|
||||||
|
# at TracingChannel.traceSync (node:diagnostics_channel:328:14)
|
||||||
|
# at wrapModuleLoad (node:internal/modules/cjs/loader:237:24)
|
||||||
|
# at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:171:5)
|
||||||
|
# at node:internal/main/run_main_module:36:49 {
|
||||||
|
# code: 'MODULE_NOT_FOUND',
|
||||||
|
# requireStack: []
|
||||||
|
# }
|
||||||
|
# Node.js v22.22.0
|
||||||
|
# Subtest: tests
|
||||||
|
not ok 2 - tests
|
||||||
|
---
|
||||||
|
duration_ms: 50.740646
|
||||||
|
type: 'test'
|
||||||
|
location: '/app/tests:1:1'
|
||||||
|
failureType: 'testCodeFailure'
|
||||||
|
exitCode: 1
|
||||||
|
signal: ~
|
||||||
|
error: 'test failed'
|
||||||
|
code: 'ERR_TEST_FAILURE'
|
||||||
|
...
|
||||||
|
1..2
|
||||||
|
# tests 2
|
||||||
|
# suites 0
|
||||||
|
# pass 0
|
||||||
|
# fail 2
|
||||||
|
# cancelled 0
|
||||||
|
# skipped 0
|
||||||
|
# todo 0
|
||||||
|
# duration_ms 66.495879
|
||||||
|
# start of coverage report
|
||||||
|
# ----------------------------------------------------------
|
||||||
|
# file | line % | branch % | funcs % | uncovered lines
|
||||||
|
# ----------------------------------------------------------
|
||||||
|
# ----------------------------------------------------------
|
||||||
|
# all files | 100.00 | 100.00 | 100.00 |
|
||||||
|
# ----------------------------------------------------------
|
||||||
|
# end of coverage report
|
||||||
@@ -0,0 +1,366 @@
|
|||||||
|
{
|
||||||
|
"parseArgs should parse basic command": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"parseArgs should parse agent and cadence": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"parseArgs should parse equals sign syntax": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"parseArgs should parse dry-run flag": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"parseArgs should parse port with default": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"parseArgs should parse custom port": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"parseArgs should parse ignore-logs flag": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"parseArgs should parse memory flags": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"parseArgs should parse memory flags with equals": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"src/lib.mjs": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"ci-resilience lock acquisition under unstable relays": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"publishLock retry behavior": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"lock-ops": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"listMemories, inspectMemory, pinMemory, and unpinMemory support admin hooks": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"triggerPruneDryRun returns candidate metadata without deleting records": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"memoryStats reports counts/rates and retrieval telemetry remains redaction-safe": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"runPruneCycle respects prune feature-flag modes": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"stores runtime events in signer/session namespace and clears scope": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"expires runtime events and records expiration metrics": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"enforces LRU bounds and tracks eviction metrics": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"blocks decrypted/session-sensitive data from durable promotion without sanitizer pass": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"createEmbedderAdapter selects in-memory backend and supports vector CRUD": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"createEmbedderAdapter reads backend from environment": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"memory feature flags parse defaults and canary allow-lists": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"memory prune flag supports off, dry-run, and active modes": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"ingestMemoryWindow gathers sources, redacts pii, chunks, embeds, links, and emits telemetry": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"ingestMemoryWindow dedupes overlapping windows by hashed content + agent + timestamp bucket": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"createLifecyclePlan respects delete safety constraints and logs reasons": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"createLifecyclePlan condenses near-duplicates and archives merged source records": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"applyLifecycleActions executes keep/archive/delete and merge markers": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"filterAndRankMemories applies metadata filters and composite score": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"formatMemoriesForPrompt returns prompt blocks and enforces token budget trimming": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"isMemoryRetrievalEnabled supports global toggle and agent canary list": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"renderPromptWithMemoryContext injects bounded memory block before base prompt": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"createDbAdvisoryLockProvider acquires and releases postgres advisory locks": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"startMemoryMaintenanceScheduler runs all jobs and emits metrics including retries": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"startMemoryMaintenanceScheduler skips jobs when feature flags disable memory": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"normalizeMemoryItem applies durable schema defaults and coercions": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"validateMemoryItem reports field-level errors": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"ingestEvents rejects invalid writes and logs structured validation details": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"loadMemoryPromptTemplates loads all memory prompt templates from disk": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"summarizeEvents retries with repair prompt when first response is malformed JSON": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"summarizeEvents falls back to deterministic minimal summary with conservative importance when parsing fails twice": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"CLI Smoke Test": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"cmdInit creates directory structure and files": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"cmdInit fails if directory exists without force": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"cmdUpdate preserves modified prompt": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"cmdUpdate overwrites prompt with force": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"cmdUpdate creates backup": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"torch-lock check respects local roster": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"cmdInit creates torch-config.json with random namespace": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"relay fanout quorum integration": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"raises all-relays-down duration alert after threshold": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"raises success-rate alert when rolling health falls below threshold": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"fails required memory policy when retrieval/storage evidence is missing": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"accepts required memory policy when markers or artifacts are produced": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"records backend failure metadata when lock command exits with code 2": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"defers backend failures in non-strict mode and reuses idempotency key on successful retry": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"fails after exceeding non-strict deferral budget": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"strict lock mode fails immediately without deferral": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"skips lock health preflight by default when not enabled": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"fails scheduler with preflight metadata when lock health preflight is enabled": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"categorizes unreadable prompt files as prompt_parse_error": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"categorizes invalid prompt schema as prompt_schema_error": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"categorizes handoff runtime failures as execution_error": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"retries lock acquisition for backend error and succeeds on a later attempt": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"stops retrying after configured lock backend retries are exhausted": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"does not use backend retry flow for exit code 3 lock conflicts": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"runs selected scheduler prompt through configured runner command": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"propagates non-zero runner exit code": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"scheduler lock backend failure artifact matches required frontmatter schema": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"schema contract catches missing required lock-backend field": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"schema contract catches misnamed lock-backend key": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"lock preflight e2e: successful lock writes completed status snapshot": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"lock preflight e2e: exit code 2 quorum failure persists failed backend status and prompt-not-started marker": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"lock preflight e2e: non-lock failure exits failed without prompt parse/schema classification": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"Scheduler Ratchet Logic (Log Checking)": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"Scheduler cycle ordering guarantees": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"summarizes recent lock reliability by platform/cadence/backend/relay": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"loadTorchConfig (Unit Tests with Mocked FS)": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"torch-config": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"Date Utilities": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"passes when artifacts include metadata and failure IDs map to unresolved known issue entries": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"fails when required metadata is missing from artifacts": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"fails when failure identifiers are not cross-linked to known issues or incidents": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"schema validation covers required fields, coercion, and rejection paths": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"summarizer parser accepts valid json and recovers after malformed json response": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"ingest integration persists memory records and links embeddings from event batches": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"retrieval ranking blends semantic, importance, and recency signals": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"pruner policy protects pins, merges duplicates, and picks archive/delete outcomes": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
},
|
||||||
|
"scheduler smoke test handles mocked clock and lock contention": {
|
||||||
|
"pass": 5,
|
||||||
|
"fail": 0
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
import { spawn } from 'node:child_process';
|
||||||
|
import { writeFileSync } from 'node:fs';
|
||||||
|
|
||||||
|
const RUNS = 5;
|
||||||
|
const results = {};
|
||||||
|
|
||||||
|
async function runTests(i) {
|
||||||
|
console.log(`Run ${i + 1}/${RUNS}...`);
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
const child = spawn('node', ['--test', ...process.argv.slice(2)], {
|
||||||
|
stdio: ['ignore', 'pipe', 'pipe']
|
||||||
|
});
|
||||||
|
|
||||||
|
let stdout = '';
|
||||||
|
|
||||||
|
child.stdout.on('data', (d) => stdout += d.toString());
|
||||||
|
|
||||||
|
child.on('close', (code) => {
|
||||||
|
const lines = stdout.split('\n');
|
||||||
|
for (const line of lines) {
|
||||||
|
if (line.startsWith('not ok ') && !line.includes('# skip') && !line.includes('# todo')) {
|
||||||
|
const name = line.substring(line.indexOf('-') + 1).trim();
|
||||||
|
if (!results[name]) results[name] = { pass: 0, fail: 0 };
|
||||||
|
results[name].fail++;
|
||||||
|
} else if (line.startsWith('ok ') && !line.includes('# skip') && !line.includes('# todo')) {
|
||||||
|
const name = line.substring(line.indexOf('-') + 1).trim();
|
||||||
|
if (!results[name]) results[name] = { pass: 0, fail: 0 };
|
||||||
|
results[name].pass++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
for (let i = 0; i < RUNS; i++) {
|
||||||
|
await runTests(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
writeFileSync('test-audit/flakiness-matrix.json', JSON.stringify(results, null, 2));
|
||||||
|
console.log('Flakiness matrix written.');
|
||||||
|
}
|
||||||
|
|
||||||
|
main();
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
import { readFileSync, writeFileSync, readdirSync, statSync } from 'node:fs';
|
||||||
|
import { join } from 'node:path';
|
||||||
|
|
||||||
|
function getFiles(dir, files = []) {
|
||||||
|
try {
|
||||||
|
const fileList = readdirSync(dir);
|
||||||
|
for (const file of fileList) {
|
||||||
|
const name = join(dir, file);
|
||||||
|
if (statSync(name).isDirectory()) {
|
||||||
|
getFiles(name, files);
|
||||||
|
} else if (name.endsWith('.mjs') || name.endsWith('.js')) {
|
||||||
|
files.push(name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
// ignore if dir doesn't exist
|
||||||
|
}
|
||||||
|
return files;
|
||||||
|
}
|
||||||
|
|
||||||
|
const testFiles = [...getFiles('test'), ...getFiles('tests')];
|
||||||
|
const suspicious = [];
|
||||||
|
|
||||||
|
for (const file of testFiles) {
|
||||||
|
const content = readFileSync(file, 'utf8');
|
||||||
|
const issues = [];
|
||||||
|
|
||||||
|
if (content.includes('.only(')) issues.push('Found .only()');
|
||||||
|
if (content.includes('.skip(')) issues.push('Found .skip()');
|
||||||
|
if (content.includes('setTimeout(')) issues.push('Found setTimeout()');
|
||||||
|
if (content.includes('sleep(')) issues.push('Found sleep()');
|
||||||
|
|
||||||
|
// Naive check for assertions: look for assert., expect(, t.
|
||||||
|
// Many tests use 't' context from node:test
|
||||||
|
if (!content.includes('assert.') && !content.includes('expect(') && !content.includes('t.plan') && !content.includes('strictEqual')) {
|
||||||
|
// This is very naive, might false positive on 't.ok' etc.
|
||||||
|
// Let's look for common assertion keywords.
|
||||||
|
const assertionKeywords = ['assert', 'expect', 'strictEqual', 'deepStrictEqual', 'ok', 'equal'];
|
||||||
|
const hasAssertion = assertionKeywords.some(k => content.includes(k));
|
||||||
|
if (!hasAssertion) issues.push('No obvious assertions');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (issues.length > 0) {
|
||||||
|
suspicious.push({ file, issues });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
writeFileSync('test-audit/suspicious-tests.json', JSON.stringify(suspicious, null, 2));
|
||||||
|
console.log(`Found ${suspicious.length} suspicious files.`);
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"file": "test/memory-cache.test.mjs",
|
||||||
|
"issues": [
|
||||||
|
"Found setTimeout()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"file": "test/memory-scheduler.test.mjs",
|
||||||
|
"issues": [
|
||||||
|
"Found setTimeout()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"file": "test/relay-fanout-quorum.integration.test.mjs",
|
||||||
|
"issues": [
|
||||||
|
"Found setTimeout()"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"file": "tests/fixtures/memory-fixtures.js",
|
||||||
|
"issues": [
|
||||||
|
"No obvious assertions"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"file": "tests/memory.test.js",
|
||||||
|
"issues": [
|
||||||
|
"Found setTimeout()"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
Reference in new issue
Block a user