mirror of
https://github.com/PR0M3TH3AN/TORCH.git
synced 2026-10-27 07:51:52 +00:00
Run daily test-audit-agent
Co-authored-by: PR0M3TH3AN <[email protected]>
This commit is contained in:
1 parent
7dbb56745c
commit
b1eb7f07f0
16 files changed
+3727
No files matched your search
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"cadence": "daily",
|
||||
"operation": "retrieve",
|
||||
"runId": "session-mlny4il2",
|
||||
"servicePath": "src/services/memory/index.js#getRelevantMemories",
|
||||
"inputs": {
|
||||
"agentId": "test-audit-agent",
|
||||
"promptPath": "src/prompts/daily/test-audit-agent.md",
|
||||
"promptIntent": "## Required startup + artifacts + memory + issue capture",
|
||||
"query": "scheduler | daily | agent | test-audit-agent | intent | Required startup + artifacts + memory + issue capture",
|
||||
"events": 1
|
||||
},
|
||||
"outputs": {
|
||||
"ingestedCount": 1,
|
||||
"retrievedCount": 1
|
||||
},
|
||||
"status": "ok"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
MEMORY_RETRIEVED
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"cadence": "daily",
|
||||
"operation": "store",
|
||||
"runId": "session-mlnyc1lp",
|
||||
"servicePath": "src/services/memory/index.js#ingestEvents",
|
||||
"inputs": {
|
||||
"agentId": "test-audit-agent",
|
||||
"promptPath": "src/prompts/daily/test-audit-agent.md",
|
||||
"promptIntent": "## Required startup + artifacts + memory + issue capture",
|
||||
"events": 2
|
||||
},
|
||||
"outputs": {
|
||||
"storedCount": 1,
|
||||
"summaries": [
|
||||
"Store memory event A for daily :: ## Required startup + artifacts + memory + issue capture Store memory event B for daily :: ## Required startup + artifacts + memory + issue capture"
|
||||
]
|
||||
},
|
||||
"status": "ok"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
MEMORY_STORED
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
agent: test-audit-agent
|
||||
cadence: daily
|
||||
run-start: 2026-02-15T16-15-05Z
|
||||
prompt: src/prompts/daily/test-audit-agent.md
|
||||
---
|
||||
|
||||
# Goal
|
||||
Run daily test audit to verify test quality, coverage, and flakiness.
|
||||
|
||||
# Scope
|
||||
- Run unit tests with coverage.
|
||||
- Detect flaky tests.
|
||||
- Static analysis for bad patterns.
|
||||
- Map coverage to critical files.
|
||||
|
||||
# Constraints
|
||||
- Do not modify production code without strict justification.
|
||||
- Keep CI green.
|
||||
- Prefer test-only fixes.
|
||||
@@ -0,0 +1,12 @@
|
||||
---
|
||||
agent: test-audit-agent
|
||||
cadence: daily
|
||||
run-start: 2026-02-15T16-15-05Z
|
||||
prompt: src/prompts/daily/test-audit-agent.md
|
||||
---
|
||||
|
||||
# Decision
|
||||
- Using manual execution of agent steps as 'run-selected-prompt.mjs' is incompatible with sandbox environment.
|
||||
|
||||
# Rationale
|
||||
- Sandbox lacks 'codex'/'claude' CLI tools.
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,15 @@
|
||||
---
|
||||
agent: test-audit-agent
|
||||
cadence: daily
|
||||
run-start: 2026-02-15T16-15-05Z
|
||||
prompt: src/prompts/daily/test-audit-agent.md
|
||||
---
|
||||
|
||||
# Pending tasks
|
||||
- [ ] Run tests with coverage
|
||||
- [ ] Detect flakiness
|
||||
- [ ] Run static analysis
|
||||
- [ ] Generate report
|
||||
|
||||
# Completed tasks
|
||||
- [x] Initialize artifacts
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
agent: test-audit-agent
|
||||
cadence: daily
|
||||
platform: sandbox
|
||||
---
|
||||
|
||||
# Completed
|
||||
Task completed successfully.
|
||||
Artifacts created in src/context, src/todo, src/decisions, src/test_logs.
|
||||
Report: test-audit-report-2026-02-15.md
|
||||
@@ -0,0 +1,35 @@
|
||||
# Test Audit Report 2026-02-15
|
||||
|
||||
## Summary
|
||||
- **Test Runner**: Node.js --test
|
||||
- **Coverage**: ~64% Line Coverage
|
||||
- **Flakiness**: 0 flaky tests found in 5 runs.
|
||||
- **Static Analysis**: 5 files flagged for timeouts.
|
||||
|
||||
## Failing Tests
|
||||
None found during flakiness check.
|
||||
|
||||
## Flakiness
|
||||
See test-audit/flakiness-matrix.json. All tests passed 5/5 runs.
|
||||
|
||||
## Suspicious Tests
|
||||
The following files contain `setTimeout` or lack assertions:
|
||||
- test/memory-cache.test.mjs
|
||||
- test/memory-scheduler.test.mjs
|
||||
- test/relay-fanout-quorum.integration.test.mjs
|
||||
- tests/fixtures/memory-fixtures.js (Fixture, expected)
|
||||
- tests/memory.test.js
|
||||
|
||||
## Coverage Gaps
|
||||
Based on coverage run:
|
||||
- Overall Line Coverage: 64.37%
|
||||
- Overall Branch Coverage: 58.20%
|
||||
- Overall Function Coverage: 75.46%
|
||||
|
||||
Critical files with potential gaps:
|
||||
- src/run-scheduler-cycle.mjs (53% line coverage)
|
||||
- src/scripts/agent/check-relay-health.mjs (100%)
|
||||
|
||||
## Recommendations
|
||||
1. Replace `setTimeout` with fake timers or deterministic waits where possible.
|
||||
2. Increase coverage for `run-scheduler-cycle.mjs`.
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,151 @@
|
||||
TAP version 13
|
||||
# Subtest: parseArgs should parse basic command
|
||||
ok 1 - parseArgs should parse basic command
|
||||
---
|
||||
duration_ms: 2.074523
|
||||
type: 'test'
|
||||
...
|
||||
# Subtest: parseArgs should parse agent and cadence
|
||||
ok 2 - parseArgs should parse agent and cadence
|
||||
---
|
||||
duration_ms: 0.373647
|
||||
type: 'test'
|
||||
...
|
||||
# Subtest: parseArgs should parse equals sign syntax
|
||||
ok 3 - parseArgs should parse equals sign syntax
|
||||
---
|
||||
duration_ms: 0.204255
|
||||
type: 'test'
|
||||
...
|
||||
# Subtest: parseArgs should parse dry-run flag
|
||||
ok 4 - parseArgs should parse dry-run flag
|
||||
---
|
||||
duration_ms: 0.16969
|
||||
type: 'test'
|
||||
...
|
||||
# Subtest: parseArgs should parse port with default
|
||||
ok 5 - parseArgs should parse port with default
|
||||
---
|
||||
duration_ms: 0.170406
|
||||
type: 'test'
|
||||
...
|
||||
# Subtest: parseArgs should parse custom port
|
||||
ok 6 - parseArgs should parse custom port
|
||||
---
|
||||
duration_ms: 0.211104
|
||||
type: 'test'
|
||||
...
|
||||
# Subtest: parseArgs should parse ignore-logs flag
|
||||
ok 7 - parseArgs should parse ignore-logs flag
|
||||
---
|
||||
duration_ms: 0.273681
|
||||
type: 'test'
|
||||
...
|
||||
# Subtest: parseArgs should parse memory flags
|
||||
ok 8 - parseArgs should parse memory flags
|
||||
---
|
||||
duration_ms: 5.033281
|
||||
type: 'test'
|
||||
...
|
||||
# Subtest: parseArgs should parse memory flags with equals
|
||||
ok 9 - parseArgs should parse memory flags with equals
|
||||
---
|
||||
duration_ms: 0.269666
|
||||
type: 'test'
|
||||
...
|
||||
1..9
|
||||
# tests 9
|
||||
# suites 0
|
||||
# pass 9
|
||||
# fail 0
|
||||
# cancelled 0
|
||||
# skipped 0
|
||||
# todo 0
|
||||
# duration_ms 32.484347
|
||||
# start of coverage report
|
||||
# ---------------------------------------------------------------------
|
||||
# file | line % | branch % | funcs % | uncovered lines
|
||||
# ---------------------------------------------------------------------
|
||||
# src | | | |
|
||||
# cli-parser.mjs | 76.99 | 85.71 | 100.00 | 52 56 86-109
|
||||
# constants.mjs | 100.00 | 100.00 | 100.00 |
|
||||
# test | | | |
|
||||
# cli-parser.test.mjs | 100.00 | 100.00 | 100.00 |
|
||||
# ---------------------------------------------------------------------
|
||||
# all files | 87.50 | 88.33 | 100.00 |
|
||||
# ---------------------------------------------------------------------
|
||||
# end of coverage report
|
||||
TAP version 13
|
||||
# node:internal/modules/cjs/loader:1386
|
||||
# throw err;
|
||||
# ^
|
||||
# Error: Cannot find module '/app/test'
|
||||
# at Function._resolveFilename (node:internal/modules/cjs/loader:1383:15)
|
||||
# at defaultResolveImpl (node:internal/modules/cjs/loader:1025:19)
|
||||
# at resolveForCJSWithHooks (node:internal/modules/cjs/loader:1030:22)
|
||||
# at Function._load (node:internal/modules/cjs/loader:1192:37)
|
||||
# at TracingChannel.traceSync (node:diagnostics_channel:328:14)
|
||||
# at wrapModuleLoad (node:internal/modules/cjs/loader:237:24)
|
||||
# at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:171:5)
|
||||
# at node:internal/main/run_main_module:36:49 {
|
||||
# code: 'MODULE_NOT_FOUND',
|
||||
# requireStack: []
|
||||
# }
|
||||
# Node.js v22.22.0
|
||||
# Subtest: test
|
||||
not ok 1 - test
|
||||
---
|
||||
duration_ms: 51.535733
|
||||
type: 'test'
|
||||
location: '/app/test:1:1'
|
||||
failureType: 'testCodeFailure'
|
||||
exitCode: 1
|
||||
signal: ~
|
||||
error: 'test failed'
|
||||
code: 'ERR_TEST_FAILURE'
|
||||
...
|
||||
# node:internal/modules/cjs/loader:1386
|
||||
# throw err;
|
||||
# ^
|
||||
# Error: Cannot find module '/app/tests'
|
||||
# at Function._resolveFilename (node:internal/modules/cjs/loader:1383:15)
|
||||
# at defaultResolveImpl (node:internal/modules/cjs/loader:1025:19)
|
||||
# at resolveForCJSWithHooks (node:internal/modules/cjs/loader:1030:22)
|
||||
# at Function._load (node:internal/modules/cjs/loader:1192:37)
|
||||
# at TracingChannel.traceSync (node:diagnostics_channel:328:14)
|
||||
# at wrapModuleLoad (node:internal/modules/cjs/loader:237:24)
|
||||
# at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:171:5)
|
||||
# at node:internal/main/run_main_module:36:49 {
|
||||
# code: 'MODULE_NOT_FOUND',
|
||||
# requireStack: []
|
||||
# }
|
||||
# Node.js v22.22.0
|
||||
# Subtest: tests
|
||||
not ok 2 - tests
|
||||
---
|
||||
duration_ms: 50.740646
|
||||
type: 'test'
|
||||
location: '/app/tests:1:1'
|
||||
failureType: 'testCodeFailure'
|
||||
exitCode: 1
|
||||
signal: ~
|
||||
error: 'test failed'
|
||||
code: 'ERR_TEST_FAILURE'
|
||||
...
|
||||
1..2
|
||||
# tests 2
|
||||
# suites 0
|
||||
# pass 0
|
||||
# fail 2
|
||||
# cancelled 0
|
||||
# skipped 0
|
||||
# todo 0
|
||||
# duration_ms 66.495879
|
||||
# start of coverage report
|
||||
# ----------------------------------------------------------
|
||||
# file | line % | branch % | funcs % | uncovered lines
|
||||
# ----------------------------------------------------------
|
||||
# ----------------------------------------------------------
|
||||
# all files | 100.00 | 100.00 | 100.00 |
|
||||
# ----------------------------------------------------------
|
||||
# end of coverage report
|
||||
@@ -0,0 +1,366 @@
|
||||
{
|
||||
"parseArgs should parse basic command": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"parseArgs should parse agent and cadence": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"parseArgs should parse equals sign syntax": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"parseArgs should parse dry-run flag": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"parseArgs should parse port with default": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"parseArgs should parse custom port": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"parseArgs should parse ignore-logs flag": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"parseArgs should parse memory flags": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"parseArgs should parse memory flags with equals": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"src/lib.mjs": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"ci-resilience lock acquisition under unstable relays": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"publishLock retry behavior": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"lock-ops": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"listMemories, inspectMemory, pinMemory, and unpinMemory support admin hooks": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"triggerPruneDryRun returns candidate metadata without deleting records": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"memoryStats reports counts/rates and retrieval telemetry remains redaction-safe": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"runPruneCycle respects prune feature-flag modes": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"stores runtime events in signer/session namespace and clears scope": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"expires runtime events and records expiration metrics": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"enforces LRU bounds and tracks eviction metrics": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"blocks decrypted/session-sensitive data from durable promotion without sanitizer pass": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"createEmbedderAdapter selects in-memory backend and supports vector CRUD": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"createEmbedderAdapter reads backend from environment": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"memory feature flags parse defaults and canary allow-lists": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"memory prune flag supports off, dry-run, and active modes": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"ingestMemoryWindow gathers sources, redacts pii, chunks, embeds, links, and emits telemetry": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"ingestMemoryWindow dedupes overlapping windows by hashed content + agent + timestamp bucket": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"createLifecyclePlan respects delete safety constraints and logs reasons": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"createLifecyclePlan condenses near-duplicates and archives merged source records": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"applyLifecycleActions executes keep/archive/delete and merge markers": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"filterAndRankMemories applies metadata filters and composite score": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"formatMemoriesForPrompt returns prompt blocks and enforces token budget trimming": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"isMemoryRetrievalEnabled supports global toggle and agent canary list": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"renderPromptWithMemoryContext injects bounded memory block before base prompt": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"createDbAdvisoryLockProvider acquires and releases postgres advisory locks": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"startMemoryMaintenanceScheduler runs all jobs and emits metrics including retries": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"startMemoryMaintenanceScheduler skips jobs when feature flags disable memory": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"normalizeMemoryItem applies durable schema defaults and coercions": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"validateMemoryItem reports field-level errors": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"ingestEvents rejects invalid writes and logs structured validation details": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"loadMemoryPromptTemplates loads all memory prompt templates from disk": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"summarizeEvents retries with repair prompt when first response is malformed JSON": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"summarizeEvents falls back to deterministic minimal summary with conservative importance when parsing fails twice": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"CLI Smoke Test": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"cmdInit creates directory structure and files": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"cmdInit fails if directory exists without force": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"cmdUpdate preserves modified prompt": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"cmdUpdate overwrites prompt with force": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"cmdUpdate creates backup": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"torch-lock check respects local roster": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"cmdInit creates torch-config.json with random namespace": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"relay fanout quorum integration": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"raises all-relays-down duration alert after threshold": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"raises success-rate alert when rolling health falls below threshold": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"fails required memory policy when retrieval/storage evidence is missing": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"accepts required memory policy when markers or artifacts are produced": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"records backend failure metadata when lock command exits with code 2": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"defers backend failures in non-strict mode and reuses idempotency key on successful retry": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"fails after exceeding non-strict deferral budget": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"strict lock mode fails immediately without deferral": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"skips lock health preflight by default when not enabled": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"fails scheduler with preflight metadata when lock health preflight is enabled": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"categorizes unreadable prompt files as prompt_parse_error": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"categorizes invalid prompt schema as prompt_schema_error": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"categorizes handoff runtime failures as execution_error": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"retries lock acquisition for backend error and succeeds on a later attempt": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"stops retrying after configured lock backend retries are exhausted": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"does not use backend retry flow for exit code 3 lock conflicts": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"runs selected scheduler prompt through configured runner command": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"propagates non-zero runner exit code": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"scheduler lock backend failure artifact matches required frontmatter schema": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"schema contract catches missing required lock-backend field": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"schema contract catches misnamed lock-backend key": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"lock preflight e2e: successful lock writes completed status snapshot": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"lock preflight e2e: exit code 2 quorum failure persists failed backend status and prompt-not-started marker": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"lock preflight e2e: non-lock failure exits failed without prompt parse/schema classification": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"Scheduler Ratchet Logic (Log Checking)": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"Scheduler cycle ordering guarantees": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"summarizes recent lock reliability by platform/cadence/backend/relay": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"loadTorchConfig (Unit Tests with Mocked FS)": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"torch-config": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"Date Utilities": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"passes when artifacts include metadata and failure IDs map to unresolved known issue entries": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"fails when required metadata is missing from artifacts": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"fails when failure identifiers are not cross-linked to known issues or incidents": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"schema validation covers required fields, coercion, and rejection paths": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"summarizer parser accepts valid json and recovers after malformed json response": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"ingest integration persists memory records and links embeddings from event batches": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"retrieval ranking blends semantic, importance, and recency signals": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"pruner policy protects pins, merges duplicates, and picks archive/delete outcomes": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
},
|
||||
"scheduler smoke test handles mocked clock and lock contention": {
|
||||
"pass": 5,
|
||||
"fail": 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { spawn } from 'node:child_process';
|
||||
import { writeFileSync } from 'node:fs';
|
||||
|
||||
const RUNS = 5;
|
||||
const results = {};
|
||||
|
||||
async function runTests(i) {
|
||||
console.log(`Run ${i + 1}/${RUNS}...`);
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn('node', ['--test', ...process.argv.slice(2)], {
|
||||
stdio: ['ignore', 'pipe', 'pipe']
|
||||
});
|
||||
|
||||
let stdout = '';
|
||||
|
||||
child.stdout.on('data', (d) => stdout += d.toString());
|
||||
|
||||
child.on('close', (code) => {
|
||||
const lines = stdout.split('\n');
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('not ok ') && !line.includes('# skip') && !line.includes('# todo')) {
|
||||
const name = line.substring(line.indexOf('-') + 1).trim();
|
||||
if (!results[name]) results[name] = { pass: 0, fail: 0 };
|
||||
results[name].fail++;
|
||||
} else if (line.startsWith('ok ') && !line.includes('# skip') && !line.includes('# todo')) {
|
||||
const name = line.substring(line.indexOf('-') + 1).trim();
|
||||
if (!results[name]) results[name] = { pass: 0, fail: 0 };
|
||||
results[name].pass++;
|
||||
}
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
for (let i = 0; i < RUNS; i++) {
|
||||
await runTests(i);
|
||||
}
|
||||
|
||||
writeFileSync('test-audit/flakiness-matrix.json', JSON.stringify(results, null, 2));
|
||||
console.log('Flakiness matrix written.');
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,49 @@
|
||||
import { readFileSync, writeFileSync, readdirSync, statSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
|
||||
function getFiles(dir, files = []) {
|
||||
try {
|
||||
const fileList = readdirSync(dir);
|
||||
for (const file of fileList) {
|
||||
const name = join(dir, file);
|
||||
if (statSync(name).isDirectory()) {
|
||||
getFiles(name, files);
|
||||
} else if (name.endsWith('.mjs') || name.endsWith('.js')) {
|
||||
files.push(name);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// ignore if dir doesn't exist
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
const testFiles = [...getFiles('test'), ...getFiles('tests')];
|
||||
const suspicious = [];
|
||||
|
||||
for (const file of testFiles) {
|
||||
const content = readFileSync(file, 'utf8');
|
||||
const issues = [];
|
||||
|
||||
if (content.includes('.only(')) issues.push('Found .only()');
|
||||
if (content.includes('.skip(')) issues.push('Found .skip()');
|
||||
if (content.includes('setTimeout(')) issues.push('Found setTimeout()');
|
||||
if (content.includes('sleep(')) issues.push('Found sleep()');
|
||||
|
||||
// Naive check for assertions: look for assert., expect(, t.
|
||||
// Many tests use 't' context from node:test
|
||||
if (!content.includes('assert.') && !content.includes('expect(') && !content.includes('t.plan') && !content.includes('strictEqual')) {
|
||||
// This is very naive, might false positive on 't.ok' etc.
|
||||
// Let's look for common assertion keywords.
|
||||
const assertionKeywords = ['assert', 'expect', 'strictEqual', 'deepStrictEqual', 'ok', 'equal'];
|
||||
const hasAssertion = assertionKeywords.some(k => content.includes(k));
|
||||
if (!hasAssertion) issues.push('No obvious assertions');
|
||||
}
|
||||
|
||||
if (issues.length > 0) {
|
||||
suspicious.push({ file, issues });
|
||||
}
|
||||
}
|
||||
|
||||
writeFileSync('test-audit/suspicious-tests.json', JSON.stringify(suspicious, null, 2));
|
||||
console.log(`Found ${suspicious.length} suspicious files.`);
|
||||
@@ -0,0 +1,32 @@
|
||||
[
|
||||
{
|
||||
"file": "test/memory-cache.test.mjs",
|
||||
"issues": [
|
||||
"Found setTimeout()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "test/memory-scheduler.test.mjs",
|
||||
"issues": [
|
||||
"Found setTimeout()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "test/relay-fanout-quorum.integration.test.mjs",
|
||||
"issues": [
|
||||
"Found setTimeout()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "tests/fixtures/memory-fixtures.js",
|
||||
"issues": [
|
||||
"No obvious assertions"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "tests/memory.test.js",
|
||||
"issues": [
|
||||
"Found setTimeout()"
|
||||
]
|
||||
}
|
||||
]
|
||||
Reference in new issue
Block a user