TORCH/test/test-audit-run-flaky-check.test.mjs

import { test } from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
import { spawn } from 'node:child_process';

const SCRIPT = path.resolve('scripts/test-audit/run-flaky-check.mjs');

function runFlakyCheck(args) {
  return new Promise((resolve) => {
    const childEnv = { ...process.env };
    delete childEnv.NODE_TEST_CONTEXT;
    delete childEnv.NODE_UNIQUE_ID;

    const child = spawn(process.execPath, [SCRIPT, ...args], {
      cwd: process.cwd(),
      stdio: ['ignore', 'pipe', 'pipe'],
      env: childEnv,
    });

    let stdout = '';
    let stderr = '';
    child.stdout.on('data', (chunk) => { stdout += chunk.toString(); });
    child.stderr.on('data', (chunk) => { stderr += chunk.toString(); });
    child.on('close', (code) => resolve({ code: code ?? 1, stdout, stderr }));
  });
}

test('run-flaky-check reports pass/fail counts from TAP output files', async () => {
  const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), 'flaky-check-report-'));
  const passFixture = 'test/fixtures/flaky-check/pass.fixture.test.mjs';
  const failFixture = 'test/fixtures/flaky-check/fail.fixture.test.mjs';

  const result = await runFlakyCheck(['--output-dir', outputDir, passFixture, failFixture]);
  assert.equal(result.code, 0, result.stderr || result.stdout);

  const matrix = JSON.parse(await fs.readFile(path.join(outputDir, 'flakiness-matrix.json'), 'utf8'));

  // The tool keys results by test name, not filename
  assert.deepEqual(matrix['pass fixture stays green'], { pass: 5, fail: 0 });
  assert.deepEqual(matrix['fail fixture stays red'], { pass: 0, fail: 5 });

  const diagnostics = JSON.parse(await fs.readFile(path.join(outputDir, 'flakiness-runs.json'), 'utf8'));
  assert.equal(diagnostics.length, 5);
  assert.ok(diagnostics.every((run) => run.observed >= 2));
});