diff --git a/.github/workflows/mutation-testing.yml b/.github/workflows/mutation-testing.yml new file mode 100644 index 00000000..1d0a23b5 --- /dev/null +++ b/.github/workflows/mutation-testing.yml @@ -0,0 +1,68 @@ +name: Mutation Testing + +# WHY THIS IS SCHEDULED AND NOT ON EVERY PUSH +# +# A full sweep is ~132 mutants and each one runs `tsc` plus a vitest suite, +# so it takes roughly an hour. That is far too slow to gate a push, and the +# thing it measures does not change push-to-push: it answers "would the suite +# NOTICE if a security check broke?", which drifts slowly as tests and +# checks are added, not with every commit. +# +# The failure it catches is specific and otherwise invisible: someone adds a +# security check with no test, or deletes the only test defending one, and +# every other check stays green because the code still works on the cases +# somebody wrote down. Nothing else in this repository can see that. + +on: + schedule: + # Mondays, 06:17 UTC. Off the hour on purpose — scheduled jobs at :00 + # queue behind everyone else's. + - cron: "17 6 * * 1" + workflow_dispatch: + inputs: + target: + description: "Which target to sweep (agent, routes, server, sinks, approvals, highrisk, or all)" + required: false + default: "all" + +permissions: + contents: read + +jobs: + mutate: + runs-on: ubuntu-latest + # Comfortably above the ~60 minute full sweep, so a slow runner reports a + # result rather than a timeout that looks like an infrastructure problem. + timeout-minutes: 150 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Enable pnpm via corepack + run: corepack enable + - name: Install JS dependencies + run: pnpm install --frozen-lockfile + working-directory: js + - name: Sweep + # Exit 1 = a mutant survived: a security check whose removal no test + # notices. Exit 2 = a target generated no mutants at all, which is + # NOT a pass — it means the generator cannot reach that file and the + # sweep measured nothing there. + run: | + python scripts/mutation_test.py --target "${{ github.event.inputs.target || 'all' }}" + - name: Confirm no mutant was left in the tree + # The script restores after every mutant and verifies the restore, and + # traps SIGINT/SIGTERM/SIGHUP to do the same on a kill. This is the + # belt-and-braces check: a job that ended with deliberately broken + # security code in the workspace must be loud about it, because the + # whole technique depends on that never escaping. + if: always() + run: | + if ! git diff --exit-code; then + echo "::error::A mutation was left in the working tree. Do not trust this run." + exit 1 + fi diff --git a/TODO.md b/TODO.md index 57f7208f..60bf8ee4 100644 --- a/TODO.md +++ b/TODO.md @@ -139,10 +139,19 @@ everything still open, in the order it should be tackled. converge to different vaults. Also found — via mutation testing turned on the fuzzer itself — that it could not reach the tombstone or subject caps; fixed and re-verified. - - [ ] **Wire the fuzzer and `cross_impl_check.py` into CI.** Neither - runs automatically, and this branch has a documented history of - CI going unwatched (ts-parity red for 33 runs). A check nobody - runs is not a check. + - [x] **Wire the fuzzer and `cross_impl_check.py` into CI.** Done — + `.github/workflows/ts-parity.yml` runs both: `cross-implementation` + and `differential-fuzz` (three fixed seeds for bisectable + regressions, plus one exploratory seed derived from the run id so + the explored input space grows instead of freezing). + - [x] **Mutation testing in CI**, added 2026-08-19 as + `.github/workflows/mutation-testing.yml`. Weekly rather than + per-push: a full sweep is ~1 hour, and what it measures ("would + the suite notice if a security check broke?") drifts with test + and check changes, not with every commit. The script now exits 1 + on a survivor and 2 on a target that generated no mutants, so + "not measured" cannot report as "clean". The job also fails if + any mutation is left in the tree at the end. - [x] **Systematic mutation testing** of the security-critical modules that have NO differential oracle because Python has no equivalent: the session agent, the API HTTP layer, high-risk diff --git a/scripts/mutation_test.py b/scripts/mutation_test.py index 1950a7db..0908f8eb 100644 --- a/scripts/mutation_test.py +++ b/scripts/mutation_test.py @@ -369,6 +369,17 @@ def main() -> int: print(f" was: {mutant.original.strip()[:110]}") print(f" now: {mutant.mutated.strip()[:110]}") print() + + # Exit non-zero so this can gate CI. A survivor is not automatically a + # defect -- it may be an equivalent mutant -- but it is always something a + # human has to look at and then either fix or annotate with + # `mutation-equivalent:`, which is exactly what a red check should mean. + # A target that generated nothing gets its own code, because "not + # measured" and "measured and clean" must not be the same signal. + if overall_survivors: + return 1 + if unmeasured: + return 2 return 0