mirror of
https://github.com/PR0M3TH3AN/SeedPass.git
synced 2026-10-27 07:52:03 +00:00
test: differential fuzzing against Python, as the substitute for outside review
The cutover plan's last open gate is an independent security review, and there
is no third party to hand the branch to. Self-review has a known failure mode:
you cannot see what you did not think of, which is why two earlier rounds each
found criticals that reading had missed. Me reviewing this session's code has
exactly that defect.
The substitute for another pair of eyes is not more reading. It is an oracle
that is not the author's judgement — and this project has the best one
available: two implementations required to agree. Feed both the same randomly
generated input and any disagreement is a bug in one of them, established
without anyone knowing in advance which answer is right or what to look for.
scripts/differential_fuzz.py covers the surfaces where a disagreement would be
silent and expensive: canonical JSON and hashing (underpins everything else),
password derivation with random policies, the sync CRDT (a mismatch means two
clients converge to DIFFERENT vaults), index0, recovery shares, the semantic
index, and entry hashing. Results are compared as canonical JSON so the
comparison cannot itself paper over an ordering difference. ~10,000 cases
across four seeds currently agree.
IT FOUND A REAL DEFECT, and then found a defect in itself.
canonical.ts asserted that the int/float ambiguity "does not arise" because
SeedPass data never carries float-typed whole numbers. That is true of data
SeedPass writes and false of data it carries: unknown-kind records, hand-edited
vaults, third-party tools. Python distinguishes int from float; JavaScript does
not, so `JSON.parse("0.0")` and `JSON.parse("0")` are the same value and TS
emits "0" where CPython emits "0.0". Entry hashes break same-timestamp merge
ties, so two clients could converge to different vaults — silently. No amount
of formatting fixes it: the information is destroyed at parse time.
The comment is now accurate rather than reassuring, and `findAmbiguousNumbers`
detects the condition on raw JSON text, which is the only moment the
distinction still exists. A second, separate limitation is documented in the
same place: values outside the safe-integer range are refused where Python
encodes them, which is the SAFE direction, because JSON.parse has already
rounded such a value and any hash of it would be a hash of the wrong number.
Then mutation testing was turned on the fuzzer itself, and it failed: two
deliberate off-by-ones in the tombstone-retention and checkpoint-subject caps
went undetected, because random small payloads never reach a bound of 2048 or
64. Two generator bugs behind it — tombstone keys drawn from 31 distinct
values, so 2050 writes landed in 31 slots; and event subjects that peaked at
63 distinct against a cap of 64, one short. Both fixed by generating well past
each bound with margin, and both mutations are now caught. Bounded code is
only tested at its bound.
A fuzzer that cannot fail is worthless, so that is verified too: inverting the
merge tie-break is caught in 20 of 120 cases.
Known number-representation divergences are classified separately rather than
filtered, and the classifier is deliberately narrow — "known limitation" is
exactly the category that quietly grows to hide real bugs.
Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01Vtx94nXDmNgXEeThUZP4QN
This commit is contained in:
thePR0M3TH3ANandClaude Opus 5 committed
1 parent
dd60eabbcf
commit
991c71189e
6 files changed
+876
-7
No files matched your search
@@ -0,0 +1,120 @@
|
||||
/**
|
||||
* TypeScript side of scripts/differential_fuzz.py.
|
||||
*
|
||||
* Reads a JSON array of requests, returns a JSON array of results in the same
|
||||
* order. Errors are returned as `ERROR:<Name>` strings rather than thrown, so
|
||||
* that "both implementations refuse this input" counts as agreement — a
|
||||
* divergence where one accepts and the other refuses is exactly the kind of
|
||||
* bug this is looking for.
|
||||
*/
|
||||
|
||||
import { readFileSync } from "node:fs";
|
||||
import {
|
||||
canonicalJson,
|
||||
entryEventHash,
|
||||
mergeIndexPayloads,
|
||||
generatePassword,
|
||||
passwordPolicyFromRecord,
|
||||
Bip85,
|
||||
appendIndex0Event,
|
||||
compactIndex0Payload,
|
||||
recoverSecret,
|
||||
buildSemanticRecords,
|
||||
searchSemanticRecords,
|
||||
} from "./src/index.js";
|
||||
|
||||
type Request = Record<string, any>;
|
||||
|
||||
function attempt(fn: () => unknown): unknown {
|
||||
try {
|
||||
return fn();
|
||||
} catch (e) {
|
||||
// Mirrors the Python side's `ERROR:<ExceptionName>`. Python raises
|
||||
// ValueError where TS throws Error, so the names are normalized to the
|
||||
// one distinction that matters: refused vs accepted.
|
||||
void e;
|
||||
return "ERROR:Refused";
|
||||
}
|
||||
}
|
||||
|
||||
const requests = JSON.parse(readFileSync(process.argv[2]!, "utf8")) as Request[];
|
||||
const results: unknown[] = [];
|
||||
|
||||
for (const request of requests) {
|
||||
switch (request["op"]) {
|
||||
case "noop":
|
||||
results.push(null);
|
||||
break;
|
||||
|
||||
case "canonical":
|
||||
results.push(attempt(() => canonicalJson(request["value"])));
|
||||
break;
|
||||
|
||||
case "entryHash":
|
||||
results.push(attempt(() => entryEventHash(request["entry"])));
|
||||
break;
|
||||
|
||||
case "password":
|
||||
results.push(
|
||||
attempt(() =>
|
||||
generatePassword(Bip85.fromMnemonic(request["mnemonic"]), {
|
||||
length: request["length"],
|
||||
index: request["index"],
|
||||
genVersion: request["genVersion"],
|
||||
policy: passwordPolicyFromRecord(request["policy"]),
|
||||
}),
|
||||
),
|
||||
);
|
||||
break;
|
||||
|
||||
case "merge":
|
||||
results.push(
|
||||
attempt(() =>
|
||||
mergeIndexPayloads(request["current"], request["incoming"], request["sourceTag"]),
|
||||
),
|
||||
);
|
||||
break;
|
||||
|
||||
case "index0":
|
||||
results.push(
|
||||
attempt(() => {
|
||||
let payload: unknown = { schema_version: 4, entries: request["entries"] };
|
||||
for (const spec of request["events"] as Request[]) {
|
||||
payload = appendIndex0Event(payload, {
|
||||
eventType: spec["event_type"],
|
||||
subjectType: spec["subject_type"],
|
||||
subjectId: spec["subject_id"],
|
||||
subjectKind: spec["subject_kind"],
|
||||
modifiedTs: spec["modified_ts"],
|
||||
fingerprintDir: request["fingerprintDir"],
|
||||
tags: spec["tags"],
|
||||
summary: spec["summary"],
|
||||
});
|
||||
}
|
||||
return compactIndex0Payload(payload, { fingerprintDir: request["fingerprintDir"] });
|
||||
}),
|
||||
);
|
||||
break;
|
||||
|
||||
case "recover":
|
||||
results.push(attempt(() => recoverSecret(request["shares"])));
|
||||
break;
|
||||
|
||||
case "semantic":
|
||||
results.push(
|
||||
attempt(() => {
|
||||
const records = buildSemanticRecords(request["entries"]);
|
||||
return {
|
||||
records,
|
||||
hits: searchSemanticRecords(records, request["query"], { k: 10 }),
|
||||
};
|
||||
}),
|
||||
);
|
||||
break;
|
||||
|
||||
default:
|
||||
results.push(`ERROR:UnknownOp:${String(request["op"])}`);
|
||||
}
|
||||
}
|
||||
|
||||
process.stdout.write(JSON.stringify(results));
|
||||
@@ -208,9 +208,10 @@ export {
|
||||
type EntryLink,
|
||||
type ResolvedLink,
|
||||
} from "./vault/entryMod.js";
|
||||
export { canonicalJson, canonicalHash } from "./sync/canonical.js";
|
||||
export { findAmbiguousNumbers, canonicalJson, canonicalHash } from "./sync/canonical.js";
|
||||
export {
|
||||
mergeIndexPayloads,
|
||||
entryEventHash,
|
||||
emptyIndex0,
|
||||
safeInt,
|
||||
TOMBSTONE_RETENTION_CAP,
|
||||
|
||||
@@ -54,11 +54,30 @@ function encodeString(s: string): string {
|
||||
* 1e23 1e+23 1e+23 (agrees)
|
||||
* -0 0 -0.0
|
||||
*
|
||||
* Integral values are emitted without a decimal point, matching Python ints;
|
||||
* SeedPass data carries integers (timestamps, lengths, counts) and never
|
||||
* float-typed whole numbers, so the int/float ambiguity CPython would
|
||||
* otherwise expose does not arise. A non-integral or non-safe value would
|
||||
* be ambiguous, so it is rejected rather than guessed at.
|
||||
* TWO KNOWN LIMITATIONS, both found by differential fuzzing against Python
|
||||
* (scripts/differential_fuzz.py) rather than by reading:
|
||||
*
|
||||
* 1. INTEGRAL FLOATS CANNOT BE REPRODUCED. Python distinguishes int from
|
||||
* float; JavaScript does not. `JSON.parse("0.0")` and `JSON.parse("0")`
|
||||
* yield the same JS value, so this function emits "0" where CPython emits
|
||||
* "0.0". No amount of formatting fixes it — the information is destroyed
|
||||
* at parse time.
|
||||
*
|
||||
* Data SeedPass writes is unaffected: timestamps, lengths and counts are
|
||||
* Python ints and JS integers on both sides. It bites on data SeedPass did
|
||||
* not write — an unknown-kind record carried through verbatim, a
|
||||
* hand-edited vault, a third-party tool — which the port explicitly
|
||||
* supports carrying. The consequence is a different entry hash, and entry
|
||||
* hashes break same-timestamp merge ties, so two clients could converge to
|
||||
* different vaults. Use `findAmbiguousNumbers` on the raw JSON text to
|
||||
* detect it while the distinction still exists.
|
||||
*
|
||||
* 2. VALUES OUTSIDE THE SAFE-INTEGER RANGE ARE REFUSED, where Python encodes
|
||||
* them. That is deliberate and is the safe direction: `JSON.parse` has
|
||||
* already rounded such a value (10000000000000001 becomes
|
||||
* 10000000000000000), so the number in hand is not the number on disk and
|
||||
* any hash computed from it would be wrong. Refusing loudly beats hashing
|
||||
* a corrupted value.
|
||||
*/
|
||||
function encodeNumber(n: number): string {
|
||||
if (!Number.isFinite(n)) throw new Error("non-finite number in canonical JSON");
|
||||
@@ -139,3 +158,61 @@ export function canonicalJson(value: unknown): string {
|
||||
export function canonicalHash(value: unknown): string {
|
||||
return bytesToHex(sha256(utf8(canonicalJson(value))));
|
||||
}
|
||||
|
||||
/**
|
||||
* Number literals in `jsonText` whose canonical form this module cannot
|
||||
* reproduce byte-for-byte.
|
||||
*
|
||||
* Must be run on the raw JSON TEXT, before `JSON.parse`: the whole problem is
|
||||
* that parsing destroys the int/float distinction Python preserves. Returns
|
||||
* the offending literals (deduplicated, in order of first appearance), or an
|
||||
* empty array when the text is safe.
|
||||
*
|
||||
* Reports two classes, matching the limitations documented above:
|
||||
* - integral floats: `0.0`, `-0.0`, `1.0`, `2e3` — Python writes a decimal
|
||||
* point or exponent, this module cannot know to.
|
||||
* - out-of-safe-range values: already rounded by the time JS sees them.
|
||||
*
|
||||
* Non-integral floats like `1.5` are NOT reported: those survive parsing and
|
||||
* are encoded to match CPython by `pythonFloatRepr`.
|
||||
*/
|
||||
export function findAmbiguousNumbers(jsonText: string): string[] {
|
||||
const found: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
// JSON number grammar, anchored so it cannot match digits inside a string.
|
||||
// Strings are skipped explicitly rather than by lookaround, because an
|
||||
// escaped quote inside a string would defeat a naive pattern.
|
||||
let inString = false;
|
||||
let escaped = false;
|
||||
for (let i = 0; i < jsonText.length; i++) {
|
||||
const ch = jsonText[i]!;
|
||||
if (inString) {
|
||||
if (escaped) escaped = false;
|
||||
else if (ch === "\\") escaped = true;
|
||||
else if (ch === '"') inString = false;
|
||||
continue;
|
||||
}
|
||||
if (ch === '"') {
|
||||
inString = true;
|
||||
continue;
|
||||
}
|
||||
if (ch !== "-" && (ch < "0" || ch > "9")) continue;
|
||||
const match = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/.exec(jsonText.slice(i));
|
||||
if (!match) continue;
|
||||
const literal = match[0];
|
||||
i += literal.length - 1;
|
||||
|
||||
const value = Number(literal);
|
||||
const hasFractionOrExponent = /[.eE]/.test(literal);
|
||||
const ambiguous =
|
||||
// An integral value written as a float: Python keeps the decimal point.
|
||||
(hasFractionOrExponent && Number.isInteger(value)) ||
|
||||
// Already rounded by JSON.parse; the value in hand is not the one on disk.
|
||||
(Number.isInteger(value) && !Number.isSafeInteger(value));
|
||||
if (ambiguous && !seen.has(literal)) {
|
||||
seen.add(literal);
|
||||
found.push(literal);
|
||||
}
|
||||
}
|
||||
return found;
|
||||
}
|
||||
@@ -76,7 +76,7 @@ function entryHash(entry: Dict): string {
|
||||
return canonicalHash(entry);
|
||||
}
|
||||
|
||||
function entryEventHash(entry: Dict): string {
|
||||
export function entryEventHash(entry: Dict): string {
|
||||
return canonicalHash({ entry, kind: "entry" });
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Numbers whose canonical form cannot be reproduced from a parsed JS value.
|
||||
*
|
||||
* Found by differential fuzzing against Python, not by reading the code — the
|
||||
* module's own comment previously asserted this ambiguity "does not arise".
|
||||
* It does, for data SeedPass did not write, which the port explicitly
|
||||
* supports carrying through.
|
||||
*
|
||||
* The detector runs on raw TEXT because that is the only moment the
|
||||
* distinction still exists: JSON.parse("0.0") and JSON.parse("0") produce the
|
||||
* same JS value.
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { findAmbiguousNumbers, canonicalJson } from "@seedpass/core";
|
||||
|
||||
describe("the limitation is real", () => {
|
||||
it("cannot tell an integral float from an integer once parsed", () => {
|
||||
// This is the whole problem in one assertion. Python emits "0.0" for the
|
||||
// first and "0" for the second; JS has one value for both.
|
||||
expect(JSON.parse("0.0")).toBe(JSON.parse("0"));
|
||||
expect(canonicalJson(JSON.parse("0.0"))).toBe("0");
|
||||
});
|
||||
|
||||
it("refuses a value JSON.parse has already rounded", () => {
|
||||
// 10000000000000001 does not survive parsing, so any hash computed from
|
||||
// it would be a hash of the wrong number. Refusing is the safe direction.
|
||||
const rounded = JSON.parse("10000000000000001");
|
||||
expect(rounded).toBe(10000000000000000);
|
||||
expect(() => canonicalJson(rounded)).toThrow(/safe range/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("findAmbiguousNumbers", () => {
|
||||
it("flags integral floats", () => {
|
||||
expect(findAmbiguousNumbers('{"a":0.0}')).toEqual(["0.0"]);
|
||||
expect(findAmbiguousNumbers('{"a":-0.0}')).toEqual(["-0.0"]);
|
||||
expect(findAmbiguousNumbers('{"a":1.0,"b":2.00}')).toEqual(["1.0", "2.00"]);
|
||||
expect(findAmbiguousNumbers('{"a":2e3}')).toEqual(["2e3"]);
|
||||
expect(findAmbiguousNumbers('{"a":1e+16}')).toEqual(["1e+16"]);
|
||||
});
|
||||
|
||||
it("flags values outside the safe-integer range", () => {
|
||||
expect(findAmbiguousNumbers('{"a":10000000000000001}')).toEqual(["10000000000000001"]);
|
||||
});
|
||||
|
||||
it("stays quiet on everything SeedPass actually writes", () => {
|
||||
// Timestamps, lengths, counts, negative numbers and genuine fractions all
|
||||
// round-trip identically; flagging them would make the detector useless.
|
||||
const realistic = JSON.stringify({
|
||||
schema_version: 4,
|
||||
entries: {
|
||||
"0": { modified_ts: 1700000000, length: 16, index: 0, archived: false },
|
||||
"1": { period: 30, digits: 6, tags: ["a"], notes: "" },
|
||||
},
|
||||
_sync_meta: { next_index: 12, last_merge_ts: 1700000123 },
|
||||
});
|
||||
expect(findAmbiguousNumbers(realistic)).toEqual([]);
|
||||
expect(findAmbiguousNumbers('{"a":1.5,"b":-2.25,"c":1e-7}')).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not mistake digits inside strings for numbers", () => {
|
||||
// A label like "1.0" is text, not a number, and flagging it would train
|
||||
// the reader to ignore the warning.
|
||||
expect(findAmbiguousNumbers('{"label":"1.0","note":"0.0 and 1e+16"}')).toEqual([]);
|
||||
// Including when the string contains an escaped quote before the digits.
|
||||
expect(findAmbiguousNumbers('{"a":"say \\"1.0\\" now"}')).toEqual([]);
|
||||
});
|
||||
|
||||
it("deduplicates and preserves first-appearance order", () => {
|
||||
expect(findAmbiguousNumbers('{"a":1.0,"b":0.0,"c":1.0}')).toEqual(["1.0", "0.0"]);
|
||||
});
|
||||
|
||||
it("handles an empty or number-free document", () => {
|
||||
expect(findAmbiguousNumbers("{}")).toEqual([]);
|
||||
expect(findAmbiguousNumbers('{"a":"b"}')).toEqual([]);
|
||||
expect(findAmbiguousNumbers("")).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,592 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Differential fuzzing: Python SeedPass vs the TypeScript port.
|
||||
|
||||
WHY THIS EXISTS
|
||||
---------------
|
||||
The cutover plan's last open gate is an independent security review. There is
|
||||
no third party to hand the branch to, and the author reviewing their own work
|
||||
has a known failure mode: you cannot see what you did not think of. Two
|
||||
earlier review rounds each found criticals that self-review had missed.
|
||||
|
||||
The substitute for another pair of eyes is not more self-review. It is an
|
||||
ORACLE THAT IS NOT THE AUTHOR'S JUDGEMENT. This project happens to have the
|
||||
best such oracle available: two independent implementations that are required
|
||||
to agree. Feed both the same randomly generated input and any disagreement is
|
||||
a bug in one of them, established without anyone having to know in advance
|
||||
which answer is right or what to look for.
|
||||
|
||||
Fixtures already prove agreement on inputs a human chose. This proves it on
|
||||
inputs nobody chose, which is the whole point — the bugs that survive to
|
||||
production are the ones outside the cases someone thought to write down.
|
||||
|
||||
WHAT IT COVERS
|
||||
--------------
|
||||
The surfaces where a disagreement would be silent and expensive:
|
||||
|
||||
canonical canonical JSON + hashing. Underpins every other hash, so a
|
||||
divergence here is a divergence everywhere.
|
||||
password deterministic password derivation, v1 and v2, random policies.
|
||||
A mismatch means a user gets the wrong password after migrating.
|
||||
merge the sync CRDT. A mismatch means two clients converge to
|
||||
DIFFERENT vaults, permanently.
|
||||
index0 the activity ledger's hashes, checkpoints, views and merge.
|
||||
recovery threshold share split/recover, cross-implementation.
|
||||
semantic retrieval index records and ranking.
|
||||
entry entry normalization and modification.
|
||||
|
||||
Usage:
|
||||
.venv/bin/python scripts/differential_fuzz.py [--cases 200] [--seed 1]
|
||||
.venv/bin/python scripts/differential_fuzz.py --only merge --cases 5000
|
||||
|
||||
Exits non-zero on any divergence, printing the seed and the exact input needed
|
||||
to reproduce it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
import string
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO / "src"))
|
||||
|
||||
CORE_DIR = REPO / "js" / "packages" / "core"
|
||||
RUNNER = CORE_DIR / "_fuzz_runner.ts"
|
||||
|
||||
# Characters chosen to exercise the encoders rather than to look tidy:
|
||||
# non-ASCII (ensure_ascii escaping), a surrogate pair (UTF-16 vs code-point
|
||||
# ordering), quotes and backslashes (JSON escaping), and control characters.
|
||||
INTERESTING_TEXT = [
|
||||
"",
|
||||
" ",
|
||||
"a",
|
||||
"simple",
|
||||
"with space",
|
||||
'quote"inside',
|
||||
"back\\slash",
|
||||
"new\nline",
|
||||
"tab\there",
|
||||
"\x00null-byte",
|
||||
"\x1b[31mansi",
|
||||
"\u00fcn\u00efcod\u00e9",
|
||||
"\u65e5\u672c\u8a9e",
|
||||
"\u2713",
|
||||
"\U0001f600",
|
||||
"\ufffd",
|
||||
" pad ",
|
||||
"0",
|
||||
"-1",
|
||||
"1e5",
|
||||
"null",
|
||||
"True",
|
||||
]
|
||||
|
||||
# Bounds the generator deliberately reaches. Kept in sync with the
|
||||
# implementations by the caps test in js/packages/core/test/.
|
||||
TOMBSTONE_CAP = 2048
|
||||
SUBJECT_CAP = 64
|
||||
|
||||
KINDS = [
|
||||
"password",
|
||||
"totp",
|
||||
"key_value",
|
||||
"document",
|
||||
"ssh",
|
||||
"pgp",
|
||||
"nostr",
|
||||
"seed",
|
||||
"managed_account",
|
||||
]
|
||||
|
||||
|
||||
def rand_text(rng: random.Random) -> str:
|
||||
if rng.random() < 0.5:
|
||||
return rng.choice(INTERESTING_TEXT)
|
||||
return "".join(
|
||||
rng.choice(string.ascii_letters + string.digits + " -_.") for _ in range(rng.randint(0, 12))
|
||||
)
|
||||
|
||||
|
||||
def rand_scalar(rng: random.Random) -> Any:
|
||||
roll = rng.random()
|
||||
if roll < 0.30:
|
||||
return rand_text(rng)
|
||||
if roll < 0.50:
|
||||
return rng.randint(-(2**31), 2**31)
|
||||
if roll < 0.60:
|
||||
return rng.choice([True, False])
|
||||
if roll < 0.68:
|
||||
return None
|
||||
if roll < 0.76:
|
||||
# Floats are where CPython's repr and JS's toString diverge most.
|
||||
return rng.choice([0.0, -0.0, 1.5, 1e16, 1e-7, 3.141592653589793])
|
||||
if roll < 0.88:
|
||||
return [rand_text(rng) for _ in range(rng.randint(0, 3))]
|
||||
return {rand_text(rng): rand_text(rng) for _ in range(rng.randint(0, 3))}
|
||||
|
||||
|
||||
def rand_entry(rng: random.Random) -> dict[str, Any]:
|
||||
kind = rng.choice(KINDS)
|
||||
entry: dict[str, Any] = {
|
||||
"kind": kind,
|
||||
"type": kind,
|
||||
"label": rand_text(rng),
|
||||
"notes": rand_text(rng),
|
||||
"tags": [rand_text(rng) for _ in range(rng.randint(0, 4))],
|
||||
"archived": rng.choice([True, False]),
|
||||
"modified_ts": rng.randint(0, 2_000_000_000),
|
||||
}
|
||||
if rng.random() < 0.5:
|
||||
entry["username"] = rand_text(rng)
|
||||
if rng.random() < 0.4:
|
||||
entry["url"] = rand_text(rng)
|
||||
if rng.random() < 0.4:
|
||||
entry["length"] = rng.randint(8, 128)
|
||||
if rng.random() < 0.3:
|
||||
entry["index"] = rng.randint(0, 50)
|
||||
if rng.random() < 0.3:
|
||||
entry["links"] = [
|
||||
{
|
||||
"target_id": str(rng.randint(0, 20)),
|
||||
"relation": rand_text(rng),
|
||||
"note": rand_text(rng),
|
||||
}
|
||||
for _ in range(rng.randint(0, 3))
|
||||
]
|
||||
if rng.random() < 0.2:
|
||||
entry["custom_fields"] = [{"label": rand_text(rng), "value": rand_text(rng)}]
|
||||
if rng.random() < 0.15:
|
||||
# A field neither implementation knows: both must carry it through.
|
||||
entry[rand_text(rng) or "x"] = rand_scalar(rng)
|
||||
return entry
|
||||
|
||||
|
||||
def rand_entries(rng: random.Random, max_entries: int = 6) -> dict[str, Any]:
|
||||
out: dict[str, Any] = {}
|
||||
for _ in range(rng.randint(0, max_entries)):
|
||||
# Mostly numeric ids, occasionally not — non-numeric ids are exactly
|
||||
# the input that used to break view building.
|
||||
key = str(rng.randint(0, 30)) if rng.random() < 0.85 else rand_text(rng) or "k"
|
||||
out[key] = rand_entry(rng)
|
||||
return out
|
||||
|
||||
|
||||
def rand_payload(rng: random.Random) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {
|
||||
"schema_version": 4,
|
||||
"entries": rand_entries(rng),
|
||||
}
|
||||
if rng.random() < 0.7:
|
||||
tombstones = {}
|
||||
# Occasionally generate past the retention cap. Without this the
|
||||
# eviction path is never reached: a mutation testing run proved the
|
||||
# fuzzer could not see an off-by-one in the cap, because random small
|
||||
# payloads never trip it. Bounded code is only tested at its bound.
|
||||
count = (
|
||||
rng.randint(TOMBSTONE_CAP - 2, TOMBSTONE_CAP + 4)
|
||||
if rng.random() < 0.04
|
||||
else rng.randint(0, 3)
|
||||
)
|
||||
for n in range(count):
|
||||
# Keys must be DISTINCT to reach the cap: drawing from a small id
|
||||
# range produced 2050 writes into 31 slots, so the cap was never
|
||||
# crossed and the eviction path stayed unreachable.
|
||||
key = str(rng.randint(0, 30)) if count <= 3 else str(n)
|
||||
tombstones[key] = {
|
||||
"deleted_ts": rng.randint(0, 2_000_000_000),
|
||||
"entry_hash": rand_text(rng),
|
||||
"event_hash": rand_text(rng),
|
||||
"source": rand_text(rng),
|
||||
}
|
||||
payload["_sync_meta"] = {
|
||||
"next_index": rng.randint(0, 40),
|
||||
"last_merge_ts": rng.randint(0, 2_000_000_000),
|
||||
"tombstones": tombstones,
|
||||
}
|
||||
return payload
|
||||
|
||||
|
||||
def rand_policy(rng: random.Random) -> dict[str, Any]:
|
||||
policy: dict[str, Any] = {}
|
||||
for key in ("min_uppercase", "min_lowercase", "min_digits", "min_special"):
|
||||
if rng.random() < 0.6:
|
||||
policy[key] = rng.randint(0, 4)
|
||||
if rng.random() < 0.4:
|
||||
policy["include_special_chars"] = rng.choice([True, False])
|
||||
if rng.random() < 0.3:
|
||||
policy["exclude_ambiguous"] = rng.choice([True, False])
|
||||
if rng.random() < 0.25:
|
||||
policy["allowed_special_chars"] = rng.choice(["!@#", "$%^", "!@#$%^*-_+=?"])
|
||||
return policy
|
||||
|
||||
|
||||
# --------------------------------------------------------------- case builders
|
||||
# Each builder returns (name, python_result, ts_request). The TS request is
|
||||
# executed by the node runner; results are compared as canonical JSON so the
|
||||
# comparison itself cannot paper over an ordering difference.
|
||||
|
||||
|
||||
def case_canonical(rng: random.Random) -> tuple[str, Any, dict[str, Any]]:
|
||||
from utils.checksum import canonical_json_dumps
|
||||
|
||||
value = rand_scalar(rng) if rng.random() < 0.4 else rand_entry(rng)
|
||||
return (
|
||||
"canonical",
|
||||
canonical_json_dumps(value),
|
||||
{"op": "canonical", "value": value},
|
||||
)
|
||||
|
||||
|
||||
def case_password(rng: random.Random) -> tuple[str, Any, dict[str, Any]]:
|
||||
from bip_utils import Bip39SeedGenerator
|
||||
from local_bip85.bip85 import BIP85
|
||||
from seedpass.core.password_generation import PasswordGenerator, PasswordPolicy
|
||||
|
||||
class _Deriver:
|
||||
def derive_seed_from_mnemonic(self, mnemonic, passphrase=""):
|
||||
return Bip39SeedGenerator(mnemonic).Generate(passphrase)
|
||||
|
||||
mnemonic = (
|
||||
"abandon abandon abandon abandon abandon abandon "
|
||||
"abandon abandon abandon abandon abandon about"
|
||||
)
|
||||
policy_kwargs = rand_policy(rng)
|
||||
length = rng.randint(8, 64)
|
||||
index = rng.randint(0, 5000)
|
||||
version = rng.choice([1, 2])
|
||||
bip85 = BIP85(Bip39SeedGenerator(mnemonic).Generate(""))
|
||||
pg = PasswordGenerator(_Deriver(), mnemonic, bip85, policy=PasswordPolicy(**policy_kwargs))
|
||||
kwargs = {} if version == 1 else {"gen_version": version}
|
||||
try:
|
||||
result: Any = pg.generate_password(length=length, index=index, **kwargs)
|
||||
except Exception as exc: # a refusal is a result too, and must match
|
||||
result = "ERROR:Refused"
|
||||
return (
|
||||
"password",
|
||||
result,
|
||||
{
|
||||
"op": "password",
|
||||
"mnemonic": mnemonic,
|
||||
"length": length,
|
||||
"index": index,
|
||||
"genVersion": version,
|
||||
"policy": policy_kwargs,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def case_merge(rng: random.Random) -> tuple[str, Any, dict[str, Any]]:
|
||||
from seedpass.core.sync_conflict import merge_index_payloads
|
||||
|
||||
current = rand_payload(rng)
|
||||
incoming = rand_payload(rng)
|
||||
source = rand_text(rng)
|
||||
try:
|
||||
result: Any = merge_index_payloads(current, incoming, source_tag=source)
|
||||
except Exception as exc:
|
||||
result = "ERROR:Refused"
|
||||
return (
|
||||
"merge",
|
||||
result,
|
||||
{"op": "merge", "current": current, "incoming": incoming, "sourceTag": source},
|
||||
)
|
||||
|
||||
|
||||
def case_index0(rng: random.Random) -> tuple[str, Any, dict[str, Any]]:
|
||||
from seedpass.core import index0 as index0_mod
|
||||
|
||||
fp_dir = f"/tmp/seedpass/{''.join(rng.choice('0123456789ABCDEF') for _ in range(16))}"
|
||||
payload: dict[str, Any] = {"schema_version": 4, "entries": rand_entries(rng)}
|
||||
events = []
|
||||
# Same reasoning as the tombstone cap: occasionally produce enough events,
|
||||
# with enough distinct subjects, to reach INDEX0_CHECKPOINT_SUBJECT_CAP
|
||||
# and the per-writer checkpoint retention limit.
|
||||
big = rng.random() < 0.10
|
||||
# Well PAST the cap, not at it. A first attempt generated 62-72 events and
|
||||
# peaked at 63 distinct subjects across 250 cases -- one short -- so the
|
||||
# eviction path stayed unreachable and a deliberate off-by-one there went
|
||||
# undetected. Bounded code has to be driven past its bound, with margin.
|
||||
event_count = rng.randint(SUBJECT_CAP + 20, SUBJECT_CAP + 60) if big else rng.randint(0, 5)
|
||||
# A large run is clustered into one day half the time, so both the subject
|
||||
# cap (per checkpoint) and the checkpoint retention limit get reached.
|
||||
day_spread = not big or rng.random() < 0.5
|
||||
for _i in range(event_count):
|
||||
events.append(
|
||||
{
|
||||
"event_type": rand_text(rng) or "entry_created",
|
||||
"subject_type": "entry",
|
||||
# Distinct by construction for the large runs: collisions in a
|
||||
# random id space are what kept the distinct count below the cap.
|
||||
"subject_id": str(rng.randint(0, 400)) if not big else f"s{_i}",
|
||||
"subject_kind": rng.choice(KINDS),
|
||||
# Two shapes on purpose. Spread across days exercises
|
||||
# checkpoint grouping and the per-writer retention limit;
|
||||
# clustered into ONE day is what reaches the subject cap,
|
||||
# since subjects are capped per checkpoint and a day with one
|
||||
# event has one subject.
|
||||
"modified_ts": (
|
||||
rng.randint(1, 2_000_000_000)
|
||||
if day_spread
|
||||
else 1_700_000_000 + rng.randint(0, 86_000)
|
||||
),
|
||||
"tags": [rand_text(rng) for _ in range(rng.randint(0, 3))],
|
||||
"summary": rand_text(rng),
|
||||
}
|
||||
)
|
||||
try:
|
||||
for spec in events:
|
||||
payload = index0_mod.append_index0_event(
|
||||
payload,
|
||||
event_type=spec["event_type"],
|
||||
subject_type=spec["subject_type"],
|
||||
subject_id=spec["subject_id"],
|
||||
subject_kind=spec["subject_kind"],
|
||||
modified_ts=spec["modified_ts"],
|
||||
fingerprint_dir=fp_dir,
|
||||
tags=spec["tags"],
|
||||
summary=spec["summary"],
|
||||
)
|
||||
result: Any = index0_mod.compact_index0_payload(payload, fingerprint_dir=fp_dir)
|
||||
except Exception as exc:
|
||||
result = "ERROR:Refused"
|
||||
return (
|
||||
"index0",
|
||||
result,
|
||||
{"op": "index0", "fingerprintDir": fp_dir, "entries": payload.get("entries", {}), "events": events},
|
||||
)
|
||||
|
||||
|
||||
def case_recovery(rng: random.Random) -> tuple[str, Any, dict[str, Any]]:
|
||||
"""Split in Python, recover in TypeScript.
|
||||
|
||||
Shares are randomized by design, so the two implementations cannot produce
|
||||
identical tokens. What must hold is that each can recover the other's --
|
||||
which is the property that actually matters.
|
||||
"""
|
||||
from seedpass.core.agent_recovery import split_secret
|
||||
|
||||
secret = rand_text(rng) or "fallback secret"
|
||||
threshold = rng.randint(2, 4)
|
||||
total = rng.randint(threshold, threshold + 3)
|
||||
try:
|
||||
shares = split_secret(secret, total_shares=total, threshold=threshold)
|
||||
chosen = rng.sample(shares, threshold)
|
||||
except Exception as exc:
|
||||
return ("recovery", "ERROR:Refused", {"op": "noop"})
|
||||
return ("recovery", secret, {"op": "recover", "shares": chosen})
|
||||
|
||||
|
||||
def case_semantic(rng: random.Random) -> tuple[str, Any, dict[str, Any]]:
|
||||
from seedpass.core.semantic_index import SemanticIndex
|
||||
|
||||
entries = []
|
||||
for entry_id in range(rng.randint(0, 6)):
|
||||
entry = rand_entry(rng)
|
||||
entry["id"] = entry_id
|
||||
entries.append(entry)
|
||||
query = rand_text(rng)
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
index = SemanticIndex(Path(tmpdir))
|
||||
try:
|
||||
index.build(entries)
|
||||
records = json.loads(
|
||||
(Path(tmpdir) / "semantic_index" / "records.json").read_text(encoding="utf-8")
|
||||
)
|
||||
hits = index.search(query, k=10)
|
||||
except Exception as exc:
|
||||
return ("semantic", "ERROR:Refused", {"op": "noop"})
|
||||
return (
|
||||
"semantic",
|
||||
{"records": records, "hits": hits},
|
||||
{"op": "semantic", "entries": entries, "query": query},
|
||||
)
|
||||
|
||||
|
||||
def case_entry(rng: random.Random) -> tuple[str, Any, dict[str, Any]]:
|
||||
"""Entry hashing, which decides merge winners."""
|
||||
from seedpass.core.sync_conflict import _entry_event_hash # type: ignore
|
||||
|
||||
entry = rand_entry(rng)
|
||||
try:
|
||||
result: Any = _entry_event_hash(entry)
|
||||
except Exception as exc:
|
||||
result = "ERROR:Refused"
|
||||
return ("entry", result, {"op": "entryHash", "entry": entry})
|
||||
|
||||
|
||||
BUILDERS: dict[str, Callable[[random.Random], tuple[str, Any, dict[str, Any]]]] = {
|
||||
"canonical": case_canonical,
|
||||
"password": case_password,
|
||||
"merge": case_merge,
|
||||
"index0": case_index0,
|
||||
"recovery": case_recovery,
|
||||
"semantic": case_semantic,
|
||||
"entry": case_entry,
|
||||
}
|
||||
|
||||
|
||||
def _float_literals(value: Any) -> bool:
|
||||
"""Does this value contain a float that JavaScript cannot distinguish?
|
||||
|
||||
Python's json keeps int and float apart; JavaScript has one number type.
|
||||
An integral float (0.0, 1.0, 2e3) therefore cannot survive the round trip,
|
||||
and neither can a value outside the safe-integer range, which JSON.parse
|
||||
has already rounded. Both are documented limitations of
|
||||
js/packages/core/src/sync/canonical.ts, not new bugs -- but they must be
|
||||
recognized precisely, or "known limitation" becomes a place to hide real
|
||||
divergences.
|
||||
"""
|
||||
if isinstance(value, bool):
|
||||
return False
|
||||
if isinstance(value, float):
|
||||
return value.is_integer() or abs(value) > 2**53
|
||||
if isinstance(value, int):
|
||||
return abs(value) > 2**53
|
||||
if isinstance(value, dict):
|
||||
return any(_float_literals(v) for v in value.values()) or any(
|
||||
_float_literals(k) for k in value
|
||||
)
|
||||
if isinstance(value, list):
|
||||
return any(_float_literals(v) for v in value)
|
||||
return False
|
||||
|
||||
|
||||
def is_known_limitation(python_result: Any, request: Any) -> bool:
|
||||
"""True when a divergence is one of the two documented number limitations.
|
||||
|
||||
Checks the INPUT as well as the Python result, because some surfaces
|
||||
return a string (canonical JSON) in which the offending float is no longer
|
||||
a float. Deliberately narrow otherwise: a divergence with no
|
||||
unrepresentable number anywhere in it is a real finding and must never be
|
||||
filtered away -- "known limitation" is exactly the kind of category that
|
||||
quietly grows to hide real bugs.
|
||||
"""
|
||||
return _float_literals(request) or _float_literals(python_result)
|
||||
|
||||
|
||||
def canonical(value: Any) -> str:
|
||||
from utils.checksum import canonical_json_dumps
|
||||
|
||||
return canonical_json_dumps(value)
|
||||
|
||||
|
||||
def run_ts(requests: list[dict[str, Any]]) -> list[Any]:
|
||||
"""Execute every TS-side request in one node process."""
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fh:
|
||||
json.dump(requests, fh)
|
||||
input_path = fh.name
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["npx", "tsx", str(RUNNER), input_path],
|
||||
cwd=CORE_DIR,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=1800,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
print(proc.stdout[-4000:], file=sys.stderr)
|
||||
print(proc.stderr[-4000:], file=sys.stderr)
|
||||
raise SystemExit("TypeScript runner failed")
|
||||
return json.loads(proc.stdout)
|
||||
finally:
|
||||
Path(input_path).unlink(missing_ok=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--cases", type=int, default=200, help="cases per surface")
|
||||
parser.add_argument("--seed", type=int, default=1, help="RNG seed (reproduces a run)")
|
||||
parser.add_argument(
|
||||
"--only",
|
||||
action="append",
|
||||
choices=sorted(BUILDERS),
|
||||
help="restrict to one or more surfaces (repeatable)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
surfaces = args.only or sorted(BUILDERS)
|
||||
rng = random.Random(args.seed)
|
||||
|
||||
print(f"differential fuzz: seed={args.seed} cases={args.cases} surfaces={','.join(surfaces)}")
|
||||
|
||||
expected: list[tuple[str, int, Any]] = []
|
||||
requests: list[dict[str, Any]] = []
|
||||
for surface in surfaces:
|
||||
builder = BUILDERS[surface]
|
||||
for i in range(args.cases):
|
||||
name, python_result, ts_request = builder(rng)
|
||||
expected.append((name, i, python_result))
|
||||
requests.append(ts_request)
|
||||
|
||||
print(f" generated {len(requests)} cases; running the TypeScript side...")
|
||||
ts_results = run_ts(requests)
|
||||
if len(ts_results) != len(expected):
|
||||
raise SystemExit(f"runner returned {len(ts_results)} results for {len(expected)} cases")
|
||||
|
||||
divergences: list[tuple[str, int, str, str, dict[str, Any]]] = []
|
||||
known: list[tuple[str, int]] = []
|
||||
skipped = 0
|
||||
for (name, idx, python_result), ts_result, request in zip(expected, ts_results, requests):
|
||||
if request.get("op") == "noop":
|
||||
skipped += 1
|
||||
continue
|
||||
py_canon = canonical(python_result)
|
||||
ts_canon = canonical(ts_result)
|
||||
if py_canon == ts_canon:
|
||||
continue
|
||||
if is_known_limitation(python_result, request):
|
||||
known.append((name, idx))
|
||||
continue
|
||||
divergences.append((name, idx, py_canon, ts_canon, request))
|
||||
|
||||
by_surface: dict[str, int] = {}
|
||||
for name, _, _ in expected:
|
||||
by_surface[name] = by_surface.get(name, 0) + 1
|
||||
for surface in sorted(by_surface):
|
||||
failures = sum(1 for d in divergences if d[0] == surface)
|
||||
mark = "PASS" if failures == 0 else "FAIL"
|
||||
print(f" [{mark}] {surface}: {by_surface[surface] - failures}/{by_surface[surface]} agree")
|
||||
|
||||
if skipped:
|
||||
print(f" ({skipped} cases skipped: the Python side refused the input)")
|
||||
if known:
|
||||
by_known: dict[str, int] = {}
|
||||
for name, _ in known:
|
||||
by_known[name] = by_known.get(name, 0) + 1
|
||||
detail = ", ".join(f"{k}={v}" for k, v in sorted(by_known.items()))
|
||||
print(
|
||||
f" ({len(known)} known number-representation limitations: {detail} "
|
||||
f"-- see canonical.ts, findAmbiguousNumbers)"
|
||||
)
|
||||
|
||||
if divergences:
|
||||
print(f"\n{len(divergences)} DIVERGENCE(S) — reproduce with --seed {args.seed}\n")
|
||||
dump = Path("/tmp/claude-1000/-home-user/4efd790e-427f-430a-b379-ef8391e6744f/scratchpad/divergences.json")
|
||||
dump.write_text(json.dumps([
|
||||
{"surface": n, "case": i, "python": p_, "ts": t, "input": r}
|
||||
for n, i, p_, t, r in divergences
|
||||
], indent=1))
|
||||
print(f" (full list written to {dump})\n")
|
||||
for name, idx, py_canon, ts_canon, request in divergences[:5]:
|
||||
print(f"--- {name} case {idx} ---")
|
||||
print(f" input : {json.dumps(request)[:600]}")
|
||||
print(f" python: {py_canon[:600]}")
|
||||
print(f" ts : {ts_canon[:600]}")
|
||||
print()
|
||||
if len(divergences) > 5:
|
||||
print(f" ... and {len(divergences) - 5} more")
|
||||
return 1
|
||||
|
||||
print(f"\nno divergences across {len(requests) - skipped} compared cases")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user