From e84a7e1c0ee9004411fb92395ecb5d50da6b2885 Mon Sep 17 00:00:00 2001 From: thePR0M3TH3AN <53631862+PR0M3TH3AN@users.noreply.github.com> Date: Fri, 14 Aug 2026 14:51:18 -0400 Subject: [PATCH] =?UTF-8?q?TS=20port=20slice=209:=20relay=20transport=20+?= =?UTF-8?q?=20nostr=20sync/restore=20=E2=80=94=20Milestone=205=20complete?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - core RelayPool over the platform WebSocket (global in Node 22+/browsers, no dependency): multi-relay publish with per-relay OK collection, fetch aggregation until EOSE with dedupe and signature verification on ingest, connect timeouts, dead-relay tolerance. - Sync flows mirroring nostr/snapshot.py: publishSnapshot (chunk events -> manifest with HMAC identity + nonce), fetchLatestSnapshot (newest-first, per-chunk hash verify, falls through incomplete manifests), publishDelta, fetchDeltasSince (created_at replay order). - CLI nostr group: get-pubkey (matches app-1237 fixture npub), list/add/ remove-relay (config-backed, ws:// validation, last-relay guard), sync (publish local vault), restore (snapshot + delta merge -> local vault). - Tested against an in-process NIP-01 mock relay: core round trip incl. corrupted-chunk rejection, and CLI end-to-end sync -> destroy local vault -> restore with secrets intact. Transport tests are Node-only (jsdom WebSocket can't connect; real browsers come with Milestone 6). - 151 core + 32 CLI tests green. Milestone 5 CLI surface is complete. Co-Authored-By: Claude Fable 5 --- docs/typescript_port_compatibility_matrix.md | 9 +- js/packages/cli/src/program.ts | 148 +++++++++++++++ js/packages/cli/test/nostr.test.ts | 103 +++++++++++ js/packages/cli/tsconfig.json | 3 +- js/packages/core/package.json | 4 +- js/packages/core/src/index.ts | 9 + js/packages/core/src/sync/relay.ts | 180 +++++++++++++++++++ js/packages/core/src/sync/syncFlows.ts | 163 +++++++++++++++++ js/packages/core/src/util/bytes.ts | 6 + js/packages/core/test/mockRelay.ts | 73 ++++++++ js/packages/core/test/relay.test.ts | 117 ++++++++++++ js/pnpm-lock.yaml | 27 +++ 12 files changed, 835 insertions(+), 7 deletions(-) create mode 100644 js/packages/cli/test/nostr.test.ts create mode 100644 js/packages/core/src/sync/relay.ts create mode 100644 js/packages/core/src/sync/syncFlows.ts create mode 100644 js/packages/core/test/mockRelay.ts create mode 100644 js/packages/core/test/relay.test.ts diff --git a/docs/typescript_port_compatibility_matrix.md b/docs/typescript_port_compatibility_matrix.md index 196faa65..cf608599 100644 --- a/docs/typescript_port_compatibility_matrix.md +++ b/docs/typescript_port_compatibility_matrix.md @@ -31,7 +31,8 @@ until a later milestone. | KDF metadata parsing | P0 | green | `kdf_metadata.json`, `legacy_payloads.json` | KdfConfig schema with Python-default fallbacks. | | SSH key derivation | P1 | todo | — | Ed25519 from BIP-85 app 32; PEM serialization parity risk. | | PGP key derivation | P1/P2 | todo | — | Highest parity risk (PGPy serialization); import/roundtrip first. | -| Nostr snapshot chunk/manifest model | P0 | green | `nostr_snapshot.json` | Kinds 30070/30071/30072; gzip chunking, chunk-hash verify, reassembly of Python-produced chunks, manifest JSON parse, manifest id HMAC (key_index chain). Compressed bytes are deliberately not byte-pinned (deflate encoders differ); cross-decompression verified both ways. Relay WebSocket adapter: todo. | +| Nostr snapshot chunk/manifest model | P0 | green | `nostr_snapshot.json` | Kinds 30070/30071/30072; gzip chunking, chunk-hash verify, reassembly of Python-produced chunks, manifest JSON parse, manifest id HMAC (key_index chain). Compressed bytes are deliberately not byte-pinned (deflate encoders differ); cross-decompression verified both ways. | +| Relay transport + sync flows | P0 | green | — (live mock relay) | RelayPool over platform WebSocket (multi-relay publish/fetch, dedupe, signature verification on ingest); publish/fetch snapshot with chunk-hash verify, delta publish/replay. End-to-end tested against an in-process NIP-01 relay incl. corrupted-chunk rejection, and at the CLI level: sync -> destroy local vault -> restore. jsdom skips transport tests (real-browser run: Milestone 6 CI). | | Deterministic conflict merge | P0 | green | `sync_merge.json` | Full `merge_index_payloads` port: ts precedence, canonical-hash tie-breaks (Python-compatible ensure_ascii canonical JSON), equal-ts field union incl. the `custom_fields: null` quirk, tombstone lifecycle, retention cap, `_sync_meta`. | | Tombstones / delta replay | P0 | green | `delta_replay.json` | Encrypted V3 deltas decrypted and merged in order with `source_tag = sha256(payload)[:16]`, final state matches Python. | | index0 content merge (`_system.index0`) | P2 | partial | — | Empty-skeleton normalization matches Python; merging populated index0 event logs throws loudly instead of silently dropping data. Full port belongs to the atlas milestone. | @@ -40,7 +41,7 @@ until a later milestone. | Entry creation (provision) | P0 | green | `entries_index.json` | TS `add*Entry` ops rebuild the Python `EntryManager` fixture index byte-for-byte (pinned clock): field shapes, id allocation, independent TOTP derivation-index allocation, ISO timestamp format. Not ported: index0 event emission. | | Entry modify/archive/links | P0 | green | `entry_mods.json` | TS replays a Python `EntryManager` op sequence (modify with kind-checked field matrix + policy merge, archive/restore, link add/remove with normalization and dedupe) byte-for-byte, including timestamp touching. | | Entry secret retrieval | P0 | green | `entry_secrets.json` | Reveal values match Python retrieval for password (v2), TOTP-at-time, nostr entry nsec (BIP-85 app 39 — NOT the sync client's 1237), seed and managed-account mnemonics. | -| CLI (Milestone 5, agent-blind MVP) | P1 | partial | — | `packages/cli` seedpass-js: capabilities, entry list/get/search (reference-first, `sp://entry/`, secrets replaced by `has_*` flags), `entry add password/totp/key-value/document/seed/managed-account/nostr` (provision-blind: returns refs, never the created secret), `entry reveal` as the only plaintext egress, `use --clipboard/--exec/--stdin-to` sinks (env-var injection, never argv/stdout), vault export/import. Tests assert secrets never appear in default or provisioning output. Also: modify, archive/unarchive, links, totp-codes, util generate-password. Not yet: relay sync, leases/tokens. | +| CLI (Milestone 5, agent-blind MVP) | P1 | partial | — | `packages/cli` seedpass-js: capabilities, entry list/get/search (reference-first, `sp://entry/`, secrets replaced by `has_*` flags), `entry add password/totp/key-value/document/seed/managed-account/nostr` (provision-blind: returns refs, never the created secret), `entry reveal` as the only plaintext egress, `use --clipboard/--exec/--stdin-to` sinks (env-var injection, never argv/stdout), vault export/import. Tests assert secrets never appear in default or provisioning output. Also: modify, archive/unarchive, links, totp-codes, util generate-password, nostr get-pubkey/list-relays/add-relay/remove-relay/sync/restore. Milestone 5 command surface complete except ssh/pgp adds and document import/export; leases/tokens are Milestone 8. | | Profiles + config (Python `~/.seedpass` layout) | P1 | green* | — | `fingerprint list/add/switch/remove`, `config get/set` over the Python directory layout (fingerprints.json, parent_seed.enc kdf/ct wrapper, index-key-encrypted config with ConfigManager defaults). *Layout-compatible and tested end-to-end in TS; opening a real Python-created profile is verified indirectly via the parent-seed wrapper fixture — direct cross-open test TODO. | | Session agent (vault unlock/lock) | new (TS-only) | green | — | ssh-agent-style unix-socket daemon holding seeds with TTL (0600 socket, in-memory only); seed resolution env -> agent; full lifecycle tested with no mnemonic in the environment. Designed enforcement point for the section-9.3 lease/token layer. Python has no equivalent (its lock is in-process TUI state). | @@ -48,8 +49,8 @@ until a later milestone. | Environment | Status | |---|---| -| Node 22 (vitest) | green — 148/148 core + 29 CLI | -| jsdom browser-like env | green — 148/148 core | +| Node 22 (vitest) | green — 151/151 core + 32 CLI | +| jsdom browser-like env | green — 148/148 core (3 transport tests Node-only) | | Real Chromium/Firefox (vitest browser mode) | todo — lands with Milestone 6 web app CI | ## Dependency notes (to expand into docs/typescript_dependency_review.md) diff --git a/js/packages/cli/src/program.ts b/js/packages/cli/src/program.ts index 3d2c7003..7e0ef639 100644 --- a/js/packages/cli/src/program.ts +++ b/js/packages/cli/src/program.ts @@ -28,6 +28,17 @@ import { Bip85, totpCodeAt, deriveTotpSecret, + deriveNostrKeys, + deriveKeyIndex, + deriveIndexKeyBytes, + RelayPool, + publishSnapshot, + fetchLatestSnapshot, + fetchDeltasSince, + decryptPayload, + mergeIndexPayloads, + parseVaultIndex, + sha256Hex, type ModifyChanges, type PasswordPolicy, addPasswordEntry, @@ -591,6 +602,143 @@ export function buildProgram(io: ProgramIo = defaultIo): Command { io.out(JSON.stringify({ [key]: parsed })); }); + const nostr = program.command("nostr").description("Nostr relay sync"); + + async function nostrContext(opts: GlobalOpts) { + const app = new AppDir(resolveAppDir(opts.appDir)); + const fp = await currentFingerprint(app, opts); + const mnemonic = await resolveMnemonic(app, opts); + const cfg = await loadConfig(app.profileDir(fp), mnemonic); + const relays = cfg["relays"] as string[]; + const keys = deriveNostrKeys(Bip85.fromMnemonic(mnemonic), 0); + return { app, fp, mnemonic, cfg, relays, keys }; + } + + nostr + .command("get-pubkey") + .description("display the active profile's npub") + .action(async () => { + const { keys } = await nostrContext(program.opts()); + io.out(keys.npub); + }); + + nostr + .command("list-relays") + .description("show configured relays") + .action(async () => { + const { relays } = await nostrContext(program.opts()); + io.out(JSON.stringify(relays, null, 2)); + }); + + nostr + .command("add-relay ") + .description("add a relay URL") + .action(async (url: string) => { + if (!/^wss?:\/\//.test(url)) throw new Error("relay URL must start with ws:// or wss://"); + const { app, fp, mnemonic, cfg } = await nostrContext(program.opts()); + const relays = cfg["relays"] as string[]; + if (!relays.includes(url)) relays.push(url); + await saveConfig(app.profileDir(fp), mnemonic, cfg); + io.out(JSON.stringify({ relays })); + }); + + nostr + .command("remove-relay ") + .description("remove a relay by 1-based index") + .action(async (indexStr: string) => { + const { app, fp, mnemonic, cfg } = await nostrContext(program.opts()); + const relays = cfg["relays"] as string[]; + const i = Number(indexStr) - 1; + if (!Number.isInteger(i) || i < 0 || i >= relays.length) { + throw new Error(`index out of range 1..${relays.length}`); + } + if (relays.length === 1) throw new Error("at least one relay must remain"); + relays.splice(i, 1); + await saveConfig(app.profileDir(fp), mnemonic, cfg); + io.out(JSON.stringify({ relays })); + }); + + nostr + .command("sync") + .description("publish the local vault as a snapshot to the configured relays") + .option("--chunk-limit ", "max chunk size", "50000") + .action(async (o: { chunkLimit: string }) => { + const opts = program.opts() as GlobalOpts; + const { app, fp, mnemonic, relays, keys } = await nostrContext(opts); + const vaultPath = opts.vault ?? join(app.profileDir(fp), INDEX_FILENAME); + const encrypted = new Uint8Array(await readFile(vaultPath)); + const pool = new RelayPool(relays); + try { + const published = await publishSnapshot( + pool, + keys.privateKeyHex, + deriveKeyIndex(mnemonic), + encrypted, + { limit: Number(o.chunkLimit) }, + ); + io.out( + JSON.stringify( + { + manifest_id: published.manifestId, + manifest_event_id: published.manifestEventId, + chunk_event_ids: published.chunkEventIds, + relays, + }, + null, + 2, + ), + ); + } finally { + await pool.close(); + } + }); + + nostr + .command("restore") + .description("fetch the latest snapshot + deltas from relays and merge into the local vault") + .action(async () => { + const opts = program.opts() as GlobalOpts; + const { app, fp, mnemonic, relays, keys } = await nostrContext(opts); + const pool = new RelayPool(relays); + try { + const fetched = await fetchLatestSnapshot(pool, keys.privateKeyHex); + if (!fetched) throw new Error("no snapshot found on the configured relays"); + const indexKey = deriveIndexKeyBytes(mnemonic); + let state = JSON.parse( + new TextDecoder().decode(await decryptPayload(indexKey, fetched.encrypted)), + ) as Record; + + let deltaCount = 0; + if (fetched.manifest.delta_since) { + const deltas = await fetchDeltasSince( + pool, + keys.privateKeyHex, + fetched.manifest.delta_since, + ); + for (const delta of deltas) { + const incoming = JSON.parse( + new TextDecoder().decode(await decryptPayload(indexKey, delta)), + ); + state = mergeIndexPayloads(state, incoming, sha256Hex(delta).slice(0, 16)); + deltaCount++; + } + } + + const index = parseVaultIndex(state); + const vaultPath = opts.vault ?? join(app.profileDir(fp), INDEX_FILENAME); + await saveVault({ index, mnemonic, path: vaultPath }); + io.out( + JSON.stringify({ + restored: vaultPath, + entry_count: Object.keys(index.entries).length, + deltas_applied: deltaCount, + }), + ); + } finally { + await pool.close(); + } + }); + const agent = program.command("agent").description("session agent (holds unlocked seeds)"); agent diff --git a/js/packages/cli/test/nostr.test.ts b/js/packages/cli/test/nostr.test.ts new file mode 100644 index 00000000..defd1a58 --- /dev/null +++ b/js/packages/cli/test/nostr.test.ts @@ -0,0 +1,103 @@ +/** + * End-to-end CLI relay sync: create a profile, add entries, sync to an + * in-process relay, destroy the local vault, restore it from the relay. + */ + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import process from "node:process"; +import { mnemonics, nostrKeyCases } from "@seedpass/test-vectors"; +import { generateFingerprint } from "@seedpass/core"; +import { buildProgram, type ProgramIo } from "../src/index.js"; +import { MockRelay } from "../../core/test/mockRelay.js"; + +const MNEMONIC = mnemonics["abandon12"]!; +const FINGERPRINT = generateFingerprint(MNEMONIC); + +let appDir: string; +let relay: MockRelay; + +async function run(...argv: string[]): Promise<{ stdout: string; error?: unknown }> { + const out: string[] = []; + const io: ProgramIo = { out: (l: string) => out.push(l), err: () => {} }; + let error: unknown; + try { + await buildProgram(io).parseAsync(["node", "seedpass-js", "--app-dir", appDir, ...argv]); + } catch (e) { + error = e; + } + return { stdout: out.join("\n"), error }; +} + +beforeAll(async () => { + appDir = await mkdtemp(join(tmpdir(), "seedpass-nostr-")); + process.env["SEEDPASS_MNEMONIC"] = MNEMONIC; + process.env["SEEDPASS_PASSWORD"] = "nostr-test-pw"; + relay = new MockRelay(); + await relay.start(); + await run("fingerprint", "add", "--name", "sync-test"); +}); + +afterAll(async () => { + await relay.stop(); + delete process.env["SEEDPASS_PASSWORD"]; +}); + +describe("nostr CLI", () => { + it("get-pubkey matches the app-1237 fixture npub", async () => { + const fixture = nostrKeyCases.find( + (c) => c.mnemonic_id === "abandon12" && c.account_index === 0, + )!; + const r = await run("nostr", "get-pubkey"); + expect(r.stdout).toBe(fixture.npub); + }); + + it("manages the relay list in config", async () => { + await run("nostr", "add-relay", relay.url); + const list = JSON.parse((await run("nostr", "list-relays")).stdout) as string[]; + expect(list).toContain(relay.url); + + // Keep only the mock relay so sync never touches the network + while (true) { + const current = JSON.parse((await run("nostr", "list-relays")).stdout) as string[]; + const removable = current.findIndex((u) => u !== relay.url); + if (removable === -1) break; + await run("nostr", "remove-relay", String(removable + 1)); + } + const final = JSON.parse((await run("nostr", "list-relays")).stdout) as string[]; + expect(final).toEqual([relay.url]); + + const bad = await run("nostr", "add-relay", "https://not-a-relay"); + expect(String((bad.error as Error).message)).toContain("ws://"); + }); + + it("syncs the vault to the relay and restores after local destruction", async () => { + await run("entry", "add", "key-value", "synced-secret", "k", "value-roundtrip"); + await run("entry", "add", "password", "synced-site", "--length", "20"); + + const sync = await run("nostr", "sync", "--chunk-limit", "300"); + const syncRow = JSON.parse(sync.stdout); + expect(syncRow.manifest_id).toMatch(/^[0-9a-f]{64}$/); + expect(syncRow.chunk_event_ids.length).toBeGreaterThan(0); + expect(relay.events.some((e) => e.kind === 30070)).toBe(true); + + // Destroy the local vault + const vaultPath = join(appDir, FINGERPRINT, "seedpass_entries_db.json.enc"); + await writeFile(vaultPath, "garbage"); + const broken = await run("entry", "list"); + expect(broken.error).toBeTruthy(); + + // Restore from the relay alone + const restore = await run("nostr", "restore"); + const restored = JSON.parse(restore.stdout); + expect(restored.entry_count).toBe(2); + expect(restored.deltas_applied).toBe(0); + + const list = JSON.parse((await run("entry", "list")).stdout) as { label: string }[]; + expect(list.map((e) => e.label).sort()).toEqual(["synced-secret", "synced-site"]); + const reveal = await run("entry", "reveal", "synced-secret"); + expect(reveal.stdout).toBe("value-roundtrip"); + }); +}); diff --git a/js/packages/cli/tsconfig.json b/js/packages/cli/tsconfig.json index 87de85e8..d730de45 100644 --- a/js/packages/cli/tsconfig.json +++ b/js/packages/cli/tsconfig.json @@ -1,9 +1,8 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { - "rootDir": ".", "noEmit": true, "types": ["node"] }, - "include": ["src", "test", "bin"] + "include": ["src", "test", "bin", "../core/test/mockRelay.ts"] } diff --git a/js/packages/core/package.json b/js/packages/core/package.json index c356fb88..a38a5f2d 100644 --- a/js/packages/core/package.json +++ b/js/packages/core/package.json @@ -23,8 +23,10 @@ }, "devDependencies": { "@seedpass/test-vectors": "workspace:*", + "@types/ws": "^8.18.1", "jsdom": "^30.0.1", "typescript": "^5.7.0", - "vitest": "^3.0.0" + "vitest": "^3.0.0", + "ws": "^8.21.3" } } diff --git a/js/packages/core/src/index.ts b/js/packages/core/src/index.ts index 389a2c51..627251b7 100644 --- a/js/packages/core/src/index.ts +++ b/js/packages/core/src/index.ts @@ -136,4 +136,13 @@ export { type Filter, type RelayMessage, } from "./sync/events.js"; +export { RelayPool, type PublishResult, type RelayPoolOptions } from "./sync/relay.js"; +export { + publishSnapshot, + fetchLatestSnapshot, + publishDelta, + fetchDeltasSince, + type PublishedSnapshot, + type FetchedSnapshot, +} from "./sync/syncFlows.js"; export * from "./util/bytes.js"; diff --git a/js/packages/core/src/sync/relay.ts b/js/packages/core/src/sync/relay.ts new file mode 100644 index 00000000..649ab689 --- /dev/null +++ b/js/packages/core/src/sync/relay.ts @@ -0,0 +1,180 @@ +/** + * Relay transport: a small pool over the platform WebSocket (global in + * Node >= 22 and all target browsers — no dependency). + * + * Publishes to every relay and aggregates fetches across them, deduping by + * event id and dropping events whose id or signature fails verification. + * Parity reference for behavior: src/nostr/connection.py + snapshot.py + * (timeouts, per-chunk retry with backoff). + */ + +import { + eventMessage, + parseRelayMessage, + reqMessage, + verifyEvent, + type Filter, + type NostrEvent, +} from "./events.js"; + +export interface PublishResult { + relay: string; + ok: boolean; + message: string; +} + +export interface RelayPoolOptions { + /** Per-operation timeout in milliseconds. */ + timeoutMs?: number; +} + +interface Conn { + url: string; + socket: WebSocket; + ready: Promise; +} + +let subCounter = 0; + +export class RelayPool { + private readonly timeoutMs: number; + private conns: Conn[] | null = null; + + constructor( + public readonly urls: string[], + options: RelayPoolOptions = {}, + ) { + if (urls.length === 0) throw new Error("no relays configured"); + this.timeoutMs = options.timeoutMs ?? 10_000; + } + + private async connect(): Promise { + if (this.conns) return this.conns; + const attempts = this.urls.map((url) => { + const socket = new WebSocket(url); + const ready = new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`connect timeout: ${url}`)), + this.timeoutMs, + ); + socket.addEventListener("open", () => { + clearTimeout(timer); + resolve(); + }); + socket.addEventListener("error", () => { + clearTimeout(timer); + reject(new Error(`connect failed: ${url}`)); + }); + }); + return { url, socket, ready }; + }); + const settled = await Promise.allSettled(attempts.map((c) => c.ready)); + const alive = attempts.filter((_, i) => settled[i]!.status === "fulfilled"); + for (let i = 0; i < attempts.length; i++) { + if (settled[i]!.status === "rejected") { + try { + attempts[i]!.socket.close(); + } catch { + // already dead + } + } + } + if (alive.length === 0) { + throw new Error(`could not connect to any relay: ${this.urls.join(", ")}`); + } + this.conns = alive; + return alive; + } + + /** Publish an event to every connected relay; resolves with per-relay OKs. */ + async publish(event: NostrEvent): Promise { + const conns = await this.connect(); + const msg = eventMessage(event); + return Promise.all( + conns.map( + (conn) => + new Promise((resolve) => { + const timer = setTimeout(() => { + cleanup(); + resolve({ relay: conn.url, ok: false, message: "timeout waiting for OK" }); + }, this.timeoutMs); + const onMessage = (raw: MessageEvent) => { + try { + const parsed = parseRelayMessage(String(raw.data)); + if (parsed.type === "OK" && parsed.eventId === event.id) { + cleanup(); + resolve({ relay: conn.url, ok: parsed.accepted, message: parsed.message }); + } + } catch { + // ignore unrelated frames + } + }; + const cleanup = () => { + clearTimeout(timer); + conn.socket.removeEventListener("message", onMessage); + }; + conn.socket.addEventListener("message", onMessage); + conn.socket.send(msg); + }), + ), + ); + } + + /** + * Fetch events matching the filters from every relay until EOSE (or + * timeout), deduped by id; invalid ids/signatures are dropped. + */ + async fetch(...filters: Filter[]): Promise { + const conns = await this.connect(); + const byId = new Map(); + await Promise.all( + conns.map( + (conn) => + new Promise((resolve) => { + const subId = `sp-${++subCounter}-${Math.floor(Math.random() * 1e6)}`; + const timer = setTimeout(() => finish(), this.timeoutMs); + const onMessage = (raw: MessageEvent) => { + try { + const parsed = parseRelayMessage(String(raw.data)); + if (parsed.type === "EVENT" && parsed.subscriptionId === subId) { + const ev = parsed.event; + if (!byId.has(ev.id) && verifyEvent(ev)) byId.set(ev.id, ev); + } else if ( + (parsed.type === "EOSE" || parsed.type === "CLOSED") && + parsed.subscriptionId === subId + ) { + finish(); + } + } catch { + // ignore malformed frames + } + }; + const finish = () => { + clearTimeout(timer); + conn.socket.removeEventListener("message", onMessage); + try { + conn.socket.send(JSON.stringify(["CLOSE", subId])); + } catch { + // socket may be gone + } + resolve(); + }; + conn.socket.addEventListener("message", onMessage); + conn.socket.send(reqMessage(subId, ...filters)); + }), + ), + ); + return [...byId.values()]; + } + + async close(): Promise { + for (const conn of this.conns ?? []) { + try { + conn.socket.close(); + } catch { + // already closed + } + } + this.conns = null; + } +} diff --git a/js/packages/core/src/sync/syncFlows.ts b/js/packages/core/src/sync/syncFlows.ts new file mode 100644 index 00000000..3596e613 --- /dev/null +++ b/js/packages/core/src/sync/syncFlows.ts @@ -0,0 +1,163 @@ +/** + * High-level sync flows over a RelayPool, mirroring src/nostr/snapshot.py: + * publish a chunked snapshot + manifest, fetch the latest snapshot with + * chunk-hash verification, publish and replay deltas. + */ + +import { sha256 } from "@noble/hashes/sha2.js"; +import { base64 } from "@scure/base"; +import { bytesToHex } from "../util/bytes.js"; +import { + buildChunkEvent, + buildDeltaEvent, + buildManifestEvent, + signEvent, + signerPublicKeyHex, + type NostrEvent, +} from "./events.js"; +import { + KIND_DELTA, + KIND_MANIFEST, + KIND_SNAPSHOT_CHUNK, + newManifestId, + parseManifest, + prepareSnapshot, + reassembleSnapshot, + type Manifest, +} from "./snapshot.js"; +import type { RelayPool } from "./relay.js"; + +function nowUnix(): number { + return Math.floor(Date.now() / 1000); +} + +export interface PublishedSnapshot { + manifest: Manifest; + manifestId: string; + manifestEventId: string; + chunkEventIds: string[]; +} + +/** Compress, chunk, sign, and publish a snapshot plus its manifest. */ +export async function publishSnapshot( + pool: RelayPool, + privateKeyHex: string, + keyIndex: Uint8Array, + encryptedBytes: Uint8Array, + options: { limit?: number; createdAt?: number } = {}, +): Promise { + const createdAt = options.createdAt ?? nowUnix(); + const { manifest, chunks } = await prepareSnapshot(encryptedBytes, options.limit ?? 50_000); + + const chunkEventIds: string[] = []; + for (let i = 0; i < chunks.length; i++) { + const meta = manifest.chunks[i]!; + const event = signEvent( + privateKeyHex, + buildChunkEvent(meta.id, base64.encode(chunks[i]!), createdAt), + ); + const results = await pool.publish(event); + if (!results.some((r) => r.ok)) { + throw new Error(`no relay accepted chunk ${meta.id}`); + } + meta.event_id = event.id; + chunkEventIds.push(event.id); + } + + const { id: manifestId, nonce } = newManifestId(keyIndex); + manifest.nonce = base64.encode(nonce); + manifest.delta_since = createdAt; + + const manifestJson = JSON.stringify({ + ver: manifest.ver, + algo: manifest.algo, + chunks: manifest.chunks, + delta_since: manifest.delta_since, + nonce: manifest.nonce, + index0: manifest.index0, + }); + const manifestEvent = signEvent( + privateKeyHex, + buildManifestEvent(manifestId, manifestJson, createdAt), + ); + const results = await pool.publish(manifestEvent); + if (!results.some((r) => r.ok)) throw new Error("no relay accepted the manifest"); + + return { manifest, manifestId, manifestEventId: manifestEvent.id, chunkEventIds }; +} + +export interface FetchedSnapshot { + manifest: Manifest; + manifestEvent: NostrEvent; + /** Decompressed, still-encrypted vault payload. */ + encrypted: Uint8Array; +} + +/** Fetch the newest manifest and all its chunks, verifying every hash. */ +export async function fetchLatestSnapshot( + pool: RelayPool, + privateKeyHex: string, +): Promise { + const pubkey = signerPublicKeyHex(privateKeyHex); + const manifests = await pool.fetch({ authors: [pubkey], kinds: [KIND_MANIFEST] }); + if (manifests.length === 0) return null; + manifests.sort((a, b) => b.created_at - a.created_at); + + for (const manifestEvent of manifests) { + let manifest: Manifest; + try { + manifest = parseManifest(manifestEvent.content); + } catch { + continue; + } + const chunks: Uint8Array[] = []; + let complete = true; + for (const meta of manifest.chunks) { + const filter = meta.event_id + ? { ids: [meta.event_id], authors: [pubkey], kinds: [KIND_SNAPSHOT_CHUNK] } + : { authors: [pubkey], kinds: [KIND_SNAPSHOT_CHUNK], "#d": [meta.id], limit: 1 }; + const events = await pool.fetch(filter); + const match = events.find( + (ev) => bytesToHex(sha256(base64.decode(ev.content))) === meta.hash, + ); + if (!match) { + complete = false; + break; + } + chunks.push(base64.decode(match.content)); + } + if (!complete) continue; + const encrypted = await reassembleSnapshot(manifest, chunks); + return { manifest, manifestEvent, encrypted }; + } + return null; +} + +/** Publish an encrypted delta referencing the manifest event. */ +export async function publishDelta( + pool: RelayPool, + privateKeyHex: string, + manifestEventId: string, + deltaBytes: Uint8Array, + options: { createdAt?: number } = {}, +): Promise { + const event = signEvent( + privateKeyHex, + buildDeltaEvent(manifestEventId, base64.encode(deltaBytes), options.createdAt ?? nowUnix()), + ); + const results = await pool.publish(event); + if (!results.some((r) => r.ok)) throw new Error("no relay accepted the delta"); + return event.id; +} + +/** Fetch deltas since a timestamp, sorted by created_at (replay order). */ +export async function fetchDeltasSince( + pool: RelayPool, + privateKeyHex: string, + since: number, +): Promise { + const pubkey = signerPublicKeyHex(privateKeyHex); + const events = await pool.fetch({ authors: [pubkey], kinds: [KIND_DELTA], since }); + events.sort((a, b) => a.created_at - b.created_at); + return events.map((ev) => base64.decode(ev.content)); +} diff --git a/js/packages/core/src/util/bytes.ts b/js/packages/core/src/util/bytes.ts index 2d32c8cc..7e2f8859 100644 --- a/js/packages/core/src/util/bytes.ts +++ b/js/packages/core/src/util/bytes.ts @@ -1,5 +1,11 @@ /** Small byte/string helpers shared across the core. No Node or DOM APIs. */ +import { sha256 } from "@noble/hashes/sha2.js"; + +export function sha256Hex(data: Uint8Array): string { + return bytesToHex(sha256(data)); +} + export function utf8(s: string): Uint8Array { return new TextEncoder().encode(s); } diff --git a/js/packages/core/test/mockRelay.ts b/js/packages/core/test/mockRelay.ts new file mode 100644 index 00000000..90d0928d --- /dev/null +++ b/js/packages/core/test/mockRelay.ts @@ -0,0 +1,73 @@ +/** + * In-process NIP-01 relay for tests: stores events, answers REQ with + * matching EVENTs + EOSE, acks EVENT with OK. Test-only (uses `ws`). + */ + +import { WebSocketServer, type WebSocket as WsSocket } from "ws"; +import type { AddressInfo } from "node:net"; +import type { Filter, NostrEvent } from "@seedpass/core"; + +function matches(event: NostrEvent, filter: Filter): boolean { + if (filter.ids && !filter.ids.includes(event.id)) return false; + if (filter.authors && !filter.authors.includes(event.pubkey)) return false; + if (filter.kinds && !filter.kinds.includes(event.kind)) return false; + if (filter.since !== undefined && event.created_at < filter.since) return false; + if (filter.until !== undefined && event.created_at > filter.until) return false; + for (const tagName of ["#d", "#e"] as const) { + const wanted = filter[tagName]; + if (wanted) { + const values = event.tags.filter((t) => t[0] === tagName[1]).map((t) => t[1]); + if (!wanted.some((w) => values.includes(w))) return false; + } + } + return true; +} + +export class MockRelay { + readonly events: NostrEvent[] = []; + private server: WebSocketServer | null = null; + url = ""; + + async start(): Promise { + this.server = new WebSocketServer({ port: 0 }); + await new Promise((resolve) => this.server!.once("listening", resolve)); + const { port } = this.server.address() as AddressInfo; + this.url = `ws://127.0.0.1:${port}`; + this.server.on("connection", (socket: WsSocket) => { + socket.on("message", (raw) => { + let msg: unknown[]; + try { + msg = JSON.parse(String(raw)) as unknown[]; + } catch { + return; + } + if (msg[0] === "EVENT") { + const event = msg[1] as NostrEvent; + this.events.push(event); + socket.send(JSON.stringify(["OK", event.id, true, ""])); + } else if (msg[0] === "REQ") { + const subId = String(msg[1]); + const filters = msg.slice(2) as Filter[]; + let sent = 0; + for (const event of this.events) { + if (filters.some((f) => matches(event, f))) { + const limit = Math.min(...filters.map((f) => f.limit ?? Infinity)); + if (sent >= limit) break; + socket.send(JSON.stringify(["EVENT", subId, event])); + sent++; + } + } + socket.send(JSON.stringify(["EOSE", subId])); + } + // CLOSE is a no-op for the mock + }); + }); + return this.url; + } + + async stop(): Promise { + for (const client of this.server?.clients ?? []) client.terminate(); + await new Promise((resolve) => this.server?.close(() => resolve())); + this.server = null; + } +} diff --git a/js/packages/core/test/relay.test.ts b/js/packages/core/test/relay.test.ts new file mode 100644 index 00000000..e023da07 --- /dev/null +++ b/js/packages/core/test/relay.test.ts @@ -0,0 +1,117 @@ +/** + * End-to-end relay sync over a live (in-process) NIP-01 relay: publish a + * chunked snapshot, restore it from the relay alone, publish/replay deltas, + * and survive a corrupted-chunk relay response. + */ + +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { base64 } from "@scure/base"; +import { mnemonics, nostrKeyCases, vaultV3Payload } from "@seedpass/test-vectors"; +import { + RelayPool, + publishSnapshot, + fetchLatestSnapshot, + publishDelta, + fetchDeltasSince, + deriveKeyIndex, + deriveIndexKeyBytes, + decryptV3, + encryptV3, + mergeIndexPayloads, + utf8, + bytesToHex, +} from "@seedpass/core"; +import { sha256 } from "@noble/hashes/sha2.js"; +import { MockRelay } from "./mockRelay.js"; + +// jsdom's WebSocket shim cannot make real connections; transport tests run +// in Node here and in real browsers with the Milestone 6 web-app CI. +const IS_JSDOM = typeof navigator !== "undefined" && navigator.userAgent.includes("jsdom"); + +const MNEMONIC = mnemonics["abandon12"]!; +const PRIVKEY = nostrKeyCases.find( + (c) => c.mnemonic_id === "abandon12" && c.account_index === 0, +)!.private_key_hex; + +let relay: MockRelay; +let pool: RelayPool; + +beforeAll(async () => { + if (IS_JSDOM) return; + relay = new MockRelay(); + await relay.start(); + pool = new RelayPool([relay.url], { timeoutMs: 3000 }); +}); + +afterAll(async () => { + if (IS_JSDOM) return; + await pool.close(); + await relay.stop(); +}); + +describe.skipIf(IS_JSDOM)("relay snapshot round trip", () => { + const encrypted = () => base64.decode(vaultV3Payload.payload_b64); + + it("publishes a chunked snapshot and restores it from the relay", async () => { + const keyIndex = deriveKeyIndex(MNEMONIC); + const published = await publishSnapshot(pool, PRIVKEY, keyIndex, encrypted(), { + limit: 400, + }); + expect(published.chunkEventIds.length).toBeGreaterThan(1); + expect(relay.events.filter((e) => e.kind === 30071).length).toBe( + published.chunkEventIds.length, + ); + + const fetched = await fetchLatestSnapshot(pool, PRIVKEY); + expect(fetched).not.toBeNull(); + expect(base64.encode(fetched!.encrypted)).toBe(vaultV3Payload.payload_b64); + expect(fetched!.manifest.delta_since).toBe(published.manifest.delta_since); + + // The restored payload decrypts with the index key like any local vault + const key = deriveIndexKeyBytes(MNEMONIC); + const plain = await decryptV3(key, fetched!.encrypted); + expect(bytesToHex(sha256(plain))).toBe(vaultV3Payload.plaintext_sha256); + }); + + it("publishes deltas and replays them onto the snapshot state", async () => { + const key = deriveIndexKeyBytes(MNEMONIC); + const t = 1700000000; + const deltaIndex = { + schema_version: 4, + entries: { + "99": { + type: "password", kind: "password", label: "from-delta", + length: 16, archived: false, notes: "", tags: [], modified_ts: t + 50, + }, + }, + }; + const deltaBytes = await encryptV3(key, utf8(JSON.stringify(deltaIndex))); + const fetched = await fetchLatestSnapshot(pool, PRIVKEY); + await publishDelta(pool, PRIVKEY, fetched!.manifestEvent.id, deltaBytes, { + createdAt: (fetched!.manifest.delta_since ?? 0) + 10, + }); + + const deltas = await fetchDeltasSince(pool, PRIVKEY, fetched!.manifest.delta_since ?? 0); + expect(deltas).toHaveLength(1); + + // Replay: decrypt snapshot state, merge each delta deterministically + let state = JSON.parse( + new TextDecoder().decode(await decryptV3(key, fetched!.encrypted)), + ) as Record; + for (const delta of deltas) { + const incoming = JSON.parse(new TextDecoder().decode(await decryptV3(key, delta))); + state = mergeIndexPayloads(state, incoming, bytesToHex(sha256(delta)).slice(0, 16)); + } + const entries = state["entries"] as Record; + expect(entries["99"]!.label).toBe("from-delta"); + }); + + it("rejects snapshots whose chunks fail hash verification", async () => { + // Corrupt every stored chunk's content in the relay + for (const ev of relay.events) { + if (ev.kind === 30071) ev.content = base64.encode(utf8("corrupted")); + } + const fetched = await fetchLatestSnapshot(pool, PRIVKEY); + expect(fetched).toBeNull(); + }); +}); diff --git a/js/pnpm-lock.yaml b/js/pnpm-lock.yaml index c592532c..0d9e69c3 100644 --- a/js/pnpm-lock.yaml +++ b/js/pnpm-lock.yaml @@ -60,6 +60,9 @@ importers: '@seedpass/test-vectors': specifier: workspace:* version: link:../test-vectors + '@types/ws': + specifier: ^8.18.1 + version: 8.18.1 jsdom: specifier: ^30.0.1 version: 30.0.1(@noble/hashes@2.3.0) @@ -69,6 +72,9 @@ importers: vitest: specifier: ^3.0.0 version: 3.2.7(@types/node@26.2.0)(jsdom@30.0.1(@noble/hashes@2.3.0))(tsx@4.23.12) + ws: + specifier: ^8.21.3 + version: 8.21.3 packages/test-vectors: devDependencies: @@ -468,6 +474,9 @@ packages: '@types/node@26.2.0': resolution: {integrity: sha512-5IviulTZeRNp2vAJ514cc/HUlY5nZ9fCbq9DMyC52BrhFZACo3nI0R7qBxhQmo/d27NFe96ur/b7Wwxklda+kg==} + '@types/ws@8.18.1': + resolution: {integrity: sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==} + '@vitest/expect@3.2.7': resolution: {integrity: sha512-E8eBXaKibuvH2pSZErOjdVb5vF4PbKYcrnluBTYxEk1l/VhhwZg1kZQsdtjq+CsF5CFydf2Rdkz7jDHKSisi3w==} @@ -826,6 +835,18 @@ packages: engines: {node: '>=8'} hasBin: true + ws@8.21.3: + resolution: {integrity: sha512-201TZ/kPWxoPr/OKWjquZR1SWKXcvxdH+e1xrx89b3YbmzLMFCLfnaG1HFIgWzJOEWZ7MvpK++odZufgYR50Rw==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + xml-name-validator@5.0.0: resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==} engines: {node: '>=18'} @@ -1074,6 +1095,10 @@ snapshots: dependencies: undici-types: 8.3.0 + '@types/ws@8.18.1': + dependencies: + '@types/node': 26.2.0 + '@vitest/expect@3.2.7': dependencies: '@types/chai': 5.2.3 @@ -1470,6 +1495,8 @@ snapshots: siginfo: 2.0.0 stackback: 0.0.2 + ws@8.21.3: {} + xml-name-validator@5.0.0: {} xmlchars@2.2.0: {}