From 733cb5937b2d84984b24f203eb8d982bc70bfc5f Mon Sep 17 00:00:00 2001 From: thePR0M3TH3AN <53631862+PR0M3TH3AN@users.noreply.github.com> Date: Mon, 22 Jun 2026 18:24:05 -0400 Subject: [PATCH] nostr: sanitize hex-only filter fields (fix relay "from_hex" REQ rejects) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Strict relays (primal) reject a whole REQ with "uneven size input to from_hex" when any ids/authors/#e/#p/#q entry isn't 64-char hex — silently dropping that subscription's results. One malformed value anywhere in a filter poisoned the request. Sanitize at the universal pool choke point (normalizeFilterList, used by both the shimmed pool.sub and pool.list): lowercase/trim and drop non- 64-hex entries from the hex-only fields; if a once-populated hex field ends up empty, drop that filter rather than silently broadening the query. Non-hex fields (#a/#d/#t/kinds/…) are untouched. Dev-logs what it strips (with the filter's kinds) so the malformed source can be traced. Also records pre-launch TODO items: NIP-71 inbound ingest is now live (Phase 2, incl. the resolve-posted-at history-fetch fix), and a new launch-blocker — channel-profile walls fetch only kind-30078 so a creator's NIP-71 videos don't show on their profile. Tests: filter-hex-sanitize (strip/keep/lowercase/drop-empty/leave-non-hex); mutation-verified. 1244/1245 unit (known UploadModal flake). Co-Authored-By: Claude Opus 4.8 --- js/nostr/toolkit.js | 63 +++++++++++++++++++++++++++-- tests/filter-hex-sanitize.test.mjs | 64 ++++++++++++++++++++++++++++++ todo/TODO_2026-06-20_pre-launch.md | 28 ++++++++++++- 3 files changed, 150 insertions(+), 5 deletions(-) create mode 100644 tests/filter-hex-sanitize.test.mjs diff --git a/js/nostr/toolkit.js b/js/nostr/toolkit.js index 96361159..94f46e5a 100644 --- a/js/nostr/toolkit.js +++ b/js/nostr/toolkit.js @@ -147,17 +147,74 @@ function normalizeRelayList(relays) { .filter((relay) => relay); } +// Filter fields that relays hex-decode (from_hex) and therefore MUST contain +// 64-char lowercase hex. A single malformed/odd-length entry makes strict relays +// (e.g. primal) reject the whole REQ with "uneven size input to from_hex", +// silently dropping that subscription's results. +const HEX64_FILTER_FIELDS = ["ids", "authors", "#e", "#p", "#q"]; +const HEX64_PATTERN = /^[0-9a-f]{64}$/; + +// Strip invalid hex from the hex-only filter fields so one bad value can't get a +// whole REQ rejected. Returns the sanitized filter, or null if a hex field that +// had entries ends up empty (the query would be meaningless/over-broad without +// them, so the filter is dropped rather than silently widened). +export function sanitizeHexFilterFields(filter) { + if (!filter || typeof filter !== "object") { + return filter; + } + let sanitized = filter; + for (const field of HEX64_FILTER_FIELDS) { + const values = filter[field]; + if (!Array.isArray(values) || values.length === 0) { + continue; + } + const kept = []; + for (const value of values) { + const normalized = typeof value === "string" ? value.trim().toLowerCase() : ""; + if (HEX64_PATTERN.test(normalized)) { + kept.push(normalized); + } + } + const changed = + kept.length !== values.length || kept.some((value, i) => value !== values[i]); + if (!changed) { + continue; + } + const droppedCount = values.length - kept.length; + if (droppedCount > 0) { + devLogger.warn( + `[toolkit] dropped ${droppedCount} invalid hex value(s) from filter "${field}"` + + ` (kinds: ${Array.isArray(filter.kinds) ? filter.kinds.join(",") : "?"})`, + ); + } + if (kept.length === 0) { + return null; // a once-populated hex field is now empty — drop the filter + } + if (sanitized === filter) { + sanitized = { ...filter }; + } + sanitized[field] = kept; + } + return sanitized; +} + function normalizeFilterList(filters) { if (!filters) { return []; } + let list; if (!Array.isArray(filters)) { if (typeof filters === "object" && filters !== null) { - return [filters]; + list = [filters]; + } else { + return []; } - return []; + } else { + list = filters.filter((candidate) => candidate && typeof candidate === "object"); } - return filters.filter((candidate) => candidate && typeof candidate === "object"); + return list + .map((filter) => sanitizeHexFilterFields(filter)) + .filter((filter) => filter && typeof filter === "object"); } function safeInvoke(handler, args, contextLabel) { diff --git a/tests/filter-hex-sanitize.test.mjs b/tests/filter-hex-sanitize.test.mjs new file mode 100644 index 00000000..a3701af7 --- /dev/null +++ b/tests/filter-hex-sanitize.test.mjs @@ -0,0 +1,64 @@ +// Sanitizing hex-only filter fields (ids/authors/#e/#p/#q) before they reach a +// relay. One odd-length/non-hex value otherwise makes strict relays reject the +// whole REQ ("uneven size input to from_hex"), silently dropping results. + +import assert from "node:assert/strict"; +import test from "node:test"; +import { sanitizeHexFilterFields } from "../js/nostr/toolkit.js"; + +const HEX_A = "a".repeat(64); +const HEX_B = "b".repeat(64); +const ODD = "abc"; // odd length -> from_hex failure +const SHORT = "a".repeat(63); // wrong length + +test("strips invalid hex from authors but keeps valid entries", () => { + const out = sanitizeHexFilterFields({ kinds: [0], authors: [HEX_A, ODD, HEX_B] }); + assert.deepEqual(out.authors, [HEX_A, HEX_B]); + assert.deepEqual(out.kinds, [0], "non-hex fields untouched"); +}); + +test("lowercases and trims valid hex", () => { + const out = sanitizeHexFilterFields({ ids: [` ${HEX_A.toUpperCase()} `] }); + assert.deepEqual(out.ids, [HEX_A]); +}); + +test("drops the whole filter when a populated hex field becomes empty", () => { + // If every author was invalid, removing the key would broaden the query to + // "all authors" — drop the filter instead. + const out = sanitizeHexFilterFields({ kinds: [1], authors: [ODD, SHORT] }); + assert.equal(out, null); +}); + +test("sanitizes event/pubkey tag filters (#e/#p/#q)", () => { + const out = sanitizeHexFilterFields({ + kinds: [1984], + "#e": [HEX_A, ODD], + "#p": [HEX_B], + }); + assert.deepEqual(out["#e"], [HEX_A]); + assert.deepEqual(out["#p"], [HEX_B]); +}); + +test("leaves non-hex tag fields (#a/#d/#t) alone", () => { + const filter = { + kinds: [21, 22], + "#a": ["30078:" + HEX_A + ":some-d-tag"], + "#d": ["subscriptions"], + "#t": ["music"], + }; + const out = sanitizeHexFilterFields(filter); + assert.deepEqual(out["#a"], filter["#a"]); + assert.deepEqual(out["#d"], filter["#d"]); + assert.deepEqual(out["#t"], filter["#t"]); +}); + +test("returns the same object when nothing needs sanitizing (no needless copy)", () => { + const filter = { kinds: [0], authors: [HEX_A] }; + const out = sanitizeHexFilterFields(filter); + assert.equal(out, filter); +}); + +test("ignores empty/absent hex fields", () => { + const out = sanitizeHexFilterFields({ kinds: [1], authors: [] }); + assert.deepEqual(out, { kinds: [1], authors: [] }); +}); diff --git a/todo/TODO_2026-06-20_pre-launch.md b/todo/TODO_2026-06-20_pre-launch.md index 707c1eeb..ce7d4949 100644 --- a/todo/TODO_2026-06-20_pre-launch.md +++ b/todo/TODO_2026-06-20_pre-launch.md @@ -283,5 +283,29 @@ notes below. Summary: OFF/dormant — superseded by the opt-in 34235/36 mirror; clean up later. - [ ] Phase 1.5: NIP-89 handler reg (kind 31990 → "Open in bitvid" elsewhere); NIP-51 kind 30005 portable playlists. -- [ ] Phase 2: inbound ingest of external NIP-71 videos (dedup, moderation, trust, - `origin` attribution, distinct discovery surface). +- [x] Phase 2: inbound ingest of external NIP-71 videos — LIVE on unstable. + - `js/nostr/nip71IngestAdapter.js` (foreign 21/22/34235/34236 → bitvid video; + content-warning→isNsfw; skips bitvid mirrors; surfaces nip71.publishedAt), + `js/services/nip71IngestService.js` (whitelist-scoped subscription, + hydration retry, deferred-until-feed-ready, throttled refresh), + wired in applicationBootstrap. Admin toggle `FEATURE_NIP71_INGEST` + (instance-config) default ON. + - Whitelist model: ingest is scoped to whitelisted authors and reuses the + existing render-time filter (whitelist/NSFW/blacklist/private). When the + admin disables whitelist mode, ingest opens to all authors (capped). + - KEY FIX: the feed's resolve-posted-at stage was firing a per-video + hydrateVideoHistory() network fetch for each ingested video (no kind-30078 + history exists) → relay storm + feed hang. Fixed by surfacing + nip71.publishedAt so the stage short-circuits. + - Verified live: Goblinbox (Nostube) content renders; liveness check works. + +### 17b. Channel Profile wall shows only bitvid videos, not NIP-71 (LAUNCH-BLOCKER) +- [ ] The channel-profile grid (`js/channelProfile.js`, ~line 5387) fetches ONLY + `{ kinds:[30078], authors:[pubkey], "#t":["video"] }`, so a creator's + NIP-71 videos (kinds 21/22/34235/34236 — incl. their Nostube uploads) never + appear on their profile wall even though they show in the main feed via + ingest. Fix: also fetch the author's NIP-71 video kinds and merge them + (convert via `buildVideoFromNip71Event`, skip bitvid mirrors, dedupe by + videoRootId, native-30078 wins a shared root — same precedence as ingest). + Consider reusing the ingested videos already in `nostrClient.allEvents` for + that author rather than a fresh fetch.