nostr: sanitize hex-only filter fields (fix relay "from_hex" REQ rejects)

Strict relays (primal) reject a whole REQ with "uneven size input to
from_hex" when any ids/authors/#e/#p/#q entry isn't 64-char hex — silently
dropping that subscription's results. One malformed value anywhere in a
filter poisoned the request.

Sanitize at the universal pool choke point (normalizeFilterList, used by
both the shimmed pool.sub and pool.list): lowercase/trim and drop non-
64-hex entries from the hex-only fields; if a once-populated hex field ends
up empty, drop that filter rather than silently broadening the query.
Non-hex fields (#a/#d/#t/kinds/…) are untouched. Dev-logs what it strips
(with the filter's kinds) so the malformed source can be traced.

Also records pre-launch TODO items: NIP-71 inbound ingest is now live
(Phase 2, incl. the resolve-posted-at history-fetch fix), and a new
launch-blocker — channel-profile walls fetch only kind-30078 so a
creator's NIP-71 videos don't show on their profile.

Tests: filter-hex-sanitize (strip/keep/lowercase/drop-empty/leave-non-hex);
mutation-verified. 1244/1245 unit (known UploadModal flake).

Co-Authored-By: Claude Opus 4.8 <[email protected]>

js/nostr/toolkit.js

@@ -147,17 +147,74 @@ function normalizeRelayList(relays) {
     .filter((relay) => relay);
 }
 
+// Filter fields that relays hex-decode (from_hex) and therefore MUST contain
+// 64-char lowercase hex. A single malformed/odd-length entry makes strict relays
+// (e.g. primal) reject the whole REQ with "uneven size input to from_hex",
+// silently dropping that subscription's results.
+const HEX64_FILTER_FIELDS = ["ids", "authors", "#e", "#p", "#q"];
+const HEX64_PATTERN = /^[0-9a-f]{64}$/;
+
+// Strip invalid hex from the hex-only filter fields so one bad value can't get a
+// whole REQ rejected. Returns the sanitized filter, or null if a hex field that
+// had entries ends up empty (the query would be meaningless/over-broad without
+// them, so the filter is dropped rather than silently widened).
+export function sanitizeHexFilterFields(filter) {
+  if (!filter || typeof filter !== "object") {
+    return filter;
+  }
+  let sanitized = filter;
+  for (const field of HEX64_FILTER_FIELDS) {
+    const values = filter[field];
+    if (!Array.isArray(values) || values.length === 0) {
+      continue;
+    }
+    const kept = [];
+    for (const value of values) {
+      const normalized = typeof value === "string" ? value.trim().toLowerCase() : "";
+      if (HEX64_PATTERN.test(normalized)) {
+        kept.push(normalized);
+      }
+    }
+    const changed =
+      kept.length !== values.length || kept.some((value, i) => value !== values[i]);
+    if (!changed) {
+      continue;
+    }
+    const droppedCount = values.length - kept.length;
+    if (droppedCount > 0) {
+      devLogger.warn(
+        `[toolkit] dropped ${droppedCount} invalid hex value(s) from filter "${field}"` +
+          ` (kinds: ${Array.isArray(filter.kinds) ? filter.kinds.join(",") : "?"})`,
+      );
+    }
+    if (kept.length === 0) {
+      return null; // a once-populated hex field is now empty — drop the filter
+    }
+    if (sanitized === filter) {
+      sanitized = { ...filter };
+    }
+    sanitized[field] = kept;
+  }
+  return sanitized;
+}
+
 function normalizeFilterList(filters) {
   if (!filters) {
     return [];
   }
+  let list;
   if (!Array.isArray(filters)) {
     if (typeof filters === "object" && filters !== null) {
-      return [filters];
+      list = [filters];
+    } else {
+      return [];
     }
-    return [];
+  } else {
+    list = filters.filter((candidate) => candidate && typeof candidate === "object");
   }
-  return filters.filter((candidate) => candidate && typeof candidate === "object");
+  return list
+    .map((filter) => sanitizeHexFilterFields(filter))
+    .filter((filter) => filter && typeof filter === "object");
 }
 
 function safeInvoke(handler, args, contextLabel) {

tests/filter-hex-sanitize.test.mjs

@@ -0,0 +1,64 @@
+// Sanitizing hex-only filter fields (ids/authors/#e/#p/#q) before they reach a
+// relay. One odd-length/non-hex value otherwise makes strict relays reject the
+// whole REQ ("uneven size input to from_hex"), silently dropping results.
+
+import assert from "node:assert/strict";
+import test from "node:test";
+import { sanitizeHexFilterFields } from "../js/nostr/toolkit.js";
+
+const HEX_A = "a".repeat(64);
+const HEX_B = "b".repeat(64);
+const ODD = "abc"; // odd length -> from_hex failure
+const SHORT = "a".repeat(63); // wrong length
+
+test("strips invalid hex from authors but keeps valid entries", () => {
+  const out = sanitizeHexFilterFields({ kinds: [0], authors: [HEX_A, ODD, HEX_B] });
+  assert.deepEqual(out.authors, [HEX_A, HEX_B]);
+  assert.deepEqual(out.kinds, [0], "non-hex fields untouched");
+});
+
+test("lowercases and trims valid hex", () => {
+  const out = sanitizeHexFilterFields({ ids: [` ${HEX_A.toUpperCase()} `] });
+  assert.deepEqual(out.ids, [HEX_A]);
+});
+
+test("drops the whole filter when a populated hex field becomes empty", () => {
+  // If every author was invalid, removing the key would broaden the query to
+  // "all authors" — drop the filter instead.
+  const out = sanitizeHexFilterFields({ kinds: [1], authors: [ODD, SHORT] });
+  assert.equal(out, null);
+});
+
+test("sanitizes event/pubkey tag filters (#e/#p/#q)", () => {
+  const out = sanitizeHexFilterFields({
+    kinds: [1984],
+    "#e": [HEX_A, ODD],
+    "#p": [HEX_B],
+  });
+  assert.deepEqual(out["#e"], [HEX_A]);
+  assert.deepEqual(out["#p"], [HEX_B]);
+});
+
+test("leaves non-hex tag fields (#a/#d/#t) alone", () => {
+  const filter = {
+    kinds: [21, 22],
+    "#a": ["30078:" + HEX_A + ":some-d-tag"],
+    "#d": ["subscriptions"],
+    "#t": ["music"],
+  };
+  const out = sanitizeHexFilterFields(filter);
+  assert.deepEqual(out["#a"], filter["#a"]);
+  assert.deepEqual(out["#d"], filter["#d"]);
+  assert.deepEqual(out["#t"], filter["#t"]);
+});
+
+test("returns the same object when nothing needs sanitizing (no needless copy)", () => {
+  const filter = { kinds: [0], authors: [HEX_A] };
+  const out = sanitizeHexFilterFields(filter);
+  assert.equal(out, filter);
+});
+
+test("ignores empty/absent hex fields", () => {
+  const out = sanitizeHexFilterFields({ kinds: [1], authors: [] });
+  assert.deepEqual(out, { kinds: [1], authors: [] });
+});

todo/TODO_2026-06-20_pre-launch.md

@@ -283,5 +283,29 @@ notes below. Summary:
         OFF/dormant — superseded by the opt-in 34235/36 mirror; clean up later.
 - [ ] Phase 1.5: NIP-89 handler reg (kind 31990 → "Open in bitvid" elsewhere);
       NIP-51 kind 30005 portable playlists.
-- [ ] Phase 2: inbound ingest of external NIP-71 videos (dedup, moderation, trust,
-      `origin` attribution, distinct discovery surface).
+- [x] Phase 2: inbound ingest of external NIP-71 videos — LIVE on unstable.
+      - `js/nostr/nip71IngestAdapter.js` (foreign 21/22/34235/34236 → bitvid video;
+        content-warning→isNsfw; skips bitvid mirrors; surfaces nip71.publishedAt),
+        `js/services/nip71IngestService.js` (whitelist-scoped subscription,
+        hydration retry, deferred-until-feed-ready, throttled refresh),
+        wired in applicationBootstrap. Admin toggle `FEATURE_NIP71_INGEST`
+        (instance-config) default ON.
+      - Whitelist model: ingest is scoped to whitelisted authors and reuses the
+        existing render-time filter (whitelist/NSFW/blacklist/private). When the
+        admin disables whitelist mode, ingest opens to all authors (capped).
+      - KEY FIX: the feed's resolve-posted-at stage was firing a per-video
+        hydrateVideoHistory() network fetch for each ingested video (no kind-30078
+        history exists) → relay storm + feed hang. Fixed by surfacing
+        nip71.publishedAt so the stage short-circuits.
+      - Verified live: Goblinbox (Nostube) content renders; liveness check works.
+
+### 17b. Channel Profile wall shows only bitvid videos, not NIP-71 (LAUNCH-BLOCKER)
+- [ ] The channel-profile grid (`js/channelProfile.js`, ~line 5387) fetches ONLY
+      `{ kinds:[30078], authors:[pubkey], "#t":["video"] }`, so a creator's
+      NIP-71 videos (kinds 21/22/34235/34236 — incl. their Nostube uploads) never
+      appear on their profile wall even though they show in the main feed via
+      ingest. Fix: also fetch the author's NIP-71 video kinds and merge them
+      (convert via `buildVideoFromNip71Event`, skip bitvid mirrors, dedupe by
+      videoRootId, native-30078 wins a shared root — same precedence as ingest).
+      Consider reusing the ingested videos already in `nostrClient.allEvents` for
+      that author rather than a fresh fetch.