From a8e9f5202a054b8da90b86cc27c176e0f553d2b8 Mon Sep 17 00:00:00 2001 From: thePR0M3TH3AN <53631862+PR0M3TH3AN@users.noreply.github.com> Date: Mon, 22 Jun 2026 18:47:00 -0400 Subject: [PATCH] adminLists: batch community-blacklist fetch (kill cold-start kind-30000 storm) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cold-start fired one kind-30000 REQ per community-blacklist curator (loadCommunityBlacklistEntries did Promise.all over references, each a fetchLatestListEvent) — ~28-32 parallel REQs that dominated the login relay-REQ spike and starved the feed. A Nostr filter ORs `authors` and `#d`, so all curators collapse into ONE batched REQ. New js/adminListBatch.js (pure, mutation-tested): buildBatchedReferenceFilter + selectNewestEventsForReferences (newest per author:d-tag, discards author×d-tag cross-products) + fetchBatchedReferenceEvents (routes through the subscription manager's list — health-gated/timeout-bounded, not raw pool). Helpers extracted to their own module so adminListStore stays under its size cap. Best-effort: on batch failure we return nothing and the cached admin state carries over (readCachedAdminState). Addresses pre-launch TODO #9 (cold-login REQ storm). Tests + mutation- verified (newest selection, cross-product discard). 1249/1250 unit (known UploadModal flake). Co-Authored-By: Claude Opus 4.8 --- js/adminListBatch.js | 88 +++++++++++++++++++++++++ js/adminListStore.js | 38 +++++------ tests/admin-list-batched-fetch.test.mjs | 73 ++++++++++++++++++++ 3 files changed, 180 insertions(+), 19 deletions(-) create mode 100644 js/adminListBatch.js create mode 100644 tests/admin-list-batched-fetch.test.mjs diff --git a/js/adminListBatch.js b/js/adminListBatch.js new file mode 100644 index 00000000..83f7a385 --- /dev/null +++ b/js/adminListBatch.js @@ -0,0 +1,88 @@ +// Pure helpers for batching the community-blacklist curator fetch into a single +// multi-author REQ instead of one kind-30000 REQ per curator (the cold-start +// relay storm). Kept dependency-free so they're cheat-resistant to test. + +// Build a single batched filter for many (authorHex, dTag) references. A Nostr +// filter ORs `authors` and ORs `#d`, so one REQ replaces N per-curator REQs. +// Over-fetches author×dtag cross-products, which selectNewestEventsForReferences +// discards when matching back to exact references. +export function buildBatchedReferenceFilter(references, { kind = 30000 } = {}) { + const authors = new Set(); + const dTags = new Set(); + for (const ref of Array.isArray(references) ? references : []) { + const author = + typeof ref?.authorHex === "string" ? ref.authorHex.trim().toLowerCase() : ""; + const dTag = typeof ref?.dTag === "string" ? ref.dTag.trim() : ""; + if (author) authors.add(author); + if (dTag) dTags.add(dTag); + } + if (!authors.size || !dTags.size) { + return null; + } + return { + kinds: [kind], + authors: Array.from(authors), + "#d": Array.from(dTags), + }; +} + +function dTagOf(event) { + const tags = Array.isArray(event?.tags) ? event.tags : []; + for (const tag of tags) { + if (Array.isArray(tag) && tag[0] === "d") { + return typeof tag[1] === "string" ? tag[1] : ""; + } + } + return ""; +} + +// Group batched events to the newest per (author, dTag), then return the events +// matching the requested references (in reference order). Cross-product events +// (author×dtag combinations that weren't requested) are discarded. +export function selectNewestEventsForReferences(events, references) { + const newest = new Map(); // key `${pubkey}:${dtag}` -> event + for (const event of Array.isArray(events) ? events : []) { + const pubkey = typeof event?.pubkey === "string" ? event.pubkey.toLowerCase() : ""; + const dTag = dTagOf(event); + if (!pubkey || !dTag) { + continue; + } + const key = `${pubkey}:${dTag}`; + const prev = newest.get(key); + if ( + !prev || + event.created_at > prev.created_at || + (event.created_at === prev.created_at && event.id > prev.id) + ) { + newest.set(key, event); + } + } + const matched = []; + for (const ref of Array.isArray(references) ? references : []) { + const author = typeof ref?.authorHex === "string" ? ref.authorHex.toLowerCase() : ""; + const dTag = typeof ref?.dTag === "string" ? ref.dTag : ""; + const event = newest.get(`${author}:${dTag}`); + if (event) { + matched.push(event); + } + } + return matched; +} + +// Fetch all curator blacklists in ONE batched REQ and return the newest event +// matched to each reference. `listEvents` is injected (the subscription +// manager's health-gated, timeout-bounded list) so this stays off the raw pool. +// Returns [] when there's nothing to batch; rejects on fetch failure. +export async function fetchBatchedReferenceEvents({ + references, + listEvents, + relays, + timeoutMs, +}) { + const filter = buildBatchedReferenceFilter(references); + if (!filter || typeof listEvents !== "function") { + return []; + } + const events = await listEvents({ filters: [filter], relays, timeoutMs }); + return selectNewestEventsForReferences(events, references); +} diff --git a/js/adminListStore.js b/js/adminListStore.js index 83eddb5e..140c1696 100644 --- a/js/adminListStore.js +++ b/js/adminListStore.js @@ -22,6 +22,7 @@ import { } from "./nostrEventSchemas.js"; import { CACHE_POLICIES, STORAGE_TIERS } from "./nostr/cachePolicies.js"; import { publishEventToRelay } from "./nostrPublish.js"; +import { fetchBatchedReferenceEvents } from "./adminListBatch.js"; const ADMIN_STATE_CACHE_VERSION = 1; const ADMIN_STATE_CACHE_KEY = `bitvid_admin_state_v${ADMIN_STATE_CACHE_VERSION}`; @@ -688,25 +689,24 @@ async function loadCommunityBlacklistEntries() { return []; } - const results = await Promise.all( - references.map(async (reference) => { - try { - const event = await fetchLatestListEvent( - { "#d": [reference.dTag], authors: [reference.authorHex] }, - reference.dTag, - ); - return event ? extractNpubsFromEvent(event) : []; - } catch (error) { - devLogger.warn( - `[adminListStore] Failed to load community blacklist ${reference.dTag}:`, - error, - ); - return []; - } - }) - ); - - return results.flat(); + // Batch all curator blacklists into ONE REQ instead of one-per-curator — this + // was the cold-start kind-30000 relay storm. Best-effort: on failure we return + // nothing and the cached admin state (readCachedAdminState) carries over. + try { + const matched = await fetchBatchedReferenceEvents({ + references, + listEvents: (args) => requireNostrClient().getSubscriptionManager().list(args), + relays: ensureNostrReady(), + timeoutMs: SHORT_TIMEOUT_MS, + }); + return matched.flatMap((event) => extractNpubsFromEvent(event)); + } catch (error) { + devLogger.warn( + "[adminListStore] Batched community blacklist fetch failed:", + error, + ); + return []; + } } function buildListEvent(listKey, npubs, actorHex) { diff --git a/tests/admin-list-batched-fetch.test.mjs b/tests/admin-list-batched-fetch.test.mjs new file mode 100644 index 00000000..8da47e4f --- /dev/null +++ b/tests/admin-list-batched-fetch.test.mjs @@ -0,0 +1,73 @@ +// Batching the community-blacklist curator fetch into a single multi-author REQ +// (instead of one kind-30000 REQ per curator — the cold-start relay storm). +// Tests the pure filter-builder + result-selector. + +import assert from "node:assert/strict"; +import test from "node:test"; +import { + buildBatchedReferenceFilter, + selectNewestEventsForReferences, +} from "../js/adminListBatch.js"; + +const A = "a".repeat(64); +const B = "b".repeat(64); + +function evt({ id, pubkey, d, created_at }) { + return { id, pubkey, created_at, tags: [["d", d]] }; +} + +test("builds one filter with all unique authors + d-tags", () => { + const filter = buildBatchedReferenceFilter([ + { authorHex: A, dTag: "community:blacklist" }, + { authorHex: B, dTag: "community:blacklist" }, + { authorHex: A, dTag: "community:blacklist" }, // dup + ]); + assert.deepEqual(filter.kinds, [30000]); + assert.deepEqual(filter.authors.sort(), [A, B]); + assert.deepEqual(filter["#d"], ["community:blacklist"]); +}); + +test("returns null when there are no usable references", () => { + assert.equal(buildBatchedReferenceFilter([]), null); + assert.equal(buildBatchedReferenceFilter([{ authorHex: "", dTag: "" }]), null); +}); + +test("selects the newest event per (author, d-tag) reference", () => { + const references = [ + { authorHex: A, dTag: "list-1" }, + { authorHex: B, dTag: "list-1" }, + ]; + const events = [ + evt({ id: "a-old", pubkey: A, d: "list-1", created_at: 100 }), + evt({ id: "a-new", pubkey: A, d: "list-1", created_at: 200 }), + evt({ id: "b-only", pubkey: B, d: "list-1", created_at: 50 }), + ]; + const matched = selectNewestEventsForReferences(events, references); + assert.deepEqual(matched.map((e) => e.id), ["a-new", "b-only"]); +}); + +test("discards cross-product events that aren't real references", () => { + // The batched filter (authors A,B × d-tags d1,d2) can return (A,d2) even + // though only (A,d1) and (B,d2) were requested. Those must be dropped. + const references = [ + { authorHex: A, dTag: "d1" }, + { authorHex: B, dTag: "d2" }, + ]; + const events = [ + evt({ id: "a-d1", pubkey: A, d: "d1", created_at: 10 }), + evt({ id: "a-d2-crossproduct", pubkey: A, d: "d2", created_at: 99 }), + evt({ id: "b-d2", pubkey: B, d: "d2", created_at: 10 }), + ]; + const matched = selectNewestEventsForReferences(events, references); + assert.deepEqual(matched.map((e) => e.id).sort(), ["a-d1", "b-d2"]); +}); + +test("tie-break on created_at uses higher event id (deterministic)", () => { + const references = [{ authorHex: A, dTag: "d1" }]; + const events = [ + evt({ id: "id-aaa", pubkey: A, d: "d1", created_at: 100 }), + evt({ id: "id-zzz", pubkey: A, d: "d1", created_at: 100 }), + ]; + const matched = selectNewestEventsForReferences(events, references); + assert.deepEqual(matched.map((e) => e.id), ["id-zzz"]); +});