From bf908c3b05085a8cd904493377c799ec581a83ab Mon Sep 17 00:00:00 2001 From: thePR0M3TH3AN <53631862+PR0M3TH3AN@users.noreply.github.com> Date: Sat, 18 Oct 2025 21:46:27 -0400 Subject: [PATCH] Optimize subscription author lookup --- js/feedEngine/sources.js | 17 +++++-- js/services/nostrService.js | 84 +++++++++++++++++++++++++++++++ tests/subscriptions-feed.test.mjs | 63 ++++++++++++++++++++++- 3 files changed, 159 insertions(+), 5 deletions(-) diff --git a/js/feedEngine/sources.js b/js/feedEngine/sources.js index 494773fc..418d7af7 100644 --- a/js/feedEngine/sources.js +++ b/js/feedEngine/sources.js @@ -100,11 +100,22 @@ export function createSubscriptionAuthorsSource({ service } = {}) { : () => false, }; + const authorList = Array.from(authors); + const hasTargetedLookup = + resolvedService && + typeof resolvedService.getActiveVideosByAuthors === "function"; + let videos = []; try { - videos = await Promise.resolve( - resolvedService.getFilteredActiveVideos(options) - ); + if (hasTargetedLookup) { + videos = await Promise.resolve( + resolvedService.getActiveVideosByAuthors(authorList, options) + ); + } else { + videos = await Promise.resolve( + resolvedService.getFilteredActiveVideos(options) + ); + } } catch (error) { context?.log?.( "[subscriptions-source] Failed to resolve videos from nostrService", diff --git a/js/services/nostrService.js b/js/services/nostrService.js index 09a5b962..1a45ec0b 100644 --- a/js/services/nostrService.js +++ b/js/services/nostrService.js @@ -114,6 +114,8 @@ export class NostrService { } }); this.videosMap = null; + this.videosByAuthorIndex = null; + this.authorIndexDirty = false; this.moderationService = moderationService || null; try { if (this.moderationService && typeof this.moderationService.setNostrClient === "function") { @@ -175,6 +177,7 @@ export class NostrService { } this.videosMap = map; + this.markAuthorIndexDirty(); return this.videosMap; } @@ -215,16 +218,50 @@ export class NostrService { } } setStoredVideosMap(map); + this.markAuthorIndexDirty(); this.emit("videos:cache", { size: map.size }); } resetVideosCache() { const map = new Map(); this.videosMap = map; + this.videosByAuthorIndex = new Map(); + this.authorIndexDirty = false; setStoredVideosMap(map); this.emit("videos:cache", { size: 0 }); } + markAuthorIndexDirty() { + this.authorIndexDirty = true; + } + + ensureVideosByAuthorIndex() { + const videosMap = this.ensureVideosMap(); + + if (this.videosByAuthorIndex instanceof Map && !this.authorIndexDirty) { + return this.videosByAuthorIndex; + } + + const index = new Map(); + for (const video of videosMap.values()) { + if (!video || typeof video !== "object") { + continue; + } + const author = normalizeHexPubkey(video.pubkey); + if (!author) { + continue; + } + if (!index.has(author)) { + index.set(author, []); + } + index.get(author).push(video); + } + + this.videosByAuthorIndex = index; + this.authorIndexDirty = false; + return this.videosByAuthorIndex; + } + shouldIncludeVideo(video, { blacklistedEventIds = new Set(), isAuthorBlocked = () => false, @@ -306,6 +343,53 @@ export class NostrService { return this.filterVideos(all, options); } + getActiveVideosByAuthors(authors = [], options = {}) { + const candidates = ensureSet(authors); + const normalizedAuthors = new Set(); + for (const candidate of candidates) { + const normalized = normalizeHexPubkey(candidate); + if (normalized) { + normalizedAuthors.add(normalized); + } + } + + if (!normalizedAuthors.size) { + return this.getFilteredActiveVideos(options); + } + + const index = this.ensureVideosByAuthorIndex(); + const collected = []; + const seen = new Set(); + + for (const author of normalizedAuthors) { + const entries = index.get(author); + if (!Array.isArray(entries)) { + continue; + } + for (const video of entries) { + if (!video || typeof video !== "object") { + continue; + } + const id = typeof video.id === "string" ? video.id : ""; + if (!id || seen.has(id)) { + continue; + } + seen.add(id); + collected.push(video); + } + } + + const filtered = this.filterVideos(collected, options); + + return filtered.sort((a, b) => { + const aCreatedCandidate = Number(a?.created_at); + const bCreatedCandidate = Number(b?.created_at); + const aCreated = Number.isFinite(aCreatedCandidate) ? aCreatedCandidate : 0; + const bCreated = Number.isFinite(bCreatedCandidate) ? bCreatedCandidate : 0; + return bCreated - aCreated; + }); + } + async loadVideos({ forceFetch = false, blacklistedEventIds, diff --git a/tests/subscriptions-feed.test.mjs b/tests/subscriptions-feed.test.mjs index 71cc90f4..a4ace784 100644 --- a/tests/subscriptions-feed.test.mjs +++ b/tests/subscriptions-feed.test.mjs @@ -45,10 +45,15 @@ async function testBlockedCreatorsFiltered() { { id: "ignored-author", pubkey: "other", created_at: 250 }, ]; + const calls = []; const service = { - async getFilteredActiveVideos() { + async getActiveVideosByAuthors(authors, options) { + calls.push({ authors, options }); return videos; }, + async getFilteredActiveVideos() { + throw new Error("fallback lookup should not run when authors exist"); + }, }; engine.registerFeed(feedName, { @@ -79,6 +84,17 @@ async function testBlockedCreatorsFiltered() { }, }); + assert.equal( + calls.length, + 1, + "subscription source should request a targeted author lookup" + ); + assert.deepEqual( + new Set(calls[0].authors), + new Set(["pub1", "blocked"]), + "targeted lookup should receive the subscribed authors" + ); + assert.deepEqual( result.videos.map((video) => video.id), ["keep"], @@ -104,10 +120,15 @@ async function testDuplicateRootsFiltered() { { id: "other", pubkey: "pub1", created_at: 150, videoRootId: "root-b" }, ]; + const calls = []; const service = { - async getFilteredActiveVideos() { + async getActiveVideosByAuthors(authors) { + calls.push({ authors }); return videos; }, + async getFilteredActiveVideos() { + throw new Error("should not reach filtered active fallback"); + }, }; engine.registerFeed(feedName, { @@ -128,6 +149,12 @@ async function testDuplicateRootsFiltered() { }, }); + assert.equal( + calls.length, + 1, + "dedupe scenario should use the targeted author lookup" + ); + assert.deepEqual( result.videos.map((video) => video.id), ["newer", "other"], @@ -141,7 +168,39 @@ async function testDuplicateRootsFiltered() { assert.equal(dedupeReasons[0].videoId, "older"); } +async function testNoAuthorsSkipsLookup() { + const engine = createFeedEngine(); + const feedName = "subscriptions-empty"; + + const service = { + async getActiveVideosByAuthors() { + throw new Error("should not request targeted lookup when authors missing"); + }, + async getFilteredActiveVideos() { + throw new Error("should not fetch active videos when authors missing"); + }, + }; + + engine.registerFeed(feedName, { + source: createSubscriptionAuthorsSource({ service }), + stages: [createBlacklistFilterStage({ shouldIncludeVideo: () => true })], + sorter: createChronologicalSorter(), + }); + + const result = await engine.runFeed(feedName, { + runtime: { + subscriptionAuthors: [], + authors: [], + blacklistedEventIds: new Set(), + isAuthorBlocked: () => false, + }, + }); + + assert.deepEqual(result.videos, [], "no authors should yield an empty feed"); +} + await testBlockedCreatorsFiltered(); await testDuplicateRootsFiltered(); +await testNoAuthorsSkipsLookup(); console.log("subscriptions-feed tests passed");