diff --git a/js/nostr/watchHistory.js b/js/nostr/watchHistory.js index 04e9ecfa..4bdf89bc 100644 --- a/js/nostr/watchHistory.js +++ b/js/nostr/watchHistory.js @@ -24,6 +24,15 @@ import { DEFAULT_NIP07_PERMISSION_METHODS } from "./nip07Permissions.js"; import { devLogger, userLogger } from "../utils/logger.js"; import { profileCache } from "../state/profileCache.js"; import { queueSignEvent } from "./signRequestQueue.js"; +import { + normalizePointerInput, + normalizePointerTag, + pointerKey, + clonePointerItem, + mergePointerDetails, +} from "../utils/pointerNormalization.js"; + +export { normalizePointerInput, pointerKey }; /** * Domain utilities for watch-history interactions. This module owns pointer @@ -41,166 +50,6 @@ const WATCH_HISTORY_ENCRYPTION_FALLBACK_ORDER = Object.freeze([ "nip04", ]); -function mergePointerDetails(target, source) { - if (!target || typeof target !== "object" || !source || typeof source !== "object") { - return target; - } - if (source.session === true) { - target.session = true; - } - if (Number.isFinite(source.resumeAt)) { - target.resumeAt = Math.max(0, Math.floor(source.resumeAt)); - } - if (source.completed === true) { - target.completed = true; - } - if (Number.isFinite(source.watchedAt)) { - target.watchedAt = Math.max(0, Math.floor(source.watchedAt)); - } - if (typeof source.relay === "string" && source.relay.trim()) { - if (!target.relay || !target.relay.trim()) { - target.relay = source.relay.trim(); - } - } - return target; -} - -function clonePointerItem(pointer) { - if (!pointer || typeof pointer !== "object") { - return null; - } - - const cloned = { - type: pointer.type === "a" ? "a" : "e", - value: typeof pointer.value === "string" ? pointer.value.trim() : "", - }; - - if (!cloned.value) { - return null; - } - - if (typeof pointer.relay === "string" && pointer.relay.trim()) { - cloned.relay = pointer.relay.trim(); - } - - if (Number.isFinite(pointer.watchedAt)) { - cloned.watchedAt = Math.max(0, Math.floor(pointer.watchedAt)); - } - - if (Number.isFinite(pointer.resumeAt)) { - cloned.resumeAt = Math.max(0, Math.floor(pointer.resumeAt)); - } - - if (pointer.completed === true) { - cloned.completed = true; - } - - if (pointer.session === true) { - cloned.session = true; - } - - return cloned; -} - -export function pointerKey(pointer) { - if (!pointer) { - return ""; - } - const type = pointer.type === "a" ? "a" : "e"; - const value = typeof pointer.value === "string" ? pointer.value.trim().toLowerCase() : ""; - if (!type || !value) { - return ""; - } - return `${type}:${value}`; -} - -function normalizePointerTag(tag) { - if (!Array.isArray(tag) || tag.length < 2) { - return null; - } - const type = tag[0] === "a" ? "a" : tag[0] === "e" ? "e" : ""; - if (!type) { - return null; - } - const value = typeof tag[1] === "string" ? tag[1].trim() : ""; - if (!value) { - return null; - } - const relay = - tag.length > 2 && typeof tag[2] === "string" && tag[2].trim() - ? tag[2].trim() - : null; - return { type, value, relay }; -} - -export function normalizePointerInput(pointer) { - if (!pointer) { - return null; - } - if (Array.isArray(pointer)) { - return normalizePointerTag(pointer); - } - if (typeof pointer === "object") { - if (typeof pointer.type === "string" && typeof pointer.value === "string") { - return clonePointerItem(pointer); - } - if (Array.isArray(pointer.tag)) { - return normalizePointerTag(pointer.tag); - } - } - if (typeof pointer !== "string") { - return null; - } - const trimmed = pointer.trim(); - if (!trimmed) { - return null; - } - if (trimmed.startsWith("naddr") || trimmed.startsWith("nevent")) { - try { - const decoder = getCachedNostrTools()?.nip19?.decode; - if (typeof decoder === "function") { - const decoded = decoder(trimmed); - if (decoded?.type === "naddr" && decoded.data) { - const { kind, pubkey, identifier, relays } = decoded.data; - if ( - typeof kind === "number" && - typeof pubkey === "string" && - typeof identifier === "string" - ) { - const relay = - Array.isArray(relays) && relays.length && typeof relays[0] === "string" - ? relays[0] - : null; - return { - type: "a", - value: `${kind}:${pubkey}:${identifier}`, - relay, - }; - } - } - if (decoded?.type === "nevent" && decoded.data) { - const { id, relays } = decoded.data; - if (typeof id === "string" && id.trim()) { - const relay = - Array.isArray(relays) && relays.length && typeof relays[0] === "string" - ? relays[0] - : null; - return { - type: "e", - value: id.trim(), - relay, - }; - } - } - } - } catch (error) { - devLogger.warn(`[nostr] Failed to decode pointer ${trimmed}:`, error); - } - } - const type = trimmed.includes(":") ? "a" : "e"; - return { type, value: trimmed, relay: null }; -} - function extractPointerItemsFromEvent(event) { const items = []; const seen = new Set(); diff --git a/js/services/exploreDataService.js b/js/services/exploreDataService.js index da35d17c..c8cb38b7 100644 --- a/js/services/exploreDataService.js +++ b/js/services/exploreDataService.js @@ -1,64 +1,34 @@ import { devLogger } from "../utils/logger.js"; -import { normalizeHashtag } from "../utils/hashtagNormalization.js"; -import { normalizePointerInput } from "../nostr/watchHistory.js"; -import { buildVideoAddressPointer } from "../utils/videoPointer.js"; const DEFAULT_IDF_REFRESH_INTERVAL_MS = 5 * 60 * 1000; const DEFAULT_HISTORY_REFRESH_INTERVAL_MS = 60 * 1000; const DEFAULT_REFRESH_DEBOUNCE_MS = 200; -const YIELD_CHUNK_SIZE = 500; -const yieldToMain = () => new Promise((resolve) => setTimeout(resolve, 0)); - -function normalizeAddressKey(value) { - if (typeof value !== "string") { - return ""; +let workerInstance = null; +function getWorker() { + if (!workerInstance) { + workerInstance = new Worker(new URL("../workers/exploreData.worker.js", import.meta.url), { type: "module" }); } - const trimmed = value.trim(); - if (!trimmed) { - return ""; - } - return trimmed.toLowerCase(); + return workerInstance; } -const collectVideoTagsCache = new WeakMap(); - -function collectVideoTags(video) { - if (!video || typeof video !== "object") { - return new Set(); - } - - if (collectVideoTagsCache.has(video)) { - return collectVideoTagsCache.get(video); - } - - const tags = new Set(); - - if (Array.isArray(video.tags)) { - for (const tag of video.tags) { - if (Array.isArray(tag) && tag[0] === "t" && typeof tag[1] === "string") { - const normalized = normalizeHashtag(tag[1]); - if (normalized) { - tags.add(normalized); +function runWorkerTask(type, payload) { + return new Promise((resolve, reject) => { + const worker = getWorker(); + const id = crypto.randomUUID(); + const handler = (e) => { + if (e.data.id === id) { + worker.removeEventListener("message", handler); + if (e.data.error) { + reject(new Error(e.data.error)); + } else { + resolve(e.data.result); } } - } - } - - if (Array.isArray(video.nip71?.hashtags)) { - for (const tag of video.nip71.hashtags) { - if (typeof tag !== "string") { - continue; - } - const normalized = normalizeHashtag(tag); - if (normalized) { - tags.add(normalized); - } - } - } - - collectVideoTagsCache.set(video, tags); - return tags; + }; + worker.addEventListener("message", handler); + worker.postMessage({ type, id, payload }); + }); } export async function buildWatchHistoryTagCounts({ @@ -85,102 +55,28 @@ export async function buildWatchHistoryTagCounts({ ? nostrService.getVideosMap() : new Map(); - const needsAddressIndex = Array.isArray(items) - ? items.some((item) => { - const pointer = normalizePointerInput(item?.pointer || item); - return pointer?.type === "a"; - }) - : false; - - let addressIndex = null; - if (needsAddressIndex) { - addressIndex = new Map(); - for (const video of videosMap.values()) { - const address = buildVideoAddressPointer(video); - const key = normalizeAddressKey(address); - if (key) { - addressIndex.set(key, video); - } - } + try { + const counts = await runWorkerTask('CALC_HISTORY_COUNTS', { items, videosMap }); + return counts instanceof Map ? counts : new Map(); + } catch (error) { + devLogger.warn("[exploreData] Worker failed to calculate history counts:", error); + return new Map(); } - - const counts = new Map(); - if (!Array.isArray(items)) { - return counts; - } - - for (let i = 0; i < items.length; i++) { - const entry = items[i]; - const pointer = normalizePointerInput(entry?.pointer || entry); - let video = entry?.video || entry?.metadata?.video || null; - - if (pointer?.type === "e" && pointer.value) { - const eventId = typeof pointer.value === "string" ? pointer.value.trim() : ""; - if (eventId) { - video = videosMap.get(eventId) || videosMap.get(eventId.toLowerCase()) || video; - } - } else if (pointer?.type === "a" && pointer.value && addressIndex) { - const key = normalizeAddressKey(pointer.value); - if (key && addressIndex.has(key)) { - video = addressIndex.get(key); - } - } - - if (!video) { - continue; - } - - const tags = collectVideoTags(video); - for (const tag of tags) { - counts.set(tag, (counts.get(tag) || 0) + 1); - } - - if (i > 0 && i % YIELD_CHUNK_SIZE === 0) { - await yieldToMain(); - } - } - - return counts; } export async function buildTagIdf({ videos } = {}) { - const idf = new Map(); const list = Array.isArray(videos) ? videos : []; if (!list.length) { - return idf; + return new Map(); } - const docFrequency = new Map(); - const totalDocs = list.length; - - for (let i = 0; i < list.length; i++) { - const video = list[i]; - const tags = collectVideoTags(video); - if (tags.size) { - for (const tag of tags) { - docFrequency.set(tag, (docFrequency.get(tag) || 0) + 1); - } - } - - if (i > 0 && i % YIELD_CHUNK_SIZE === 0) { - await yieldToMain(); - } + try { + const idf = await runWorkerTask('CALC_IDF', { videos: list }); + return idf instanceof Map ? idf : new Map(); + } catch (error) { + devLogger.warn("[exploreData] Worker failed to calculate IDF:", error); + return new Map(); } - - let processedCount = 0; - for (const [tag, df] of docFrequency.entries()) { - const ratio = (totalDocs + 1) / (df + 1); - const value = 1 + Math.log(ratio); - if (Number.isFinite(value) && value > 0) { - idf.set(tag, value); - } - processedCount++; - if (processedCount % YIELD_CHUNK_SIZE === 0) { - await yieldToMain(); - } - } - - return idf; } export default class ExploreDataService { diff --git a/js/utils/pointerNormalization.js b/js/utils/pointerNormalization.js new file mode 100644 index 00000000..4804c5d4 --- /dev/null +++ b/js/utils/pointerNormalization.js @@ -0,0 +1,164 @@ +import { devLogger } from "./logger.js"; +import { getCachedNostrTools } from "../nostr/toolkit.js"; + +export function pointerKey(pointer) { + if (!pointer) { + return ""; + } + const type = pointer.type === "a" ? "a" : "e"; + const value = typeof pointer.value === "string" ? pointer.value.trim().toLowerCase() : ""; + if (!type || !value) { + return ""; + } + return `${type}:${value}`; +} + +export function normalizePointerTag(tag) { + if (!Array.isArray(tag) || tag.length < 2) { + return null; + } + const type = tag[0] === "a" ? "a" : tag[0] === "e" ? "e" : ""; + if (!type) { + return null; + } + const value = typeof tag[1] === "string" ? tag[1].trim() : ""; + if (!value) { + return null; + } + const relay = + tag.length > 2 && typeof tag[2] === "string" && tag[2].trim() + ? tag[2].trim() + : null; + return { type, value, relay }; +} + +function clonePointerItem(pointer) { + if (!pointer || typeof pointer !== "object") { + return null; + } + + const cloned = { + type: pointer.type === "a" ? "a" : "e", + value: typeof pointer.value === "string" ? pointer.value.trim() : "", + }; + + if (!cloned.value) { + return null; + } + + if (typeof pointer.relay === "string" && pointer.relay.trim()) { + cloned.relay = pointer.relay.trim(); + } + + if (Number.isFinite(pointer.watchedAt)) { + cloned.watchedAt = Math.max(0, Math.floor(pointer.watchedAt)); + } + + if (Number.isFinite(pointer.resumeAt)) { + cloned.resumeAt = Math.max(0, Math.floor(pointer.resumeAt)); + } + + if (pointer.completed === true) { + cloned.completed = true; + } + + if (pointer.session === true) { + cloned.session = true; + } + + return cloned; +} + +export { clonePointerItem }; + +export function mergePointerDetails(target, source) { + if (!target || typeof target !== "object" || !source || typeof source !== "object") { + return target; + } + if (source.session === true) { + target.session = true; + } + if (Number.isFinite(source.resumeAt)) { + target.resumeAt = Math.max(0, Math.floor(source.resumeAt)); + } + if (source.completed === true) { + target.completed = true; + } + if (Number.isFinite(source.watchedAt)) { + target.watchedAt = Math.max(0, Math.floor(source.watchedAt)); + } + if (typeof source.relay === "string" && source.relay.trim()) { + if (!target.relay || !target.relay.trim()) { + target.relay = source.relay.trim(); + } + } + return target; +} + +export function normalizePointerInput(pointer) { + if (!pointer) { + return null; + } + if (Array.isArray(pointer)) { + return normalizePointerTag(pointer); + } + if (typeof pointer === "object") { + if (typeof pointer.type === "string" && typeof pointer.value === "string") { + return clonePointerItem(pointer); + } + if (Array.isArray(pointer.tag)) { + return normalizePointerTag(pointer.tag); + } + } + if (typeof pointer !== "string") { + return null; + } + const trimmed = pointer.trim(); + if (!trimmed) { + return null; + } + if (trimmed.startsWith("naddr") || trimmed.startsWith("nevent")) { + try { + const decoder = getCachedNostrTools()?.nip19?.decode; + if (typeof decoder === "function") { + const decoded = decoder(trimmed); + if (decoded?.type === "naddr" && decoded.data) { + const { kind, pubkey, identifier, relays } = decoded.data; + if ( + typeof kind === "number" && + typeof pubkey === "string" && + typeof identifier === "string" + ) { + const relay = + Array.isArray(relays) && relays.length && typeof relays[0] === "string" + ? relays[0] + : null; + return { + type: "a", + value: `${kind}:${pubkey}:${identifier}`, + relay, + }; + } + } + if (decoded?.type === "nevent" && decoded.data) { + const { id, relays } = decoded.data; + if (typeof id === "string" && id.trim()) { + const relay = + Array.isArray(relays) && relays.length && typeof relays[0] === "string" + ? relays[0] + : null; + return { + type: "e", + value: id.trim(), + relay, + }; + } + } + } + } catch (error) { + devLogger.warn(`[nostr] Failed to decode pointer ${trimmed}:`, error); + } + } + const type = trimmed.includes(":") ? "a" : "e"; + return { type, value: trimmed, relay: null }; +} diff --git a/js/workers/exploreData.worker.js b/js/workers/exploreData.worker.js new file mode 100644 index 00000000..f414cdca --- /dev/null +++ b/js/workers/exploreData.worker.js @@ -0,0 +1,145 @@ +// js/workers/exploreData.worker.js + +// Configure local NostrTools entrypoint for the bootstrap logic +self.__BITVID_LOCAL_NOSTR_TOOLS_ENTRY__ = "../../vendor/nostr-tools.bundle.min.js"; + +import { collectVideoTags } from "../utils/videoTags.js"; +import { normalizePointerInput } from "../utils/pointerNormalization.js"; +import { buildVideoAddressPointer } from "../utils/videoPointer.js"; +import { normalizeHashtag } from "../utils/hashtagNormalization.js"; + +function normalizeAddressKey(value) { + if (typeof value !== "string") { + return ""; + } + const trimmed = value.trim(); + if (!trimmed) { + return ""; + } + return trimmed.toLowerCase(); +} + +async function buildWatchHistoryTagCounts({ items, videosMap, videosArray }) { + // videosMap is passed as an object or Map. Serialization converts Map to object? + // Structured Clone Algorithm supports Map since recently. + // Assuming browser supports Map transfer/clone. + // If items are raw objects, we process them. + + const counts = new Map(); + if (!Array.isArray(items)) { + return counts; + } + + // Reconstruct Map if needed, but if we pass Map, it works. + // However, videosMap might be huge. + // If we pass array of videos, we can build a temporary map. + + let lookupMap = videosMap; + if (Array.isArray(videosMap)) { + lookupMap = new Map(videosMap.map(v => [v.id, v])); + } else if (!(videosMap instanceof Map) && videosArray) { + lookupMap = new Map(videosArray.map(v => [v.id, v])); + } + + const needsAddressIndex = items.some((item) => { + const pointer = normalizePointerInput(item?.pointer || item); + return pointer?.type === "a"; + }); + + let addressIndex = null; + if (needsAddressIndex && lookupMap) { + addressIndex = new Map(); + for (const video of lookupMap.values()) { + const address = buildVideoAddressPointer(video); + const key = normalizeAddressKey(address); + if (key) { + addressIndex.set(key, video); + } + } + } + + for (let i = 0; i < items.length; i++) { + const entry = items[i]; + const pointer = normalizePointerInput(entry?.pointer || entry); + let video = entry?.video || entry?.metadata?.video || null; + + if (pointer?.type === "e" && pointer.value) { + const eventId = typeof pointer.value === "string" ? pointer.value.trim() : ""; + if (eventId && lookupMap) { + video = lookupMap.get(eventId) || lookupMap.get(eventId.toLowerCase()) || video; + } + } else if (pointer?.type === "a" && pointer.value && addressIndex) { + const key = normalizeAddressKey(pointer.value); + if (key && addressIndex.has(key)) { + video = addressIndex.get(key); + } + } + + if (!video) { + continue; + } + + const tags = collectVideoTags(video); + for (const tag of tags) { + const normalized = normalizeHashtag(tag); + if (normalized) { + counts.set(normalized, (counts.get(normalized) || 0) + 1); + } + } + } + + return counts; +} + +async function buildTagIdf({ videos }) { + const idf = new Map(); + const list = Array.isArray(videos) ? videos : []; + if (!list.length) { + return idf; + } + + const docFrequency = new Map(); + const totalDocs = list.length; + + for (let i = 0; i < list.length; i++) { + const video = list[i]; + const tags = collectVideoTags(video); + if (tags && (tags.length || tags.size)) { + for (const tag of tags) { + const normalized = normalizeHashtag(tag); + if (normalized) { + docFrequency.set(normalized, (docFrequency.get(normalized) || 0) + 1); + } + } + } + } + + for (const [tag, df] of docFrequency.entries()) { + const ratio = (totalDocs + 1) / (df + 1); + const value = 1 + Math.log(ratio); + if (Number.isFinite(value) && value > 0) { + idf.set(tag, value); + } + } + + return idf; +} + +self.onmessage = async (e) => { + const { type, id, payload } = e.data; + + try { + let result; + if (type === 'CALC_HISTORY_COUNTS') { + result = await buildWatchHistoryTagCounts(payload); + } else if (type === 'CALC_IDF') { + result = await buildTagIdf(payload); + } else { + throw new Error(`Unknown message type: ${type}`); + } + + self.postMessage({ id, result }); + } catch (error) { + self.postMessage({ id, error: error.message || String(error) }); + } +}; diff --git a/playwright.bench.config.ts b/playwright.bench.config.ts new file mode 100644 index 00000000..8119bfaf --- /dev/null +++ b/playwright.bench.config.ts @@ -0,0 +1,29 @@ +import { defineConfig } from "@playwright/test"; + +const HOST = "127.0.0.1"; +const PORT = 4173; +const BASE_URL = `http://${HOST}:${PORT}`; + +export default defineConfig({ + testDir: "./tests", + outputDir: "artifacts/test-results-bench", + timeout: 60_000, + reporter: "list", + use: { + baseURL: BASE_URL, + viewport: { width: 1280, height: 720 }, + ignoreHTTPSErrors: true, + }, + projects: [ + { + name: "benchmark", + testMatch: "**/*.spec.mjs", + }, + ], + webServer: { + command: `npx http-server . -p ${PORT} -a ${HOST} -c-1 --silent`, + url: `${BASE_URL}/docs/kitchen-sink.html`, + reuseExistingServer: true, + timeout: 120_000, + }, +}); diff --git a/tests/benchmark-explore-data.spec.mjs b/tests/benchmark-explore-data.spec.mjs new file mode 100644 index 00000000..eee0fa4f --- /dev/null +++ b/tests/benchmark-explore-data.spec.mjs @@ -0,0 +1,92 @@ +import { test, expect } from '@playwright/test'; + +test.describe('Explore Data Performance Benchmark', () => { + test('benchmark buildWatchHistoryTagCounts and buildTagIdf', async ({ page }) => { + // Go to the app to ensure environment is loaded + await page.goto('/'); + + // Wait for modules to load (optional, but good practice) + await page.waitForLoadState('networkidle'); + + const result = await page.evaluate(async () => { + // Dynamic import of the service + const { buildWatchHistoryTagCounts, buildTagIdf } = await import('./js/services/exploreDataService.js'); + + // --- Mock Data Generation --- + const VIDEO_COUNT = 5000; + const HISTORY_COUNT = 1000; + + const videosMap = new Map(); + const videosArray = []; + const tagsPool = ['bitcoin', 'nostr', 'art', 'music', 'coding', 'performance', 'web', 'worker', 'nature', 'space']; + + for (let i = 0; i < VIDEO_COUNT; i++) { + const id = `video-${i}`; + const tags = []; + const numTags = Math.floor(Math.random() * 5) + 1; + for (let t = 0; t < numTags; t++) { + tags.push(['t', tagsPool[Math.floor(Math.random() * tagsPool.length)]]); + } + + const video = { + id, + kind: 30078, + pubkey: '0000000000000000000000000000000000000000000000000000000000000000', + tags: [['d', id], ...tags], + }; + videosMap.set(id, video); + videosArray.push(video); + } + + // Mock NostrService + const mockNostrService = { + getVideosMap: () => videosMap, + getFilteredActiveVideos: () => videosArray, + }; + + // Mock WatchHistoryService + const historyItems = []; + for (let i = 0; i < HISTORY_COUNT; i++) { + const videoId = `video-${Math.floor(Math.random() * VIDEO_COUNT)}`; + historyItems.push({ + pointer: { type: 'a', value: `30078:0000000000000000000000000000000000000000000000000000000000000000:${videoId}` }, + video: videosMap.get(videoId), + }); + } + + const mockWatchHistoryService = { + loadLatest: () => Promise.resolve(historyItems), + }; + + // --- Benchmark Counts --- + const startCounts = performance.now(); + const counts = await buildWatchHistoryTagCounts({ + watchHistoryService: mockWatchHistoryService, + nostrService: mockNostrService, + actor: 'test-actor', + }); + const endCounts = performance.now(); + + // --- Benchmark IDF --- + const startIdf = performance.now(); + const idf = await buildTagIdf({ videos: videosArray }); + const endIdf = performance.now(); + + return { + countsTime: endCounts - startCounts, + idfTime: endIdf - startIdf, + countsSize: counts.size, + idfSize: idf.size + }; + }); + + console.log(`[Benchmark] buildWatchHistoryTagCounts: ${result.countsTime.toFixed(2)}ms`); + console.log(`[Benchmark] buildTagIdf: ${result.idfTime.toFixed(2)}ms`); + console.log(`[Benchmark] Counts Size: ${result.countsSize}`); + console.log(`[Benchmark] IDF Size: ${result.idfSize}`); + + // Assertions to ensure it actually ran and produced results + expect(result.countsSize).toBeGreaterThan(0); + expect(result.idfSize).toBeGreaterThan(0); + }); +});