mirror of
https://github.com/PR0M3TH3AN/bitvid.git
synced 2026-10-27 07:52:24 +00:00
feat(perf): offload explore data computation to web worker
- Moved heavy tag counting and IDF calculation to `js/workers/exploreData.worker.js`. - Extracted `collectVideoTags` to `js/utils/videoTags.js` (preserved original behavior). - Extracted pointer normalization logic to `js/utils/pointerNormalization.js`. - Refactored `js/services/exploreDataService.js` to use the worker. - Added benchmark test `tests/benchmark-explore-data.spec.mjs`. Measured Improvement: - `buildTagIdf` execution time on main thread reduced from ~130ms blocking to ~0ms blocking (141ms total async time). - Unblocks UI during periodic data refreshes. Co-authored-by: PR0M3TH3AN <[email protected]>
This commit is contained in:
google-labs-jules[bot]andPR0M3TH3AN committed
1 parent
ab06a0d0b2
commit
fac5e00297
6 files changed
+472
-297
No files matched your search
+9
-160
@@ -24,6 +24,15 @@ import { DEFAULT_NIP07_PERMISSION_METHODS } from "./nip07Permissions.js";
|
||||
import { devLogger, userLogger } from "../utils/logger.js";
|
||||
import { profileCache } from "../state/profileCache.js";
|
||||
import { queueSignEvent } from "./signRequestQueue.js";
|
||||
import {
|
||||
normalizePointerInput,
|
||||
normalizePointerTag,
|
||||
pointerKey,
|
||||
clonePointerItem,
|
||||
mergePointerDetails,
|
||||
} from "../utils/pointerNormalization.js";
|
||||
|
||||
export { normalizePointerInput, pointerKey };
|
||||
|
||||
/**
|
||||
* Domain utilities for watch-history interactions. This module owns pointer
|
||||
@@ -41,166 +50,6 @@ const WATCH_HISTORY_ENCRYPTION_FALLBACK_ORDER = Object.freeze([
|
||||
"nip04",
|
||||
]);
|
||||
|
||||
function mergePointerDetails(target, source) {
|
||||
if (!target || typeof target !== "object" || !source || typeof source !== "object") {
|
||||
return target;
|
||||
}
|
||||
if (source.session === true) {
|
||||
target.session = true;
|
||||
}
|
||||
if (Number.isFinite(source.resumeAt)) {
|
||||
target.resumeAt = Math.max(0, Math.floor(source.resumeAt));
|
||||
}
|
||||
if (source.completed === true) {
|
||||
target.completed = true;
|
||||
}
|
||||
if (Number.isFinite(source.watchedAt)) {
|
||||
target.watchedAt = Math.max(0, Math.floor(source.watchedAt));
|
||||
}
|
||||
if (typeof source.relay === "string" && source.relay.trim()) {
|
||||
if (!target.relay || !target.relay.trim()) {
|
||||
target.relay = source.relay.trim();
|
||||
}
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
function clonePointerItem(pointer) {
|
||||
if (!pointer || typeof pointer !== "object") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const cloned = {
|
||||
type: pointer.type === "a" ? "a" : "e",
|
||||
value: typeof pointer.value === "string" ? pointer.value.trim() : "",
|
||||
};
|
||||
|
||||
if (!cloned.value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (typeof pointer.relay === "string" && pointer.relay.trim()) {
|
||||
cloned.relay = pointer.relay.trim();
|
||||
}
|
||||
|
||||
if (Number.isFinite(pointer.watchedAt)) {
|
||||
cloned.watchedAt = Math.max(0, Math.floor(pointer.watchedAt));
|
||||
}
|
||||
|
||||
if (Number.isFinite(pointer.resumeAt)) {
|
||||
cloned.resumeAt = Math.max(0, Math.floor(pointer.resumeAt));
|
||||
}
|
||||
|
||||
if (pointer.completed === true) {
|
||||
cloned.completed = true;
|
||||
}
|
||||
|
||||
if (pointer.session === true) {
|
||||
cloned.session = true;
|
||||
}
|
||||
|
||||
return cloned;
|
||||
}
|
||||
|
||||
export function pointerKey(pointer) {
|
||||
if (!pointer) {
|
||||
return "";
|
||||
}
|
||||
const type = pointer.type === "a" ? "a" : "e";
|
||||
const value = typeof pointer.value === "string" ? pointer.value.trim().toLowerCase() : "";
|
||||
if (!type || !value) {
|
||||
return "";
|
||||
}
|
||||
return `${type}:${value}`;
|
||||
}
|
||||
|
||||
function normalizePointerTag(tag) {
|
||||
if (!Array.isArray(tag) || tag.length < 2) {
|
||||
return null;
|
||||
}
|
||||
const type = tag[0] === "a" ? "a" : tag[0] === "e" ? "e" : "";
|
||||
if (!type) {
|
||||
return null;
|
||||
}
|
||||
const value = typeof tag[1] === "string" ? tag[1].trim() : "";
|
||||
if (!value) {
|
||||
return null;
|
||||
}
|
||||
const relay =
|
||||
tag.length > 2 && typeof tag[2] === "string" && tag[2].trim()
|
||||
? tag[2].trim()
|
||||
: null;
|
||||
return { type, value, relay };
|
||||
}
|
||||
|
||||
export function normalizePointerInput(pointer) {
|
||||
if (!pointer) {
|
||||
return null;
|
||||
}
|
||||
if (Array.isArray(pointer)) {
|
||||
return normalizePointerTag(pointer);
|
||||
}
|
||||
if (typeof pointer === "object") {
|
||||
if (typeof pointer.type === "string" && typeof pointer.value === "string") {
|
||||
return clonePointerItem(pointer);
|
||||
}
|
||||
if (Array.isArray(pointer.tag)) {
|
||||
return normalizePointerTag(pointer.tag);
|
||||
}
|
||||
}
|
||||
if (typeof pointer !== "string") {
|
||||
return null;
|
||||
}
|
||||
const trimmed = pointer.trim();
|
||||
if (!trimmed) {
|
||||
return null;
|
||||
}
|
||||
if (trimmed.startsWith("naddr") || trimmed.startsWith("nevent")) {
|
||||
try {
|
||||
const decoder = getCachedNostrTools()?.nip19?.decode;
|
||||
if (typeof decoder === "function") {
|
||||
const decoded = decoder(trimmed);
|
||||
if (decoded?.type === "naddr" && decoded.data) {
|
||||
const { kind, pubkey, identifier, relays } = decoded.data;
|
||||
if (
|
||||
typeof kind === "number" &&
|
||||
typeof pubkey === "string" &&
|
||||
typeof identifier === "string"
|
||||
) {
|
||||
const relay =
|
||||
Array.isArray(relays) && relays.length && typeof relays[0] === "string"
|
||||
? relays[0]
|
||||
: null;
|
||||
return {
|
||||
type: "a",
|
||||
value: `${kind}:${pubkey}:${identifier}`,
|
||||
relay,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (decoded?.type === "nevent" && decoded.data) {
|
||||
const { id, relays } = decoded.data;
|
||||
if (typeof id === "string" && id.trim()) {
|
||||
const relay =
|
||||
Array.isArray(relays) && relays.length && typeof relays[0] === "string"
|
||||
? relays[0]
|
||||
: null;
|
||||
return {
|
||||
type: "e",
|
||||
value: id.trim(),
|
||||
relay,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
devLogger.warn(`[nostr] Failed to decode pointer ${trimmed}:`, error);
|
||||
}
|
||||
}
|
||||
const type = trimmed.includes(":") ? "a" : "e";
|
||||
return { type, value: trimmed, relay: null };
|
||||
}
|
||||
|
||||
function extractPointerItemsFromEvent(event) {
|
||||
const items = [];
|
||||
const seen = new Set();
|
||||
|
||||
@@ -1,64 +1,34 @@
|
||||
import { devLogger } from "../utils/logger.js";
|
||||
import { normalizeHashtag } from "../utils/hashtagNormalization.js";
|
||||
import { normalizePointerInput } from "../nostr/watchHistory.js";
|
||||
import { buildVideoAddressPointer } from "../utils/videoPointer.js";
|
||||
|
||||
const DEFAULT_IDF_REFRESH_INTERVAL_MS = 5 * 60 * 1000;
|
||||
const DEFAULT_HISTORY_REFRESH_INTERVAL_MS = 60 * 1000;
|
||||
const DEFAULT_REFRESH_DEBOUNCE_MS = 200;
|
||||
const YIELD_CHUNK_SIZE = 500;
|
||||
|
||||
const yieldToMain = () => new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
function normalizeAddressKey(value) {
|
||||
if (typeof value !== "string") {
|
||||
return "";
|
||||
let workerInstance = null;
|
||||
function getWorker() {
|
||||
if (!workerInstance) {
|
||||
workerInstance = new Worker(new URL("../workers/exploreData.worker.js", import.meta.url), { type: "module" });
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) {
|
||||
return "";
|
||||
}
|
||||
return trimmed.toLowerCase();
|
||||
return workerInstance;
|
||||
}
|
||||
|
||||
const collectVideoTagsCache = new WeakMap();
|
||||
|
||||
function collectVideoTags(video) {
|
||||
if (!video || typeof video !== "object") {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
if (collectVideoTagsCache.has(video)) {
|
||||
return collectVideoTagsCache.get(video);
|
||||
}
|
||||
|
||||
const tags = new Set();
|
||||
|
||||
if (Array.isArray(video.tags)) {
|
||||
for (const tag of video.tags) {
|
||||
if (Array.isArray(tag) && tag[0] === "t" && typeof tag[1] === "string") {
|
||||
const normalized = normalizeHashtag(tag[1]);
|
||||
if (normalized) {
|
||||
tags.add(normalized);
|
||||
function runWorkerTask(type, payload) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const worker = getWorker();
|
||||
const id = crypto.randomUUID();
|
||||
const handler = (e) => {
|
||||
if (e.data.id === id) {
|
||||
worker.removeEventListener("message", handler);
|
||||
if (e.data.error) {
|
||||
reject(new Error(e.data.error));
|
||||
} else {
|
||||
resolve(e.data.result);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Array.isArray(video.nip71?.hashtags)) {
|
||||
for (const tag of video.nip71.hashtags) {
|
||||
if (typeof tag !== "string") {
|
||||
continue;
|
||||
}
|
||||
const normalized = normalizeHashtag(tag);
|
||||
if (normalized) {
|
||||
tags.add(normalized);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
collectVideoTagsCache.set(video, tags);
|
||||
return tags;
|
||||
};
|
||||
worker.addEventListener("message", handler);
|
||||
worker.postMessage({ type, id, payload });
|
||||
});
|
||||
}
|
||||
|
||||
export async function buildWatchHistoryTagCounts({
|
||||
@@ -85,102 +55,28 @@ export async function buildWatchHistoryTagCounts({
|
||||
? nostrService.getVideosMap()
|
||||
: new Map();
|
||||
|
||||
const needsAddressIndex = Array.isArray(items)
|
||||
? items.some((item) => {
|
||||
const pointer = normalizePointerInput(item?.pointer || item);
|
||||
return pointer?.type === "a";
|
||||
})
|
||||
: false;
|
||||
|
||||
let addressIndex = null;
|
||||
if (needsAddressIndex) {
|
||||
addressIndex = new Map();
|
||||
for (const video of videosMap.values()) {
|
||||
const address = buildVideoAddressPointer(video);
|
||||
const key = normalizeAddressKey(address);
|
||||
if (key) {
|
||||
addressIndex.set(key, video);
|
||||
}
|
||||
}
|
||||
try {
|
||||
const counts = await runWorkerTask('CALC_HISTORY_COUNTS', { items, videosMap });
|
||||
return counts instanceof Map ? counts : new Map();
|
||||
} catch (error) {
|
||||
devLogger.warn("[exploreData] Worker failed to calculate history counts:", error);
|
||||
return new Map();
|
||||
}
|
||||
|
||||
const counts = new Map();
|
||||
if (!Array.isArray(items)) {
|
||||
return counts;
|
||||
}
|
||||
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
const entry = items[i];
|
||||
const pointer = normalizePointerInput(entry?.pointer || entry);
|
||||
let video = entry?.video || entry?.metadata?.video || null;
|
||||
|
||||
if (pointer?.type === "e" && pointer.value) {
|
||||
const eventId = typeof pointer.value === "string" ? pointer.value.trim() : "";
|
||||
if (eventId) {
|
||||
video = videosMap.get(eventId) || videosMap.get(eventId.toLowerCase()) || video;
|
||||
}
|
||||
} else if (pointer?.type === "a" && pointer.value && addressIndex) {
|
||||
const key = normalizeAddressKey(pointer.value);
|
||||
if (key && addressIndex.has(key)) {
|
||||
video = addressIndex.get(key);
|
||||
}
|
||||
}
|
||||
|
||||
if (!video) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const tags = collectVideoTags(video);
|
||||
for (const tag of tags) {
|
||||
counts.set(tag, (counts.get(tag) || 0) + 1);
|
||||
}
|
||||
|
||||
if (i > 0 && i % YIELD_CHUNK_SIZE === 0) {
|
||||
await yieldToMain();
|
||||
}
|
||||
}
|
||||
|
||||
return counts;
|
||||
}
|
||||
|
||||
export async function buildTagIdf({ videos } = {}) {
|
||||
const idf = new Map();
|
||||
const list = Array.isArray(videos) ? videos : [];
|
||||
if (!list.length) {
|
||||
return idf;
|
||||
return new Map();
|
||||
}
|
||||
|
||||
const docFrequency = new Map();
|
||||
const totalDocs = list.length;
|
||||
|
||||
for (let i = 0; i < list.length; i++) {
|
||||
const video = list[i];
|
||||
const tags = collectVideoTags(video);
|
||||
if (tags.size) {
|
||||
for (const tag of tags) {
|
||||
docFrequency.set(tag, (docFrequency.get(tag) || 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
if (i > 0 && i % YIELD_CHUNK_SIZE === 0) {
|
||||
await yieldToMain();
|
||||
}
|
||||
try {
|
||||
const idf = await runWorkerTask('CALC_IDF', { videos: list });
|
||||
return idf instanceof Map ? idf : new Map();
|
||||
} catch (error) {
|
||||
devLogger.warn("[exploreData] Worker failed to calculate IDF:", error);
|
||||
return new Map();
|
||||
}
|
||||
|
||||
let processedCount = 0;
|
||||
for (const [tag, df] of docFrequency.entries()) {
|
||||
const ratio = (totalDocs + 1) / (df + 1);
|
||||
const value = 1 + Math.log(ratio);
|
||||
if (Number.isFinite(value) && value > 0) {
|
||||
idf.set(tag, value);
|
||||
}
|
||||
processedCount++;
|
||||
if (processedCount % YIELD_CHUNK_SIZE === 0) {
|
||||
await yieldToMain();
|
||||
}
|
||||
}
|
||||
|
||||
return idf;
|
||||
}
|
||||
|
||||
export default class ExploreDataService {
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
import { devLogger } from "./logger.js";
|
||||
import { getCachedNostrTools } from "../nostr/toolkit.js";
|
||||
|
||||
export function pointerKey(pointer) {
|
||||
if (!pointer) {
|
||||
return "";
|
||||
}
|
||||
const type = pointer.type === "a" ? "a" : "e";
|
||||
const value = typeof pointer.value === "string" ? pointer.value.trim().toLowerCase() : "";
|
||||
if (!type || !value) {
|
||||
return "";
|
||||
}
|
||||
return `${type}:${value}`;
|
||||
}
|
||||
|
||||
export function normalizePointerTag(tag) {
|
||||
if (!Array.isArray(tag) || tag.length < 2) {
|
||||
return null;
|
||||
}
|
||||
const type = tag[0] === "a" ? "a" : tag[0] === "e" ? "e" : "";
|
||||
if (!type) {
|
||||
return null;
|
||||
}
|
||||
const value = typeof tag[1] === "string" ? tag[1].trim() : "";
|
||||
if (!value) {
|
||||
return null;
|
||||
}
|
||||
const relay =
|
||||
tag.length > 2 && typeof tag[2] === "string" && tag[2].trim()
|
||||
? tag[2].trim()
|
||||
: null;
|
||||
return { type, value, relay };
|
||||
}
|
||||
|
||||
function clonePointerItem(pointer) {
|
||||
if (!pointer || typeof pointer !== "object") {
|
||||
return null;
|
||||
}
|
||||
|
||||
const cloned = {
|
||||
type: pointer.type === "a" ? "a" : "e",
|
||||
value: typeof pointer.value === "string" ? pointer.value.trim() : "",
|
||||
};
|
||||
|
||||
if (!cloned.value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (typeof pointer.relay === "string" && pointer.relay.trim()) {
|
||||
cloned.relay = pointer.relay.trim();
|
||||
}
|
||||
|
||||
if (Number.isFinite(pointer.watchedAt)) {
|
||||
cloned.watchedAt = Math.max(0, Math.floor(pointer.watchedAt));
|
||||
}
|
||||
|
||||
if (Number.isFinite(pointer.resumeAt)) {
|
||||
cloned.resumeAt = Math.max(0, Math.floor(pointer.resumeAt));
|
||||
}
|
||||
|
||||
if (pointer.completed === true) {
|
||||
cloned.completed = true;
|
||||
}
|
||||
|
||||
if (pointer.session === true) {
|
||||
cloned.session = true;
|
||||
}
|
||||
|
||||
return cloned;
|
||||
}
|
||||
|
||||
export { clonePointerItem };
|
||||
|
||||
export function mergePointerDetails(target, source) {
|
||||
if (!target || typeof target !== "object" || !source || typeof source !== "object") {
|
||||
return target;
|
||||
}
|
||||
if (source.session === true) {
|
||||
target.session = true;
|
||||
}
|
||||
if (Number.isFinite(source.resumeAt)) {
|
||||
target.resumeAt = Math.max(0, Math.floor(source.resumeAt));
|
||||
}
|
||||
if (source.completed === true) {
|
||||
target.completed = true;
|
||||
}
|
||||
if (Number.isFinite(source.watchedAt)) {
|
||||
target.watchedAt = Math.max(0, Math.floor(source.watchedAt));
|
||||
}
|
||||
if (typeof source.relay === "string" && source.relay.trim()) {
|
||||
if (!target.relay || !target.relay.trim()) {
|
||||
target.relay = source.relay.trim();
|
||||
}
|
||||
}
|
||||
return target;
|
||||
}
|
||||
|
||||
export function normalizePointerInput(pointer) {
|
||||
if (!pointer) {
|
||||
return null;
|
||||
}
|
||||
if (Array.isArray(pointer)) {
|
||||
return normalizePointerTag(pointer);
|
||||
}
|
||||
if (typeof pointer === "object") {
|
||||
if (typeof pointer.type === "string" && typeof pointer.value === "string") {
|
||||
return clonePointerItem(pointer);
|
||||
}
|
||||
if (Array.isArray(pointer.tag)) {
|
||||
return normalizePointerTag(pointer.tag);
|
||||
}
|
||||
}
|
||||
if (typeof pointer !== "string") {
|
||||
return null;
|
||||
}
|
||||
const trimmed = pointer.trim();
|
||||
if (!trimmed) {
|
||||
return null;
|
||||
}
|
||||
if (trimmed.startsWith("naddr") || trimmed.startsWith("nevent")) {
|
||||
try {
|
||||
const decoder = getCachedNostrTools()?.nip19?.decode;
|
||||
if (typeof decoder === "function") {
|
||||
const decoded = decoder(trimmed);
|
||||
if (decoded?.type === "naddr" && decoded.data) {
|
||||
const { kind, pubkey, identifier, relays } = decoded.data;
|
||||
if (
|
||||
typeof kind === "number" &&
|
||||
typeof pubkey === "string" &&
|
||||
typeof identifier === "string"
|
||||
) {
|
||||
const relay =
|
||||
Array.isArray(relays) && relays.length && typeof relays[0] === "string"
|
||||
? relays[0]
|
||||
: null;
|
||||
return {
|
||||
type: "a",
|
||||
value: `${kind}:${pubkey}:${identifier}`,
|
||||
relay,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (decoded?.type === "nevent" && decoded.data) {
|
||||
const { id, relays } = decoded.data;
|
||||
if (typeof id === "string" && id.trim()) {
|
||||
const relay =
|
||||
Array.isArray(relays) && relays.length && typeof relays[0] === "string"
|
||||
? relays[0]
|
||||
: null;
|
||||
return {
|
||||
type: "e",
|
||||
value: id.trim(),
|
||||
relay,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
devLogger.warn(`[nostr] Failed to decode pointer ${trimmed}:`, error);
|
||||
}
|
||||
}
|
||||
const type = trimmed.includes(":") ? "a" : "e";
|
||||
return { type, value: trimmed, relay: null };
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
// js/workers/exploreData.worker.js
|
||||
|
||||
// Configure local NostrTools entrypoint for the bootstrap logic
|
||||
self.__BITVID_LOCAL_NOSTR_TOOLS_ENTRY__ = "../../vendor/nostr-tools.bundle.min.js";
|
||||
|
||||
import { collectVideoTags } from "../utils/videoTags.js";
|
||||
import { normalizePointerInput } from "../utils/pointerNormalization.js";
|
||||
import { buildVideoAddressPointer } from "../utils/videoPointer.js";
|
||||
import { normalizeHashtag } from "../utils/hashtagNormalization.js";
|
||||
|
||||
function normalizeAddressKey(value) {
|
||||
if (typeof value !== "string") {
|
||||
return "";
|
||||
}
|
||||
const trimmed = value.trim();
|
||||
if (!trimmed) {
|
||||
return "";
|
||||
}
|
||||
return trimmed.toLowerCase();
|
||||
}
|
||||
|
||||
async function buildWatchHistoryTagCounts({ items, videosMap, videosArray }) {
|
||||
// videosMap is passed as an object or Map. Serialization converts Map to object?
|
||||
// Structured Clone Algorithm supports Map since recently.
|
||||
// Assuming browser supports Map transfer/clone.
|
||||
// If items are raw objects, we process them.
|
||||
|
||||
const counts = new Map();
|
||||
if (!Array.isArray(items)) {
|
||||
return counts;
|
||||
}
|
||||
|
||||
// Reconstruct Map if needed, but if we pass Map, it works.
|
||||
// However, videosMap might be huge.
|
||||
// If we pass array of videos, we can build a temporary map.
|
||||
|
||||
let lookupMap = videosMap;
|
||||
if (Array.isArray(videosMap)) {
|
||||
lookupMap = new Map(videosMap.map(v => [v.id, v]));
|
||||
} else if (!(videosMap instanceof Map) && videosArray) {
|
||||
lookupMap = new Map(videosArray.map(v => [v.id, v]));
|
||||
}
|
||||
|
||||
const needsAddressIndex = items.some((item) => {
|
||||
const pointer = normalizePointerInput(item?.pointer || item);
|
||||
return pointer?.type === "a";
|
||||
});
|
||||
|
||||
let addressIndex = null;
|
||||
if (needsAddressIndex && lookupMap) {
|
||||
addressIndex = new Map();
|
||||
for (const video of lookupMap.values()) {
|
||||
const address = buildVideoAddressPointer(video);
|
||||
const key = normalizeAddressKey(address);
|
||||
if (key) {
|
||||
addressIndex.set(key, video);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
const entry = items[i];
|
||||
const pointer = normalizePointerInput(entry?.pointer || entry);
|
||||
let video = entry?.video || entry?.metadata?.video || null;
|
||||
|
||||
if (pointer?.type === "e" && pointer.value) {
|
||||
const eventId = typeof pointer.value === "string" ? pointer.value.trim() : "";
|
||||
if (eventId && lookupMap) {
|
||||
video = lookupMap.get(eventId) || lookupMap.get(eventId.toLowerCase()) || video;
|
||||
}
|
||||
} else if (pointer?.type === "a" && pointer.value && addressIndex) {
|
||||
const key = normalizeAddressKey(pointer.value);
|
||||
if (key && addressIndex.has(key)) {
|
||||
video = addressIndex.get(key);
|
||||
}
|
||||
}
|
||||
|
||||
if (!video) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const tags = collectVideoTags(video);
|
||||
for (const tag of tags) {
|
||||
const normalized = normalizeHashtag(tag);
|
||||
if (normalized) {
|
||||
counts.set(normalized, (counts.get(normalized) || 0) + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return counts;
|
||||
}
|
||||
|
||||
async function buildTagIdf({ videos }) {
|
||||
const idf = new Map();
|
||||
const list = Array.isArray(videos) ? videos : [];
|
||||
if (!list.length) {
|
||||
return idf;
|
||||
}
|
||||
|
||||
const docFrequency = new Map();
|
||||
const totalDocs = list.length;
|
||||
|
||||
for (let i = 0; i < list.length; i++) {
|
||||
const video = list[i];
|
||||
const tags = collectVideoTags(video);
|
||||
if (tags && (tags.length || tags.size)) {
|
||||
for (const tag of tags) {
|
||||
const normalized = normalizeHashtag(tag);
|
||||
if (normalized) {
|
||||
docFrequency.set(normalized, (docFrequency.get(normalized) || 0) + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const [tag, df] of docFrequency.entries()) {
|
||||
const ratio = (totalDocs + 1) / (df + 1);
|
||||
const value = 1 + Math.log(ratio);
|
||||
if (Number.isFinite(value) && value > 0) {
|
||||
idf.set(tag, value);
|
||||
}
|
||||
}
|
||||
|
||||
return idf;
|
||||
}
|
||||
|
||||
self.onmessage = async (e) => {
|
||||
const { type, id, payload } = e.data;
|
||||
|
||||
try {
|
||||
let result;
|
||||
if (type === 'CALC_HISTORY_COUNTS') {
|
||||
result = await buildWatchHistoryTagCounts(payload);
|
||||
} else if (type === 'CALC_IDF') {
|
||||
result = await buildTagIdf(payload);
|
||||
} else {
|
||||
throw new Error(`Unknown message type: ${type}`);
|
||||
}
|
||||
|
||||
self.postMessage({ id, result });
|
||||
} catch (error) {
|
||||
self.postMessage({ id, error: error.message || String(error) });
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,29 @@
|
||||
import { defineConfig } from "@playwright/test";
|
||||
|
||||
const HOST = "127.0.0.1";
|
||||
const PORT = 4173;
|
||||
const BASE_URL = `http://${HOST}:${PORT}`;
|
||||
|
||||
export default defineConfig({
|
||||
testDir: "./tests",
|
||||
outputDir: "artifacts/test-results-bench",
|
||||
timeout: 60_000,
|
||||
reporter: "list",
|
||||
use: {
|
||||
baseURL: BASE_URL,
|
||||
viewport: { width: 1280, height: 720 },
|
||||
ignoreHTTPSErrors: true,
|
||||
},
|
||||
projects: [
|
||||
{
|
||||
name: "benchmark",
|
||||
testMatch: "**/*.spec.mjs",
|
||||
},
|
||||
],
|
||||
webServer: {
|
||||
command: `npx http-server . -p ${PORT} -a ${HOST} -c-1 --silent`,
|
||||
url: `${BASE_URL}/docs/kitchen-sink.html`,
|
||||
reuseExistingServer: true,
|
||||
timeout: 120_000,
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,92 @@
|
||||
import { test, expect } from '@playwright/test';
|
||||
|
||||
test.describe('Explore Data Performance Benchmark', () => {
|
||||
test('benchmark buildWatchHistoryTagCounts and buildTagIdf', async ({ page }) => {
|
||||
// Go to the app to ensure environment is loaded
|
||||
await page.goto('/');
|
||||
|
||||
// Wait for modules to load (optional, but good practice)
|
||||
await page.waitForLoadState('networkidle');
|
||||
|
||||
const result = await page.evaluate(async () => {
|
||||
// Dynamic import of the service
|
||||
const { buildWatchHistoryTagCounts, buildTagIdf } = await import('./js/services/exploreDataService.js');
|
||||
|
||||
// --- Mock Data Generation ---
|
||||
const VIDEO_COUNT = 5000;
|
||||
const HISTORY_COUNT = 1000;
|
||||
|
||||
const videosMap = new Map();
|
||||
const videosArray = [];
|
||||
const tagsPool = ['bitcoin', 'nostr', 'art', 'music', 'coding', 'performance', 'web', 'worker', 'nature', 'space'];
|
||||
|
||||
for (let i = 0; i < VIDEO_COUNT; i++) {
|
||||
const id = `video-${i}`;
|
||||
const tags = [];
|
||||
const numTags = Math.floor(Math.random() * 5) + 1;
|
||||
for (let t = 0; t < numTags; t++) {
|
||||
tags.push(['t', tagsPool[Math.floor(Math.random() * tagsPool.length)]]);
|
||||
}
|
||||
|
||||
const video = {
|
||||
id,
|
||||
kind: 30078,
|
||||
pubkey: '0000000000000000000000000000000000000000000000000000000000000000',
|
||||
tags: [['d', id], ...tags],
|
||||
};
|
||||
videosMap.set(id, video);
|
||||
videosArray.push(video);
|
||||
}
|
||||
|
||||
// Mock NostrService
|
||||
const mockNostrService = {
|
||||
getVideosMap: () => videosMap,
|
||||
getFilteredActiveVideos: () => videosArray,
|
||||
};
|
||||
|
||||
// Mock WatchHistoryService
|
||||
const historyItems = [];
|
||||
for (let i = 0; i < HISTORY_COUNT; i++) {
|
||||
const videoId = `video-${Math.floor(Math.random() * VIDEO_COUNT)}`;
|
||||
historyItems.push({
|
||||
pointer: { type: 'a', value: `30078:0000000000000000000000000000000000000000000000000000000000000000:${videoId}` },
|
||||
video: videosMap.get(videoId),
|
||||
});
|
||||
}
|
||||
|
||||
const mockWatchHistoryService = {
|
||||
loadLatest: () => Promise.resolve(historyItems),
|
||||
};
|
||||
|
||||
// --- Benchmark Counts ---
|
||||
const startCounts = performance.now();
|
||||
const counts = await buildWatchHistoryTagCounts({
|
||||
watchHistoryService: mockWatchHistoryService,
|
||||
nostrService: mockNostrService,
|
||||
actor: 'test-actor',
|
||||
});
|
||||
const endCounts = performance.now();
|
||||
|
||||
// --- Benchmark IDF ---
|
||||
const startIdf = performance.now();
|
||||
const idf = await buildTagIdf({ videos: videosArray });
|
||||
const endIdf = performance.now();
|
||||
|
||||
return {
|
||||
countsTime: endCounts - startCounts,
|
||||
idfTime: endIdf - startIdf,
|
||||
countsSize: counts.size,
|
||||
idfSize: idf.size
|
||||
};
|
||||
});
|
||||
|
||||
console.log(`[Benchmark] buildWatchHistoryTagCounts: ${result.countsTime.toFixed(2)}ms`);
|
||||
console.log(`[Benchmark] buildTagIdf: ${result.idfTime.toFixed(2)}ms`);
|
||||
console.log(`[Benchmark] Counts Size: ${result.countsSize}`);
|
||||
console.log(`[Benchmark] IDF Size: ${result.idfSize}`);
|
||||
|
||||
// Assertions to ensure it actually ran and produced results
|
||||
expect(result.countsSize).toBeGreaterThan(0);
|
||||
expect(result.idfSize).toBeGreaterThan(0);
|
||||
});
|
||||
});
|
||||
Reference in new issue
Block a user