Merge pull request #2322 from PR0M3TH3AN/perf-feed-tag-caching-7931834874140558598

⚡ Optimize feed engine tag extraction with caching

benchmarks/feed_tags.mjs

@@ -0,0 +1,81 @@
+
+import { normalizeHashtag } from "../js/utils/hashtagNormalization.js";
+import { getVideoTags } from "../js/feedEngine/utils.js";
+
+// Mock video object
+function createMockVideo(tagCount) {
+  const tags = [];
+  for (let i = 0; i < tagCount; i++) {
+    tags.push(["t", `tag${i}`]);
+  }
+  return {
+    id: "video-" + Math.random(),
+    tags: tags,
+    nip71: {
+      hashtags: ["extra1", "extra2"]
+    }
+  };
+}
+
+// Legacy Uncached collectVideoTags (for comparison)
+function collectVideoTags(video) {
+  const videoTags = new Set();
+
+  if (Array.isArray(video.tags)) {
+    for (const tag of video.tags) {
+      if (Array.isArray(tag) && tag[0] === "t" && typeof tag[1] === "string") {
+        const normalized = normalizeHashtag(tag[1]);
+        if (normalized) {
+          videoTags.add(normalized);
+        }
+      }
+    }
+  }
+
+  if (Array.isArray(video.nip71?.hashtags)) {
+    for (const tag of video.nip71.hashtags) {
+      if (typeof tag === "string") {
+        const normalized = normalizeHashtag(tag);
+        if (normalized) {
+          videoTags.add(normalized);
+        }
+      }
+    }
+  }
+
+  return videoTags;
+}
+
+// Benchmark
+const VIDEOS_COUNT = 10000;
+const TAGS_PER_VIDEO = 10;
+const ITERATIONS = 100;
+
+const videos = [];
+for (let i = 0; i < VIDEOS_COUNT; i++) {
+  videos.push(createMockVideo(TAGS_PER_VIDEO));
+}
+
+console.log(`Benchmarking with ${VIDEOS_COUNT} videos, ${ITERATIONS} iterations.`);
+
+console.time("Uncached (collectVideoTags)");
+for (let i = 0; i < ITERATIONS; i++) {
+  for (const video of videos) {
+    collectVideoTags(video);
+  }
+}
+console.timeEnd("Uncached (collectVideoTags)");
+
+console.time("Cached (getVideoTags) - First Run");
+for (const video of videos) {
+  getVideoTags(video);
+}
+console.timeEnd("Cached (getVideoTags) - First Run");
+
+console.time("Cached (getVideoTags) - Second Run");
+for (let i = 0; i < ITERATIONS - 1; i++) {
+  for (const video of videos) {
+    getVideoTags(video);
+  }
+}
+console.timeEnd("Cached (getVideoTags) - Second Run");

js/feedEngine/kidsScoring.js

@@ -1,7 +1,7 @@
 // js/feedEngine/kidsScoring.js
 
 import { normalizeHashtag } from "../utils/hashtagNormalization.js";
-import { isPlainObject, toSet } from "./utils.js";
+import { isPlainObject, toSet, getVideoTags } from "./utils.js";
 
 const DEFAULT_WEIGHTS = Object.freeze({
   w_age: 0.35,
@@ -14,8 +14,6 @@ const DEFAULT_WEIGHTS = Object.freeze({
 
 const DEFAULT_FRESHNESS_HALF_LIFE_DAYS = 14;
 
-const videoTagCache = new WeakMap();
-
 const AGE_GROUP_DEFAULTS = Object.freeze({
   toddler: {
     maxDurationSeconds: 5 * 60,
@@ -52,45 +50,6 @@ function clamp01(value) {
   return value;
 }
 
-function normalizeTagSetFromVideo(video) {
-  if (!video || typeof video !== "object") {
-    return new Set();
-  }
-
-  if (videoTagCache.has(video)) {
-    return videoTagCache.get(video);
-  }
-
-  const normalized = new Set();
-
-  const addTag = (rawTag) => {
-    if (typeof rawTag !== "string") {
-      return;
-    }
-    const tag = normalizeHashtag(rawTag);
-    if (tag) {
-      normalized.add(tag);
-    }
-  };
-
-  if (Array.isArray(video.tags)) {
-    for (const tag of video.tags) {
-      if (Array.isArray(tag) && tag[0] === "t") {
-        addTag(tag[1]);
-      }
-    }
-  }
-
-  if (Array.isArray(video.nip71?.hashtags)) {
-    for (const tag of video.nip71.hashtags) {
-      addTag(tag);
-    }
-  }
-
-  videoTagCache.set(video, normalized);
-  return normalized;
-}
-
 function normalizeTagSet(values) {
   const normalized = new Set();
   if (!values) {
@@ -323,7 +282,7 @@ export function createKidsScorerStage({
         continue;
       }
 
-      const tags = normalizeTagSetFromVideo(video);
+      const tags = getVideoTags(video);
       const duration = resolveDuration(video);
       let durationScore = 0.5;
       if (Number.isFinite(duration) && duration > 0) {

js/feedEngine/stages.js

@@ -11,7 +11,7 @@ import moderationService from "../services/moderationService.js";
 import logger from "../utils/logger.js";
 import { dedupeToNewestByRoot } from "../utils/videoDeduper.js";
 import { normalizeHashtag } from "../utils/hashtagNormalization.js";
-import { isPlainObject, toSet, markAsNormalized } from "./utils.js";
+import { isPlainObject, toSet, markAsNormalized, getVideoTags } from "./utils.js";
 
 const FEED_HIDE_BYPASS_NAMES = new Set(["home", "recent"]);
 
@@ -306,34 +306,6 @@ export function createDisinterestFilterStage({
   };
 }
 
-function collectVideoTags(video) {
-  const videoTags = new Set();
-
-  if (Array.isArray(video.tags)) {
-    for (const tag of video.tags) {
-      if (Array.isArray(tag) && tag[0] === "t" && typeof tag[1] === "string") {
-        const normalized = normalizeHashtag(tag[1]);
-        if (normalized) {
-          videoTags.add(normalized);
-        }
-      }
-    }
-  }
-
-  if (Array.isArray(video.nip71?.hashtags)) {
-    for (const tag of video.nip71.hashtags) {
-      if (typeof tag === "string") {
-        const normalized = normalizeHashtag(tag);
-        if (normalized) {
-          videoTags.add(normalized);
-        }
-      }
-    }
-  }
-
-  return videoTags;
-}
-
 function filterByTagPreferences({
   items = [],
   context = {},
@@ -364,7 +336,7 @@ function filterByTagPreferences({
       continue;
     }
 
-    const videoTags = collectVideoTags(video);
+    const videoTags = getVideoTags(video);
 
     let disinterested = false;
     if (disinterests.size) {

js/feedEngine/utils.js

@@ -4,6 +4,8 @@ import { normalizeHashtag } from "../utils/hashtagNormalization.js";
 
 export const NORMALIZED_ARRAY_MARKER = Symbol("FeedItemsNormalized");
 
+const videoTagCache = new WeakMap();
+
 export function markAsNormalized(items) {
   if (Array.isArray(items)) {
     items[NORMALIZED_ARRAY_MARKER] = true;
@@ -39,33 +41,54 @@ export function toArray(value) {
   return [value];
 }
 
+export function getVideoTags(video) {
+  if (!video || typeof video !== "object") {
+    return new Set();
+  }
+
+  if (videoTagCache.has(video)) {
+    return videoTagCache.get(video);
+  }
+
+  const normalized = new Set();
+
+  const addTag = (rawTag) => {
+    if (typeof rawTag !== "string") {
+      return;
+    }
+    const tag = normalizeHashtag(rawTag);
+    if (tag) {
+      normalized.add(tag);
+    }
+  };
+
+  if (Array.isArray(video.tags)) {
+    for (const tag of video.tags) {
+      if (Array.isArray(tag) && tag[0] === "t" && typeof tag[1] === "string") {
+        addTag(tag[1]);
+      }
+    }
+  }
+
+  if (Array.isArray(video.nip71?.hashtags)) {
+    for (const tag of video.nip71.hashtags) {
+      addTag(tag);
+    }
+  }
+
+  videoTagCache.set(video, normalized);
+  return normalized;
+}
+
 export function hasDisinterestedTag(video, disinterestsSet) {
   if (!video || !disinterestsSet || !(disinterestsSet instanceof Set) || disinterestsSet.size === 0) {
     return false;
   }
 
-  const matchesDisinterest = (tag) => {
-    const normalized = normalizeHashtag(tag);
-    return Boolean(normalized && disinterestsSet.has(normalized));
-  };
-
-  // Check raw tags
-  if (Array.isArray(video.tags)) {
-    for (const tag of video.tags) {
-      if (Array.isArray(tag) && tag[0] === "t" && typeof tag[1] === "string") {
-        if (matchesDisinterest(tag[1])) {
-          return true;
-        }
-      }
-    }
-  }
-
-  // Check NIP-71 metadata hashtags if available
-  if (video.nip71 && Array.isArray(video.nip71.hashtags)) {
-    for (const tag of video.nip71.hashtags) {
-      if (typeof tag === "string" && matchesDisinterest(tag)) {
-        return true;
-      }
+  const tags = getVideoTags(video);
+  for (const tag of tags) {
+    if (disinterestsSet.has(tag)) {
+      return true;
     }
   }