diff --git a/docs/feed-engine.md b/docs/feed-engine.md index 8545b1e8..7bcdddce 100644 --- a/docs/feed-engine.md +++ b/docs/feed-engine.md @@ -71,6 +71,7 @@ constructed today behave exactly like the existing UI. | `createTagPreferenceFilterStage` | Normalizes interest/disinterest tags via `normalizeHashtag`, drops videos that match disinterests, and (by default) filters out videos that do not match interest tags. Set `enforceInterests: false` to keep items even when they lack interest tags. | | `createDisinterestFilterStage` | Normalizes tag preferences via `normalizeHashtag` and drops videos that match disinterests without excluding items that lack interest tags (useful for Explore-style feeds). | | `createWatchHistorySuppressionStage` | Invokes feed-provided hooks to optionally suppress watched items. Useful for per-feed watch history preferences. | +| `createExploreScorerStage` | Computes `metadata.exploreScore` and `metadata.exploreComponents` using novelty, freshness, tag overlap, and popularity signals. Emits `addWhy()` entries for dominant positives and disinterest overlap. | Stages receive `(items, context)` and should return the transformed list. They can rely on `context.addWhy(...)` to annotate decisions without mutating the @@ -79,8 +80,30 @@ items in place. ## Sorting & Decorators `createChronologicalSorter` is the baseline sorter that orders DTOs by -`video.created_at` (newest first by default). Additional decorators can run -after sorting to attach extra metadata or inject presentation hints. +`video.created_at` (newest first by default). Explore feeds can apply +`createExploreDiversitySorter` to Maximal Marginal Relevance (MMR) re-rank +items for tag diversity while still respecting `metadata.exploreScore`. +Additional decorators can run after sorting to attach extra metadata or inject +presentation hints. + +## Explore Feed Pipeline + +Explore builds on the standard feed engine stages with a scoring + diversity +pass. The current Explore pipeline is: + +1. `createDisinterestFilterStage` — drop disinterested tags early. +2. `createBlacklistFilterStage` — enforce NSFW/privacy/author block policies. +3. `createDedupeByRootStage` — keep the newest root version. +4. `createModerationStage` — apply moderation blur/autoplay/hide rules. +5. `createResolvePostedAtStage` — hydrate `rootCreatedAt` timestamps. +6. `createExploreScorerStage` — calculate explore scores/components. +7. `createExploreDiversitySorter` — re-rank by MMR to improve tag variety. + +Explore expects additional runtime inputs: + +- `tagPreferences` — `{ interests, disinterests }` arrays for preference-aware scoring. +- `watchHistoryTagCounts` — map/object of tag counts for novelty + history similarity. +- `exploreTagIdf` — IDF weights for tag vectors (Map or object). ## Configuration Hooks diff --git a/docs/video-grid-filtering.md b/docs/video-grid-filtering.md index e27a01ad..6d1c4a13 100644 --- a/docs/video-grid-filtering.md +++ b/docs/video-grid-filtering.md @@ -36,6 +36,9 @@ Once the feed engine is involved, additional filtering happens inside - Invokes `nostrService.shouldIncludeVideo` again within the pipeline so that blacklist/author-block/NSFW/privacy gates remain enforced for feed-sourced items. +- **`createDisinterestFilterStage()`** + - Drops videos that match a viewer’s disinterested tags. Explore uses this + stage to remove “not interested” matches before scoring and reranking. - **`hasDisinterestedTag(video, tagPreferences)`** - Interprets **tag disinterest** preferences, returning `true` when a video’s tags match a user’s “not interested” preferences. @@ -73,6 +76,8 @@ videos that have already survived the filtering stages above. 3. **Sorter** - The feed’s configured sorter (for example, chronological or relevance) orders the surviving entries. + - Explore uses `createExploreDiversitySorter` to re-rank scored items so + top results include a broader spread of tags. 4. **Render** - The feed engine hands the final list to `VideoListView`, which renders grids in `views/most-recent-videos.html`, `views/subscriptions.html`, and diff --git a/tests/feed-engine/explore-diversity.test.mjs b/tests/feed-engine/explore-diversity.test.mjs new file mode 100644 index 00000000..ab418a49 --- /dev/null +++ b/tests/feed-engine/explore-diversity.test.mjs @@ -0,0 +1,85 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { createExploreDiversitySorter } from "../../js/feedEngine/index.js"; + +const buildItem = ({ id, score, tags }) => ({ + video: { + id, + tags: tags.map((tag) => ["t", tag]), + rootCreatedAt: 1000, + }, + metadata: { + exploreScore: score, + }, +}); + +test("explore diversity sorter increases tag diversity in the top results", async () => { + const sorter = createExploreDiversitySorter({ lambda: 0.6 }); + + const items = [ + buildItem({ id: "a", score: 0.9, tags: ["cats"] }), + buildItem({ id: "b", score: 0.85, tags: ["cats"] }), + buildItem({ id: "c", score: 0.83, tags: ["dogs"] }), + ]; + + const sorted = await sorter(items, { + addWhy() {}, + }); + + assert.equal(sorted[0].video.id, "a"); + assert.equal(sorted[1].video.id, "c"); + + const topTags = new Set( + sorted + .slice(0, 2) + .flatMap((item) => item.video.tags.map((tag) => tag[1].toLowerCase())), + ); + assert.equal(topTags.size, 2); +}); + +test("explore diversity sorter logs why when MMR re-orders similar candidates", async () => { + const sorter = createExploreDiversitySorter({ lambda: 0.7 }); + + const items = [ + buildItem({ id: "a", score: 0.9, tags: ["cats", "dogs"] }), + buildItem({ id: "b", score: 0.88, tags: ["cats"] }), + buildItem({ + id: "c", + score: 0.86, + tags: [ + "cats", + "birds", + "foxes", + "lions", + "otters", + "pandas", + "bears", + "mice", + "horses", + "snakes", + "geckos", + "whales", + "turtles", + ], + }), + buildItem({ id: "d", score: 0.5, tags: ["neon"] }), + ]; + + const why = []; + const sorted = await sorter(items, { + addWhy: (detail) => why.push(detail), + }); + + assert.equal(sorted[0].video.id, "a"); + assert.equal(sorted[1].video.id, "c"); + + assert.ok(why.length >= 1); + const entry = why[0]; + assert.equal(entry.stage, "explore-diversity-sorter"); + assert.equal(entry.type, "diversity"); + assert.equal(entry.reason, "explore-diversity"); + assert.ok(Array.isArray(entry.competitorIds)); + assert.ok(entry.selectedId); + assert.ok(Number.isFinite(entry.similarity)); +}); diff --git a/tests/feed-engine/explore-scorer.test.mjs b/tests/feed-engine/explore-scorer.test.mjs new file mode 100644 index 00000000..872a391e --- /dev/null +++ b/tests/feed-engine/explore-scorer.test.mjs @@ -0,0 +1,160 @@ +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + createDisinterestFilterStage, + createExploreScorerStage, +} from "../../js/feedEngine/index.js"; + +test("explore disinterest filter drops videos with disinterested tags", async () => { + const stage = createDisinterestFilterStage(); + const items = [ + { + video: { + id: "keep", + pubkey: "a".repeat(64), + tags: [["t", "Cats"]], + }, + metadata: {}, + }, + { + video: { + id: "drop", + pubkey: "b".repeat(64), + tags: [["t", "Dogs"]], + }, + metadata: {}, + }, + ]; + + const why = []; + + const result = await stage(items, { + runtime: { + tagPreferences: { + interests: [], + disinterests: ["dogs"], + }, + }, + addWhy: (detail) => why.push(detail), + }); + + assert.equal(result.length, 1); + assert.equal(result[0].video.id, "keep"); + assert.equal(why.length, 1); + assert.equal(why[0].reason, "disinterested-tag"); + assert.equal(why[0].videoId, "drop"); +}); + +test("explore scorer penalizes disinterest overlap and logs why metadata", async () => { + const stage = createExploreScorerStage(); + + const items = [ + { + video: { + id: "neutral", + pubkey: "c".repeat(64), + tags: [["t", "Cats"]], + viewCount: 100, + }, + metadata: {}, + }, + { + video: { + id: "disinterest", + pubkey: "d".repeat(64), + tags: [["t", "Dogs"]], + viewCount: 100, + }, + metadata: {}, + }, + ]; + + const why = []; + + await stage(items, { + runtime: { + tagPreferences: { + interests: [], + disinterests: ["dogs"], + }, + exploreWeights: { + novelty: 0, + freshness: 0, + historySimilarity: 0, + newTagFraction: 0, + popularityNorm: 1, + disinterestOverlap: 0.5, + }, + explorePopularityMax: 100, + }, + addWhy: (detail) => why.push(detail), + }); + + const neutralScore = items[0].metadata.exploreScore; + const disinterestScore = items[1].metadata.exploreScore; + + assert.ok(neutralScore > disinterestScore); + assert.equal(neutralScore, 1); + assert.equal(disinterestScore, 0.5); + + const disinterestWhy = why.filter((entry) => entry.videoId === "disinterest"); + assert.ok(disinterestWhy.length >= 2); + for (const entry of disinterestWhy) { + assert.equal(entry.stage, "explore-scorer"); + assert.equal(entry.type, "score"); + assert.ok(entry.score >= 0); + assert.equal(entry.videoId, "disinterest"); + assert.equal(entry.pubkey, "d".repeat(64)); + } +}); + +test("explore scorer rewards novelty and new tag fraction", async () => { + const stage = createExploreScorerStage(); + + const items = [ + { + video: { + id: "familiar", + pubkey: "e".repeat(64), + tags: [["t", "Cats"]], + }, + metadata: {}, + }, + { + video: { + id: "novel", + pubkey: "f".repeat(64), + tags: [["t", "Dogs"]], + }, + metadata: {}, + }, + ]; + + await stage(items, { + runtime: { + watchHistoryTagCounts: { + cats: 5, + }, + exploreWeights: { + novelty: 0.7, + freshness: 0, + historySimilarity: 0, + newTagFraction: 0.3, + popularityNorm: 0, + disinterestOverlap: 0, + }, + }, + addWhy() {}, + }); + + const familiar = items[0].metadata.exploreComponents; + const novel = items[1].metadata.exploreComponents; + + assert.equal(items[0].metadata.exploreScore, 0); + assert.equal(items[1].metadata.exploreScore, 1); + assert.equal(familiar.novelty, 0); + assert.equal(familiar.newTagFraction, 0); + assert.equal(novel.novelty, 1); + assert.equal(novel.newTagFraction, 1); +});