Add explore feed engine tests and docs

docs/feed-engine.md

@@ -71,6 +71,7 @@ constructed today behave exactly like the existing UI.
 | `createTagPreferenceFilterStage` | Normalizes interest/disinterest tags via `normalizeHashtag`, drops videos that match disinterests, and (by default) filters out videos that do not match interest tags. Set `enforceInterests: false` to keep items even when they lack interest tags. |
 | `createDisinterestFilterStage` | Normalizes tag preferences via `normalizeHashtag` and drops videos that match disinterests without excluding items that lack interest tags (useful for Explore-style feeds). |
 | `createWatchHistorySuppressionStage` | Invokes feed-provided hooks to optionally suppress watched items. Useful for per-feed watch history preferences. |
+| `createExploreScorerStage` | Computes `metadata.exploreScore` and `metadata.exploreComponents` using novelty, freshness, tag overlap, and popularity signals. Emits `addWhy()` entries for dominant positives and disinterest overlap. |
 
 Stages receive `(items, context)` and should return the transformed list. They
 can rely on `context.addWhy(...)` to annotate decisions without mutating the
@@ -79,8 +80,30 @@ items in place.
 ## Sorting & Decorators
 
 `createChronologicalSorter` is the baseline sorter that orders DTOs by
-`video.created_at` (newest first by default). Additional decorators can run
-after sorting to attach extra metadata or inject presentation hints.
+`video.created_at` (newest first by default). Explore feeds can apply
+`createExploreDiversitySorter` to Maximal Marginal Relevance (MMR) re-rank
+items for tag diversity while still respecting `metadata.exploreScore`.
+Additional decorators can run after sorting to attach extra metadata or inject
+presentation hints.
+
+## Explore Feed Pipeline
+
+Explore builds on the standard feed engine stages with a scoring + diversity
+pass. The current Explore pipeline is:
+
+1. `createDisinterestFilterStage` — drop disinterested tags early.
+2. `createBlacklistFilterStage` — enforce NSFW/privacy/author block policies.
+3. `createDedupeByRootStage` — keep the newest root version.
+4. `createModerationStage` — apply moderation blur/autoplay/hide rules.
+5. `createResolvePostedAtStage` — hydrate `rootCreatedAt` timestamps.
+6. `createExploreScorerStage` — calculate explore scores/components.
+7. `createExploreDiversitySorter` — re-rank by MMR to improve tag variety.
+
+Explore expects additional runtime inputs:
+
+- `tagPreferences` — `{ interests, disinterests }` arrays for preference-aware scoring.
+- `watchHistoryTagCounts` — map/object of tag counts for novelty + history similarity.
+- `exploreTagIdf` — IDF weights for tag vectors (Map or object).
 
 ## Configuration Hooks
 

docs/video-grid-filtering.md

@@ -36,6 +36,9 @@ Once the feed engine is involved, additional filtering happens inside
   - Invokes `nostrService.shouldIncludeVideo` again within the pipeline so that
     blacklist/author-block/NSFW/privacy gates remain enforced for feed-sourced
     items.
+- **`createDisinterestFilterStage()`**
+  - Drops videos that match a viewer’s disinterested tags. Explore uses this
+    stage to remove “not interested” matches before scoring and reranking.
 - **`hasDisinterestedTag(video, tagPreferences)`**
   - Interprets **tag disinterest** preferences, returning `true` when a video’s
     tags match a user’s “not interested” preferences.
@@ -73,6 +76,8 @@ videos that have already survived the filtering stages above.
 3. **Sorter**
    - The feed’s configured sorter (for example, chronological or relevance)
      orders the surviving entries.
+   - Explore uses `createExploreDiversitySorter` to re-rank scored items so
+     top results include a broader spread of tags.
 4. **Render**
    - The feed engine hands the final list to `VideoListView`, which renders
      grids in `views/most-recent-videos.html`, `views/subscriptions.html`, and

tests/feed-engine/explore-diversity.test.mjs

@@ -0,0 +1,85 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import { createExploreDiversitySorter } from "../../js/feedEngine/index.js";
+
+const buildItem = ({ id, score, tags }) => ({
+  video: {
+    id,
+    tags: tags.map((tag) => ["t", tag]),
+    rootCreatedAt: 1000,
+  },
+  metadata: {
+    exploreScore: score,
+  },
+});
+
+test("explore diversity sorter increases tag diversity in the top results", async () => {
+  const sorter = createExploreDiversitySorter({ lambda: 0.6 });
+
+  const items = [
+    buildItem({ id: "a", score: 0.9, tags: ["cats"] }),
+    buildItem({ id: "b", score: 0.85, tags: ["cats"] }),
+    buildItem({ id: "c", score: 0.83, tags: ["dogs"] }),
+  ];
+
+  const sorted = await sorter(items, {
+    addWhy() {},
+  });
+
+  assert.equal(sorted[0].video.id, "a");
+  assert.equal(sorted[1].video.id, "c");
+
+  const topTags = new Set(
+    sorted
+      .slice(0, 2)
+      .flatMap((item) => item.video.tags.map((tag) => tag[1].toLowerCase())),
+  );
+  assert.equal(topTags.size, 2);
+});
+
+test("explore diversity sorter logs why when MMR re-orders similar candidates", async () => {
+  const sorter = createExploreDiversitySorter({ lambda: 0.7 });
+
+  const items = [
+    buildItem({ id: "a", score: 0.9, tags: ["cats", "dogs"] }),
+    buildItem({ id: "b", score: 0.88, tags: ["cats"] }),
+    buildItem({
+      id: "c",
+      score: 0.86,
+      tags: [
+        "cats",
+        "birds",
+        "foxes",
+        "lions",
+        "otters",
+        "pandas",
+        "bears",
+        "mice",
+        "horses",
+        "snakes",
+        "geckos",
+        "whales",
+        "turtles",
+      ],
+    }),
+    buildItem({ id: "d", score: 0.5, tags: ["neon"] }),
+  ];
+
+  const why = [];
+  const sorted = await sorter(items, {
+    addWhy: (detail) => why.push(detail),
+  });
+
+  assert.equal(sorted[0].video.id, "a");
+  assert.equal(sorted[1].video.id, "c");
+
+  assert.ok(why.length >= 1);
+  const entry = why[0];
+  assert.equal(entry.stage, "explore-diversity-sorter");
+  assert.equal(entry.type, "diversity");
+  assert.equal(entry.reason, "explore-diversity");
+  assert.ok(Array.isArray(entry.competitorIds));
+  assert.ok(entry.selectedId);
+  assert.ok(Number.isFinite(entry.similarity));
+});

tests/feed-engine/explore-scorer.test.mjs

@@ -0,0 +1,160 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  createDisinterestFilterStage,
+  createExploreScorerStage,
+} from "../../js/feedEngine/index.js";
+
+test("explore disinterest filter drops videos with disinterested tags", async () => {
+  const stage = createDisinterestFilterStage();
+  const items = [
+    {
+      video: {
+        id: "keep",
+        pubkey: "a".repeat(64),
+        tags: [["t", "Cats"]],
+      },
+      metadata: {},
+    },
+    {
+      video: {
+        id: "drop",
+        pubkey: "b".repeat(64),
+        tags: [["t", "Dogs"]],
+      },
+      metadata: {},
+    },
+  ];
+
+  const why = [];
+
+  const result = await stage(items, {
+    runtime: {
+      tagPreferences: {
+        interests: [],
+        disinterests: ["dogs"],
+      },
+    },
+    addWhy: (detail) => why.push(detail),
+  });
+
+  assert.equal(result.length, 1);
+  assert.equal(result[0].video.id, "keep");
+  assert.equal(why.length, 1);
+  assert.equal(why[0].reason, "disinterested-tag");
+  assert.equal(why[0].videoId, "drop");
+});
+
+test("explore scorer penalizes disinterest overlap and logs why metadata", async () => {
+  const stage = createExploreScorerStage();
+
+  const items = [
+    {
+      video: {
+        id: "neutral",
+        pubkey: "c".repeat(64),
+        tags: [["t", "Cats"]],
+        viewCount: 100,
+      },
+      metadata: {},
+    },
+    {
+      video: {
+        id: "disinterest",
+        pubkey: "d".repeat(64),
+        tags: [["t", "Dogs"]],
+        viewCount: 100,
+      },
+      metadata: {},
+    },
+  ];
+
+  const why = [];
+
+  await stage(items, {
+    runtime: {
+      tagPreferences: {
+        interests: [],
+        disinterests: ["dogs"],
+      },
+      exploreWeights: {
+        novelty: 0,
+        freshness: 0,
+        historySimilarity: 0,
+        newTagFraction: 0,
+        popularityNorm: 1,
+        disinterestOverlap: 0.5,
+      },
+      explorePopularityMax: 100,
+    },
+    addWhy: (detail) => why.push(detail),
+  });
+
+  const neutralScore = items[0].metadata.exploreScore;
+  const disinterestScore = items[1].metadata.exploreScore;
+
+  assert.ok(neutralScore > disinterestScore);
+  assert.equal(neutralScore, 1);
+  assert.equal(disinterestScore, 0.5);
+
+  const disinterestWhy = why.filter((entry) => entry.videoId === "disinterest");
+  assert.ok(disinterestWhy.length >= 2);
+  for (const entry of disinterestWhy) {
+    assert.equal(entry.stage, "explore-scorer");
+    assert.equal(entry.type, "score");
+    assert.ok(entry.score >= 0);
+    assert.equal(entry.videoId, "disinterest");
+    assert.equal(entry.pubkey, "d".repeat(64));
+  }
+});
+
+test("explore scorer rewards novelty and new tag fraction", async () => {
+  const stage = createExploreScorerStage();
+
+  const items = [
+    {
+      video: {
+        id: "familiar",
+        pubkey: "e".repeat(64),
+        tags: [["t", "Cats"]],
+      },
+      metadata: {},
+    },
+    {
+      video: {
+        id: "novel",
+        pubkey: "f".repeat(64),
+        tags: [["t", "Dogs"]],
+      },
+      metadata: {},
+    },
+  ];
+
+  await stage(items, {
+    runtime: {
+      watchHistoryTagCounts: {
+        cats: 5,
+      },
+      exploreWeights: {
+        novelty: 0.7,
+        freshness: 0,
+        historySimilarity: 0,
+        newTagFraction: 0.3,
+        popularityNorm: 0,
+        disinterestOverlap: 0,
+      },
+    },
+    addWhy() {},
+  });
+
+  const familiar = items[0].metadata.exploreComponents;
+  const novel = items[1].metadata.exploreComponents;
+
+  assert.equal(items[0].metadata.exploreScore, 0);
+  assert.equal(items[1].metadata.exploreScore, 1);
+  assert.equal(familiar.novelty, 0);
+  assert.equal(familiar.newTagFraction, 0);
+  assert.equal(novel.novelty, 1);
+  assert.equal(novel.newTagFraction, 1);
+});