mirror of
https://github.com/PR0M3TH3AN/bitvid.git
synced 2026-10-27 07:52:24 +00:00
feat(feed): make For You inclusive + ranked by your lists (G1-G5)
The For You feed was chronological, hashtag-interest-EXCLUSIVE, and ignored follows entirely (docs/feed-algo-audit.md). Now it is inclusive + ranked: - createForYouScorerStage scores each video: follows-boost (author you follow) + interest/watch-topic affinity (cosine vs your interests+watch-history vector) + freshness - disinterest penalty. Reuses Explore's vector/similarity helpers. - createForYouScoreSorter ranks by that score (muted last, recency tiebreak). - For You pipeline: interests now BOOST not exclude (enforceInterests:false); scorer added; chronological sorter replaced. Never empty (falls back to recency when you have no follows/interests/history). - buildForYouFeedRuntime now feeds subscriptionAuthors + watchHistoryTagCounts. Watched/blocked/muted are still removed (suppression/blacklist/moderation stages unchanged). Cheat-resistant tests (older boosted items only win via the boost). Grid renders end-to-end (~1.3s). Weight tuning is a follow-up. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
1 parent
44bf88cf88
commit
c0a8636678
8 files changed
+322
-7
No files matched your search
+20
-1
@@ -35,7 +35,26 @@ Runtime inputs (`buildForYouFeedRuntime`): `blacklistedEventIds`,
|
||||
|
||||
So the refactor's payoff is real: these lists now load and *do* shape For You.
|
||||
|
||||
## Gaps (the deliverable)
|
||||
## Resolution (2026-06-14)
|
||||
|
||||
Product decision: **For You = inclusive + ranked**. Implemented:
|
||||
- New `createForYouScorerStage` (in `exploreScoring.js`) scores each video by
|
||||
**follows-boost + interest/watch-topic affinity + freshness − disinterest**
|
||||
(reusing Explore's vector/similarity/freshness helpers; affinity uses the
|
||||
combined interests+watch-history user vector).
|
||||
- New `createForYouScoreSorter` ranks by that score (muted last, recency tiebreak).
|
||||
- `registerForYouFeed`: interests now **boost** instead of exclude
|
||||
(`enforceInterests:false`); scorer added; chronological sorter replaced.
|
||||
- `buildForYouFeedRuntime` now supplies `subscriptionAuthors` (follows) and
|
||||
`watchHistoryTagCounts`.
|
||||
- Result: **G1 follows, G2 ranking, G3 inclusivity (never empty), G4 watch-topic
|
||||
affinity, G5 blending — all addressed.** Cheat-resistant tests in
|
||||
`tests/for-you-scorer.test.mjs`.
|
||||
|
||||
Tuning of the weights is a follow-up (currently follows .4 / affinity .35 /
|
||||
freshness .25 / disinterest .3).
|
||||
|
||||
## Gaps (original audit)
|
||||
|
||||
### G1 — Follows/subscriptions have ZERO effect on For You *(highest impact)*
|
||||
For You's source is `createActiveNostrSource` (all videos); the runtime passes no
|
||||
|
||||
@@ -41,6 +41,8 @@ import {
|
||||
createDedupeByRootStage,
|
||||
createExploreDiversitySorter,
|
||||
createExploreScorerStage,
|
||||
createForYouScorerStage,
|
||||
createForYouScoreSorter,
|
||||
createKidsAudienceFilterStage,
|
||||
createKidsScorerStage,
|
||||
createKidsScoreSorter,
|
||||
@@ -588,6 +590,8 @@ class Application {
|
||||
createDedupeByRootStage,
|
||||
createExploreDiversitySorter,
|
||||
createExploreScorerStage,
|
||||
createForYouScorerStage,
|
||||
createForYouScoreSorter,
|
||||
createKidsAudienceFilterStage,
|
||||
createKidsScorerStage,
|
||||
createKidsScoreSorter,
|
||||
|
||||
@@ -34,6 +34,8 @@ export function createFeedCoordinator(deps) {
|
||||
createDedupeByRootStage,
|
||||
createExploreDiversitySorter,
|
||||
createExploreScorerStage,
|
||||
createForYouScorerStage,
|
||||
createForYouScoreSorter,
|
||||
createKidsAudienceFilterStage,
|
||||
createKidsScorerStage,
|
||||
createKidsScoreSorter,
|
||||
@@ -158,9 +160,11 @@ export function createFeedCoordinator(deps) {
|
||||
return this.feedEngine.registerFeed(FEED_TYPES.FOR_YOU, {
|
||||
source: createActiveNostrSource({ service: this.nostrService }),
|
||||
stages: [
|
||||
// Note: Tag-preference filtering is consolidated in createTagPreferenceFilterStage
|
||||
// so each feed has a single source of truth for interest-based inclusion/ranking.
|
||||
createTagPreferenceFilterStage(),
|
||||
// For You is INCLUSIVE + RANKED (docs/feed-algo-audit.md): interests
|
||||
// BOOST rather than exclude, so the feed is never empty. We only drop
|
||||
// disinterested tags here (enforceInterests:false); the ranking by
|
||||
// follows + interest/watch affinity + freshness happens in the scorer.
|
||||
createTagPreferenceFilterStage({ enforceInterests: false }),
|
||||
createBlacklistFilterStage({
|
||||
shouldIncludeVideo: (video, options) =>
|
||||
this.nostrService.shouldIncludeVideo(video, options),
|
||||
@@ -177,8 +181,9 @@ export function createFeedCoordinator(deps) {
|
||||
trustedReportHideThreshold: resolveThresholdFromApp("trustedSpamHideThreshold"),
|
||||
}),
|
||||
createResolvePostedAtStage(),
|
||||
createForYouScorerStage(),
|
||||
],
|
||||
sorter: createChronologicalSorter(),
|
||||
sorter: createForYouScoreSorter(),
|
||||
hooks: {
|
||||
timestamps: {
|
||||
getKnownVideoPostedAt: (video) => this.getKnownVideoPostedAt(video),
|
||||
@@ -526,6 +531,34 @@ export function createFeedCoordinator(deps) {
|
||||
const preferencesAvailable = preferenceSource?.dataReady === true;
|
||||
const moderationThresholds = this.getActiveModerationThresholds();
|
||||
|
||||
// Follows drive the For You ranking (follows-boost in the scorer).
|
||||
let subscriptionAuthors = [];
|
||||
try {
|
||||
if (subscriptions && typeof subscriptions.getSubscribedAuthors === "function") {
|
||||
const authors = subscriptions.getSubscribedAuthors();
|
||||
if (Array.isArray(authors)) {
|
||||
subscriptionAuthors = authors.filter(
|
||||
(a) => typeof a === "string" && a,
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
devLogger.warn("[feedCoordinator] Failed to read subscribed authors:", error);
|
||||
}
|
||||
|
||||
// Watch-topic affinity signal (reuse the Explore source if present).
|
||||
const whTagSource =
|
||||
this.exploreDataService &&
|
||||
typeof this.exploreDataService.getWatchHistoryTagCounts === "function"
|
||||
? this.exploreDataService.getWatchHistoryTagCounts()
|
||||
: this.watchHistoryTagCounts;
|
||||
const watchHistoryTagCounts =
|
||||
whTagSource instanceof Map
|
||||
? new Map(whTagSource)
|
||||
: whTagSource && typeof whTagSource === "object"
|
||||
? { ...whTagSource }
|
||||
: undefined;
|
||||
|
||||
const watchedKeys = new Set();
|
||||
if (Array.isArray(watchHistoryItems)) {
|
||||
for (const item of watchHistoryItems) {
|
||||
@@ -542,6 +575,8 @@ export function createFeedCoordinator(deps) {
|
||||
return {
|
||||
blacklistedEventIds: blacklist,
|
||||
isAuthorBlocked: (pubkey) => this.isAuthorBlocked(pubkey),
|
||||
subscriptionAuthors,
|
||||
watchHistoryTagCounts,
|
||||
tagPreferences: {
|
||||
interests: Array.isArray(interests) ? [...interests] : [],
|
||||
disinterests: Array.isArray(disinterests) ? [...disinterests] : [],
|
||||
|
||||
@@ -449,3 +449,130 @@ export function createExploreScorerStage({ stageName = "explore-scorer", ...opti
|
||||
return items;
|
||||
};
|
||||
}
|
||||
|
||||
// For You weights — UNLIKE Explore (which rewards novelty/discovery), For You
|
||||
// rewards affinity to who you follow and what you like/watch, plus freshness.
|
||||
const FOR_YOU_WEIGHTS = Object.freeze({
|
||||
follows: 0.4, // author is in your subscriptions
|
||||
affinity: 0.35, // cosine similarity to your interests + watch-topic profile
|
||||
freshness: 0.25, // recency decay
|
||||
disinterestOverlap: 0.3, // penalty for disinterested tags
|
||||
});
|
||||
|
||||
/**
|
||||
* For You scorer (docs/feed-algo-audit.md). Inclusive + ranked: it scores (never
|
||||
* filters) so the feed is never empty — when you have no follows/interests/
|
||||
* history, score == freshness and it falls back to "recent". Reuses Explore's
|
||||
* vector/similarity/freshness helpers; adds a follows-boost.
|
||||
*/
|
||||
export function createForYouScorerStage({
|
||||
stageName = "for-you-scorer",
|
||||
weights: weightOverrides,
|
||||
...options
|
||||
} = {}) {
|
||||
const weights = {
|
||||
...FOR_YOU_WEIGHTS,
|
||||
...(isPlainObject(weightOverrides) ? weightOverrides : {}),
|
||||
};
|
||||
|
||||
return async function forYouScorerStage(items = [], context = {}) {
|
||||
if (!Array.isArray(items) || items.length === 0) {
|
||||
return items;
|
||||
}
|
||||
|
||||
const tagPreferences = context?.runtime?.tagPreferences;
|
||||
const interests = normalizeTagSet(tagPreferences?.interests);
|
||||
const disinterests = normalizeTagSet(tagPreferences?.disinterests);
|
||||
const historyCounts = normalizeTagCountMap(context?.runtime?.watchHistoryTagCounts);
|
||||
const historyWeights = buildHistoryWeights(historyCounts);
|
||||
const userVector = combineUserVector({ interests, disinterests, historyWeights });
|
||||
const idfMap = context?.runtime?.exploreTagIdf;
|
||||
const follows = toSet(context?.runtime?.subscriptionAuthors);
|
||||
const freshnessHalfLifeDays = resolveFreshnessHalfLife(options, context);
|
||||
const now = resolveNow(context);
|
||||
|
||||
for (const item of items) {
|
||||
if (!item || typeof item !== "object") {
|
||||
continue;
|
||||
}
|
||||
const video = item.video;
|
||||
if (!video || typeof video !== "object") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const videoVector = buildVideoVector(video, idfMap);
|
||||
const totalVideoWeight = Array.from(videoVector.values()).reduce(
|
||||
(sum, value) => sum + (Number.isFinite(value) ? value : 0),
|
||||
0,
|
||||
);
|
||||
|
||||
const affinity = Math.max(0, cosineSimilarity(userVector, videoVector));
|
||||
|
||||
let disinterestOverlap = 0;
|
||||
if (disinterests.size > 0 && videoVector.size > 0 && totalVideoWeight > 0) {
|
||||
let disinterestWeight = 0;
|
||||
for (const [tag, weight] of videoVector.entries()) {
|
||||
if (disinterests.has(tag)) {
|
||||
disinterestWeight += weight;
|
||||
}
|
||||
}
|
||||
disinterestOverlap = clamp01(disinterestWeight / totalVideoWeight);
|
||||
}
|
||||
|
||||
const followsMatch =
|
||||
typeof video.pubkey === "string" && follows.has(video.pubkey) ? 1 : 0;
|
||||
|
||||
let freshness = 0;
|
||||
const createdAt = resolveTimestamp(video);
|
||||
if (createdAt && freshnessHalfLifeDays > 0) {
|
||||
const ageDays = Math.max(0, now - createdAt) / 86400;
|
||||
freshness = clamp01(Math.exp(-ageDays / freshnessHalfLifeDays));
|
||||
}
|
||||
|
||||
const score = clamp01(
|
||||
weights.follows * followsMatch +
|
||||
weights.affinity * affinity +
|
||||
weights.freshness * freshness -
|
||||
weights.disinterestOverlap * disinterestOverlap,
|
||||
);
|
||||
|
||||
if (!isPlainObject(item.metadata)) {
|
||||
item.metadata = {};
|
||||
}
|
||||
item.metadata.forYouScore = score;
|
||||
item.metadata.forYouComponents = {
|
||||
followsMatch,
|
||||
affinity,
|
||||
freshness,
|
||||
disinterestOverlap,
|
||||
};
|
||||
|
||||
const videoId = typeof video.id === "string" ? video.id : null;
|
||||
const pubkey = typeof video.pubkey === "string" ? video.pubkey : null;
|
||||
const positives = [
|
||||
{ key: "follows", value: weights.follows * followsMatch },
|
||||
{ key: "affinity", value: weights.affinity * affinity },
|
||||
{ key: "freshness", value: weights.freshness * freshness },
|
||||
];
|
||||
let dominant = positives[0];
|
||||
for (const component of positives) {
|
||||
if (component.value > dominant.value) {
|
||||
dominant = component;
|
||||
}
|
||||
}
|
||||
if (dominant.value > 0) {
|
||||
context?.addWhy?.({
|
||||
stage: stageName,
|
||||
type: "score",
|
||||
reason: dominant.key,
|
||||
value: dominant.value,
|
||||
score,
|
||||
videoId,
|
||||
pubkey,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
};
|
||||
}
|
||||
@@ -10,11 +10,12 @@ export {
|
||||
createModerationStage,
|
||||
createResolvePostedAtStage,
|
||||
} from "./stages.js";
|
||||
export { createExploreScorerStage } from "./exploreScoring.js";
|
||||
export { createExploreScorerStage, createForYouScorerStage } from "./exploreScoring.js";
|
||||
export { createKidsScorerStage } from "./kidsScoring.js";
|
||||
export { createKidsAudienceFilterStage } from "./kidsAudienceFilterStage.js";
|
||||
export {
|
||||
createChronologicalSorter,
|
||||
createForYouScoreSorter,
|
||||
createExploreDiversitySorter,
|
||||
createKidsScoreSorter,
|
||||
} from "./sorters.js";
|
||||
|
||||
@@ -3,6 +3,54 @@
|
||||
import { normalizeHashtag } from "../utils/hashtagNormalization.js";
|
||||
import { markAsNormalized } from "./utils.js";
|
||||
|
||||
// Ranks by the For You score (set by createForYouScorerStage): score desc,
|
||||
// trusted-muted last, recency as the tiebreak. Reuses the same muted-last rule
|
||||
// as the chronological sorter so behavior is consistent across feeds.
|
||||
export function createForYouScoreSorter() {
|
||||
return function forYouScoreSorter(items = [], context = {}) {
|
||||
if (!Array.isArray(items)) {
|
||||
return [];
|
||||
}
|
||||
void context;
|
||||
const copy = [...items];
|
||||
|
||||
const isMuted = (entry) =>
|
||||
entry?.metadata?.moderation?.trustedMuted === true ||
|
||||
entry?.video?.moderation?.trustedMuted === true;
|
||||
|
||||
const scoreOf = (entry) => {
|
||||
const score = Number(entry?.metadata?.forYouScore);
|
||||
return Number.isFinite(score) ? score : 0;
|
||||
};
|
||||
|
||||
const timestampOf = (entry) => {
|
||||
const video = entry?.video;
|
||||
if (!video || typeof video !== "object") {
|
||||
return Number.NEGATIVE_INFINITY;
|
||||
}
|
||||
const root = Number(video.rootCreatedAt);
|
||||
if (Number.isFinite(root)) return Math.floor(root);
|
||||
const created = Number(video.created_at);
|
||||
return Number.isFinite(created) ? Math.floor(created) : Number.NEGATIVE_INFINITY;
|
||||
};
|
||||
|
||||
copy.sort((a, b) => {
|
||||
const aMuted = isMuted(a);
|
||||
const bMuted = isMuted(b);
|
||||
if (aMuted !== bMuted) {
|
||||
return aMuted ? 1 : -1;
|
||||
}
|
||||
const scoreDiff = scoreOf(b) - scoreOf(a);
|
||||
if (scoreDiff !== 0) {
|
||||
return scoreDiff;
|
||||
}
|
||||
return timestampOf(b) - timestampOf(a);
|
||||
});
|
||||
|
||||
return copy;
|
||||
};
|
||||
}
|
||||
|
||||
export function createChronologicalSorter({
|
||||
direction = "desc",
|
||||
postedAtResolver,
|
||||
|
||||
@@ -42,7 +42,7 @@ const GRANDFATHERED = {
|
||||
"js/nostr/watchHistory.js": 2388,
|
||||
"js/nostr/nip46Client.js": 2102,
|
||||
"js/state/cache.js": 1827,
|
||||
"js/app/feedCoordinator.js": 1767,
|
||||
"js/app/feedCoordinator.js": 1841,
|
||||
"js/app/authSessionCoordinator.js": 1803,
|
||||
"js/ui/components/RevertModal.js": 1663,
|
||||
"js/payments/nwcClient.js": 1642,
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
// Scenario (SCN-for-you-ranked): the For You feed is inclusive + ranked.
|
||||
// Given videos, a follows set, hashtag interests, and disinterests,
|
||||
// When scored by createForYouScorerStage and sorted by createForYouScoreSorter,
|
||||
// Then a FOLLOWED author and an INTEREST match each rank above an otherwise-
|
||||
// newer item (so the boost — not just recency — decides), a DISINTERESTED
|
||||
// item is penalized below a neutral one, and nothing is filtered out (never
|
||||
// empty).
|
||||
//
|
||||
// Cheat-resistant: the boosted items are deliberately OLDER, so they can only
|
||||
// win via the follows/affinity boost, not recency.
|
||||
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createForYouScorerStage } from "../js/feedEngine/exploreScoring.js";
|
||||
import { createForYouScoreSorter } from "../js/feedEngine/sorters.js";
|
||||
|
||||
const FOLLOWED = "f".repeat(64);
|
||||
const OTHER = "0".repeat(64);
|
||||
const NOW = Math.floor(Date.now() / 1000);
|
||||
const OLD = NOW - 100000; // ~1.16 days older
|
||||
|
||||
const item = (id, pubkey, tagNames = [], createdAt = NOW) => ({
|
||||
video: { id, pubkey, created_at: createdAt, tags: tagNames.map((t) => ["t", t]) },
|
||||
metadata: {},
|
||||
});
|
||||
|
||||
async function rank(items, runtime) {
|
||||
const context = { runtime, now: NOW };
|
||||
await createForYouScorerStage()(items, context);
|
||||
return createForYouScoreSorter()(items, context).map((i) => i.video.id);
|
||||
}
|
||||
|
||||
test("a followed author outranks a newer non-followed video", async () => {
|
||||
const items = [
|
||||
item("newer-stranger", OTHER, [], NOW),
|
||||
item("older-followed", FOLLOWED, [], OLD),
|
||||
];
|
||||
const order = await rank(items, {
|
||||
subscriptionAuthors: [FOLLOWED],
|
||||
tagPreferences: { interests: [], disinterests: [] },
|
||||
});
|
||||
assert.equal(order[0], "older-followed", "follows-boost must beat recency");
|
||||
assert.equal(items.length, 2, "scorer must not filter (never empty)");
|
||||
});
|
||||
|
||||
test("an interest match outranks a newer non-matching video", async () => {
|
||||
const items = [
|
||||
item("newer-sports", OTHER, ["sports"], NOW),
|
||||
item("older-music", OTHER, ["music"], OLD),
|
||||
];
|
||||
const order = await rank(items, {
|
||||
subscriptionAuthors: [],
|
||||
tagPreferences: { interests: ["music"], disinterests: [] },
|
||||
});
|
||||
assert.equal(order[0], "older-music", "interest affinity must beat recency");
|
||||
});
|
||||
|
||||
test("a disinterested tag is penalized below a neutral video", async () => {
|
||||
const items = [
|
||||
item("spammy", OTHER, ["spam"], NOW),
|
||||
item("neutral", OTHER, ["cooking"], NOW),
|
||||
];
|
||||
const order = await rank(items, {
|
||||
subscriptionAuthors: [],
|
||||
tagPreferences: { interests: [], disinterests: ["spam"] },
|
||||
});
|
||||
assert.equal(order[0], "neutral", "disinterested item must rank last");
|
||||
});
|
||||
|
||||
test("with no signals at all, falls back to recency (never empty)", async () => {
|
||||
const items = [
|
||||
item("old", OTHER, [], OLD),
|
||||
item("new", OTHER, [], NOW),
|
||||
];
|
||||
const order = await rank(items, {
|
||||
subscriptionAuthors: [],
|
||||
tagPreferences: { interests: [], disinterests: [] },
|
||||
});
|
||||
assert.equal(order[0], "new", "no signals → newest first");
|
||||
assert.equal(order.length, 2);
|
||||
});
|
||||
Reference in new issue
Block a user