diff --git a/js/feedEngine/index.js b/js/feedEngine/index.js index 57cbb3dd..f02a15ce 100644 --- a/js/feedEngine/index.js +++ b/js/feedEngine/index.js @@ -10,7 +10,10 @@ export { createResolvePostedAtStage, } from "./stages.js"; export { createExploreScorerStage } from "./exploreScoring.js"; -export { createChronologicalSorter } from "./sorters.js"; +export { + createChronologicalSorter, + createExploreDiversitySorter, +} from "./sorters.js"; export { createActiveNostrSource, createSubscriptionAuthorsSource, diff --git a/js/feedEngine/sorters.js b/js/feedEngine/sorters.js index 05475e74..459d396a 100644 --- a/js/feedEngine/sorters.js +++ b/js/feedEngine/sorters.js @@ -1,5 +1,7 @@ // js/feedEngine/sorters.js +import { normalizeHashtag } from "../utils/hashtagNormalization.js"; + export function createChronologicalSorter({ direction = "desc", postedAtResolver, @@ -122,3 +124,290 @@ export function createChronologicalSorter({ return copy; }; } + +function buildTagVector(item) { + const vector = new Map(); + const video = item?.video; + if (!video || typeof video !== "object") { + return vector; + } + + const addTag = (rawTag) => { + if (typeof rawTag !== "string") { + return; + } + const tag = normalizeHashtag(rawTag); + if (!tag) { + return; + } + vector.set(tag, (vector.get(tag) || 0) + 1); + }; + + if (Array.isArray(video.tags)) { + for (const tag of video.tags) { + if (Array.isArray(tag) && tag[0] === "t") { + addTag(tag[1]); + } + } + } + + if (Array.isArray(video.nip71?.hashtags)) { + for (const tag of video.nip71.hashtags) { + addTag(tag); + } + } + + return vector; +} + +function cosineSimilarity(vectorA, vectorB) { + if (!vectorA || !vectorB || vectorA.size === 0 || vectorB.size === 0) { + return 0; + } + + let dot = 0; + let normA = 0; + let normB = 0; + + for (const value of vectorA.values()) { + normA += value * value; + } + for (const value of vectorB.values()) { + normB += value * value; + } + + if (!Number.isFinite(normA) || !Number.isFinite(normB) || normA <= 0 || normB <= 0) { + return 0; + } + + const smaller = vectorA.size <= vectorB.size ? vectorA : vectorB; + const larger = smaller === vectorA ? vectorB : vectorA; + + for (const [tag, value] of smaller.entries()) { + const other = larger.get(tag); + if (Number.isFinite(other)) { + dot += value * other; + } + } + + const similarity = dot / (Math.sqrt(normA) * Math.sqrt(normB)); + if (!Number.isFinite(similarity)) { + return 0; + } + if (similarity > 1) { + return 1; + } + if (similarity < -1) { + return -1; + } + return similarity; +} + +function resolveItemTimestamp(item) { + const rootCreatedAt = Number(item?.video?.rootCreatedAt); + if (Number.isFinite(rootCreatedAt)) { + return Math.floor(rootCreatedAt); + } + + const createdAt = Number(item?.video?.created_at); + if (Number.isFinite(createdAt)) { + return Math.floor(createdAt); + } + + return Number.NEGATIVE_INFINITY; +} + +function stableVideoId(item) { + return typeof item?.video?.id === "string" ? item.video.id : ""; +} + +function compareByTimestampId(a, b) { + const aTs = resolveItemTimestamp(a); + const bTs = resolveItemTimestamp(b); + if (aTs !== bTs) { + return bTs - aTs; + } + return stableVideoId(a).localeCompare(stableVideoId(b)); +} + +export function createExploreDiversitySorter({ + lambda = 0.7, + stageName = "explore-diversity-sorter", +} = {}) { + const normalizedLambda = Number.isFinite(lambda) + ? Math.min(1, Math.max(0, lambda)) + : 0.7; + + return function exploreDiversitySorter(items = [], context = {}) { + if (!Array.isArray(items)) { + return []; + } + + const candidates = []; + const rest = []; + + for (const item of items) { + const score = item?.metadata?.exploreScore; + if (Number.isFinite(score)) { + candidates.push(item); + } else { + rest.push(item); + } + } + + if (candidates.length === 0) { + return [...rest].sort(compareByTimestampId); + } + + let maxScore = Number.NEGATIVE_INFINITY; + let minScore = Number.POSITIVE_INFINITY; + for (const item of candidates) { + const score = Number(item?.metadata?.exploreScore); + if (!Number.isFinite(score)) { + continue; + } + if (score > maxScore) { + maxScore = score; + } + if (score < minScore) { + minScore = score; + } + } + + const scoreSpan = maxScore - minScore; + const normalizeScore = (score) => { + const numeric = Number(score); + if (!Number.isFinite(numeric)) { + return 0; + } + if (!Number.isFinite(scoreSpan) || scoreSpan <= 0) { + return 1; + } + const normalized = (numeric - minScore) / scoreSpan; + if (normalized <= 0) { + return 0; + } + if (normalized >= 1) { + return 1; + } + return normalized; + }; + + const vectors = new Map(); + for (const item of candidates) { + vectors.set(item, buildTagVector(item)); + } + + const selected = []; + const remaining = [...candidates]; + + while (remaining.length > 0) { + let bestCandidate = null; + let bestMmr = Number.NEGATIVE_INFINITY; + let bestRaw = Number.NEGATIVE_INFINITY; + let bestSimilarity = 0; + let bestSimilarItemId = ""; + + let bestRawCandidate = null; + let bestRawScore = Number.NEGATIVE_INFINITY; + + for (const candidate of remaining) { + const rawScore = normalizeScore(candidate?.metadata?.exploreScore); + if (rawScore > bestRawScore) { + bestRawScore = rawScore; + bestRawCandidate = candidate; + } else if (rawScore === bestRawScore && bestRawCandidate) { + const tie = compareByTimestampId(candidate, bestRawCandidate); + if (tie < 0) { + bestRawCandidate = candidate; + } + } + } + + for (const candidate of remaining) { + const rawScore = normalizeScore(candidate?.metadata?.exploreScore); + let penalty = 0; + let similarItemId = ""; + + if (selected.length > 0) { + const candidateVector = vectors.get(candidate); + for (const selectedItem of selected) { + const similarity = cosineSimilarity( + candidateVector, + vectors.get(selectedItem), + ); + if (similarity > penalty) { + penalty = similarity; + similarItemId = stableVideoId(selectedItem); + } + } + } + + const mmrScore = + normalizedLambda * rawScore - (1 - normalizedLambda) * penalty; + + if (mmrScore > bestMmr) { + bestCandidate = candidate; + bestMmr = mmrScore; + bestRaw = rawScore; + bestSimilarity = penalty; + bestSimilarItemId = similarItemId; + continue; + } + + if (mmrScore === bestMmr) { + if (rawScore !== bestRaw) { + if (rawScore > bestRaw) { + bestCandidate = candidate; + bestRaw = rawScore; + bestSimilarity = penalty; + bestSimilarItemId = similarItemId; + } + continue; + } + + if (compareByTimestampId(candidate, bestCandidate) < 0) { + bestCandidate = candidate; + bestRaw = rawScore; + bestSimilarity = penalty; + bestSimilarItemId = similarItemId; + } + } + } + + if (!bestCandidate) { + break; + } + + selected.push(bestCandidate); + + if ( + bestRawCandidate && + bestRawCandidate !== bestCandidate && + bestSimilarity > 0 + ) { + const selectedId = stableVideoId(bestCandidate); + const competitorId = stableVideoId(bestRawCandidate); + if (selectedId && competitorId && selectedId !== competitorId) { + context?.addWhy?.({ + stage: stageName, + type: "diversity", + reason: "explore-diversity", + selectedId, + competitorIds: [competitorId], + similarity: bestSimilarity, + similarItemId: bestSimilarItemId || null, + }); + } + } + + const index = remaining.indexOf(bestCandidate); + if (index >= 0) { + remaining.splice(index, 1); + } + } + + const orderedRest = [...rest].sort(compareByTimestampId); + return selected.concat(orderedRest); + }; +}