diff --git a/js/app.js b/js/app.js index ad5ce4fc..59db1b8a 100644 --- a/js/app.js +++ b/js/app.js @@ -135,7 +135,7 @@ import resolveVideoPointer, { buildVideoAddressPointer, } from "./utils/videoPointer.js"; import { isValidMagnetUri } from "./utils/magnetValidators.js"; -import { dedupeToNewestByRoot } from "./utils/videoDeduper.js"; +import { dedupeToNewestByRoot, dedupeVideos } from "./utils/videoDeduper.js"; import { buildServiceWorkerFallbackStatus } from "./utils/serviceWorkerFallbackMessages.js"; import { batchFetchProfilesFromRelays } from "./utils/profileBatchFetcher.js"; import { @@ -633,6 +633,7 @@ class Application { fetchProfileMetadata, ensureProfileMetadataSubscription, dedupeToNewestByRoot, + dedupeVideos, buildServiceWorkerFallbackStatus, sanitizeProfileMediaUrl, UNSUPPORTED_BTITH_MESSAGE, diff --git a/js/app/playbackCoordinator.js b/js/app/playbackCoordinator.js index 78dbedab..e9f00649 100644 --- a/js/app/playbackCoordinator.js +++ b/js/app/playbackCoordinator.js @@ -56,6 +56,7 @@ export function createPlaybackCoordinator(deps) { fetchProfileMetadata, ensureProfileMetadataSubscription, dedupeToNewestByRoot, + dedupeVideos, buildServiceWorkerFallbackStatus, sanitizeProfileMediaUrl, UNSUPPORTED_BTITH_MESSAGE, @@ -216,7 +217,11 @@ export function createPlaybackCoordinator(deps) { if (!Array.isArray(videos) || videos.length === 0) { return []; } - return dedupeToNewestByRoot(videos); + // Root-dedupe AND collapse cross-ecosystem duplicates (prefer bitvid). This + // is the shared chokepoint for the feed stage + channel/search grids. + return typeof dedupeVideos === "function" + ? dedupeVideos(videos) + : dedupeToNewestByRoot(videos); }, autoplayModalVideo() { diff --git a/js/channelProfile.js b/js/channelProfile.js index a348d47b..c670a116 100644 --- a/js/channelProfile.js +++ b/js/channelProfile.js @@ -2,6 +2,7 @@ import { nostrClient } from "./nostrClientFacade.js"; import { convertChannelEvent, buildChannelVideoFilters } from "./channelProfileVideos.js"; +import { dedupeVideos } from "./utils/videoDeduper.js"; import { DEFAULT_RELAY_URLS } from "./nostr/toolkit.js"; import { subscriptions } from "./subscriptions.js"; import { attachHealthBadges } from "./gridHealth.js"; @@ -4512,7 +4513,7 @@ function buildRenderableChannelVideos({ events = [], app } = {}) { const newestByRoot = app?.dedupeVideosByRoot?.(uniqueVideos) ?? - dedupeToNewestByRoot(uniqueVideos); + dedupeVideos(uniqueVideos); let videos = newestByRoot.filter((video) => !video.deleted); videos = videos.filter((video) => { diff --git a/js/utils/videoDeduper.js b/js/utils/videoDeduper.js index bfbe8c05..928d820d 100644 --- a/js/utils/videoDeduper.js +++ b/js/utils/videoDeduper.js @@ -34,4 +34,125 @@ export function dedupeToNewestByRoot(videos) { return Array.from(map.values()); } +// Identity keys that mark two events as "the same video" across ecosystems +// (bitvid kind-30078 vs a NIP-71 mirror/original). Scoped per author. Content +// hashes + infohash are the reliable signals; `importedFrom` (Phase 2 on-board +// provenance) links an on-boarded bitvid note to the original foreign event via a +// shared "eid" namespace. Videos with none of these only ever match themselves. +function crossEcosystemIdentityKeys(video) { + const author = + typeof video?.pubkey === "string" ? video.pubkey.trim().toLowerCase() : ""; + if (!author) { + return []; + } + const keys = []; + const push = (ns, value) => { + const s = typeof value === "string" ? value.trim().toLowerCase() : ""; + if (s) { + keys.push(`${author}|${ns}|${s}`); + } + }; + push("sha", video.fileSha256); + push("ox", video.originalFileSha256); + push("ih", video.infoHash); + push("eid", video.id); + push("eid", video.importedFrom); + return keys; +} + +function isBitvidNative(video) { + return video?.source !== "nip71-ingest" && video?.foreign !== true; +} + +// Within a group of same-video events, prefer the bitvid-native (kind-30078) +// version; among the same kind, the newest (tie: higher id) for determinism. +function pickPreferredVideo(group) { + let best = null; + for (const video of group) { + if (!best) { + best = video; + continue; + } + const nativeV = isBitvidNative(video); + const nativeBest = isBitvidNative(best); + if (nativeV !== nativeBest) { + if (nativeV) best = video; + continue; + } + const tV = Number(video.created_at); + const tBest = Number(best.created_at); + const cV = Number.isFinite(tV) ? tV : -Infinity; + const cBest = Number.isFinite(tBest) ? tBest : -Infinity; + if (cV > cBest || (cV === cBest && String(video.id) > String(best.id))) { + best = video; + } + } + return best; +} + +/** + * Collapse duplicates of the same video that appear in BOTH ecosystems (a bitvid + * kind-30078 note and a NIP-71 mirror/original) so a grid never shows both. + * Always prefers the bitvid version. Same-ecosystem, distinct videos are + * untouched. Pure. + */ +export function collapseCrossEcosystem(videos) { + const list = Array.isArray(videos) ? videos.filter(Boolean) : []; + if (list.length < 2) { + return list.slice(); + } + + // Union-find: merge any videos that share an identity key. + const parent = list.map((_, i) => i); + const find = (i) => { + let root = i; + while (parent[root] !== root) root = parent[root]; + while (parent[i] !== root) { + const next = parent[i]; + parent[i] = root; + i = next; + } + return root; + }; + const union = (a, b) => { + const ra = find(a); + const rb = find(b); + if (ra !== rb) parent[ra] = rb; + }; + + const keyToIndex = new Map(); + list.forEach((video, i) => { + for (const key of crossEcosystemIdentityKeys(video)) { + if (keyToIndex.has(key)) { + union(keyToIndex.get(key), i); + } else { + keyToIndex.set(key, i); + } + } + }); + + const groups = new Map(); + list.forEach((video, i) => { + const root = find(i); + const group = groups.get(root); + if (group) group.push(video); + else groups.set(root, [video]); + }); + + const result = []; + for (const group of groups.values()) { + result.push(group.length === 1 ? group[0] : pickPreferredVideo(group)); + } + return result; +} + +/** + * The shared grid dedupe: collapse to newest-per-root, THEN collapse + * cross-ecosystem duplicates (prefer bitvid). Use this everywhere a video list is + * built so every grid behaves identically. + */ +export function dedupeVideos(videos) { + return collapseCrossEcosystem(dedupeToNewestByRoot(videos)); +} + export default dedupeToNewestByRoot; diff --git a/tests/cross-ecosystem-dedupe.test.mjs b/tests/cross-ecosystem-dedupe.test.mjs new file mode 100644 index 00000000..25df7501 --- /dev/null +++ b/tests/cross-ecosystem-dedupe.test.mjs @@ -0,0 +1,111 @@ +// Cross-ecosystem dedupe: when the same video exists as a bitvid kind-30078 note +// AND a NIP-71 mirror/original, show exactly one — always the bitvid version. +// This is the shared pass behind every grid (feed, channel, search, ...). + +import assert from "node:assert/strict"; +import test from "node:test"; +import { + collapseCrossEcosystem, + dedupeVideos, +} from "../js/utils/videoDeduper.js"; + +const PK = "a".repeat(64); +const SHA = "f".repeat(64); + +const bitvid = (over = {}) => ({ + id: "bitvid-id", + pubkey: PK, + videoRootId: "bitvid-root", + created_at: 100, + // native: no nip71-ingest source + ...over, +}); +const foreign = (over = {}) => ({ + id: "foreign-id", + pubkey: PK, + videoRootId: "foreign-root", + created_at: 200, + source: "nip71-ingest", + foreign: true, + ...over, +}); + +test("same file hash → keeps the bitvid version, drops the foreign one", () => { + const out = collapseCrossEcosystem([ + foreign({ fileSha256: SHA }), + bitvid({ fileSha256: SHA }), + ]); + assert.equal(out.length, 1); + assert.equal(out[0].id, "bitvid-id"); +}); + +test("prefers bitvid even when the foreign version is NEWER", () => { + const out = collapseCrossEcosystem([ + bitvid({ fileSha256: SHA, created_at: 1 }), + foreign({ fileSha256: SHA, created_at: 9999 }), + ]); + assert.equal(out.length, 1); + assert.equal(out[0].id, "bitvid-id"); +}); + +test("matches on infohash too", () => { + const ih = "1".repeat(40); + const out = collapseCrossEcosystem([ + foreign({ infoHash: ih }), + bitvid({ infoHash: ih }), + ]); + assert.deepEqual(out.map((v) => v.id), ["bitvid-id"]); +}); + +test("matches via on-board provenance link (importedFrom -> foreign event id)", () => { + const out = collapseCrossEcosystem([ + foreign({ id: "orig-evt" }), + bitvid({ id: "imported-note", importedFrom: "orig-evt" }), + ]); + assert.equal(out.length, 1); + assert.equal(out[0].id, "imported-note"); +}); + +test("distinct videos (no shared identity) are NOT merged", () => { + const out = collapseCrossEcosystem([ + bitvid({ id: "v1", fileSha256: "1".repeat(64) }), + foreign({ id: "v2", fileSha256: "2".repeat(64) }), + ]); + assert.equal(out.length, 2); +}); + +test("hash-less videos are not merged (documented limitation)", () => { + const out = collapseCrossEcosystem([ + bitvid({ id: "v1" }), + foreign({ id: "v2" }), + ]); + assert.equal(out.length, 2); +}); + +test("two foreign versions of one file collapse to the newest (no bitvid present)", () => { + const out = collapseCrossEcosystem([ + foreign({ id: "old", fileSha256: SHA, created_at: 100 }), + foreign({ id: "new", fileSha256: SHA, created_at: 200 }), + ]); + assert.equal(out.length, 1); + assert.equal(out[0].id, "new"); +}); + +test("dedupeVideos collapses by root THEN across ecosystems", () => { + // Two bitvid events same root (older + newer) + a foreign mirror by hash. + const out = dedupeVideos([ + bitvid({ id: "old", videoRootId: "r", created_at: 1, fileSha256: SHA }), + bitvid({ id: "new", videoRootId: "r", created_at: 2, fileSha256: SHA }), + foreign({ id: "mirror", fileSha256: SHA, created_at: 3 }), + ]); + assert.equal(out.length, 1); + assert.equal(out[0].id, "new", "newest bitvid per root wins, foreign mirror dropped"); +}); + +test("different authors with the same hash are NOT merged (identity is per-author)", () => { + const out = collapseCrossEcosystem([ + bitvid({ id: "mine", pubkey: PK, fileSha256: SHA }), + foreign({ id: "theirs", pubkey: "b".repeat(64), fileSha256: SHA }), + ]); + assert.equal(out.length, 2); +}); diff --git a/todo/TODO_2026-06-20_pre-launch.md b/todo/TODO_2026-06-20_pre-launch.md index 1a24eb57..5dbff124 100644 --- a/todo/TODO_2026-06-20_pre-launch.md +++ b/todo/TODO_2026-06-20_pre-launch.md @@ -322,12 +322,14 @@ Make NIP-71 interop two-way and guarantee a video is never shown twice. creator-initiated self-import (model A — can't forge others' notes), dedup on explicit import-link + content hash/infohash (no fuzzy url/title), import grants CDN url + WebTorrent (bitvid extras come free), one-shot import + re-run button. -- [ ] **Phase 1 (do first): cross-ecosystem dedup** — `collapseCrossEcosystem` - keyed on `author + (importLink ‖ fileSha256 ‖ ox ‖ infoHash)`, prefer the - bitvid (kind-30078) version, applied at the shared dedupe chokepoint so - EVERY grid (feed, channel profile, My Videos, search, playlists, watch - history) collapses dupes. Pure + mutation-tested. Needed regardless of - on-boarding (independent dual-posts already double up). +- [x] **Phase 1: cross-ecosystem dedup** (`js/utils/videoDeduper.js`): + `collapseCrossEcosystem` keyed per-author on `fileSha256/ox/infoHash` + an + `eid` namespace linking on-board provenance (`importedFrom`); prefers the + bitvid (kind-30078) version. Composed into `dedupeVideos` (root-dedup THEN + cross-ecosystem) and routed through the shared chokepoint + `app.dedupeVideosByRoot` → covers the feed stage + channel + search at once. + Pure + mutation-verified. (Hash-less dual-posts still can't dedup — documented + limitation; covered once on-boarded videos carry the provenance link.) - [ ] **Phase 2: creator-initiated import** — discover the logged-in creator's own NIP-71 videos lacking a bitvid version; offer FULL import (re-host to THEIR storage + optional WebTorrent) or REFERENCE import (keep external URL, no