videos: cross-ecosystem dedup (prefer bitvid, show one) across all grids

Phase 1 of the on-boarding plan, and the shared layer for unified grid
loading. A video that exists as both a bitvid kind-30078 note and a NIP-71
mirror/original previously could appear twice (e.g. independent dual-posts).

js/utils/videoDeduper.js: collapseCrossEcosystem() groups events by a
per-author identity (fileSha256 / originalFileSha256 / infoHash, plus an
`eid` namespace that links an on-boarded note's `importedFrom` to the
original foreign event) via union-find, and keeps the bitvid-native version
(else newest). dedupeVideos() = root-dedup THEN cross-ecosystem collapse.

Routed through the shared chokepoint app.dedupeVideosByRoot (playback
coordinator), so the feed stage, channel profile, and search all collapse
duplicates identically — "show one, prefer bitvid" everywhere. Channel
fallback path updated too.

Limitation (documented): hash-less independent dual-posts can't be matched
until the bitvid side carries on-board provenance (Phase 2).

Tests + mutation-verified (prefer-bitvid, per-author identity, infohash,
provenance link, root+cross composition). 1262/1263 unit (known flake).

Co-Authored-By: Claude Opus 4.8 <[email protected]>

js/app.js

@@ -135,7 +135,7 @@ import resolveVideoPointer, {
   buildVideoAddressPointer,
 } from "./utils/videoPointer.js";
 import { isValidMagnetUri } from "./utils/magnetValidators.js";
-import { dedupeToNewestByRoot } from "./utils/videoDeduper.js";
+import { dedupeToNewestByRoot, dedupeVideos } from "./utils/videoDeduper.js";
 import { buildServiceWorkerFallbackStatus } from "./utils/serviceWorkerFallbackMessages.js";
 import { batchFetchProfilesFromRelays } from "./utils/profileBatchFetcher.js";
 import {
@@ -633,6 +633,7 @@ class Application {
         fetchProfileMetadata,
         ensureProfileMetadataSubscription,
         dedupeToNewestByRoot,
+        dedupeVideos,
         buildServiceWorkerFallbackStatus,
         sanitizeProfileMediaUrl,
         UNSUPPORTED_BTITH_MESSAGE,

js/app/playbackCoordinator.js

@@ -56,6 +56,7 @@ export function createPlaybackCoordinator(deps) {
     fetchProfileMetadata,
     ensureProfileMetadataSubscription,
     dedupeToNewestByRoot,
+    dedupeVideos,
     buildServiceWorkerFallbackStatus,
     sanitizeProfileMediaUrl,
     UNSUPPORTED_BTITH_MESSAGE,
@@ -216,7 +217,11 @@ export function createPlaybackCoordinator(deps) {
       if (!Array.isArray(videos) || videos.length === 0) {
         return [];
       }
-      return dedupeToNewestByRoot(videos);
+      // Root-dedupe AND collapse cross-ecosystem duplicates (prefer bitvid). This
+      // is the shared chokepoint for the feed stage + channel/search grids.
+      return typeof dedupeVideos === "function"
+        ? dedupeVideos(videos)
+        : dedupeToNewestByRoot(videos);
     },
 
     autoplayModalVideo() {

js/channelProfile.js

@@ -2,6 +2,7 @@
 
 import { nostrClient } from "./nostrClientFacade.js";
 import { convertChannelEvent, buildChannelVideoFilters } from "./channelProfileVideos.js";
+import { dedupeVideos } from "./utils/videoDeduper.js";
 import { DEFAULT_RELAY_URLS } from "./nostr/toolkit.js";
 import { subscriptions } from "./subscriptions.js";
 import { attachHealthBadges } from "./gridHealth.js";
@@ -4512,7 +4513,7 @@ function buildRenderableChannelVideos({ events = [], app } = {}) {
 
   const newestByRoot =
     app?.dedupeVideosByRoot?.(uniqueVideos) ??
-    dedupeToNewestByRoot(uniqueVideos);
+    dedupeVideos(uniqueVideos);
 
   let videos = newestByRoot.filter((video) => !video.deleted);
   videos = videos.filter((video) => {

js/utils/videoDeduper.js

@@ -34,4 +34,125 @@ export function dedupeToNewestByRoot(videos) {
   return Array.from(map.values());
 }
 
+// Identity keys that mark two events as "the same video" across ecosystems
+// (bitvid kind-30078 vs a NIP-71 mirror/original). Scoped per author. Content
+// hashes + infohash are the reliable signals; `importedFrom` (Phase 2 on-board
+// provenance) links an on-boarded bitvid note to the original foreign event via a
+// shared "eid" namespace. Videos with none of these only ever match themselves.
+function crossEcosystemIdentityKeys(video) {
+  const author =
+    typeof video?.pubkey === "string" ? video.pubkey.trim().toLowerCase() : "";
+  if (!author) {
+    return [];
+  }
+  const keys = [];
+  const push = (ns, value) => {
+    const s = typeof value === "string" ? value.trim().toLowerCase() : "";
+    if (s) {
+      keys.push(`${author}|${ns}|${s}`);
+    }
+  };
+  push("sha", video.fileSha256);
+  push("ox", video.originalFileSha256);
+  push("ih", video.infoHash);
+  push("eid", video.id);
+  push("eid", video.importedFrom);
+  return keys;
+}
+
+function isBitvidNative(video) {
+  return video?.source !== "nip71-ingest" && video?.foreign !== true;
+}
+
+// Within a group of same-video events, prefer the bitvid-native (kind-30078)
+// version; among the same kind, the newest (tie: higher id) for determinism.
+function pickPreferredVideo(group) {
+  let best = null;
+  for (const video of group) {
+    if (!best) {
+      best = video;
+      continue;
+    }
+    const nativeV = isBitvidNative(video);
+    const nativeBest = isBitvidNative(best);
+    if (nativeV !== nativeBest) {
+      if (nativeV) best = video;
+      continue;
+    }
+    const tV = Number(video.created_at);
+    const tBest = Number(best.created_at);
+    const cV = Number.isFinite(tV) ? tV : -Infinity;
+    const cBest = Number.isFinite(tBest) ? tBest : -Infinity;
+    if (cV > cBest || (cV === cBest && String(video.id) > String(best.id))) {
+      best = video;
+    }
+  }
+  return best;
+}
+
+/**
+ * Collapse duplicates of the same video that appear in BOTH ecosystems (a bitvid
+ * kind-30078 note and a NIP-71 mirror/original) so a grid never shows both.
+ * Always prefers the bitvid version. Same-ecosystem, distinct videos are
+ * untouched. Pure.
+ */
+export function collapseCrossEcosystem(videos) {
+  const list = Array.isArray(videos) ? videos.filter(Boolean) : [];
+  if (list.length < 2) {
+    return list.slice();
+  }
+
+  // Union-find: merge any videos that share an identity key.
+  const parent = list.map((_, i) => i);
+  const find = (i) => {
+    let root = i;
+    while (parent[root] !== root) root = parent[root];
+    while (parent[i] !== root) {
+      const next = parent[i];
+      parent[i] = root;
+      i = next;
+    }
+    return root;
+  };
+  const union = (a, b) => {
+    const ra = find(a);
+    const rb = find(b);
+    if (ra !== rb) parent[ra] = rb;
+  };
+
+  const keyToIndex = new Map();
+  list.forEach((video, i) => {
+    for (const key of crossEcosystemIdentityKeys(video)) {
+      if (keyToIndex.has(key)) {
+        union(keyToIndex.get(key), i);
+      } else {
+        keyToIndex.set(key, i);
+      }
+    }
+  });
+
+  const groups = new Map();
+  list.forEach((video, i) => {
+    const root = find(i);
+    const group = groups.get(root);
+    if (group) group.push(video);
+    else groups.set(root, [video]);
+  });
+
+  const result = [];
+  for (const group of groups.values()) {
+    result.push(group.length === 1 ? group[0] : pickPreferredVideo(group));
+  }
+  return result;
+}
+
+/**
+ * The shared grid dedupe: collapse to newest-per-root, THEN collapse
+ * cross-ecosystem duplicates (prefer bitvid). Use this everywhere a video list is
+ * built so every grid behaves identically.
+ */
+export function dedupeVideos(videos) {
+  return collapseCrossEcosystem(dedupeToNewestByRoot(videos));
+}
+
 export default dedupeToNewestByRoot;

tests/cross-ecosystem-dedupe.test.mjs

@@ -0,0 +1,111 @@
+// Cross-ecosystem dedupe: when the same video exists as a bitvid kind-30078 note
+// AND a NIP-71 mirror/original, show exactly one — always the bitvid version.
+// This is the shared pass behind every grid (feed, channel, search, ...).
+
+import assert from "node:assert/strict";
+import test from "node:test";
+import {
+  collapseCrossEcosystem,
+  dedupeVideos,
+} from "../js/utils/videoDeduper.js";
+
+const PK = "a".repeat(64);
+const SHA = "f".repeat(64);
+
+const bitvid = (over = {}) => ({
+  id: "bitvid-id",
+  pubkey: PK,
+  videoRootId: "bitvid-root",
+  created_at: 100,
+  // native: no nip71-ingest source
+  ...over,
+});
+const foreign = (over = {}) => ({
+  id: "foreign-id",
+  pubkey: PK,
+  videoRootId: "foreign-root",
+  created_at: 200,
+  source: "nip71-ingest",
+  foreign: true,
+  ...over,
+});
+
+test("same file hash → keeps the bitvid version, drops the foreign one", () => {
+  const out = collapseCrossEcosystem([
+    foreign({ fileSha256: SHA }),
+    bitvid({ fileSha256: SHA }),
+  ]);
+  assert.equal(out.length, 1);
+  assert.equal(out[0].id, "bitvid-id");
+});
+
+test("prefers bitvid even when the foreign version is NEWER", () => {
+  const out = collapseCrossEcosystem([
+    bitvid({ fileSha256: SHA, created_at: 1 }),
+    foreign({ fileSha256: SHA, created_at: 9999 }),
+  ]);
+  assert.equal(out.length, 1);
+  assert.equal(out[0].id, "bitvid-id");
+});
+
+test("matches on infohash too", () => {
+  const ih = "1".repeat(40);
+  const out = collapseCrossEcosystem([
+    foreign({ infoHash: ih }),
+    bitvid({ infoHash: ih }),
+  ]);
+  assert.deepEqual(out.map((v) => v.id), ["bitvid-id"]);
+});
+
+test("matches via on-board provenance link (importedFrom -> foreign event id)", () => {
+  const out = collapseCrossEcosystem([
+    foreign({ id: "orig-evt" }),
+    bitvid({ id: "imported-note", importedFrom: "orig-evt" }),
+  ]);
+  assert.equal(out.length, 1);
+  assert.equal(out[0].id, "imported-note");
+});
+
+test("distinct videos (no shared identity) are NOT merged", () => {
+  const out = collapseCrossEcosystem([
+    bitvid({ id: "v1", fileSha256: "1".repeat(64) }),
+    foreign({ id: "v2", fileSha256: "2".repeat(64) }),
+  ]);
+  assert.equal(out.length, 2);
+});
+
+test("hash-less videos are not merged (documented limitation)", () => {
+  const out = collapseCrossEcosystem([
+    bitvid({ id: "v1" }),
+    foreign({ id: "v2" }),
+  ]);
+  assert.equal(out.length, 2);
+});
+
+test("two foreign versions of one file collapse to the newest (no bitvid present)", () => {
+  const out = collapseCrossEcosystem([
+    foreign({ id: "old", fileSha256: SHA, created_at: 100 }),
+    foreign({ id: "new", fileSha256: SHA, created_at: 200 }),
+  ]);
+  assert.equal(out.length, 1);
+  assert.equal(out[0].id, "new");
+});
+
+test("dedupeVideos collapses by root THEN across ecosystems", () => {
+  // Two bitvid events same root (older + newer) + a foreign mirror by hash.
+  const out = dedupeVideos([
+    bitvid({ id: "old", videoRootId: "r", created_at: 1, fileSha256: SHA }),
+    bitvid({ id: "new", videoRootId: "r", created_at: 2, fileSha256: SHA }),
+    foreign({ id: "mirror", fileSha256: SHA, created_at: 3 }),
+  ]);
+  assert.equal(out.length, 1);
+  assert.equal(out[0].id, "new", "newest bitvid per root wins, foreign mirror dropped");
+});
+
+test("different authors with the same hash are NOT merged (identity is per-author)", () => {
+  const out = collapseCrossEcosystem([
+    bitvid({ id: "mine", pubkey: PK, fileSha256: SHA }),
+    foreign({ id: "theirs", pubkey: "b".repeat(64), fileSha256: SHA }),
+  ]);
+  assert.equal(out.length, 2);
+});

todo/TODO_2026-06-20_pre-launch.md

@@ -322,12 +322,14 @@ Make NIP-71 interop two-way and guarantee a video is never shown twice.
 creator-initiated self-import (model A — can't forge others' notes), dedup on
 explicit import-link + content hash/infohash (no fuzzy url/title), import grants
 CDN url + WebTorrent (bitvid extras come free), one-shot import + re-run button.
-- [ ] **Phase 1 (do first): cross-ecosystem dedup** — `collapseCrossEcosystem`
+- [x] **Phase 1: cross-ecosystem dedup** (`js/utils/videoDeduper.js`):
-      keyed on `author + (importLink ‖ fileSha256 ‖ ox ‖ infoHash)`, prefer the
+      `collapseCrossEcosystem` keyed per-author on `fileSha256/ox/infoHash` + an
-      bitvid (kind-30078) version, applied at the shared dedupe chokepoint so
+      `eid` namespace linking on-board provenance (`importedFrom`); prefers the
-      EVERY grid (feed, channel profile, My Videos, search, playlists, watch
+      bitvid (kind-30078) version. Composed into `dedupeVideos` (root-dedup THEN
-      history) collapses dupes. Pure + mutation-tested. Needed regardless of
+      cross-ecosystem) and routed through the shared chokepoint
-      on-boarding (independent dual-posts already double up).
+      `app.dedupeVideosByRoot` → covers the feed stage + channel + search at once.
+      Pure + mutation-verified. (Hash-less dual-posts still can't dedup — documented
+      limitation; covered once on-boarded videos carry the provenance link.)
 - [ ] **Phase 2: creator-initiated import** — discover the logged-in creator's own
       NIP-71 videos lacking a bitvid version; offer FULL import (re-host to THEIR
       storage + optional WebTorrent) or REFERENCE import (keep external URL, no