mirror of
https://github.com/PR0M3TH3AN/bitvid.git
synced 2026-11-05 20:32:27 +00:00
nip71: inbound ingest service (whitelist-scoped, default on)
Pulls NIP-71 video events published by other Nostr apps (kinds 21/22/34235/34236), converts them via the ingest adapter, and injects them into nostrClient.activeMap/allEvents so they flow through the existing feed pipeline. Gated by FEATURE_NIP71_INGEST. Mirrors bitvid's whitelist model: since the whitelist is "authors allowed to publish/render here," ingest scopes its relay subscription to whitelisted pubkeys while whitelist mode is on (bounded, no firehose) and re-subscribes when the whitelist changes. When a deployer disables whitelist mode, ingest opens to all authors (capped by a limit). Foreign content is still gated by the same render-time filter (shouldIncludeVideo -> accessControl.canAccess): whitelist, blacklist, NSFW (content-warning -> isNsfw), private-skip, plus trust/mute/blur from the feed engine. Native bitvid videos always win a shared root; among ingested videos newest wins. Refreshes via the existing videos:updated reactive path. - accessControl.getWhitelistPubkeys(): hex whitelist for subscription scope. - Wired in applicationBootstrap (start on init, stop on teardown). - nostrService untouched (at its size cap); logic is a standalone service. Mutation-verified: native-wins, whitelist scoping, feature gate. Co-Authored-By: Claude Opus 4.8 <[email protected]>
This commit is contained in:
thePR0M3TH3ANandClaude Opus 4.8 committed
1 parent
286621dd4a
commit
c849969312
4 files changed
+498
No files matched your search
@@ -492,6 +492,12 @@ class AccessControl {
|
||||
return Array.from(this.whitelist);
|
||||
}
|
||||
|
||||
// Hex-encoded whitelist pubkeys (resolved from npubs). Used by NIP-71 ingest
|
||||
// to scope its relay subscription to authors this instance already allows.
|
||||
getWhitelistPubkeys() {
|
||||
return Array.from(this.whitelistPubkeys);
|
||||
}
|
||||
|
||||
getBlacklist() {
|
||||
return Array.from(this.blacklist);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
// Inbound NIP-71 ingest: subscribes to video events published by *other* Nostr
|
||||
// apps (kinds 21/22/34235/34236), converts them to bitvid video objects, and
|
||||
// injects them into the active-video store so they flow through the existing
|
||||
// feed pipeline (whitelist, blacklist, NSFW gate, trust/mute/blur, rendering).
|
||||
//
|
||||
// Design — ingest mirrors the whitelist model: bitvid's whitelist is "authors
|
||||
// allowed to publish/render here," so ingest pulls those same authors' NIP-71
|
||||
// content from the wider ecosystem. When whitelist mode is ON, the relay
|
||||
// subscription is scoped to whitelisted pubkeys (bounded, no firehose); when a
|
||||
// deployer disables whitelist mode, ingest opens to all authors (capped).
|
||||
//
|
||||
// All gating is reused, not re-implemented: injected videos carry
|
||||
// source:"nip71-ingest" + isNsfw (mapped from content-warning), and the
|
||||
// render-time filter (nostrService.shouldIncludeVideo -> accessControl.canAccess)
|
||||
// remains the authority. Gated by FEATURE_NIP71_INGEST so a single admin switch
|
||||
// disables it entirely.
|
||||
|
||||
import { FEATURE_NIP71_INGEST } from "../constants.js";
|
||||
import {
|
||||
buildVideoFromNip71Event,
|
||||
NIP71_KINDS,
|
||||
} from "../nostr/nip71IngestAdapter.js";
|
||||
import { devLogger } from "../utils/logger.js";
|
||||
|
||||
const FLUSH_DEBOUNCE_MS = 200;
|
||||
const DEFAULT_LIMIT = 200;
|
||||
const SUBSCRIPTION_KEY = "nip71-ingest";
|
||||
|
||||
export function createNip71IngestService({
|
||||
nostrClient,
|
||||
nostrService,
|
||||
accessControl,
|
||||
featureEnabled = FEATURE_NIP71_INGEST,
|
||||
logger = devLogger,
|
||||
flushDelayMs = FLUSH_DEBOUNCE_MS,
|
||||
limit = DEFAULT_LIMIT,
|
||||
} = {}) {
|
||||
let subscription = null;
|
||||
let buffer = [];
|
||||
let flushTimer = null;
|
||||
let offWhitelistChange = null;
|
||||
let started = false;
|
||||
|
||||
function isAvailable() {
|
||||
return (
|
||||
featureEnabled === true &&
|
||||
!!nostrClient &&
|
||||
typeof nostrClient.getSubscriptionManager === "function"
|
||||
);
|
||||
}
|
||||
|
||||
// Build the relay filter set. Returns null when there is nothing to ingest
|
||||
// (whitelist mode on but no whitelisted authors yet).
|
||||
function resolveFilters() {
|
||||
const kinds = Array.from(NIP71_KINDS);
|
||||
const whitelistOn = accessControl?.whitelistMode?.() === true;
|
||||
if (whitelistOn) {
|
||||
const authors =
|
||||
typeof accessControl?.getWhitelistPubkeys === "function"
|
||||
? accessControl.getWhitelistPubkeys().filter(Boolean)
|
||||
: [];
|
||||
if (!authors.length) {
|
||||
return null;
|
||||
}
|
||||
return [{ kinds, authors, limit }];
|
||||
}
|
||||
return [{ kinds, limit }];
|
||||
}
|
||||
|
||||
function scheduleFlush() {
|
||||
if (flushTimer) {
|
||||
return;
|
||||
}
|
||||
flushTimer = setTimeout(() => {
|
||||
flushTimer = null;
|
||||
flush();
|
||||
}, flushDelayMs);
|
||||
}
|
||||
|
||||
// Inject a converted foreign video into the active store. Native bitvid videos
|
||||
// always win for a given root; among ingested videos, newest wins.
|
||||
function injectVideo(video) {
|
||||
if (!video || video.invalid) {
|
||||
return false;
|
||||
}
|
||||
const activeMap = nostrClient?.activeMap;
|
||||
const allEvents = nostrClient?.allEvents;
|
||||
if (!(activeMap instanceof Map) || !(allEvents instanceof Map)) {
|
||||
return false;
|
||||
}
|
||||
const key =
|
||||
typeof nostrClient.getActiveKey === "function"
|
||||
? nostrClient.getActiveKey(video)
|
||||
: "";
|
||||
if (!key) {
|
||||
return false;
|
||||
}
|
||||
const existing = activeMap.get(key);
|
||||
if (existing) {
|
||||
const existingForeign = existing.source === "nip71-ingest";
|
||||
if (!existingForeign) {
|
||||
return false; // native bitvid video wins
|
||||
}
|
||||
if (Number(existing.created_at) >= Number(video.created_at)) {
|
||||
return false; // older or equal ingest
|
||||
}
|
||||
}
|
||||
activeMap.set(key, video);
|
||||
allEvents.set(video.id, video);
|
||||
return true;
|
||||
}
|
||||
|
||||
function flush() {
|
||||
if (!buffer.length) {
|
||||
return 0;
|
||||
}
|
||||
const events = buffer;
|
||||
buffer = [];
|
||||
|
||||
// Collapse to newest-per-root within the batch before injecting.
|
||||
const newestByRoot = new Map();
|
||||
for (const event of events) {
|
||||
const video = buildVideoFromNip71Event(event);
|
||||
if (video.invalid || !video.videoRootId) {
|
||||
continue;
|
||||
}
|
||||
const prev = newestByRoot.get(video.videoRootId);
|
||||
if (!prev || Number(video.created_at) > Number(prev.created_at)) {
|
||||
newestByRoot.set(video.videoRootId, video);
|
||||
}
|
||||
}
|
||||
|
||||
let injected = 0;
|
||||
for (const video of newestByRoot.values()) {
|
||||
if (injectVideo(video)) {
|
||||
injected += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (injected > 0) {
|
||||
// Re-render through the existing reactive path. The payload is the
|
||||
// post-filter active list (whitelist/NSFW/etc. already applied); the feed
|
||||
// re-reads getFilteredActiveVideos itself, so this just signals "refresh".
|
||||
let videos = [];
|
||||
try {
|
||||
videos =
|
||||
typeof nostrService?.getFilteredActiveVideos === "function"
|
||||
? nostrService.getFilteredActiveVideos()
|
||||
: [];
|
||||
} catch (error) {
|
||||
logger?.warn?.("[nip71Ingest] getFilteredActiveVideos failed", error);
|
||||
}
|
||||
if (typeof nostrService?.emit === "function") {
|
||||
nostrService.emit("videos:updated", {
|
||||
videos,
|
||||
deleted: [],
|
||||
reason: "nip71-ingest",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return injected;
|
||||
}
|
||||
|
||||
function start() {
|
||||
if (started || !isAvailable()) {
|
||||
return false;
|
||||
}
|
||||
started = true;
|
||||
|
||||
// Re-subscribe when the whitelist changes so the author scope stays current.
|
||||
if (typeof accessControl?.onWhitelistChange === "function") {
|
||||
offWhitelistChange = accessControl.onWhitelistChange(() => {
|
||||
restart();
|
||||
});
|
||||
}
|
||||
|
||||
return openSubscription();
|
||||
}
|
||||
|
||||
function openSubscription() {
|
||||
const filters = resolveFilters();
|
||||
if (!filters) {
|
||||
return false; // nothing to ingest (e.g. empty whitelist) — stay armed for re-subscribe
|
||||
}
|
||||
try {
|
||||
const manager = nostrClient.getSubscriptionManager();
|
||||
subscription = manager.subscribe({
|
||||
key: SUBSCRIPTION_KEY,
|
||||
filters,
|
||||
label: "nip71-ingest",
|
||||
onEvent: (event) => {
|
||||
if (event && NIP71_KINDS.has(Number(event.kind))) {
|
||||
buffer.push(event);
|
||||
scheduleFlush();
|
||||
}
|
||||
},
|
||||
});
|
||||
return true;
|
||||
} catch (error) {
|
||||
logger?.warn?.("[nip71Ingest] subscribe failed", error);
|
||||
subscription = null;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function closeSubscription() {
|
||||
if (flushTimer) {
|
||||
clearTimeout(flushTimer);
|
||||
flushTimer = null;
|
||||
}
|
||||
buffer = [];
|
||||
if (subscription && typeof subscription.close === "function") {
|
||||
try {
|
||||
subscription.close();
|
||||
} catch (error) {
|
||||
logger?.warn?.("[nip71Ingest] close failed", error);
|
||||
}
|
||||
}
|
||||
subscription = null;
|
||||
}
|
||||
|
||||
function restart() {
|
||||
if (!started) {
|
||||
return false;
|
||||
}
|
||||
closeSubscription();
|
||||
return openSubscription();
|
||||
}
|
||||
|
||||
function stop() {
|
||||
closeSubscription();
|
||||
if (typeof offWhitelistChange === "function") {
|
||||
try {
|
||||
offWhitelistChange();
|
||||
} catch (error) {
|
||||
// best-effort
|
||||
}
|
||||
offWhitelistChange = null;
|
||||
}
|
||||
started = false;
|
||||
}
|
||||
|
||||
return {
|
||||
isAvailable,
|
||||
start,
|
||||
stop,
|
||||
restart,
|
||||
// Exposed for tests: deterministic flush + single-video injection.
|
||||
flush,
|
||||
injectVideo,
|
||||
resolveFilters,
|
||||
};
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import NwcSettingsService from "../services/nwcSettingsService.js";
|
||||
import nostrService from "../services/nostrService.js";
|
||||
import storageService from "../services/storageService.js";
|
||||
import { initNip71MirrorSync } from "../services/nip71MirrorSync.js";
|
||||
import { createNip71IngestService } from "../services/nip71IngestService.js";
|
||||
import watchHistoryService from "../watchHistoryService.js";
|
||||
import r2Service from "../services/r2Service.js";
|
||||
import s3UploadService from "../services/s3UploadService.js";
|
||||
@@ -853,6 +854,24 @@ export default class ApplicationBootstrap {
|
||||
// Keep an opted-in NIP-71 mirror in lockstep on edit/delete (event-driven so
|
||||
// nostrService stays untouched).
|
||||
nostrUnsubscribes.push(initNip71MirrorSync(app.nostrService));
|
||||
|
||||
// Inbound NIP-71 ingest: pull video events published by other Nostr apps
|
||||
// (scoped to whitelisted authors while whitelist mode is on) and surface
|
||||
// them in the feed. Gated by FEATURE_NIP71_INGEST. Best-effort.
|
||||
const nip71IngestService = createNip71IngestService({
|
||||
nostrClient,
|
||||
nostrService: app.nostrService,
|
||||
accessControl,
|
||||
});
|
||||
app.nip71IngestService = nip71IngestService;
|
||||
if (nip71IngestService.isAvailable()) {
|
||||
try {
|
||||
nip71IngestService.start();
|
||||
} catch (error) {
|
||||
devLogger.warn("[applicationBootstrap] NIP-71 ingest start failed", error);
|
||||
}
|
||||
nostrUnsubscribes.push(() => nip71IngestService.stop());
|
||||
}
|
||||
nostrUnsubscribes.push(
|
||||
app.nostrService.on("directMessages:notification", () => {
|
||||
void app.refreshUnreadDmIndicator({ reason: "dm-notification" });
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
// Inbound NIP-71 ingest service. Scenario-style with a virtualized nostrClient
|
||||
// (in-memory activeMap/allEvents + a fake subscription manager) and a fake
|
||||
// accessControl. Asserts observable outcomes: foreign videos land in the active
|
||||
// store and trigger a refresh; native videos are never clobbered; the relay
|
||||
// subscription is scoped to the whitelist; and FEATURE_NIP71_INGEST off makes
|
||||
// the service inert.
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { createNip71IngestService } from "../js/services/nip71IngestService.js";
|
||||
|
||||
const KINDS = [21, 22, 34235, 34236];
|
||||
|
||||
function activeKey(video) {
|
||||
return video.videoRootId ? `ROOT:${video.videoRootId}` : `LEGACY:${video.id}`;
|
||||
}
|
||||
|
||||
function makeHarness({ whitelistOn = false, whitelist = [] } = {}) {
|
||||
const activeMap = new Map();
|
||||
const allEvents = new Map();
|
||||
let capturedFilters = null;
|
||||
let onEvent = null;
|
||||
let closed = false;
|
||||
let whitelistListener = null;
|
||||
|
||||
const manager = {
|
||||
subscribe(opts) {
|
||||
capturedFilters = opts.filters;
|
||||
onEvent = opts.onEvent;
|
||||
return {
|
||||
close() {
|
||||
closed = true;
|
||||
},
|
||||
update() {},
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
const emitted = [];
|
||||
const nostrClient = {
|
||||
activeMap,
|
||||
allEvents,
|
||||
getActiveKey: activeKey,
|
||||
getSubscriptionManager: () => manager,
|
||||
};
|
||||
const nostrService = {
|
||||
emit(name, detail) {
|
||||
emitted.push({ name, detail });
|
||||
},
|
||||
getFilteredActiveVideos: () => Array.from(activeMap.values()),
|
||||
};
|
||||
const accessControl = {
|
||||
whitelistMode: () => whitelistOn,
|
||||
getWhitelistPubkeys: () => whitelist,
|
||||
onWhitelistChange: (fn) => {
|
||||
whitelistListener = fn;
|
||||
return () => {
|
||||
whitelistListener = null;
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
return {
|
||||
activeMap,
|
||||
allEvents,
|
||||
emitted,
|
||||
get capturedFilters() {
|
||||
return capturedFilters;
|
||||
},
|
||||
get closed() {
|
||||
return closed;
|
||||
},
|
||||
fireEvent: (e) => onEvent?.(e),
|
||||
triggerWhitelistChange: () => whitelistListener?.(),
|
||||
nostrClient,
|
||||
nostrService,
|
||||
accessControl,
|
||||
};
|
||||
}
|
||||
|
||||
function foreignEvent(overrides = {}) {
|
||||
return {
|
||||
id: overrides.id || "evt-1",
|
||||
pubkey: overrides.pubkey || "author-1",
|
||||
kind: overrides.kind || 21,
|
||||
created_at: overrides.created_at ?? 1000,
|
||||
content: "desc",
|
||||
tags: overrides.tags || [
|
||||
["title", overrides.title || "Foreign"],
|
||||
["imeta", `url ${overrides.url || "https://e/v.mp4"}`, "m video/mp4"],
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function makeService(harness, extra = {}) {
|
||||
return createNip71IngestService({
|
||||
nostrClient: harness.nostrClient,
|
||||
nostrService: harness.nostrService,
|
||||
accessControl: harness.accessControl,
|
||||
featureEnabled: true,
|
||||
flushDelayMs: 0,
|
||||
...extra,
|
||||
});
|
||||
}
|
||||
|
||||
test("ingests a foreign NIP-71 video into the active store and signals a refresh", () => {
|
||||
const h = makeHarness();
|
||||
const svc = makeService(h);
|
||||
assert.equal(svc.start(), true);
|
||||
|
||||
h.fireEvent(foreignEvent({ id: "e1", title: "Hello" }));
|
||||
const injected = svc.flush();
|
||||
|
||||
assert.equal(injected, 1);
|
||||
const stored = h.activeMap.get("ROOT:e1");
|
||||
assert.ok(stored, "video present in activeMap");
|
||||
assert.equal(stored.title, "Hello");
|
||||
assert.equal(stored.source, "nip71-ingest");
|
||||
assert.ok(h.allEvents.has("e1"), "raw video cached in allEvents");
|
||||
|
||||
const refresh = h.emitted.find((e) => e.name === "videos:updated");
|
||||
assert.ok(refresh, "emits videos:updated");
|
||||
assert.equal(refresh.detail.reason, "nip71-ingest");
|
||||
});
|
||||
|
||||
test("never clobbers a native bitvid video sharing the same root", () => {
|
||||
const h = makeHarness();
|
||||
const native = { id: "native", videoRootId: "shared", source: "nostr", created_at: 500, title: "Native" };
|
||||
h.activeMap.set("ROOT:shared", native);
|
||||
|
||||
const svc = makeService(h);
|
||||
svc.start();
|
||||
h.fireEvent(foreignEvent({ id: "foreign", created_at: 9999, tags: [["d", "shared"], ["title", "Foreign"], ["imeta", "url https://e/v.mp4", "m video/mp4"], ], kind: 34235 }));
|
||||
const injected = svc.flush();
|
||||
|
||||
assert.equal(injected, 0, "native video is not overwritten even by a newer foreign event");
|
||||
assert.equal(h.activeMap.get("ROOT:shared"), native);
|
||||
});
|
||||
|
||||
test("among ingested videos for one root, newest wins", () => {
|
||||
const h = makeHarness();
|
||||
const svc = makeService(h);
|
||||
svc.start();
|
||||
|
||||
h.fireEvent(foreignEvent({ id: "old", kind: 34235, created_at: 100, tags: [["d", "r"], ["title", "Old"], ["imeta", "url https://e/v1.mp4", "m video/mp4"]] }));
|
||||
svc.flush();
|
||||
h.fireEvent(foreignEvent({ id: "new", kind: 34235, created_at: 200, tags: [["d", "r"], ["title", "New"], ["imeta", "url https://e/v2.mp4", "m video/mp4"]] }));
|
||||
svc.flush();
|
||||
|
||||
assert.equal(h.activeMap.get("ROOT:r").title, "New");
|
||||
|
||||
// An older event for the same root must not resurrect.
|
||||
h.fireEvent(foreignEvent({ id: "older", kind: 34235, created_at: 50, tags: [["d", "r"], ["title", "Older"], ["imeta", "url https://e/v3.mp4", "m video/mp4"]] }));
|
||||
svc.flush();
|
||||
assert.equal(h.activeMap.get("ROOT:r").title, "New", "older event does not override newer");
|
||||
});
|
||||
|
||||
test("scopes the relay subscription to whitelisted authors when whitelist mode is on", () => {
|
||||
const h = makeHarness({ whitelistOn: true, whitelist: ["aa", "bb"] });
|
||||
const svc = makeService(h);
|
||||
assert.equal(svc.start(), true);
|
||||
|
||||
const filters = h.capturedFilters;
|
||||
assert.ok(Array.isArray(filters) && filters.length === 1);
|
||||
assert.deepEqual(filters[0].kinds.slice().sort((a, b) => a - b), KINDS);
|
||||
assert.deepEqual(filters[0].authors, ["aa", "bb"], "subscribes only to whitelisted authors");
|
||||
});
|
||||
|
||||
test("with whitelist on but no authors, does not open a subscription (nothing to ingest)", () => {
|
||||
const h = makeHarness({ whitelistOn: true, whitelist: [] });
|
||||
const svc = makeService(h);
|
||||
assert.equal(svc.start(), false, "no subscription opened");
|
||||
assert.equal(h.capturedFilters, null);
|
||||
});
|
||||
|
||||
test("opens an unscoped (capped) subscription when whitelist mode is off", () => {
|
||||
const h = makeHarness({ whitelistOn: false });
|
||||
const svc = makeService(h);
|
||||
svc.start();
|
||||
const filters = h.capturedFilters;
|
||||
assert.ok(filters[0].authors === undefined, "no author scope when whitelist disabled");
|
||||
assert.ok(Number.isFinite(filters[0].limit), "still capped by a limit");
|
||||
});
|
||||
|
||||
test("re-subscribes when the whitelist changes", () => {
|
||||
const h = makeHarness({ whitelistOn: true, whitelist: ["aa"] });
|
||||
const svc = makeService(h);
|
||||
svc.start();
|
||||
assert.deepEqual(h.capturedFilters[0].authors, ["aa"]);
|
||||
|
||||
// Whitelist grows; the access-control change fires.
|
||||
h.accessControl.getWhitelistPubkeys = () => ["aa", "cc"];
|
||||
h.triggerWhitelistChange();
|
||||
assert.deepEqual(h.capturedFilters[0].authors, ["aa", "cc"], "subscription re-scoped after change");
|
||||
});
|
||||
|
||||
test("FEATURE_NIP71_INGEST off makes the service completely inert", () => {
|
||||
const h = makeHarness();
|
||||
const svc = createNip71IngestService({
|
||||
nostrClient: h.nostrClient,
|
||||
nostrService: h.nostrService,
|
||||
accessControl: h.accessControl,
|
||||
featureEnabled: false,
|
||||
flushDelayMs: 0,
|
||||
});
|
||||
assert.equal(svc.isAvailable(), false);
|
||||
assert.equal(svc.start(), false);
|
||||
assert.equal(h.capturedFilters, null, "no relay subscription when ingest is disabled");
|
||||
});
|
||||
|
||||
test("stop() closes the subscription and unhooks the whitelist listener", () => {
|
||||
const h = makeHarness({ whitelistOn: true, whitelist: ["aa"] });
|
||||
const svc = makeService(h);
|
||||
svc.start();
|
||||
svc.stop();
|
||||
assert.equal(h.closed, true, "subscription closed");
|
||||
// After stop, a whitelist change must not reopen anything.
|
||||
h.triggerWhitelistChange();
|
||||
});
|
||||
Reference in new issue
Block a user