diff --git a/config/instance-config.js b/config/instance-config.js index 8ae587ac..a439923c 100644 --- a/config/instance-config.js +++ b/config/instance-config.js @@ -63,7 +63,15 @@ export const WATCH_HISTORY_KIND = 30078; * BitVid deployments that share relays, but keep it stable once clients begin * syncing history. */ -export const WATCH_HISTORY_LIST_IDENTIFIER = "watch-history"; +export const WATCH_HISTORY_LIST_IDENTIFIER = "watch-history:v2:index"; + +/** + * Legacy identifiers that BitVid clients should continue honoring when + * fetching historical watch-history snapshots. + */ +export const WATCH_HISTORY_LEGACY_LIST_IDENTIFIERS = Object.freeze([ + "watch-history", +]); /** * Maximum number of watch-history entries to retain per user. diff --git a/docs/nostr-event-schemas.md b/docs/nostr-event-schemas.md index 28fbcad5..a5c88dd9 100644 --- a/docs/nostr-event-schemas.md +++ b/docs/nostr-event-schemas.md @@ -49,7 +49,8 @@ read/write split. | NIP-94 mirror (`NOTE_TYPES.VIDEO_MIRROR`) | `1063` | Tags forwarded from `publishVideo` (URL, mime type, thumbnail, alt text, magnet) | Plain text alt description | | Relay list (`NOTE_TYPES.RELAY_LIST`) | `10002` | Repeating `['r', ]` tags, optionally with a marker of `'read'` or `'write'` to scope the relay; marker omitted for read/write relays | Empty content | | View counter (`NOTE_TYPES.VIEW_EVENT`) | `WATCH_HISTORY_KIND` (default `30078`) | `['t','view']`, `['video', ]`, pointer tag (`['a', ...]` or `['e', ...]`), optional dedupe `['d', ]`, optional `['session','true']` when a session actor signs, plus any extra debugging tags | Optional plaintext message | -| Watch history chunk (`NOTE_TYPES.WATCH_HISTORY_CHUNK`) | `WATCH_HISTORY_KIND` | `['d', ]`, `['encrypted','nip04']`, `['snapshot', ]`, `['chunk', , ]`, pointer tags for each entry, `['head','1']` + `['a',
]` pointers on the first chunk | NIP-04 encrypted JSON chunk (`{ version, snapshot, chunkIndex, totalChunks, items[] }`) | +| Watch history index (`NOTE_TYPES.WATCH_HISTORY_INDEX`) | `WATCH_HISTORY_KIND` | `['d', WATCH_HISTORY_LIST_IDENTIFIER]`, `['snapshot', ]`, `['chunks', ]`, repeated `['a',
]` pointers to each chunk event plus schema append tags | JSON payload `{ snapshot, totalChunks }` | +| Watch history chunk (`NOTE_TYPES.WATCH_HISTORY_CHUNK`) | `WATCH_HISTORY_KIND` | `['d', ]`, `['encrypted','nip04']`, `['snapshot', ]`, `['chunk', , ]`, pointer tags for each entry, plus schema append tags | NIP-04 encrypted JSON chunk (`{ version, snapshot, chunkIndex, totalChunks, items[] }`) | | Subscription list (`NOTE_TYPES.SUBSCRIPTION_LIST`) | `30002` | `['d', 'subscriptions']` | NIP-04 encrypted JSON `{ subPubkeys: string[] }` | | User block list (`NOTE_TYPES.USER_BLOCK_LIST`) | `30002` | `['d', 'user-blocks']` | NIP-04 encrypted JSON `{ blockedPubkeys: string[] }` | | Admin moderation list (`NOTE_TYPES.ADMIN_MODERATION_LIST`) | `30000` | `['d', 'bitvid:admin:editors']`, repeated `['p', ]` entries | Empty content | @@ -59,3 +60,17 @@ read/write split. If you introduce a new Nostr feature, add its schema to `js/nostrEventSchemas.js` so that the catalogue stays complete and so existing builders inherit the same debugging knobs. + +### Watch history identifiers + +Watch-history snapshots now publish a dedicated index event with +`#d ${WATCH_HISTORY_LIST_IDENTIFIER}` (defaults to `watch-history:v2:index`). The +index advertises the active snapshot id, total chunk count, and a pointer to +each chunk via `a` tags so clients can fetch the encrypted payloads without +guessing identifiers. + +Every chunk event uses a unique `d` tag such as +`watch-history:v2//`. Legacy snapshots created before the +index event shipped stored the first chunk at `#d watch-history`; new clients +should continue reading that identifier for backward compatibility when +hydrating older histories. diff --git a/js/config.js b/js/config.js index a44c3cd9..0a102695 100644 --- a/js/config.js +++ b/js/config.js @@ -6,6 +6,7 @@ import { DEFAULT_WHITELIST_MODE_ENABLED, WATCH_HISTORY_KIND, WATCH_HISTORY_LIST_IDENTIFIER, + WATCH_HISTORY_LEGACY_LIST_IDENTIFIERS, WATCH_HISTORY_MAX_ITEMS, WATCH_HISTORY_BATCH_RESOLVE, WATCH_HISTORY_PAYLOAD_MAX_BYTES, @@ -59,6 +60,7 @@ export { ADMIN_WHITELIST_MODE_STORAGE_KEY }; export { WATCH_HISTORY_KIND }; export { WATCH_HISTORY_LIST_IDENTIFIER }; +export { WATCH_HISTORY_LEGACY_LIST_IDENTIFIERS }; export { WATCH_HISTORY_MAX_ITEMS }; export { WATCH_HISTORY_BATCH_RESOLVE }; export { WATCH_HISTORY_PAYLOAD_MAX_BYTES }; diff --git a/js/nostr.js b/js/nostr.js index ad7179c8..d82503ab 100644 --- a/js/nostr.js +++ b/js/nostr.js @@ -4,6 +4,7 @@ import { isDevMode, WATCH_HISTORY_KIND, WATCH_HISTORY_LIST_IDENTIFIER, + WATCH_HISTORY_LEGACY_LIST_IDENTIFIERS, WATCH_HISTORY_MAX_ITEMS, WATCH_HISTORY_BATCH_RESOLVE, WATCH_HISTORY_PAYLOAD_MAX_BYTES, @@ -20,6 +21,7 @@ import { buildVideoPostEvent, buildVideoMirrorEvent, buildViewEvent, + buildWatchHistoryIndexEvent, buildWatchHistoryChunkEvent, } from "./nostrEventSchemas.js"; @@ -46,6 +48,76 @@ const NIP07_LOGIN_TIMEOUT_ERROR_MESSAGE = const SESSION_ACTOR_STORAGE_KEY = "bitvid:sessionActor:v1"; const WATCH_HISTORY_CACHE_STORAGE_KEY = "bitvid:watchHistoryCache:v1"; +const WATCH_HISTORY_INDEX_IDENTIFIER_LOWER = + typeof WATCH_HISTORY_LIST_IDENTIFIER === "string" + ? WATCH_HISTORY_LIST_IDENTIFIER.trim().toLowerCase() + : ""; + +const WATCH_HISTORY_CHUNK_IDENTIFIER_PREFIX = (() => { + const rawIdentifier = + typeof WATCH_HISTORY_LIST_IDENTIFIER === "string" + ? WATCH_HISTORY_LIST_IDENTIFIER.trim() + : ""; + const suffixes = [":index", "/index"]; + for (const suffix of suffixes) { + if (rawIdentifier.toLowerCase().endsWith(suffix)) { + return rawIdentifier + .slice(0, rawIdentifier.length - suffix.length) + .replace(/\/+$/, ""); + } + } + return rawIdentifier.replace(/\/+$/, "") || "watch-history"; +})(); + +const LEGACY_WATCH_HISTORY_IDENTIFIER_SET = (() => { + const identifiers = Array.isArray(WATCH_HISTORY_LEGACY_LIST_IDENTIFIERS) + ? WATCH_HISTORY_LEGACY_LIST_IDENTIFIERS + : []; + const normalized = new Set( + identifiers + .map((value) => + typeof value === "string" ? value.trim().toLowerCase() : "" + ) + .filter(Boolean) + ); + if (!normalized.size) { + normalized.add("watch-history"); + } + return normalized; +})(); + +function deriveWatchHistoryChunkIdentifier(snapshotId, chunkIndex) { + const prefix = WATCH_HISTORY_CHUNK_IDENTIFIER_PREFIX || "watch-history"; + const sanitizedPrefix = prefix.replace(/\/+$/, ""); + const safeSnapshot = + typeof snapshotId === "string" && snapshotId ? snapshotId : "0"; + const safeIndex = Number.isFinite(chunkIndex) + ? Math.max(0, Math.floor(chunkIndex)) + : 0; + return `${sanitizedPrefix}/${safeSnapshot}/${safeIndex}`; +} + +function isIndexIdentifier(identifier) { + if (!identifier || typeof identifier !== "string") { + return false; + } + if (!WATCH_HISTORY_INDEX_IDENTIFIER_LOWER) { + return false; + } + return identifier.trim().toLowerCase() === WATCH_HISTORY_INDEX_IDENTIFIER_LOWER; +} + +function isLegacyHeadIdentifier(identifier) { + if (!identifier || typeof identifier !== "string") { + return false; + } + const normalized = identifier.trim().toLowerCase(); + if (!normalized) { + return false; + } + return LEGACY_WATCH_HISTORY_IDENTIFIER_SET.has(normalized); +} + // To limit error spam let errorLogCount = 0; const MAX_ERROR_LOGS = 100; @@ -1930,9 +2002,7 @@ class NostrClient { const totalChunks = chunkItemsList.length || 1; const chunkIdentifiers = chunkItemsList.map((_, index) => - index === 0 - ? WATCH_HISTORY_LIST_IDENTIFIER - : `watch-history:${snapshotId}:${index}` + deriveWatchHistoryChunkIdentifier(snapshotId, index) ); const chunkAddresses = chunkIdentifiers.map( (identifier) => `${WATCH_HISTORY_KIND}:${normalizedActor}:${identifier}` @@ -2002,7 +2072,8 @@ class NostrClient { return tag; }); - const chunkIdentifier = chunkIdentifiers[index] || WATCH_HISTORY_LIST_IDENTIFIER; + const chunkIdentifier = + chunkIdentifiers[index] ?? deriveWatchHistoryChunkIdentifier(snapshotId, index); // Ensure created_at stays monotonic so chunk 0 always outranks prior snapshots // even when multiple publishes land within the same wall-clock second. @@ -2014,7 +2085,6 @@ class NostrClient { chunkIndex: index, totalChunks, pointerTags, - chunkAddresses, content: encryptionResult.ciphertext, }); @@ -2065,6 +2135,50 @@ class NostrClient { }); } + const indexEvent = buildWatchHistoryIndexEvent({ + pubkey: normalizedActor, + created_at: baseTimestamp + totalChunks, + snapshotId, + totalChunks, + chunkAddresses, + }); + + let signedIndexEvent = null; + try { + if (useExtension) { + signedIndexEvent = await window.nostr.signEvent(indexEvent); + } else { + signedIndexEvent = signEventWithPrivateKey( + indexEvent, + this.sessionActor.privateKey + ); + } + } catch (error) { + console.warn( + "[nostr] Failed to sign watch history index event:", + error + ); + signedIndexEvent = null; + } + + let indexPublishResults = []; + let indexSuccess = false; + if (signedIndexEvent) { + indexPublishResults = await Promise.all( + relays.map((url) => publishEventToRelay(this.pool, url, signedIndexEvent)) + ); + indexSuccess = indexPublishResults.some((result) => result.success); + if (!indexSuccess) { + overallSuccess = false; + console.warn( + "[nostr] Failed to publish watch history index:", + indexPublishResults + ); + } + } else { + overallSuccess = false; + } + if (overallSuccess) { this.cancelWatchHistoryRepublish(normalizedActor); } else if (allowRetry && persistedItems.length) { @@ -2074,23 +2188,91 @@ class NostrClient { const baselineEntry = existingEntry || this.watchHistoryCache.get(normalizedActor) || null; - const headEvent = chunkResults[0]?.event || null; + const pointerEvent = signedIndexEvent || chunkResults[0]?.event || null; const newEntry = this.createWatchHistoryEntry( - headEvent, + pointerEvent, persistedItems, Date.now(), baselineEntry ); + if (!(newEntry.chunkEvents instanceof Map)) { + newEntry.chunkEvents = new Map(); + } else { + newEntry.chunkEvents.clear(); + } + + const chunkPointerItems = []; + const chunkPointerKeys = new Set(); + + chunkResults.forEach((chunk) => { + const address = eventToAddressPointer(chunk.event); + if (!address) { + return; + } + const pointer = clonePointerItem({ type: "a", value: address, relay: null }); + if (!pointer) { + return; + } + const key = pointerKey(pointer); + if (!key || chunkPointerKeys.has(key)) { + return; + } + chunkPointerKeys.add(key); + chunkPointerItems.push(pointer); + const clonedChunk = cloneEventForCache(chunk.event); + newEntry.chunkEvents.set(key, clonedChunk || chunk.event); + }); + + let indexPointer = null; + if (signedIndexEvent) { + const indexAddress = eventToAddressPointer(signedIndexEvent); + if (indexAddress) { + const pointer = clonePointerItem({ + type: "a", + value: indexAddress, + relay: null, + }); + if (pointer) { + const key = pointerKey(pointer); + if (key) { + chunkPointerKeys.add(key); + const clonedIndex = cloneEventForCache(signedIndexEvent); + newEntry.chunkEvents.set(key, clonedIndex || signedIndexEvent); + } + indexPointer = pointer; + } + } + } + + newEntry.chunkPointers = { + head: indexPointer || chunkPointerItems[0] || null, + index: indexPointer, + chunks: chunkPointerItems, + }; + this.watchHistoryCache.set(normalizedActor, newEntry); this.persistWatchHistoryEntry(normalizedActor, newEntry); + const combinedEvents = []; + const combinedResults = []; + if (signedIndexEvent) { + combinedEvents.push(signedIndexEvent); + combinedResults.push(indexPublishResults); + } + chunkResults.forEach((chunk) => { + combinedEvents.push(chunk.event); + combinedResults.push(chunk.results); + }); + return { ok: overallSuccess, - event: headEvent, - events: chunkResults.map((chunk) => chunk.event), + event: pointerEvent, + indexEvent: signedIndexEvent, + events: combinedEvents, chunks: chunkResults, - results: chunkResults.map((chunk) => chunk.results), + results: combinedResults, + indexResults: indexPublishResults, snapshot: snapshotId, items: newEntry.items .map((item) => clonePointerItem(item)) @@ -2819,13 +3001,34 @@ class NostrClient { : WATCH_HISTORY_FETCH_EVENT_LIMIT; const fetchLimit = Math.max(1, Math.floor(fetchLimitCandidate)); + const headIdentifierSet = new Set(); + if ( + typeof WATCH_HISTORY_LIST_IDENTIFIER === "string" && + WATCH_HISTORY_LIST_IDENTIFIER.trim() + ) { + headIdentifierSet.add(WATCH_HISTORY_LIST_IDENTIFIER.trim()); + } + if (Array.isArray(WATCH_HISTORY_LEGACY_LIST_IDENTIFIERS)) { + WATCH_HISTORY_LEGACY_LIST_IDENTIFIERS.forEach((value) => { + if (typeof value === "string" && value.trim()) { + headIdentifierSet.add(value.trim()); + } + }); + } + if (!headIdentifierSet.size && WATCH_HISTORY_INDEX_IDENTIFIER_LOWER) { + headIdentifierSet.add(WATCH_HISTORY_LIST_IDENTIFIER); + } + const filter = { kinds: [WATCH_HISTORY_KIND], authors: [actor], - "#d": [WATCH_HISTORY_LIST_IDENTIFIER], limit: fetchLimit, }; + if (headIdentifierSet.size) { + filter["#d"] = Array.from(headIdentifierSet); + } + let fetchedHeadEvents = []; try { const events = await this.pool.list(relayList, [filter]); @@ -2864,6 +3067,7 @@ class NostrClient { const chunkIdentifiers = new Set(); const snapshotIds = new Set(); + const indexSnapshots = new Map(); for (const event of fetchedHeadEvents) { if (!event || typeof event !== "object") { @@ -2871,15 +3075,29 @@ class NostrClient { } const tags = Array.isArray(event.tags) ? event.tags : []; + let identifierTagValue = ""; + let snapshotTagValue = ""; + let totalChunksTagValue = null; + const pointerIdentifiers = new Set(); + const pointerAddresses = new Set(); + for (const tag of tags) { if (!Array.isArray(tag) || tag.length < 2) { continue; } - if (tag[0] === "snapshot" && typeof tag[1] === "string" && tag[1]) { - snapshotIds.add(tag[1]); + if (tag[0] === "d" && typeof tag[1] === "string") { + identifierTagValue = tag[1]; + } else if (tag[0] === "snapshot" && typeof tag[1] === "string" && tag[1]) { + snapshotTagValue = tag[1]; + } else if (tag[0] === "chunks") { + const parsedTotal = Number.parseInt(tag[1], 10); + if (Number.isFinite(parsedTotal)) { + totalChunksTagValue = Math.max(0, parsedTotal); + } } else if (tag[0] === "a") { const pointer = normalizePointerTag(tag); - if (pointer?.type === "a") { + if (pointer?.type === "a" && typeof pointer.value === "string") { + pointerAddresses.add(pointer.value); const [kindStr, pubkey, identifier] = pointer.value.split(":"); const kind = Number.parseInt(kindStr, 10); if ( @@ -2889,13 +3107,119 @@ class NostrClient { pubkey === actor && typeof identifier === "string" && identifier && - identifier !== WATCH_HISTORY_LIST_IDENTIFIER + !isIndexIdentifier(identifier) ) { - chunkIdentifiers.add(identifier); + pointerIdentifiers.add(identifier); } } } } + + const identifier = typeof identifierTagValue === "string" ? identifierTagValue : ""; + let snapshotId = snapshotTagValue; + let totalChunks = + Number.isFinite(totalChunksTagValue) && totalChunksTagValue !== null + ? Math.max(0, Math.floor(totalChunksTagValue)) + : null; + + const isIndexEvent = isIndexIdentifier(identifier); + + if (isIndexEvent && typeof event.content === "string") { + try { + const parsed = JSON.parse(event.content); + if (!snapshotId && typeof parsed?.snapshot === "string" && parsed.snapshot) { + snapshotId = parsed.snapshot; + } + if ( + totalChunks === null && + Number.isFinite(Number.parseInt(parsed?.totalChunks, 10)) + ) { + totalChunks = Math.max(0, Math.floor(Number.parseInt(parsed.totalChunks, 10))); + } + } catch (error) { + if (isDevMode) { + console.warn( + "[nostr] Failed to parse watch history index payload:", + error + ); + } + } + } + + if (!snapshotId) { + snapshotId = typeof event.id === "string" && event.id ? event.id : ""; + } + + if (snapshotId) { + snapshotIds.add(snapshotId); + } + + pointerIdentifiers.forEach((value) => { + if (value) { + chunkIdentifiers.add(value); + } + }); + + if (identifier && !isIndexEvent) { + chunkIdentifiers.add(identifier); + } + + if (isIndexEvent && snapshotId) { + const existingMeta = indexSnapshots.get(snapshotId) || { + event: null, + totalChunks: null, + chunkIdentifiers: new Set(), + chunkAddresses: new Set(), + }; + + const existingCreatedAt = + typeof existingMeta.event?.created_at === "number" + ? existingMeta.event.created_at + : 0; + const candidateCreatedAt = + typeof event.created_at === "number" ? event.created_at : 0; + if (!existingMeta.event || candidateCreatedAt >= existingCreatedAt) { + existingMeta.event = event; + } + + if (Number.isFinite(totalChunks)) { + if ( + !Number.isFinite(existingMeta.totalChunks) || + totalChunks > existingMeta.totalChunks + ) { + existingMeta.totalChunks = totalChunks; + } + } + + pointerIdentifiers.forEach((value) => { + if (value) { + existingMeta.chunkIdentifiers.add(value); + } + }); + + pointerAddresses.forEach((value) => { + if (value) { + existingMeta.chunkAddresses.add(value); + } + }); + + indexSnapshots.set(snapshotId, existingMeta); + } else if (snapshotId) { + const existingMeta = indexSnapshots.get(snapshotId); + if (existingMeta) { + pointerIdentifiers.forEach((value) => { + if (value) { + existingMeta.chunkIdentifiers.add(value); + } + }); + pointerAddresses.forEach((value) => { + if (value) { + existingMeta.chunkAddresses.add(value); + } + }); + indexSnapshots.set(snapshotId, existingMeta); + } + } } const chunkFilters = []; @@ -2981,11 +3305,15 @@ class NostrClient { let taggedSnapshotId = ""; let taggedChunkIndex = null; let taggedTotalChunks = null; + let identifierTagValue = ""; for (const tag of tags) { if (!Array.isArray(tag) || tag.length < 2) { continue; } + if (tag[0] === "d" && typeof tag[1] === "string") { + identifierTagValue = tag[1]; + } if (tag[0] === "snapshot" && typeof tag[1] === "string") { taggedSnapshotId = tag[1]; } else if (tag[0] === "chunk") { @@ -3035,6 +3363,9 @@ class NostrClient { latestCreatedAt: typeof event.created_at === "number" ? event.created_at : 0, version: payload.version || 0, + indexEvent: null, + chunkAddresses: new Set(), + expectedChunkIdentifiers: new Set(), }; bucket.chunks.set(chunkIndex, { @@ -3054,6 +3385,58 @@ class NostrClient { bucket.latestCreatedAt = createdAt; } bucket.version = Math.max(bucket.version, payload.version || 0); + const chunkAddress = eventToAddressPointer(event); + if (chunkAddress) { + bucket.chunkAddresses.add(chunkAddress); + } + if (identifierTagValue) { + bucket.expectedChunkIdentifiers.add(identifierTagValue); + } + snapshotMap.set(snapshotId, bucket); + } + + for (const [snapshotId, meta] of indexSnapshots.entries()) { + const bucket = snapshotMap.get(snapshotId) || { + snapshotId, + chunks: new Map(), + expectedChunks: 0, + latestCreatedAt: 0, + version: 0, + indexEvent: null, + chunkAddresses: new Set(), + expectedChunkIdentifiers: new Set(), + }; + + if (meta.event) { + const createdAt = + typeof meta.event.created_at === "number" ? meta.event.created_at : 0; + if (createdAt > bucket.latestCreatedAt) { + bucket.latestCreatedAt = createdAt; + } + bucket.indexEvent = meta.event; + } + + if (Number.isFinite(meta.totalChunks)) { + const normalized = Math.max(1, Math.floor(meta.totalChunks)); + bucket.expectedChunks = Math.max(bucket.expectedChunks, normalized); + } + + if (meta.chunkAddresses instanceof Set) { + meta.chunkAddresses.forEach((value) => { + if (typeof value === "string" && value) { + bucket.chunkAddresses.add(value); + } + }); + } + + if (meta.chunkIdentifiers instanceof Set) { + meta.chunkIdentifiers.forEach((value) => { + if (typeof value === "string" && value) { + bucket.expectedChunkIdentifiers.add(value); + } + }); + } + snapshotMap.set(snapshotId, bucket); } @@ -3070,7 +3453,7 @@ class NostrClient { ); const seenKeys = new Set(); const combinedItems = []; - let pointerCandidate = null; + let pointerCandidate = bucket.indexEvent || null; for (const index of chunkIndices) { const chunk = bucket.chunks.get(index); @@ -3105,6 +3488,12 @@ class NostrClient { version: bucket.version, chunkCount: chunkIndices.length, expectedChunks: Math.max(bucket.expectedChunks, chunkIndices.length || 1), + chunkAddresses: bucket.chunkAddresses instanceof Set + ? new Set(bucket.chunkAddresses) + : new Set(), + expectedChunkIdentifiers: bucket.expectedChunkIdentifiers instanceof Set + ? new Set(bucket.expectedChunkIdentifiers) + : new Set(), }; if (!fallbackSnapshot) { @@ -3168,8 +3557,31 @@ class NostrClient { } const chunkPointerItemsForEntry = []; - let headPointerForEntry = null; const chunkPointerKeys = new Set(); + let indexPointerForEntry = null; + let firstChunkPointer = null; + + if (effectiveSnapshot?.pointerEvent) { + const indexAddress = eventToAddressPointer(effectiveSnapshot.pointerEvent); + if (indexAddress) { + const pointer = clonePointerItem({ + type: "a", + value: indexAddress, + relay: null, + }); + if (pointer) { + const key = pointerKey(pointer); + if (key) { + chunkPointerKeys.add(key); + const clonedIndex = cloneEventForCache(effectiveSnapshot.pointerEvent); + if (clonedIndex) { + entry.chunkEvents.set(key, clonedIndex); + } + indexPointerForEntry = pointer; + } + } + } + } if (effectiveSnapshot?.snapshotId) { const bucketForEntry = snapshotMap.get(effectiveSnapshot.snapshotId); @@ -3194,7 +3606,7 @@ class NostrClient { continue; } const key = pointerKey(pointer); - if (!key) { + if (!key || chunkPointerKeys.has(key)) { continue; } const clonedChunkEvent = cloneEventForCache(chunk.event); @@ -3202,25 +3614,57 @@ class NostrClient { entry.chunkEvents.set(key, clonedChunkEvent); } chunkPointerKeys.add(key); - if (chunk.payload?.chunkIndex === 0) { - headPointerForEntry = pointer; - } else { - chunkPointerItemsForEntry.push(pointer); + chunkPointerItemsForEntry.push(pointer); + if (chunk.payload?.chunkIndex === 0 && !firstChunkPointer) { + firstChunkPointer = pointer; } } } } + if (effectiveSnapshot?.chunkAddresses instanceof Set) { + effectiveSnapshot.chunkAddresses.forEach((address) => { + const pointer = clonePointerItem({ + type: "a", + value: address, + relay: null, + }); + if (!pointer) { + return; + } + const key = pointerKey(pointer); + if (!key || chunkPointerKeys.has(key)) { + return; + } + chunkPointerKeys.add(key); + chunkPointerItemsForEntry.push(pointer); + }); + } + + const headPointer = indexPointerForEntry || firstChunkPointer || null; entry.chunkPointers = { - head: headPointerForEntry, + head: headPointer, + index: indexPointerForEntry, chunks: chunkPointerItemsForEntry, }; - const headPointerKey = headPointerForEntry ? pointerKey(headPointerForEntry) : ""; - if (headPointerKey) { - chunkPointerKeys.add(headPointerKey); - if (entry.pointerEvent) { - entry.chunkEvents.set(headPointerKey, cloneEventForCache(entry.pointerEvent)); + if (headPointer) { + const headPointerKey = pointerKey(headPointer); + if (headPointerKey) { + chunkPointerKeys.add(headPointerKey); + const indexPointerKey = indexPointerForEntry + ? pointerKey(indexPointerForEntry) + : ""; + if ( + entry.pointerEvent && + indexPointerKey && + headPointerKey === indexPointerKey + ) { + entry.chunkEvents.set( + headPointerKey, + cloneEventForCache(entry.pointerEvent) + ); + } } } @@ -3320,11 +3764,20 @@ class NostrClient { if (!normalizedIdentifier) { return "none"; } - const normalizedIdentifierLower = normalizedIdentifier.toLowerCase(); - if (normalizedIdentifierLower === WATCH_HISTORY_LIST_IDENTIFIER) { + if (isIndexIdentifier(normalizedIdentifier)) { return "head"; } - if (!normalizedIdentifierLower.startsWith("watch-history:")) { + if (isLegacyHeadIdentifier(normalizedIdentifier)) { + return "head"; + } + const chunkPrefixLower = WATCH_HISTORY_CHUNK_IDENTIFIER_PREFIX + ? WATCH_HISTORY_CHUNK_IDENTIFIER_PREFIX.trim().toLowerCase() + : ""; + const normalizedIdentifierLower = normalizedIdentifier.toLowerCase(); + const matchesPrefix = chunkPrefixLower + ? normalizedIdentifierLower.startsWith(chunkPrefixLower) + : normalizedIdentifierLower.startsWith("watch-history"); + if (!matchesPrefix) { return "none"; } const normalizedOwner = diff --git a/js/nostrEventSchemas.js b/js/nostrEventSchemas.js index b8fbb4de..96ff9d25 100644 --- a/js/nostrEventSchemas.js +++ b/js/nostrEventSchemas.js @@ -10,6 +10,7 @@ export const NOTE_TYPES = Object.freeze({ VIDEO_MIRROR: "videoMirror", RELAY_LIST: "relayList", VIEW_EVENT: "viewEvent", + WATCH_HISTORY_INDEX: "watchHistoryIndex", WATCH_HISTORY_CHUNK: "watchHistoryChunk", SUBSCRIPTION_LIST: "subscriptionList", USER_BLOCK_LIST: "userBlockList", @@ -91,13 +92,30 @@ const BASE_SCHEMAS = { description: "Optional plaintext content used for diagnostics.", }, }, + [NOTE_TYPES.WATCH_HISTORY_INDEX]: { + type: NOTE_TYPES.WATCH_HISTORY_INDEX, + label: "Watch history index", + kind: WATCH_HISTORY_KIND, + identifierTag: { + name: "d", + value: WATCH_HISTORY_LIST_IDENTIFIER, + }, + snapshotTagName: "snapshot", + totalTagName: "chunks", + chunkPointerTagName: "a", + appendTags: DEFAULT_APPEND_TAGS, + content: { + format: "json", + description: + "JSON payload describing the active snapshot and chunk count.", + }, + }, [NOTE_TYPES.WATCH_HISTORY_CHUNK]: { type: NOTE_TYPES.WATCH_HISTORY_CHUNK, label: "Watch history snapshot", kind: WATCH_HISTORY_KIND, identifierTag: { name: "d", - value: WATCH_HISTORY_LIST_IDENTIFIER, }, encryptionTag: { name: "encrypted", value: "nip04" }, snapshotTagName: "snapshot", @@ -454,6 +472,73 @@ export function buildViewEvent({ }; } +export function buildWatchHistoryIndexEvent({ + pubkey, + created_at, + snapshotId, + totalChunks, + chunkAddresses = [], + additionalTags = [], + content, +}) { + const schema = getNostrEventSchema(NOTE_TYPES.WATCH_HISTORY_INDEX); + const tags = []; + const identifierName = schema?.identifierTag?.name || "d"; + const identifierValue = schema?.identifierTag?.value || WATCH_HISTORY_LIST_IDENTIFIER; + if (identifierName && identifierValue) { + tags.push([identifierName, identifierValue]); + } + const snapshotTagName = schema?.snapshotTagName || "snapshot"; + if (snapshotTagName && snapshotId) { + tags.push([snapshotTagName, snapshotId]); + } + const totalTagName = schema?.totalTagName || "chunks"; + if (totalTagName && Number.isFinite(totalChunks)) { + tags.push([totalTagName, String(Math.max(0, Math.floor(totalChunks)))]); + } + const pointerTagName = schema?.chunkPointerTagName || "a"; + chunkAddresses.forEach((address) => { + if (typeof address === "string" && address) { + tags.push([pointerTagName, address]); + } + }); + + appendSchemaTags(tags, schema); + + if (Array.isArray(additionalTags)) { + additionalTags.forEach((tag) => { + if (Array.isArray(tag) && tag.length >= 2) { + tags.push(tag.map((value) => (typeof value === "string" ? value : String(value)))); + } + }); + } + + let resolvedContent = content; + if (resolvedContent === undefined) { + const payload = {}; + if (snapshotId) { + payload.snapshot = snapshotId; + } + if (Number.isFinite(totalChunks)) { + payload.totalChunks = Math.max(0, Math.floor(totalChunks)); + } + resolvedContent = JSON.stringify(payload); + } else if (typeof resolvedContent !== "string") { + resolvedContent = JSON.stringify(resolvedContent ?? {}); + } + + return { + kind: schema?.kind ?? WATCH_HISTORY_KIND, + pubkey, + created_at, + tags, + content: + typeof resolvedContent === "string" + ? resolvedContent + : String(resolvedContent ?? ""), + }; +} + export function buildWatchHistoryChunkEvent({ pubkey, created_at, diff --git a/tests/watch-history.test.mjs b/tests/watch-history.test.mjs index a7e25b40..f39c1291 100644 --- a/tests/watch-history.test.mjs +++ b/tests/watch-history.test.mjs @@ -5,6 +5,13 @@ const { WATCH_HISTORY_LIST_IDENTIFIER, WATCH_HISTORY_KIND, } = await import("../js/config.js"); +const { + NOTE_TYPES, + getNostrEventSchema, + setNostrEventSchemaOverrides, + buildWatchHistoryIndexEvent, + buildWatchHistoryChunkEvent, +} = await import("../js/nostrEventSchemas.js"); if (typeof globalThis.window === "undefined") { globalThis.window = {}; @@ -66,6 +73,132 @@ function createDecryptClient(actorPubkey) { } const ACTOR = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; +const WATCH_HISTORY_CHUNK_PREFIX = WATCH_HISTORY_LIST_IDENTIFIER.replace( + /[:/]index$/i, + "" +); + +const indexSchema = getNostrEventSchema(NOTE_TYPES.WATCH_HISTORY_INDEX); +assert(indexSchema, "index schema should be available"); + +const chunkSchema = getNostrEventSchema(NOTE_TYPES.WATCH_HISTORY_CHUNK); +assert(chunkSchema, "chunk schema should be available"); + +const sampleChunkIdentifier = `${WATCH_HISTORY_CHUNK_PREFIX}/sample/0`; +const sampleChunkAddresses = [ + `${WATCH_HISTORY_KIND}:${ACTOR}:${sampleChunkIdentifier}`, + `${WATCH_HISTORY_KIND}:${ACTOR}:${WATCH_HISTORY_CHUNK_PREFIX}/sample/1`, +]; + +const sampleIndexEvent = buildWatchHistoryIndexEvent({ + pubkey: ACTOR, + created_at: 111, + snapshotId: "sample", + totalChunks: 2, + chunkAddresses: sampleChunkAddresses, +}); + +const sampleIndexIdentifierTag = sampleIndexEvent.tags.find( + (tag) => Array.isArray(tag) && tag[0] === "d" +); +assert(sampleIndexIdentifierTag, "index event should include identifier tag"); +assert.equal( + sampleIndexIdentifierTag[1], + WATCH_HISTORY_LIST_IDENTIFIER, + "index event should use canonical watch history identifier" +); + +const sampleChunksTag = sampleIndexEvent.tags.find( + (tag) => Array.isArray(tag) && tag[0] === "chunks" +); +assert(sampleChunksTag, "index event should include chunks tag"); +assert.equal( + Number.parseInt(sampleChunksTag[1], 10), + 2, + "chunks tag should reflect total chunks" +); + +const sampleIndexAddresses = sampleIndexEvent.tags + .filter((tag) => Array.isArray(tag) && tag[0] === "a") + .map((tag) => tag[1]); +assert.deepEqual( + sampleIndexAddresses, + sampleChunkAddresses, + "index event should list chunk addresses" +); + +const sampleChunkEvent = buildWatchHistoryChunkEvent({ + pubkey: ACTOR, + created_at: 222, + chunkIdentifier: sampleChunkIdentifier, + snapshotId: "sample", + chunkIndex: 0, + totalChunks: 2, + pointerTags: [["e", "sample-event"]], + content: "ciphertext", +}); + +const chunkIdentifierTag = sampleChunkEvent.tags.find( + (tag) => Array.isArray(tag) && tag[0] === "d" +); +assert(chunkIdentifierTag, "chunk event should include identifier tag"); +assert.equal( + chunkIdentifierTag[1], + sampleChunkIdentifier, + "chunk event should use provided identifier" +); +assert( + sampleChunkEvent.tags.some((tag) => Array.isArray(tag) && tag[0] === "snapshot"), + "chunk event should include snapshot tag" +); +assert( + sampleChunkEvent.tags.some((tag) => Array.isArray(tag) && tag[0] === "chunk"), + "chunk event should include chunk tag" +); +assert( + sampleChunkEvent.tags.some((tag) => Array.isArray(tag) && tag[0] === "encrypted"), + "chunk event should include encrypted tag" +); + +setNostrEventSchemaOverrides({ + [NOTE_TYPES.WATCH_HISTORY_INDEX]: { + identifierTag: { name: "d", value: "custom:index" }, + }, + [NOTE_TYPES.WATCH_HISTORY_CHUNK]: { + identifierTag: { name: "d", value: "custom:chunk" }, + }, +}); + +const overriddenIndexEvent = buildWatchHistoryIndexEvent({ + pubkey: ACTOR, + created_at: 333, + snapshotId: "override", + totalChunks: 1, +}); +assert( + overriddenIndexEvent.tags.some( + (tag) => Array.isArray(tag) && tag[0] === "d" && tag[1] === "custom:index" + ), + "index builder should honor overrides" +); + +const overriddenChunkEvent = buildWatchHistoryChunkEvent({ + pubkey: ACTOR, + created_at: 444, + snapshotId: "override", + chunkIndex: 0, + totalChunks: 1, + pointerTags: [], + content: "cipher", +}); +assert( + overriddenChunkEvent.tags.some( + (tag) => Array.isArray(tag) && tag[0] === "d" && tag[1] === "custom:chunk" + ), + "chunk builder should honor identifier override" +); + +setNostrEventSchemaOverrides({}); const { client: publishingClient, publishedEvents, payloads } = createPublishingClient(ACTOR); @@ -99,11 +232,11 @@ const publishResult = await publishingClient.publishWatchHistorySnapshot( ); assert.equal(publishResult.ok, true, "publish should report success"); -assert(publishResult.events.length > 1, "snapshot should chunk into multiple events"); +assert(publishResult.events.length > 1, "snapshot should include index and chunk events"); assert.equal( publishResult.events.length, - payloads.length, - "each chunk should produce an encryption payload" + payloads.length + 1, + "index event should accompany each chunk payload" ); payloads.forEach((payload) => { @@ -120,53 +253,67 @@ const snapshotIds = new Set( event.tags.find((tag) => Array.isArray(tag) && tag[0] === "snapshot")?.[1] ) ); -assert.equal(snapshotIds.size, 1, "all chunks should share a snapshot id"); +assert.equal(snapshotIds.size, 1, "all events should share a snapshot id"); -const headTag = publishResult.event.tags.find( - (tag) => Array.isArray(tag) && tag[0] === "head" +const indexEvent = publishResult.indexEvent; +assert(indexEvent, "publish result should include index event"); + +const indexIdentifierTag = indexEvent.tags.find( + (tag) => Array.isArray(tag) && tag[0] === "d" +); +assert(indexIdentifierTag, "index event should include identifier tag"); +assert.equal( + indexIdentifierTag[1], + WATCH_HISTORY_LIST_IDENTIFIER, + "index event should use canonical identifier" ); -assert(headTag, "head chunk should include head tag"); -const chunkIdentifiers = publishResult.events.map((event) => +const indexChunksTag = indexEvent.tags.find( + (tag) => Array.isArray(tag) && tag[0] === "chunks" +); +assert(indexChunksTag, "index event should include chunks tag"); +assert.equal( + Number.parseInt(indexChunksTag[1], 10), + publishResult.chunks.length, + "index event should report chunk count" +); + +const chunkEvents = publishResult.chunks.map((chunk) => chunk.event); +const chunkIdentifiers = chunkEvents.map((event) => event.tags.find((tag) => Array.isArray(tag) && tag[0] === "d")?.[1] ); assert.equal( chunkIdentifiers.length, - publishResult.events.length, + chunkEvents.length, "each chunk should expose a d tag" ); -assert.equal( - chunkIdentifiers[0], - WATCH_HISTORY_LIST_IDENTIFIER, - "head chunk should retain canonical watch history identifier" -); - const uniqueChunkIdentifiers = new Set(chunkIdentifiers); assert.equal( uniqueChunkIdentifiers.size, - publishResult.events.length, + chunkEvents.length, "each chunk should have a unique d tag" ); -chunkIdentifiers.slice(1).forEach((identifier, index) => { +chunkIdentifiers.forEach((identifier, index) => { + assert(identifier, `chunk ${index} should include identifier`); assert( - identifier && identifier.startsWith("watch-history:"), - `chunk ${index + 1} should use a watch-history namespace identifier` + identifier.startsWith(WATCH_HISTORY_CHUNK_PREFIX), + `chunk ${index} should use the chunk prefix` ); assert.notEqual( identifier, WATCH_HISTORY_LIST_IDENTIFIER, - "only the head chunk should use the canonical identifier" + "chunk identifiers should differ from index identifier" ); }); -const headATags = publishResult.event.tags +const indexAddresses = indexEvent.tags .filter((tag) => Array.isArray(tag) && tag[0] === "a") .map((tag) => tag[1]); -const expectedChunkAddresses = publishResult.events.map((event) => { +const expectedChunkAddresses = chunkEvents.map((event) => { const identifier = event.tags.find( (tag) => Array.isArray(tag) && tag[0] === "d" )?.[1]; @@ -176,8 +323,8 @@ const expectedChunkAddresses = publishResult.events.map((event) => { expectedChunkAddresses.forEach((address) => { assert( - headATags.includes(address), - `head chunk should reference ${address}` + indexAddresses.includes(address), + `index event should reference ${address}` ); }); @@ -229,9 +376,6 @@ if (typeof localStorage !== "undefined") { } const headEvent = publishResult.event; -const chunkEvents = publishResult.events.filter( - (event) => event.id !== headEvent.id -); const snapshotId = headEvent.tags.find( (tag) => Array.isArray(tag) && tag[0] === "snapshot"