Merge pull request #238 from PR0M3TH3AN/codex/update-watch_history_kind-and-related-schemas

Migrate watch history events to kind 30079

config/instance-config.js

@@ -48,12 +48,15 @@ export const DEFAULT_WHITELIST_MODE_ENABLED = true;
 /**
  * Nostr kind used when persisting watch history events.
  *
- * BitVid’s roadmap standardizes on kind 30078 so that watch events, view logs,
- * and media metadata stay in the same family of documents. Operators that want
- * to experiment with a separate list kind (for example, a NIP-51 collection)
- * can flip this number so long as their relays accept the chosen kind.
+ * BitVid’s roadmap standardizes on kind 30079 so that watch events, view logs,
+ * and media metadata stay in the same family of documents. During the rollout
+ * from the legacy 30078 payloads, clients query both kinds so historical data
+ * keeps syncing; remove the compatibility fetch once all writers emit the new
+ * kind. Operators that want to experiment with a separate list kind (for
+ * example, a NIP-51 collection) can flip this number so long as their relays
+ * accept the chosen kind.
  */
-export const WATCH_HISTORY_KIND = 30078;
+export const WATCH_HISTORY_KIND = 30079;
 
 /**
  * Identifier applied to the watch-history list when storing it on relays.

docs/nostr-event-schemas.md

@@ -48,9 +48,9 @@ read/write split.
 | Video post (`NOTE_TYPES.VIDEO_POST`) | `30078` | `['t','video']`, `['d', <stable video identifier>]` plus optional schema append tags | JSON payload using Content Schema v3 (`version`, `title`, optional `url`, `magnet`, `thumbnail`, `description`, `mode`, `videoRootId`, `deleted`, `isPrivate`, `enableComments`, `ws`, `xs`) |
 | NIP-94 mirror (`NOTE_TYPES.VIDEO_MIRROR`) | `1063` | Tags forwarded from `publishVideo` (URL, mime type, thumbnail, alt text, magnet) | Plain text alt description |
 | Relay list (`NOTE_TYPES.RELAY_LIST`) | `10002` | Repeating `['r', <relay url>]` tags, optionally with a marker of `'read'` or `'write'` to scope the relay; marker omitted for read/write relays | Empty content |
-| View counter (`NOTE_TYPES.VIEW_EVENT`) | `WATCH_HISTORY_KIND` (default `30078`) | Canonical tag set: `['t','view']` plus either `['e', <eventId>]` or `['a', <address>]`, with optional `['session','true']` when a session actor signs; schema overrides may append extra tags. `['video', ...]` is supported for legacy overrides only. | Optional plaintext message |
-| Watch history index (`NOTE_TYPES.WATCH_HISTORY_INDEX`) | `WATCH_HISTORY_KIND` | `['d', WATCH_HISTORY_LIST_IDENTIFIER]`, `['snapshot', <id>]`, `['chunks', <total>]`, repeated `['a', <address>]` pointers to each chunk event plus schema append tags | JSON payload `{ snapshot, totalChunks }` |
-| Watch history chunk (`NOTE_TYPES.WATCH_HISTORY_CHUNK`) | `WATCH_HISTORY_KIND` | `['d', <chunk identifier>]`, `['encrypted','nip04']`, `['snapshot', <id>]`, `['chunk', <index>, <total>]`, pointer tags for each entry, plus schema append tags | NIP-04 encrypted JSON chunk (`{ version, snapshot, chunkIndex, totalChunks, items[] }`) |
+| View counter (`NOTE_TYPES.VIEW_EVENT`) | `WATCH_HISTORY_KIND` (default `30079`, clients also read legacy `30078`) | Canonical tag set: `['t','view']` plus either `['e', <eventId>]` or `['a', <address>]`, with optional `['session','true']` when a session actor signs; schema overrides may append extra tags. `['video', ...]` is supported for legacy overrides only. | Optional plaintext message |
+| Watch history index (`NOTE_TYPES.WATCH_HISTORY_INDEX`) | `WATCH_HISTORY_KIND` (default `30079`, clients also read legacy `30078`) | `['d', WATCH_HISTORY_LIST_IDENTIFIER]`, `['snapshot', <id>]`, `['chunks', <total>]`, repeated `['a', <address>]` pointers to each chunk event plus schema append tags | JSON payload `{ snapshot, totalChunks }` |
+| Watch history chunk (`NOTE_TYPES.WATCH_HISTORY_CHUNK`) | `WATCH_HISTORY_KIND` (default `30079`, clients also read legacy `30078`) | `['d', <chunk identifier>]`, `['encrypted','nip04']`, `['snapshot', <id>]`, `['chunk', <index>, <total>]`, pointer tags for each entry, plus schema append tags | NIP-04 encrypted JSON chunk (`{ version, snapshot, chunkIndex, totalChunks, items[] }`) |
 | Subscription list (`NOTE_TYPES.SUBSCRIPTION_LIST`) | `30002` | `['d', 'subscriptions']` | NIP-04 encrypted JSON `{ subPubkeys: string[] }` |
 | User block list (`NOTE_TYPES.USER_BLOCK_LIST`) | `30002` | `['d', 'user-blocks']` | NIP-04 encrypted JSON `{ blockedPubkeys: string[] }` |
 | Admin moderation list (`NOTE_TYPES.ADMIN_MODERATION_LIST`) | `30000` | `['d', 'bitvid:admin:editors']`, repeated `['p', <pubkey>]` entries | Empty content |

js/nostr.js

@@ -58,6 +58,19 @@ const WATCH_HISTORY_INDEX_IDENTIFIER_LOWER =
     ? WATCH_HISTORY_LIST_IDENTIFIER.trim().toLowerCase()
     : "";
 
+// Continue reading historical payloads from kind 30078 until operators retire it.
+const LEGACY_WATCH_HISTORY_KINDS = Object.freeze([30078]);
+
+const WATCH_HISTORY_READ_KINDS = Object.freeze(
+  Array.from(
+    new Set(
+      [WATCH_HISTORY_KIND, ...LEGACY_WATCH_HISTORY_KINDS]
+        .map((value) => Number.parseInt(value, 10))
+        .filter((value) => Number.isFinite(value))
+    )
+  )
+);
+
 const WATCH_HISTORY_CHUNK_V2_PREFIX = (() => {
   const rawIdentifier =
     typeof WATCH_HISTORY_LIST_IDENTIFIER === "string"
@@ -121,6 +134,14 @@ const LEGACY_WATCH_HISTORY_IDENTIFIER_SET = (() => {
   return normalized;
 })();
 
+function isWatchHistoryKind(kind) {
+  const numericKind = Number.parseInt(kind, 10);
+  if (!Number.isFinite(numericKind)) {
+    return false;
+  }
+  return WATCH_HISTORY_READ_KINDS.includes(numericKind);
+}
+
 const viewEventPublishMemory = new Map();
 
 let ingestLocalViewEventRef = null;
@@ -582,7 +603,7 @@ function createVideoViewEventFilters(pointer) {
   }
 
   const pointerFilter = {
-    kinds: [WATCH_HISTORY_KIND],
+    kinds: WATCH_HISTORY_READ_KINDS,
     "#t": ["view"],
   };
 
@@ -596,7 +617,7 @@ function createVideoViewEventFilters(pointer) {
 
   if (VIEW_FILTER_INCLUDE_LEGACY_VIDEO) {
     filters.push({
-      kinds: [WATCH_HISTORY_KIND],
+      kinds: WATCH_HISTORY_READ_KINDS,
       "#t": ["view"],
       "#video": [resolved.value],
     });
@@ -741,7 +762,7 @@ function isVideoViewEvent(event, pointer) {
     return false;
   }
 
-  if (Number(event.kind) !== WATCH_HISTORY_KIND) {
+  if (!isWatchHistoryKind(event.kind)) {
     return false;
   }
 
@@ -3587,7 +3608,7 @@ class NostrClient {
 
     if (indexIdentifiers.length) {
       const indexFilter = {
-        kinds: [WATCH_HISTORY_KIND],
+        kinds: WATCH_HISTORY_READ_KINDS,
         authors: [actor],
         limit: fetchLimit,
         "#d": indexIdentifiers,
@@ -3610,7 +3631,7 @@ class NostrClient {
 
     if (legacyIdentifierSet.size) {
       const legacyFilter = {
-        kinds: [WATCH_HISTORY_KIND],
+        kinds: WATCH_HISTORY_READ_KINDS,
         authors: [actor],
         limit: fetchLimit,
         "#d": Array.from(legacyIdentifierSet),
@@ -3676,7 +3697,7 @@ class NostrClient {
             const kind = Number.parseInt(kindStr, 10);
             if (
               Number.isFinite(kind) &&
-              kind === WATCH_HISTORY_KIND &&
+              isWatchHistoryKind(kind) &&
               typeof pubkey === "string" &&
               pubkey === actor &&
               typeof identifier === "string" &&
@@ -3858,7 +3879,7 @@ class NostrClient {
 
     if (chunkIdentifiers.size) {
       chunkFilters.push({
-        kinds: [WATCH_HISTORY_KIND],
+        kinds: WATCH_HISTORY_READ_KINDS,
         authors: [actor],
         "#d": Array.from(chunkIdentifiers),
         limit: chunkFetchLimit,
@@ -3867,7 +3888,7 @@ class NostrClient {
 
     if (snapshotIds.size) {
       chunkFilters.push({
-        kinds: [WATCH_HISTORY_KIND],
+        kinds: WATCH_HISTORY_READ_KINDS,
         authors: [actor],
         "#snapshot": Array.from(snapshotIds),
         limit: chunkFetchLimit,
@@ -4335,10 +4356,15 @@ class NostrClient {
     }
 
     for (const key of Array.from(entry.resolved.keys())) {
-      if (
-        !key ||
-        !key.startsWith(`a:${WATCH_HISTORY_KIND}:`)
-      ) {
+      if (!key || !key.startsWith("a:")) {
+        continue;
+      }
+      const pointerValue = key.slice(2);
+      if (!pointerValue) {
+        continue;
+      }
+      const pointerKind = Number.parseInt(pointerValue.split(":")[0], 10);
+      if (!Number.isFinite(pointerKind) || !isWatchHistoryKind(pointerKind)) {
         continue;
       }
       if (!chunkPointerKeys.has(key)) {
@@ -4421,7 +4447,7 @@ class NostrClient {
       }
       const [kindStr, pubkeyRaw, ...identifierParts] = parts;
       const kind = Number.parseInt(kindStr, 10);
-      if (!Number.isFinite(kind) || kind !== WATCH_HISTORY_KIND) {
+      if (!Number.isFinite(kind) || !isWatchHistoryKind(kind)) {
         return "none";
       }
       const identifier = identifierParts.join(":");

tests/view-counter.test.mjs

@@ -4,6 +4,7 @@ import "./test-helpers/setup-localstorage.mjs";
 import assert from "node:assert/strict";
 
 const {
+  WATCH_HISTORY_KIND,
   VIEW_COUNT_DEDUPE_WINDOW_SECONDS,
   VIEW_COUNT_BACKFILL_MAX_DAYS,
   VIEW_COUNT_CACHE_TTL_MS,
@@ -318,7 +319,7 @@ async function testHydrationSkipsStaleEventsAndRollsOff() {
   harness.reset();
   harness.resetMetrics();
 
-  const pointer = { type: "a", value: "30078:pk:test-hydrate" };
+  const pointer = { type: "a", value: `${WATCH_HISTORY_KIND}:pk:test-hydrate` };
   const pointerKey = harness.pointerKeyFromInput(pointer);
   const nowSeconds = Math.floor(Date.now() / 1000);
   const horizonSeconds = VIEW_COUNT_BACKFILL_MAX_DAYS * 86_400;
@@ -513,7 +514,7 @@ async function testRecordVideoViewEmitsJsonPayload() {
 
   const pointer = {
     type: "a",
-    value: "30078:pub:view-json",
+    value: `${WATCH_HISTORY_KIND}:pub:view-json`,
     relay: "wss://relay.example",
   };
   const createdAt = Math.floor(Date.now() / 1000);

tests/watch-history.test.mjs

@@ -113,6 +113,7 @@ function createDecryptClient(actorPubkey) {
 }
 
 const ACTOR = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
+const LEGACY_WATCH_HISTORY_KIND = 30078;
 function deriveChunkPrefix(identifier) {
   if (typeof identifier !== "string") {
     return "watch-history:v2";
@@ -281,7 +282,7 @@ const longPointers = Array.from({ length: 8 }, (_, index) => {
   }
   return {
     type: "a",
-    value: `30078:${ACTOR}:history-${index}-${"y".repeat(60)}`,
+    value: `${WATCH_HISTORY_KIND}:${ACTOR}:history-${index}-${"y".repeat(60)}`,
     relay,
     watchedAt,
   };
@@ -526,7 +527,7 @@ const noSignerItems = [
   },
   {
     type: "a",
-    value: `30078:${LOGGED_ACTOR}:history`,
+    value: `${WATCH_HISTORY_KIND}:${LOGGED_ACTOR}:history`,
     watchedAt: baseWatchedAt + 456,
   },
 ];
@@ -684,6 +685,32 @@ if (typeof localStorage !== "undefined") {
 
 const headEvent = publishResult.event;
 
+const legacyPointerIdentifiers = new Set();
+const legacyIndexEvent = JSON.parse(JSON.stringify(headEvent));
+legacyIndexEvent.kind = LEGACY_WATCH_HISTORY_KIND;
+legacyIndexEvent.tags = legacyIndexEvent.tags.map((tag) => {
+  if (
+    Array.isArray(tag) &&
+    tag[0] === "a" &&
+    typeof tag[1] === "string" &&
+    !legacyPointerIdentifiers.size
+  ) {
+    const [kindStr, pubkey, ...identifierParts] = tag[1].split(":");
+    if (kindStr === String(WATCH_HISTORY_KIND) && identifierParts.length) {
+      const identifier = identifierParts.join(":");
+      legacyPointerIdentifiers.add(identifier);
+      const legacyAddress = `${LEGACY_WATCH_HISTORY_KIND}:${pubkey}:${identifier}`;
+      const clone = tag.slice();
+      clone[1] = legacyAddress;
+      return clone;
+    }
+  }
+  return tag;
+});
+const legacyChunkIdentifier = legacyPointerIdentifiers.size
+  ? Array.from(legacyPointerIdentifiers)[0]
+  : null;
+
 const snapshotId = headEvent.tags.find(
   (tag) => Array.isArray(tag) && tag[0] === "snapshot"
 )?.[1];
@@ -703,7 +730,7 @@ freshClient.pool = {
       : [];
 
     if (filters.length === 1 && dFilter.includes(WATCH_HISTORY_LIST_IDENTIFIER)) {
-      return [headEvent];
+      return [legacyIndexEvent];
     }
 
     const requestedIdentifiers = new Set();
@@ -725,6 +752,18 @@ freshClient.pool = {
     const responses = [];
 
     for (const event of publishResult.events) {
+      if (event.id === headEvent.id) {
+        continue;
+      }
+      const cloned = JSON.parse(JSON.stringify(event));
+      if (legacyChunkIdentifier) {
+        const identifierTag = cloned.tags.find(
+          (tag) => Array.isArray(tag) && tag[0] === "d"
+        );
+        if (identifierTag?.[1] === legacyChunkIdentifier) {
+          cloned.kind = LEGACY_WATCH_HISTORY_KIND;
+        }
+      }
       const identifier = event.tags.find(
         (tag) => Array.isArray(tag) && tag[0] === "d"
       )?.[1];
@@ -733,11 +772,11 @@ freshClient.pool = {
       )?.[1];
 
       if (identifier && requestedIdentifiers.has(identifier)) {
-        responses.push(event);
+        responses.push(cloned);
         continue;
       }
       if (eventSnapshot && requestedSnapshots.has(eventSnapshot)) {
-        responses.push(event);
+        responses.push(cloned);
       }
     }
 
@@ -769,6 +808,19 @@ assert.deepEqual(
   "rebuilt snapshot should preserve timestamps"
 );
 
+assert(
+  observedFilters.some((filters) =>
+    filters.some((filter) => {
+      const kinds = Array.isArray(filter?.kinds) ? filter.kinds : [];
+      return (
+        kinds.includes(WATCH_HISTORY_KIND) &&
+        kinds.includes(LEGACY_WATCH_HISTORY_KIND)
+      );
+    })
+  ),
+  "fetchWatchHistory should request both new and legacy kinds during migration"
+);
+
 assert(
   observedFilters.length >= 2,
   "fetch should include a follow-up query for chunk events"