feat(vault): port index0/atlas, and keep it current on every mutation

index0 is the per-vault activity ledger under `_system.index0`: an append-only
event stream per writer, daily checkpoints rolling those events up, and
derived canonical views (children_of, counts_by_kind, recent_activity).

It was the last piece carried through verbatim rather than computed — which
was safe for round-tripping but meant a TypeScript client never UPDATED it. A
vault written by this port went stale the moment it was touched, and a Python
client reading it afterwards saw an activity log missing everything the TS
client had done. Worse, `nostr restore` kept the local block verbatim, so a
restore silently discarded the remote's history.

Hash parity is the entire contract here: event ids, integrity hashes,
checkpoint summaries and view hashes are SHA-256 over Python's canonical JSON,
and the merge picks winners by comparing those hashes — one byte of encoding
disagreement means the two implementations permanently disagree about which
record wins. Everything routes through the existing canonicalJson (Python's
ensure_ascii escaping, code-point key ordering, CPython number formatting)
rather than JSON.stringify, and the tests compare whole structures via
canonical JSON rather than toEqual, so they pass only if the bytes that get
hashed are identical. Verified against Python fixtures covering the append
chain, compaction, manifest metadata, views and a two-replica merge —
including non-ASCII labels and tags, a non-numeric entry id, two writers
across two days, and a managed-account scope path.

Two Python quirks are replicated deliberately rather than "fixed", because
diverging would change which events exist rather than merely how they hash:
`str(None)` is "None" (so a JSON null in a required field is a non-empty
string there), and a stored integrity_hash is preserved even when it
disagrees — this normalizes, it does not verify, and silently rewriting a
mismatched hash would hide the tampering the hash exists to reveal.

Events are emitted by DIFFING the index at the single mutation funnel each
surface goes through, rather than from each call site as Python does. That is
deliberate: a call-site emitter is something a new code path can forget, and a
silently incomplete ledger is worse than an absent one because it reads as
"nothing happened". The event vocabulary matches Python's, so a stream from
either implementation reads the same. Emission is best-effort — the atlas is
derived state, and it must never fail a committed vault write.

mergeIndexPayloads now defaults to merging index0 with the same deterministic
rules; "preserve-current" and "reject" remain for callers that want them, and
`nostr restore` no longer discards the remote's history.

Also made gen_high_risk deterministic. Fernet embeds a random IV and the
current time, so that fixture regenerated differently every run and would have
failed the fixture-drift CI job each time — it now pins both, with a note that
production must never do this.

Verified end to end: the TS binary performed four mutations, and Python read
the resulting atlas, normalized it without dropping a record, and compacted it
into checkpoints and views.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01Vtx94nXDmNgXEeThUZP4QN

js/packages/core/src/index.ts

@@ -112,6 +112,42 @@ export {
   MAX_TOTAL_SHARES,
   type ParsedShare,
 } from "./vault/recoverySplit.js";
+export {
+  normalizeIndex0,
+  normalizeIndex0Event,
+  normalizeIndex0Checkpoint,
+  normalizeCanonicalView,
+  normalizeHead,
+  normalizeViewManifest,
+  ensureIndex0Payload,
+  deriveIndex0Context,
+  makeIndex0Event,
+  appendIndex0Event,
+  buildDailyCheckpoint,
+  rebuildIndex0Checkpoints,
+  compactIndex0,
+  compactIndex0Payload,
+  rebuildCanonicalViewsPayload,
+  buildManifestIndex0Metadata,
+  listCanonicalViews,
+  getCanonicalView,
+  mergeSystemIndex0,
+  recomputeIndex0Stats,
+  // Aliased: `computeEventId` is already taken by the Nostr event module,
+  // and the two hash entirely different things.
+  computeEventHash as computeIndex0EventHash,
+  computeEventId as computeIndex0EventId,
+  computeHeadHash as computeIndex0HeadHash,
+  computeCheckpointHash as computeIndex0CheckpointHash,
+  computeViewHash as computeIndex0ViewHash,
+  INDEX0_SCHEMA_VERSION,
+  INDEX0_CANONICAL_VIEW_TYPES,
+  LOCAL_ONLY_VIEW_TYPES,
+  INDEX0_MAX_CHECKPOINTS_PER_WRITER,
+  type Index0,
+  type Index0Context,
+} from "./vault/index0.js";
+export { emitEntryEvents, type EmitOptions } from "./vault/index0Events.js";
 export { encodeQr, renderQrText, EC_LEVELS, type QrMatrix, type EcLevel } from "./util/qr.js";
 export {
   buildSemanticRecords,

js/packages/core/src/sync/canonical.ts

@@ -94,7 +94,7 @@ function pythonFloatRepr(n: number): string {
 }
 
 /** Code-point ordering, matching Python's sorted() on str. */
-function compareCodePoints(a: string, b: string): number {
+export function compareCodePoints(a: string, b: string): number {
   const ai = Array.from(a);
   const bi = Array.from(b);
   const len = Math.min(ai.length, bi.length);

js/packages/core/src/sync/merge.ts

@@ -13,6 +13,7 @@
  */
 
 import { canonicalHash, canonicalJson } from "./canonical.js";
+import { mergeSystemIndex0 } from "../vault/index0.js";
 
 export const TOMBSTONE_RETENTION_CAP = 2048;
 export const MERGE_STRATEGY = "modified_ts_hash_tombstone_v2";
@@ -264,10 +265,12 @@ function assertIndex0Portable(raw: unknown): void {
 function ensureIndex0Payload(data: unknown, options: MergeOptions = {}): Dict {
   const out: Dict = isDict(data) ? { ...data } : {};
   const system: Dict = isDict(out["_system"]) ? { ...(out["_system"] as Dict) } : {};
-  if (options.index0 !== "preserve-current") {
+  if (options.index0 === "reject") {
     assertIndex0Portable(system["index0"]);
     system["index0"] = emptyIndex0();
   }
+  // "merge" and "preserve-current" both keep the block; which side wins is
+  // decided in mergeIndexPayloads, where both are in hand.
   out["_system"] = system;
   return out;
 }
@@ -332,15 +335,20 @@ export interface MergeOptions {
   report?: MergeReport;
 
   /**
-   * What to do with `_system.index0`, Python's derived atlas state.
+   * What to do with `_system.index0`, the derived atlas state.
    *
-   * Default ("reject") refuses to merge populated index0 rather than drop
-   * it silently. "preserve-current" keeps the current side's block verbatim
-   * and ignores the incoming one — correct when merging remote state into a
-   * live profile, since Python recomputes canonical views from entries on
-   * load and would otherwise be blocked entirely.
+   * Default ("merge") merges both sides with `mergeSystemIndex0`, the same
+   * deterministic rules Python uses: events union by id, checkpoints, views
+   * and heads resolve by timestamp then by their own hash. That is now the
+   * right default because index0 is ported; before it was, the only safe
+   * choices were to refuse or to keep one side.
+   *
+   * "preserve-current" keeps the current side's block verbatim and ignores
+   * the incoming one. "reject" refuses to merge a populated index0 at all,
+   * rather than dropping it silently — kept for callers that would rather
+   * fail than have derived state change under them.
    */
-  index0?: "reject" | "preserve-current";
+  index0?: "merge" | "reject" | "preserve-current";
 }
 
 function describeEntry(entry: Dict): { kind: string; label: string } {
@@ -549,8 +557,18 @@ export function mergeIndexPayloads(
   out["_sync_meta"] = meta;
 
   const outSystem: Dict = isDict(out["_system"]) ? (out["_system"] as Dict) : {};
-  if (options.index0 !== "preserve-current") {
+  const mode = options.index0 ?? "merge";
+  if (mode === "reject") {
+    // Populated blocks were already refused above; anything reaching here is
+    // empty, and is normalized so the output shape is stable.
     outSystem["index0"] = emptyIndex0();
+  } else if (mode === "merge") {
+    // Both sides' derived state, merged by the same deterministic rules
+    // Python uses. Doing this from the ORIGINAL payloads rather than from
+    // `out` matters: `out` has already had entries merged into it, and
+    // index0 records what each writer did, not what the merge concluded.
+    const incomingSystem = isDict(incoming["_system"]) ? (incoming["_system"] as Dict) : {};
+    outSystem["index0"] = mergeSystemIndex0(outSystem["index0"], incomingSystem["index0"]);
   }
   out["_system"] = outSystem;
   return out;

js/packages/core/src/vault/index0Events.ts

@@ -0,0 +1,118 @@
+/**
+ * Emit index0 events for whatever a vault mutation actually changed.
+ *
+ * Python emits from each call site in EntryManager, naming the operation it
+ * just performed. This derives the events by comparing the index before and
+ * after instead, and is wired into the single mutation funnel every surface
+ * goes through (CLI, TUI, API).
+ *
+ * That is a deliberate difference, for one reason: a call-site emitter is
+ * something a new code path can forget, and a silently incomplete activity
+ * ledger is worse than an absent one — it reads as "nothing happened". A
+ * funnel-level differ cannot be forgotten, because there is nowhere else to
+ * write a change from. The event VOCABULARY matches Python's
+ * (entry_created / entry_updated / entry_deleted), so a stream from either
+ * implementation reads the same.
+ */
+
+import { appendIndex0Event, type Dict } from "./index0.js";
+
+function isDict(value: unknown): value is Dict {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function kindOf(entry: Dict): string {
+  return String(entry["kind"] ?? entry["type"] ?? "").trim() || "entry";
+}
+
+function tsOf(entry: Dict, fallback: number): number {
+  const raw = Number(entry["modified_ts"] ?? 0);
+  return Number.isFinite(raw) && raw > 0 ? Math.trunc(raw) : fallback;
+}
+
+export interface EmitOptions {
+  /** Entries as they were before the mutation. */
+  before: Record<string, unknown>;
+  /** Entries as they are after it. */
+  after: Record<string, unknown>;
+  fingerprintDir: string;
+  /** Seconds since the epoch, for entries carrying no timestamp of their own. */
+  now: number;
+}
+
+/**
+ * Append one event per changed entry, returning the updated payload.
+ *
+ * Comparison is by canonical JSON of the whole entry, so a change to any
+ * field counts — including ones no specific call site would have thought to
+ * report.
+ */
+export function emitEntryEvents(payload: unknown, options: EmitOptions): Dict {
+  const { before, after, fingerprintDir, now } = options;
+  let out = payload as Dict;
+
+  const ids = new Set([...Object.keys(before), ...Object.keys(after)]);
+  // Sorted so the emitted order is deterministic: two replicas performing the
+  // same batch must produce the same stream.
+  for (const id of [...ids].sort((a, b) => {
+    const an = Number(a);
+    const bn = Number(b);
+    if (Number.isInteger(an) && Number.isInteger(bn)) return an - bn;
+    return a < b ? -1 : a > b ? 1 : 0;
+  })) {
+    const oldEntry = before[id];
+    const newEntry = after[id];
+    const had = isDict(oldEntry);
+    const has = isDict(newEntry);
+    if (!had && !has) continue;
+
+    if (has && !had) {
+      out = appendIndex0Event(out, {
+        eventType: "entry_created",
+        subjectType: "entry",
+        subjectId: id,
+        subjectKind: kindOf(newEntry),
+        modifiedTs: tsOf(newEntry, now),
+        fingerprintDir,
+        payloadRef: { entry_id: id },
+        links: Array.isArray(newEntry["links"]) ? (newEntry["links"] as unknown[]) : [],
+        tags: Array.isArray(newEntry["tags"]) ? (newEntry["tags"] as unknown[]) : [],
+        summary: `Created ${kindOf(newEntry)} entry`,
+        source: "entry_management",
+      });
+      continue;
+    }
+
+    if (had && !has) {
+      out = appendIndex0Event(out, {
+        eventType: "entry_deleted",
+        subjectType: "entry",
+        subjectId: id,
+        subjectKind: kindOf(oldEntry),
+        modifiedTs: now,
+        fingerprintDir,
+        payloadRef: { entry_id: id },
+        summary: `Deleted ${kindOf(oldEntry)} entry`,
+        source: "entry_management",
+      });
+      continue;
+    }
+
+    if (had && has && JSON.stringify(oldEntry) !== JSON.stringify(newEntry)) {
+      out = appendIndex0Event(out, {
+        eventType: "entry_updated",
+        subjectType: "entry",
+        subjectId: id,
+        subjectKind: kindOf(newEntry),
+        modifiedTs: tsOf(newEntry, now),
+        fingerprintDir,
+        payloadRef: { entry_id: id },
+        links: Array.isArray(newEntry["links"]) ? (newEntry["links"] as unknown[]) : [],
+        tags: Array.isArray(newEntry["tags"]) ? (newEntry["tags"] as unknown[]) : [],
+        summary: `Updated ${kindOf(newEntry)} entry`,
+        source: "entry_management",
+      });
+    }
+  }
+  return out;
+}