feat(semantic): port the retrieval index — and stop it writing secrets to disk

The "semantic (vector search)" entry in not_yet_ported was a misnomer. It is
not vector search and needs no model: MODEL_ID is "seedpass-token-overlap-v1",
and it scores by Jaccard overlap over tokenized metadata. That is why it could
be ported at all — there is nothing to embed.

Porting it surfaced two defects in the Python implementation.

**Stored secrets were written to a plaintext, 0664 file.** `_extract_text`
appended a key_value entry's `value` — the stored secret itself — and
`build()` wrote it to semantic_index/records.json in the clear, along with a
tokenized copy that leaked it just as thoroughly. The file was created at the
process umask, so on a default install it was group- and world-readable, next
to a vault that is encrypted and 0600. Anything that could read the directory
— a backup, a sync client, another user on a shared machine — got the secret
without the master password.

Fixed in Python (index the key NAME, never the value; chmod 0600 on both
files) and never done in TypeScript. The rule is worth keeping simple: nobody
searches for a secret they do not already know, so indexing one is pure
downside. MODEL_ID is bumped to v2 so an index built by the old code is
distinguishable — a v1 index on disk still contains the secrets and should be
rebuilt.

**Entry 0 was silently unsearchable.** The check was
`int(entry.get("id", 0) or 0) <= 0`, conflating "no id" with "id 0" — and
entry ids start at 0, so the first entry any user ever created was absent from
every search result with nothing to indicate why. Found by writing a test that
queried for it. Fixed in both implementations: skip an id that is absent,
unparseable or negative, which is what the check was reaching for.

The TypeScript port is wired into the CLI (`semantic build/status/search`),
the API (the /api/v1/semantic routes are now real rather than 501), and
writes through atomicWrite for a genuine 0600 inode. Fixtures come from
scripts/generate_ts_port_fixtures.py, which asserts the leaked value is absent
from its own output, so the fixture-drift job proves provenance the way it
does for every other fixture.

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01Vtx94nXDmNgXEeThUZP4QN

js/packages/cli/src/api/routes.ts

@@ -8,7 +8,7 @@
  * is the part nobody notices — which secret a given entry produces.
  */
 
-import { readFile, stat } from "node:fs/promises";
+import { readFile, mkdir } from "node:fs/promises";
 import { existsSync } from "node:fs";
 import { basename, dirname, join, resolve as resolvePath } from "node:path";
 import {
@@ -38,6 +38,11 @@ import {
   passwordPolicyFromRecord,
   removeLink,
   restoreEntry,
+  buildSemanticRecords,
+  searchSemanticRecords,
+  semanticManifest,
+  semanticStatus,
+  type SemanticRecord,
   totpCodeAt,
   Bip85,
   type Entry,
@@ -64,7 +69,6 @@ export const UNPORTED_PREFIXES: Array<{ prefix: string; feature: string }> = [
   { prefix: "/api/v1/high-risk", feature: "high-risk partitions" },
   { prefix: "/api/v1/agent/job-profiles", feature: "agent job profiles" },
   { prefix: "/api/v1/agent/recovery", feature: "agent recovery split" },
-  { prefix: "/api/v1/semantic", feature: "the semantic (vector search) index" },
 ];
 
 export interface ApiContext {
@@ -812,6 +816,84 @@ export function registerRoutes(server: ApiServer, ctx: ApiContext): void {
     };
   });
 
+  // --------------------------------------------------------------- semantic
+
+  const semanticDir = (): string => join(profileDir(ctx), "semantic_index");
+  const semanticRecordsPath = (): string => join(semanticDir(), "records.json");
+  const semanticManifestPath = (): string => join(semanticDir(), "manifest.json");
+
+  async function readSemanticRecords(): Promise<SemanticRecord[]> {
+    if (!existsSync(semanticRecordsPath())) return [];
+    try {
+      const data = JSON.parse(await readFile(semanticRecordsPath(), "utf8"));
+      return Array.isArray(data) ? (data as SemanticRecord[]) : [];
+    } catch {
+      return [];
+    }
+  }
+
+  async function buildSemantic(): Promise<number> {
+    const vault = await readVault(ctx);
+    const entries = Object.entries(vault.index.entries).map(([id, e]) => ({
+      ...(e as object),
+      id: Number(id),
+    })) as Array<Record<string, unknown>>;
+    const records = buildSemanticRecords(entries);
+    await mkdir(semanticDir(), { recursive: true });
+    // The records file is plaintext, so it holds metadata only and is written
+    // through atomicWrite for a genuine 0600 inode.
+    await atomicWrite(semanticRecordsPath(), new TextEncoder().encode(JSON.stringify(records, null, 2)));
+    await atomicWrite(
+      semanticManifestPath(),
+      new TextEncoder().encode(
+        JSON.stringify(
+          semanticManifest({
+            enabled: true,
+            built: true,
+            recordCount: records.length,
+            updatedAt: ctx.now() / 1000,
+          }),
+          null,
+          2,
+        ),
+      ),
+    );
+    return records.length;
+  }
+
+  server.route("GET", "/api/v1/semantic/status", async () => {
+    let manifest: Record<string, unknown> = {};
+    if (existsSync(semanticManifestPath())) {
+      try {
+        manifest = JSON.parse(await readFile(semanticManifestPath(), "utf8"));
+      } catch {
+        // Corrupt manifest reports as not-built; the fix is a rebuild.
+      }
+    }
+    return { json: semanticStatus(manifest, (await readSemanticRecords()).length) };
+  });
+
+  for (const path of ["/api/v1/semantic/build", "/api/v1/semantic/rebuild"]) {
+    server.route("POST", path, async () => {
+      // Build and rebuild are the same operation here: the records file is
+      // rewritten wholesale either way, so there is no stale state to clear.
+      return { json: { status: "ok", records: await buildSemantic() } };
+    });
+  }
+
+  server.route("POST", "/api/v1/semantic/search", async (req) => {
+    const body = bodyObject(req);
+    const records = await readSemanticRecords();
+    if (records.length === 0) {
+      throw new HttpError(409, "no semantic index; POST /api/v1/semantic/build first");
+    }
+    const hits = searchSemanticRecords(records, str(body["query"], "query"), {
+      k: optInt(body["k"], "k") ?? 10,
+      ...(typeof body["kind"] === "string" && { kind: body["kind"] }),
+    });
+    return { json: { results: hits.map((h) => ({ ...h, ref: refFor(String(h.entry_id)) })) } };
+  });
+
   server.route("POST", "/api/v1/shutdown", async () => {
     ctx.requestShutdown();
     return { json: { status: "shutting down" } };

js/packages/cli/src/capabilities.ts

@@ -99,7 +99,6 @@ export function capabilities(): Record<string, unknown> {
       "pgp RSA keys (ed25519 is supported; RSA generation is not reproducible)",
       "index0/atlas",
       "approval gates and high-risk partitions",
-      "semantic (vector search) command group",
       "Python's v2/v3 TUIs (interactive mode follows the legacy v1 menus)",
     ],
   };

js/packages/cli/src/program.ts

@@ -9,7 +9,7 @@
 
 import { Command } from "commander";
 import process from "node:process";
-import { entryMetadata, parseRef, resolveEntry } from "./refs.js";
+import { entryMetadata, parseRef, resolveEntry, refFor } from "./refs.js";
 import { materializeSecret } from "./secrets.js";
 import { clipboardSink, execSink, parseCommandSpec, stdinSink } from "./sinks.js";
 import { capabilities } from "./capabilities.js";
@@ -26,6 +26,11 @@ import {
   parseBackupWrapper,
   exportBackup,
   findDerivationCollisions,
+  buildSemanticRecords,
+  searchSemanticRecords,
+  semanticManifest,
+  semanticStatus,
+  type SemanticRecord,
   generateFingerprint,
   assertValidMnemonic,
   generateMnemonic,
@@ -1669,6 +1674,114 @@ export function buildProgram(io: ProgramIo = defaultIo): Command {
       io.out(JSON.stringify({ status: "shutting down", host: o.host, port }));
     });
 
+  const semantic = program
+    .command("semantic")
+    .description("local retrieval index over entry METADATA (never secrets)");
+
+  /**
+   * Records live in a plaintext file beside the vault, so the index holds
+   * only what is safe to leave unencrypted: labels, notes, tags, usernames,
+   * URLs, key names. Never a value. See buildSemanticRecords.
+   */
+  async function semanticPaths(opts: GlobalOpts) {
+    const app = new AppDir(resolveAppDir(opts.appDir));
+    const fp = await currentFingerprint(app, opts);
+    const dir = join(app.profileDir(fp), "semantic_index");
+    return { dir, records: join(dir, "records.json"), manifest: join(dir, "manifest.json") };
+  }
+
+  async function readRecords(path: string): Promise<SemanticRecord[]> {
+    if (!existsSync(path)) return [];
+    try {
+      const data = JSON.parse(await readFile(path, "utf8"));
+      return Array.isArray(data) ? (data as SemanticRecord[]) : [];
+    } catch {
+      return [];
+    }
+  }
+
+  semantic
+    .command("build")
+    .description("(re)build the index from the current vault")
+    .action(async () => {
+      const opts = program.opts() as GlobalOpts;
+      const vault = await openFromOptions(opts);
+      const paths = await semanticPaths(opts);
+      const entries = Object.entries(vault.index.entries).map(([id, e]) => ({
+        ...(e as object),
+        id: Number(id),
+      })) as Array<Record<string, unknown>>;
+      const records = buildSemanticRecords(entries);
+      await mkdir(paths.dir, { recursive: true });
+      // atomicWrite gives a fresh 0600 inode. Python created these at the
+      // process umask, which on a default install is 0664 — group and world
+      // readable, for a file sitting next to an encrypted vault.
+      await atomicWrite(
+        paths.records,
+        utf8(JSON.stringify(records, null, 2)),
+      );
+      await atomicWrite(
+        paths.manifest,
+        utf8(
+          JSON.stringify(
+            semanticManifest({
+              enabled: true,
+              built: true,
+              recordCount: records.length,
+              updatedAt: Date.now() / 1000,
+            }),
+            null,
+            2,
+          ),
+        ),
+      );
+      io.out(JSON.stringify({ built: records.length, index: paths.dir }));
+    });
+
+  semantic
+    .command("status")
+    .description("show whether the index exists and how current it is")
+    .action(async () => {
+      const opts = program.opts() as GlobalOpts;
+      const paths = await semanticPaths(opts);
+      let manifest: Record<string, unknown> = {};
+      if (existsSync(paths.manifest)) {
+        try {
+          manifest = JSON.parse(await readFile(paths.manifest, "utf8"));
+        } catch {
+          // A corrupt manifest reports as "not built" rather than failing:
+          // the fix is to rebuild, which the status is meant to prompt.
+        }
+      }
+      const records = await readRecords(paths.records);
+      io.out(JSON.stringify(semanticStatus(manifest, records.length), null, 2));
+    });
+
+  semantic
+    .command("search <query>")
+    .description("rank entries by metadata overlap (returns references, no secrets)")
+    .option("-k, --limit <n>", "maximum hits", "10")
+    .option("--kind <kind>", "restrict to one entry kind")
+    .action(async (query: string, o: { limit: string; kind?: string }) => {
+      const opts = program.opts() as GlobalOpts;
+      const paths = await semanticPaths(opts);
+      const records = await readRecords(paths.records);
+      if (records.length === 0) {
+        throw new Error("no semantic index; run 'semantic build' first");
+      }
+      const hits = searchSemanticRecords(records, query, {
+        k: parseIntOption(o.limit, "--limit", { min: 1 }),
+        ...(o.kind !== undefined && { kind: o.kind }),
+      });
+      io.out(
+        JSON.stringify(
+          hits.map((h) => ({ ...h, ref: refFor(String(h.entry_id)) })),
+          null,
+          2,
+        ),
+      );
+    });
+
   const util = program.command("util").description("utility commands");
 
   util

js/packages/cli/test/api.test.ts

@@ -10,7 +10,7 @@
  */
 
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
-import { mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises";
+import { mkdtemp, mkdir, readFile, writeFile, stat } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import process from "node:process";
@@ -430,13 +430,77 @@ describe("vault export and import", () => {
   });
 });
 
+describe("semantic index", () => {
+  it("builds, reports status, and ranks by metadata", async () => {
+    const built = await call("POST", "/api/v1/semantic/build");
+    expect(built.status).toBe(200);
+    expect(built.json.records).toBeGreaterThan(0);
+
+    const status = await call("GET", "/api/v1/semantic/status");
+    expect(status.json.built).toBe(true);
+    expect(status.json.model_id).toBe("seedpass-token-overlap-v2");
+
+    const hits = await call("POST", "/api/v1/semantic/search", { body: { query: "bank" } });
+    expect(hits.json.results.length).toBeGreaterThan(0);
+    expect(hits.json.results[0].ref).toMatch(/^sp:\/\/entry\//);
+  });
+
+  it("never writes a stored secret into the plaintext index", async () => {
+    await call("POST", "/api/v1/semantic/build");
+    // The vault holds a key_value entry whose value is this string. The
+    // index file sits unencrypted next to the vault, so the value must not
+    // reach it — nor a tokenized form of it.
+    const raw = await readFile(
+      join(app.profileDir(FINGERPRINT), "semantic_index", "records.json"),
+      "utf8",
+    );
+    expect(raw).not.toContain("kv-secret-value");
+    expect(raw.toLowerCase()).not.toContain("kv-secret-value".toLowerCase());
+    // But the entry is still findable by its name.
+    const hits = await call("POST", "/api/v1/semantic/search", { body: { query: "api-token" } });
+    expect(hits.json.results.map((r: any) => r.label)).toContain("api-token");
+  });
+
+  it("writes the index at 0600, not the process umask", async () => {
+    await call("POST", "/api/v1/semantic/build");
+    const path = join(app.profileDir(FINGERPRINT), "semantic_index", "records.json");
+    expect((await stat(path)).mode & 0o777).toBe(0o600);
+  });
+
+  it("says so rather than returning nothing when no index exists", async () => {
+    const fresh = new ApiServer({ host: "127.0.0.1", port: 0, token: TOKEN });
+    const other = await mkdtemp(join(tmpdir(), "seedpass-nosem-"));
+    const otherApp = new AppDir(other);
+    await otherApp.mutateFingerprints((d) => {
+      d.fingerprints.push(FINGERPRINT);
+      d.last_used = FINGERPRINT;
+    });
+    await mkdir(otherApp.profileDir(FINGERPRINT), { recursive: true });
+    await writeFile(
+      join(otherApp.profileDir(FINGERPRINT), INDEX_FILENAME),
+      await encryptV3(deriveIndexKeyBytes(MNEMONIC), utf8(JSON.stringify({ schema_version: 4, entries: {} }))),
+    );
+    registerRoutes(fresh, buildContext({ app: otherApp, fingerprint: FINGERPRINT, mnemonic: MNEMONIC }));
+    const bound = await fresh.listen();
+    try {
+      const res = await fetch(`http://${bound.host}:${bound.port}/api/v1/semantic/search`, {
+        method: "POST",
+        headers: { authorization: `Bearer ${TOKEN}`, "content-type": "application/json" },
+        body: JSON.stringify({ query: "anything" }),
+      });
+      expect(res.status).toBe(409);
+    } finally {
+      await fresh.close();
+    }
+  });
+});
+
 describe("the unported surface answers honestly", () => {
   it("returns 501 with the reason, not 404", async () => {
     for (const [method, path, feature] of [
       ["GET", "/api/v1/high-risk/status", "high-risk partitions"],
       ["GET", "/api/v1/agent/job-profiles", "agent job profiles"],
       ["POST", "/api/v1/agent/recovery/split", "agent recovery split"],
-      ["POST", "/api/v1/semantic/search", "semantic"],
     ] as const) {
       const res = await call(method, path);
       // 404 would read as "you typed the path wrong" and send an integrator