diff --git a/js/packages/cli/src/api/routes.ts b/js/packages/cli/src/api/routes.ts index 8dc2efef..78ca9e4f 100644 --- a/js/packages/cli/src/api/routes.ts +++ b/js/packages/cli/src/api/routes.ts @@ -8,7 +8,7 @@ * is the part nobody notices — which secret a given entry produces. */ -import { readFile, stat } from "node:fs/promises"; +import { readFile, mkdir } from "node:fs/promises"; import { existsSync } from "node:fs"; import { basename, dirname, join, resolve as resolvePath } from "node:path"; import { @@ -38,6 +38,11 @@ import { passwordPolicyFromRecord, removeLink, restoreEntry, + buildSemanticRecords, + searchSemanticRecords, + semanticManifest, + semanticStatus, + type SemanticRecord, totpCodeAt, Bip85, type Entry, @@ -64,7 +69,6 @@ export const UNPORTED_PREFIXES: Array<{ prefix: string; feature: string }> = [ { prefix: "/api/v1/high-risk", feature: "high-risk partitions" }, { prefix: "/api/v1/agent/job-profiles", feature: "agent job profiles" }, { prefix: "/api/v1/agent/recovery", feature: "agent recovery split" }, - { prefix: "/api/v1/semantic", feature: "the semantic (vector search) index" }, ]; export interface ApiContext { @@ -812,6 +816,84 @@ export function registerRoutes(server: ApiServer, ctx: ApiContext): void { }; }); + // --------------------------------------------------------------- semantic + + const semanticDir = (): string => join(profileDir(ctx), "semantic_index"); + const semanticRecordsPath = (): string => join(semanticDir(), "records.json"); + const semanticManifestPath = (): string => join(semanticDir(), "manifest.json"); + + async function readSemanticRecords(): Promise { + if (!existsSync(semanticRecordsPath())) return []; + try { + const data = JSON.parse(await readFile(semanticRecordsPath(), "utf8")); + return Array.isArray(data) ? (data as SemanticRecord[]) : []; + } catch { + return []; + } + } + + async function buildSemantic(): Promise { + const vault = await readVault(ctx); + const entries = Object.entries(vault.index.entries).map(([id, e]) => ({ + ...(e as object), + id: Number(id), + })) as Array>; + const records = buildSemanticRecords(entries); + await mkdir(semanticDir(), { recursive: true }); + // The records file is plaintext, so it holds metadata only and is written + // through atomicWrite for a genuine 0600 inode. + await atomicWrite(semanticRecordsPath(), new TextEncoder().encode(JSON.stringify(records, null, 2))); + await atomicWrite( + semanticManifestPath(), + new TextEncoder().encode( + JSON.stringify( + semanticManifest({ + enabled: true, + built: true, + recordCount: records.length, + updatedAt: ctx.now() / 1000, + }), + null, + 2, + ), + ), + ); + return records.length; + } + + server.route("GET", "/api/v1/semantic/status", async () => { + let manifest: Record = {}; + if (existsSync(semanticManifestPath())) { + try { + manifest = JSON.parse(await readFile(semanticManifestPath(), "utf8")); + } catch { + // Corrupt manifest reports as not-built; the fix is a rebuild. + } + } + return { json: semanticStatus(manifest, (await readSemanticRecords()).length) }; + }); + + for (const path of ["/api/v1/semantic/build", "/api/v1/semantic/rebuild"]) { + server.route("POST", path, async () => { + // Build and rebuild are the same operation here: the records file is + // rewritten wholesale either way, so there is no stale state to clear. + return { json: { status: "ok", records: await buildSemantic() } }; + }); + } + + server.route("POST", "/api/v1/semantic/search", async (req) => { + const body = bodyObject(req); + const records = await readSemanticRecords(); + if (records.length === 0) { + throw new HttpError(409, "no semantic index; POST /api/v1/semantic/build first"); + } + const hits = searchSemanticRecords(records, str(body["query"], "query"), { + k: optInt(body["k"], "k") ?? 10, + ...(typeof body["kind"] === "string" && { kind: body["kind"] }), + }); + return { json: { results: hits.map((h) => ({ ...h, ref: refFor(String(h.entry_id)) })) } }; + }); + server.route("POST", "/api/v1/shutdown", async () => { ctx.requestShutdown(); return { json: { status: "shutting down" } }; diff --git a/js/packages/cli/src/capabilities.ts b/js/packages/cli/src/capabilities.ts index ffc5d73e..e9618947 100644 --- a/js/packages/cli/src/capabilities.ts +++ b/js/packages/cli/src/capabilities.ts @@ -99,7 +99,6 @@ export function capabilities(): Record { "pgp RSA keys (ed25519 is supported; RSA generation is not reproducible)", "index0/atlas", "approval gates and high-risk partitions", - "semantic (vector search) command group", "Python's v2/v3 TUIs (interactive mode follows the legacy v1 menus)", ], }; diff --git a/js/packages/cli/src/program.ts b/js/packages/cli/src/program.ts index b4afa853..cc5c8c08 100644 --- a/js/packages/cli/src/program.ts +++ b/js/packages/cli/src/program.ts @@ -9,7 +9,7 @@ import { Command } from "commander"; import process from "node:process"; -import { entryMetadata, parseRef, resolveEntry } from "./refs.js"; +import { entryMetadata, parseRef, resolveEntry, refFor } from "./refs.js"; import { materializeSecret } from "./secrets.js"; import { clipboardSink, execSink, parseCommandSpec, stdinSink } from "./sinks.js"; import { capabilities } from "./capabilities.js"; @@ -26,6 +26,11 @@ import { parseBackupWrapper, exportBackup, findDerivationCollisions, + buildSemanticRecords, + searchSemanticRecords, + semanticManifest, + semanticStatus, + type SemanticRecord, generateFingerprint, assertValidMnemonic, generateMnemonic, @@ -1669,6 +1674,114 @@ export function buildProgram(io: ProgramIo = defaultIo): Command { io.out(JSON.stringify({ status: "shutting down", host: o.host, port })); }); + const semantic = program + .command("semantic") + .description("local retrieval index over entry METADATA (never secrets)"); + + /** + * Records live in a plaintext file beside the vault, so the index holds + * only what is safe to leave unencrypted: labels, notes, tags, usernames, + * URLs, key names. Never a value. See buildSemanticRecords. + */ + async function semanticPaths(opts: GlobalOpts) { + const app = new AppDir(resolveAppDir(opts.appDir)); + const fp = await currentFingerprint(app, opts); + const dir = join(app.profileDir(fp), "semantic_index"); + return { dir, records: join(dir, "records.json"), manifest: join(dir, "manifest.json") }; + } + + async function readRecords(path: string): Promise { + if (!existsSync(path)) return []; + try { + const data = JSON.parse(await readFile(path, "utf8")); + return Array.isArray(data) ? (data as SemanticRecord[]) : []; + } catch { + return []; + } + } + + semantic + .command("build") + .description("(re)build the index from the current vault") + .action(async () => { + const opts = program.opts() as GlobalOpts; + const vault = await openFromOptions(opts); + const paths = await semanticPaths(opts); + const entries = Object.entries(vault.index.entries).map(([id, e]) => ({ + ...(e as object), + id: Number(id), + })) as Array>; + const records = buildSemanticRecords(entries); + await mkdir(paths.dir, { recursive: true }); + // atomicWrite gives a fresh 0600 inode. Python created these at the + // process umask, which on a default install is 0664 — group and world + // readable, for a file sitting next to an encrypted vault. + await atomicWrite( + paths.records, + utf8(JSON.stringify(records, null, 2)), + ); + await atomicWrite( + paths.manifest, + utf8( + JSON.stringify( + semanticManifest({ + enabled: true, + built: true, + recordCount: records.length, + updatedAt: Date.now() / 1000, + }), + null, + 2, + ), + ), + ); + io.out(JSON.stringify({ built: records.length, index: paths.dir })); + }); + + semantic + .command("status") + .description("show whether the index exists and how current it is") + .action(async () => { + const opts = program.opts() as GlobalOpts; + const paths = await semanticPaths(opts); + let manifest: Record = {}; + if (existsSync(paths.manifest)) { + try { + manifest = JSON.parse(await readFile(paths.manifest, "utf8")); + } catch { + // A corrupt manifest reports as "not built" rather than failing: + // the fix is to rebuild, which the status is meant to prompt. + } + } + const records = await readRecords(paths.records); + io.out(JSON.stringify(semanticStatus(manifest, records.length), null, 2)); + }); + + semantic + .command("search ") + .description("rank entries by metadata overlap (returns references, no secrets)") + .option("-k, --limit ", "maximum hits", "10") + .option("--kind ", "restrict to one entry kind") + .action(async (query: string, o: { limit: string; kind?: string }) => { + const opts = program.opts() as GlobalOpts; + const paths = await semanticPaths(opts); + const records = await readRecords(paths.records); + if (records.length === 0) { + throw new Error("no semantic index; run 'semantic build' first"); + } + const hits = searchSemanticRecords(records, query, { + k: parseIntOption(o.limit, "--limit", { min: 1 }), + ...(o.kind !== undefined && { kind: o.kind }), + }); + io.out( + JSON.stringify( + hits.map((h) => ({ ...h, ref: refFor(String(h.entry_id)) })), + null, + 2, + ), + ); + }); + const util = program.command("util").description("utility commands"); util diff --git a/js/packages/cli/test/api.test.ts b/js/packages/cli/test/api.test.ts index 25d07251..7e2a815a 100644 --- a/js/packages/cli/test/api.test.ts +++ b/js/packages/cli/test/api.test.ts @@ -10,7 +10,7 @@ */ import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { mkdtemp, mkdir, readFile, writeFile } from "node:fs/promises"; +import { mkdtemp, mkdir, readFile, writeFile, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import process from "node:process"; @@ -430,13 +430,77 @@ describe("vault export and import", () => { }); }); +describe("semantic index", () => { + it("builds, reports status, and ranks by metadata", async () => { + const built = await call("POST", "/api/v1/semantic/build"); + expect(built.status).toBe(200); + expect(built.json.records).toBeGreaterThan(0); + + const status = await call("GET", "/api/v1/semantic/status"); + expect(status.json.built).toBe(true); + expect(status.json.model_id).toBe("seedpass-token-overlap-v2"); + + const hits = await call("POST", "/api/v1/semantic/search", { body: { query: "bank" } }); + expect(hits.json.results.length).toBeGreaterThan(0); + expect(hits.json.results[0].ref).toMatch(/^sp:\/\/entry\//); + }); + + it("never writes a stored secret into the plaintext index", async () => { + await call("POST", "/api/v1/semantic/build"); + // The vault holds a key_value entry whose value is this string. The + // index file sits unencrypted next to the vault, so the value must not + // reach it — nor a tokenized form of it. + const raw = await readFile( + join(app.profileDir(FINGERPRINT), "semantic_index", "records.json"), + "utf8", + ); + expect(raw).not.toContain("kv-secret-value"); + expect(raw.toLowerCase()).not.toContain("kv-secret-value".toLowerCase()); + // But the entry is still findable by its name. + const hits = await call("POST", "/api/v1/semantic/search", { body: { query: "api-token" } }); + expect(hits.json.results.map((r: any) => r.label)).toContain("api-token"); + }); + + it("writes the index at 0600, not the process umask", async () => { + await call("POST", "/api/v1/semantic/build"); + const path = join(app.profileDir(FINGERPRINT), "semantic_index", "records.json"); + expect((await stat(path)).mode & 0o777).toBe(0o600); + }); + + it("says so rather than returning nothing when no index exists", async () => { + const fresh = new ApiServer({ host: "127.0.0.1", port: 0, token: TOKEN }); + const other = await mkdtemp(join(tmpdir(), "seedpass-nosem-")); + const otherApp = new AppDir(other); + await otherApp.mutateFingerprints((d) => { + d.fingerprints.push(FINGERPRINT); + d.last_used = FINGERPRINT; + }); + await mkdir(otherApp.profileDir(FINGERPRINT), { recursive: true }); + await writeFile( + join(otherApp.profileDir(FINGERPRINT), INDEX_FILENAME), + await encryptV3(deriveIndexKeyBytes(MNEMONIC), utf8(JSON.stringify({ schema_version: 4, entries: {} }))), + ); + registerRoutes(fresh, buildContext({ app: otherApp, fingerprint: FINGERPRINT, mnemonic: MNEMONIC })); + const bound = await fresh.listen(); + try { + const res = await fetch(`http://${bound.host}:${bound.port}/api/v1/semantic/search`, { + method: "POST", + headers: { authorization: `Bearer ${TOKEN}`, "content-type": "application/json" }, + body: JSON.stringify({ query: "anything" }), + }); + expect(res.status).toBe(409); + } finally { + await fresh.close(); + } + }); +}); + describe("the unported surface answers honestly", () => { it("returns 501 with the reason, not 404", async () => { for (const [method, path, feature] of [ ["GET", "/api/v1/high-risk/status", "high-risk partitions"], ["GET", "/api/v1/agent/job-profiles", "agent job profiles"], ["POST", "/api/v1/agent/recovery/split", "agent recovery split"], - ["POST", "/api/v1/semantic/search", "semantic"], ] as const) { const res = await call(method, path); // 404 would read as "you typed the path wrong" and send an integrator diff --git a/js/packages/core/src/index.ts b/js/packages/core/src/index.ts index d6f61ea8..b07ca85b 100644 --- a/js/packages/core/src/index.ts +++ b/js/packages/core/src/index.ts @@ -88,6 +88,20 @@ export { SchemaMigrationError, } from "./schema/migrations.js"; export { encodeQr, renderQrText, EC_LEVELS, type QrMatrix, type EcLevel } from "./util/qr.js"; +export { + buildSemanticRecords, + searchSemanticRecords, + semanticText, + semanticManifest, + semanticStatus, + tokenize, + SEMANTIC_KINDS, + SEMANTIC_MODEL_ID, + SEMANTIC_SCHEMA_VERSION, + type SemanticRecord, + type SemanticHit, + type SemanticStatus, +} from "./vault/semanticIndex.js"; export { findDerivationCollisions, APP32_KINDS, diff --git a/js/packages/core/src/vault/semanticIndex.ts b/js/packages/core/src/vault/semanticIndex.ts new file mode 100644 index 00000000..0fb8a64b --- /dev/null +++ b/js/packages/core/src/vault/semanticIndex.ts @@ -0,0 +1,221 @@ +/** + * Local retrieval index over entry metadata — parity with + * src/seedpass/core/semantic_index.py. + * + * Despite the name this is not vector search and needs no model: it tokenizes + * an entry's descriptive fields and scores a query by Jaccard overlap + * (|intersection| / |union|) over those token sets. That is why it can be + * ported at all — there is nothing to embed. + * + * IT MUST NEVER INDEX SECRET MATERIAL. The records file is plaintext on disk, + * so anything indexed is readable by whatever can read the profile directory + * — a backup, a sync client, another user on a shared machine — without the + * master password. Python used to append a key_value entry's `value` here, + * which put the stored secret in that file in the clear along with a + * tokenized copy that leaked it just as well; that is fixed there and was + * never done here. The rule is simple and worth keeping simple: nobody + * searches for a secret they do not already know, so indexing one is pure + * downside. + */ + +const WORD_RE = /[a-z0-9_]+/g; + +export const SEMANTIC_SCHEMA_VERSION = 1; +/** Bumped when secret values stopped being indexed; a v1 index still holds them. */ +export const SEMANTIC_MODEL_ID = "seedpass-token-overlap-v2"; + +/** Kinds worth indexing. Others carry nothing searchable that is not secret. */ +export const SEMANTIC_KINDS = new Set([ + "document", + "note", + "key_value", + "password", + "stored_password", + "totp", + "nostr", + "ssh", + "pgp", +]); + +export interface SemanticRecord { + entry_id: number; + kind: string; + label: string; + text: string; + /** Sorted, for a stable on-disk form. */ + tokens: string[]; +} + +export interface SemanticHit { + entry_id: number; + kind: string; + label: string; + score: number; + excerpt: string; +} + +export interface SemanticStatus { + enabled: boolean; + built: boolean; + records: number; + schema_version: number; + model_id: string; + updated_at: number; +} + +export function tokenize(text: string): Set { + const lowered = text.trim().toLowerCase(); + if (!lowered) return new Set(); + return new Set(lowered.match(WORD_RE) ?? []); +} + +/** + * The searchable text for an entry. + * + * Every field here is a NAME or a note, never a value. Adding a field to this + * function puts it in a plaintext file, so the question to ask of any + * addition is not "would this be useful to search" but "would I be content + * for it to sit unencrypted next to the vault". + */ +export function semanticText(entry: Record, kind: string): string { + const str = (v: unknown): string => String(v ?? "").trim(); + const parts: string[] = [str(entry["label"]), str(entry["notes"])]; + + const tags = entry["tags"]; + parts.push( + Array.isArray(tags) ? tags.map((t) => str(t)).filter(Boolean).join(" ") : str(tags), + ); + + if (kind === "document" || kind === "note") { + parts.push(str(entry["content"]), str(entry["file_type"])); + } else if (kind === "password" || kind === "stored_password") { + // Username and URL, never the derived password. + parts.push(str(entry["username"]), str(entry["url"])); + } else if (kind === "key_value") { + // The key NAME only. `value` is the secret; see the file header. + parts.push(str(entry["key"])); + } else if (kind === "totp") { + parts.push(str(entry["issuer"])); + } else if (kind === "nostr") { + // The public half only. + parts.push(str(entry["npub"])); + } else if (kind === "ssh" || kind === "pgp") { + parts.push(str(entry["fingerprint"])); + } + + const links = entry["links"]; + if (Array.isArray(links)) { + for (const link of links) { + if (typeof link !== "object" || link === null) continue; + const l = link as Record; + parts.push(str(l["relation"]), str(l["note"])); + } + } + + return parts.filter((p) => p).join("\n"); +} + +/** Build records from entries, skipping kinds and entries with nothing to index. */ +export function buildSemanticRecords( + entries: Array>, +): SemanticRecord[] { + const records: SemanticRecord[] = []; + for (const entry of entries) { + // Entry 0 is a real entry — it is the FIRST one any profile creates — so + // it cannot double as "missing". Both implementations used to test + // `id <= 0`, which silently dropped it along with genuinely absent ids + // and left every user's first entry permanently unfindable, with nothing + // to indicate why. Fixed in both; see test/semanticIndex.test.ts. + const rawId = entry["id"]; + if (rawId === undefined || rawId === null) continue; + const entryId = Number(rawId); + if (!Number.isInteger(entryId) || entryId < 0) continue; + const kind = String(entry["kind"] ?? entry["type"] ?? "").trim().toLowerCase(); + if (!SEMANTIC_KINDS.has(kind)) continue; + const text = semanticText(entry, kind); + const tokens = tokenize(text); + if (!text.trim() || tokens.size === 0) continue; + records.push({ + entry_id: entryId, + kind, + label: String(entry["label"] ?? ""), + text, + tokens: [...tokens].sort(), + }); + } + return records; +} + +/** + * Score a query against built records. + * + * Jaccard overlap, ties broken by entry id so the order is stable and matches + * Python's. Scores are rounded to six places for the same reason. + */ +export function searchSemanticRecords( + records: SemanticRecord[], + query: string, + options: { k?: number; kind?: string | null } = {}, +): SemanticHit[] { + if (records.length === 0) return []; + const queryTokens = tokenize(query); + if (queryTokens.size === 0) return []; + const wantedKind = options.kind ? options.kind.trim().toLowerCase() : null; + + const scored: Array<{ score: number; record: SemanticRecord }> = []; + for (const record of records) { + const recordKind = String(record.kind ?? "").trim().toLowerCase(); + if (wantedKind && recordKind !== wantedKind) continue; + const tokens = new Set( + record.tokens.map((t) => String(t).trim().toLowerCase()).filter((t) => t), + ); + if (tokens.size === 0) continue; + let intersection = 0; + for (const token of queryTokens) if (tokens.has(token)) intersection++; + if (intersection <= 0) continue; + const union = new Set([...queryTokens, ...tokens]).size; + scored.push({ score: intersection / (union || 1), record }); + } + + scored.sort((a, b) => b.score - a.score || a.record.entry_id - b.record.entry_id); + const limit = Math.max(1, Math.trunc(options.k ?? 10)); + return scored.slice(0, limit).map(({ score, record }) => ({ + entry_id: record.entry_id, + kind: record.kind, + label: record.label, + // Six decimal places, as Python rounds it. + score: Math.round(score * 1e6) / 1e6, + excerpt: record.text.slice(0, 220), + })); +} + +/** The manifest written beside the records. */ +export function semanticManifest(options: { + enabled: boolean; + built: boolean; + recordCount: number; + updatedAt: number; +}): Record { + return { + enabled: options.enabled, + built: options.built, + schema_version: SEMANTIC_SCHEMA_VERSION, + model_id: SEMANTIC_MODEL_ID, + updated_at: options.updatedAt, + record_count: options.recordCount, + }; +} + +export function semanticStatus( + manifest: Record, + recordCount: number, +): SemanticStatus { + return { + enabled: Boolean(manifest["enabled"] ?? false), + built: Boolean(manifest["built"] ?? false), + records: recordCount, + schema_version: Number(manifest["schema_version"] ?? SEMANTIC_SCHEMA_VERSION), + model_id: String(manifest["model_id"] ?? SEMANTIC_MODEL_ID), + updated_at: Number(manifest["updated_at"] ?? 0), + }; +} diff --git a/js/packages/core/test/semanticIndex.test.ts b/js/packages/core/test/semanticIndex.test.ts new file mode 100644 index 00000000..14731df1 --- /dev/null +++ b/js/packages/core/test/semanticIndex.test.ts @@ -0,0 +1,143 @@ +/** + * The retrieval index, checked against the Python implementation. + * + * Two properties matter, and they pull in different directions: + * + * 1. It must rank identically to Python, so a query answered on one + * implementation is answered the same way on the other. Fixtures below + * are that implementation's actual output. + * 2. It must never write a secret. The records file is plaintext on disk, + * so anything indexed is readable by whatever can read the profile + * directory without the master password. Python used to index a + * key_value entry's `value`; the fixture proves it no longer does, and + * the TS side is asserted against the same rule. + */ + +import { describe, expect, it } from "vitest"; +import { + buildSemanticRecords, + searchSemanticRecords, + semanticStatus, + tokenize, + SEMANTIC_MODEL_ID, + type SemanticRecord, +} from "@seedpass/core"; +import { semanticFixture } from "@seedpass/test-vectors"; + +// Imported through the fixture package rather than read from disk, so the +// suite also runs under the jsdom environment, where there is no filesystem. +const fx = semanticFixture; + +describe("record building matches Python", () => { + it("produces the same records, in the same order, with the same tokens", () => { + expect(buildSemanticRecords(fx.entries)).toEqual(fx.records); + }); + + it("indexes entry 0, and skips only what genuinely has nothing to index", () => { + const ids = buildSemanticRecords(fx.entries).map((r) => r.entry_id); + // Entry 0 is the FIRST entry any profile creates. Both implementations + // used to test `id <= 0` and silently dropped it, so every user's first + // entry was permanently unfindable with nothing to indicate why. + expect(ids).toContain(0); + expect(ids).not.toContain(7); // 'seed' is not an indexed kind + expect(ids).not.toContain(8); // no text at all + expect(ids).toContain(9); // a link's relation and note are indexable text + }); + + it("still skips an entry with no id, which is what the old test meant", () => { + // The fixture carries one entry with no `id` key. Dropping it is correct; + // dropping id 0 alongside it was the bug. + const withoutId = fx.entries.filter((e) => e["id"] === undefined); + expect(withoutId).toHaveLength(1); + expect(buildSemanticRecords(withoutId)).toEqual([]); + }); + + it("finds the first entry a profile ever created", () => { + const hits = searchSemanticRecords(buildSemanticRecords(fx.entries), "first entry"); + expect(hits.map((h) => h.entry_id)).toContain(0); + }); +}); + +describe("no secret reaches the index", () => { + it("omits a key_value entry's value while keeping its key name", () => { + const records = buildSemanticRecords(fx.entries); + const serialized = JSON.stringify(records); + // The fixture entry carries this value; it must not survive indexing. + expect(serialized).not.toContain("SECRET-MUST-NOT-APPEAR"); + // Tokenized fragments leak it just as well as the whole string. + for (const fragment of ["secret", "appear"]) { + expect(serialized.toLowerCase()).not.toContain(fragment); + } + // Still findable by the things that are not the secret. + const kv = records.find((r) => r.entry_id === 2)!; + expect(kv.text).toContain("DEPLOY_TOKEN"); + expect(kv.text).toContain("ci pipeline"); + }); + + it("indexes only the public half of a nostr entry", () => { + const nostr = buildSemanticRecords(fx.entries).find((r) => r.entry_id === 6)!; + expect(nostr.text).toContain("npub1example"); + expect(nostr.text).not.toContain("nsec"); + }); +}); + +describe("search matches Python", () => { + for (const [query, expected] of Object.entries(fx.queries)) { + it(`ranks "${query}" identically`, () => { + const got = searchSemanticRecords(fx.records, query, { k: 10 }); + expect(got).toEqual(expected); + }); + } + + it("filters by kind", () => { + const got = searchSemanticRecords(fx.records, "ops", { k: 10, kind: "ssh" }); + expect(got.map((h) => h.entry_id)).toEqual(fx.kind_filtered.map((h) => h.entry_id)); + }); + + it("returns nothing for an empty query rather than everything", () => { + // An empty query has no tokens, so nothing overlaps it. Returning the + // whole vault would be the dangerous reading of "matches everything". + expect(searchSemanticRecords(fx.records, "")).toEqual([]); + expect(searchSemanticRecords(fx.records, " ")).toEqual([]); + }); + + it("honours k, keeping the highest scores", () => { + const all = searchSemanticRecords(fx.records, "bank account", { k: 10 }); + const one = searchSemanticRecords(fx.records, "bank account", { k: 1 }); + expect(one).toHaveLength(1); + expect(one[0]).toEqual(all[0]); + }); + + it("breaks score ties by entry id, so the order is stable", () => { + const records: SemanticRecord[] = [ + { entry_id: 9, kind: "document", label: "b", text: "alpha", tokens: ["alpha"] }, + { entry_id: 2, kind: "document", label: "a", text: "alpha", tokens: ["alpha"] }, + ]; + expect(searchSemanticRecords(records, "alpha").map((h) => h.entry_id)).toEqual([2, 9]); + }); +}); + +describe("tokenizing", () => { + it("lowercases and splits on non-word characters", () => { + // Underscore is a word character, so `Login_2FA` is one token rather than + // two — verified against Python's [a-z0-9_]+ rather than assumed. + expect([...tokenize("Bank.Example/Login_2FA")].sort()).toEqual([ + "bank", + "example", + "login_2fa", + ]); + }); + + it("is empty for whitespace", () => { + expect(tokenize(" ").size).toBe(0); + }); +}); + +describe("status", () => { + it("reports the model id that marks a post-fix index", () => { + // A v1 index was built by code that wrote secrets, so the id has to + // distinguish them. + expect(semanticStatus({ built: true, enabled: true }, 3).model_id).toBe(SEMANTIC_MODEL_ID); + expect(SEMANTIC_MODEL_ID).toBe("seedpass-token-overlap-v2"); + }); +}); diff --git a/js/packages/test-vectors/fixtures/manifest.json b/js/packages/test-vectors/fixtures/manifest.json index d35ae088..2cdf6269 100644 --- a/js/packages/test-vectors/fixtures/manifest.json +++ b/js/packages/test-vectors/fixtures/manifest.json @@ -22,6 +22,7 @@ "pgp_keys.json", "portable_backup.json", "qr.json", + "semantic.json", "ssh_keys.json", "sync_merge.json", "totp.json", @@ -29,6 +30,6 @@ ], "fixture_version": 1, "generator": "scripts/generate_ts_port_fixtures.py", - "python_commit": "46ff51f3d64a2e7a85deae90688b4312ab4f15fa", + "python_commit": "51f1358da4d9a743ca5d079c78b827fa84599c89", "redaction_policy": "All fixtures derive from public BIP-39 test mnemonics (abandon.../legal.../zoo...). Secret-shaped outputs are generated test material, never from a live profile." } diff --git a/js/packages/test-vectors/fixtures/semantic.json b/js/packages/test-vectors/fixtures/semantic.json new file mode 100644 index 00000000..e1bed35c --- /dev/null +++ b/js/packages/test-vectors/fixtures/semantic.json @@ -0,0 +1,325 @@ +{ + "description": "semantic index records and rankings", + "entries": [ + { + "id": 0, + "kind": "password", + "label": "first-ever.example", + "notes": "the very first entry a profile creates", + "tags": [ + "edge" + ], + "username": "zero" + }, + { + "id": 1, + "kind": "password", + "label": "bank.example", + "notes": "main current account", + "tags": [ + "money", + "daily" + ], + "url": "https://bank.example", + "username": "alice" + }, + { + "id": 2, + "key": "DEPLOY_TOKEN", + "kind": "key_value", + "label": "deploy-token", + "notes": "ci pipeline", + "tags": [ + "ops" + ], + "value": "SECRET-MUST-NOT-APPEAR" + }, + { + "content": "how to recover the bank account if locked out", + "id": 3, + "kind": "document", + "label": "recovery notes", + "tags": [ + "docs" + ] + }, + { + "id": 4, + "issuer": "Fastmail", + "kind": "totp", + "label": "email-2fa", + "notes": "", + "tags": [ + "email" + ] + }, + { + "fingerprint": "SHA256:abcdef", + "id": 5, + "kind": "ssh", + "label": "prod-server", + "notes": "production access", + "tags": [ + "ops" + ] + }, + { + "id": 6, + "kind": "nostr", + "label": "social", + "notes": "", + "npub": "npub1example", + "tags": [] + }, + { + "id": 7, + "kind": "seed", + "label": "child seed", + "notes": "not an indexed kind" + }, + { + "content": "", + "id": 8, + "kind": "document", + "label": "", + "notes": "", + "tags": [] + }, + { + "id": 9, + "kind": "password", + "label": "linked", + "links": [ + { + "note": "see recovery notes", + "relation": "depends_on" + } + ], + "notes": "", + "tags": [] + }, + { + "content": "skipped", + "kind": "document", + "label": "no id at all" + } + ], + "fixture_version": 1, + "kind_filtered": [ + { + "entry_id": 5, + "excerpt": "prod-server\nproduction access\nops\nSHA256:abcdef", + "kind": "ssh", + "label": "prod-server", + "score": 0.142857 + } + ], + "queries": { + "DEPLOY_TOKEN": [ + { + "entry_id": 2, + "excerpt": "deploy-token\nci pipeline\nops\nDEPLOY_TOKEN", + "kind": "key_value", + "label": "deploy-token", + "score": 0.166667 + } + ], + "bank": [ + { + "entry_id": 1, + "excerpt": "bank.example\nmain current account\nmoney daily\nalice\nhttps://bank.example", + "kind": "password", + "label": "bank.example", + "score": 0.111111 + }, + { + "entry_id": 3, + "excerpt": "recovery notes\ndocs\nhow to recover the bank account if locked out", + "kind": "document", + "label": "recovery notes", + "score": 0.083333 + } + ], + "bank account": [ + { + "entry_id": 1, + "excerpt": "bank.example\nmain current account\nmoney daily\nalice\nhttps://bank.example", + "kind": "password", + "label": "bank.example", + "score": 0.222222 + }, + { + "entry_id": 3, + "excerpt": "recovery notes\ndocs\nhow to recover the bank account if locked out", + "kind": "document", + "label": "recovery notes", + "score": 0.166667 + } + ], + "email": [ + { + "entry_id": 4, + "excerpt": "email-2fa\nemail\nFastmail", + "kind": "totp", + "label": "email-2fa", + "score": 0.333333 + } + ], + "first entry": [ + { + "entry_id": 0, + "excerpt": "first-ever.example\nthe very first entry a profile creates\nedge\nzero", + "kind": "password", + "label": "first-ever.example", + "score": 0.181818 + } + ], + "nothing matches here": [], + "ops production": [ + { + "entry_id": 5, + "excerpt": "prod-server\nproduction access\nops\nSHA256:abcdef", + "kind": "ssh", + "label": "prod-server", + "score": 0.285714 + }, + { + "entry_id": 2, + "excerpt": "deploy-token\nci pipeline\nops\nDEPLOY_TOKEN", + "kind": "key_value", + "label": "deploy-token", + "score": 0.142857 + } + ], + "recover locked": [ + { + "entry_id": 3, + "excerpt": "recovery notes\ndocs\nhow to recover the bank account if locked out", + "kind": "document", + "label": "recovery notes", + "score": 0.166667 + } + ] + }, + "records": [ + { + "entry_id": 0, + "kind": "password", + "label": "first-ever.example", + "text": "first-ever.example\nthe very first entry a profile creates\nedge\nzero", + "tokens": [ + "a", + "creates", + "edge", + "entry", + "ever", + "example", + "first", + "profile", + "the", + "very", + "zero" + ] + }, + { + "entry_id": 1, + "kind": "password", + "label": "bank.example", + "text": "bank.example\nmain current account\nmoney daily\nalice\nhttps://bank.example", + "tokens": [ + "account", + "alice", + "bank", + "current", + "daily", + "example", + "https", + "main", + "money" + ] + }, + { + "entry_id": 2, + "kind": "key_value", + "label": "deploy-token", + "text": "deploy-token\nci pipeline\nops\nDEPLOY_TOKEN", + "tokens": [ + "ci", + "deploy", + "deploy_token", + "ops", + "pipeline", + "token" + ] + }, + { + "entry_id": 3, + "kind": "document", + "label": "recovery notes", + "text": "recovery notes\ndocs\nhow to recover the bank account if locked out", + "tokens": [ + "account", + "bank", + "docs", + "how", + "if", + "locked", + "notes", + "out", + "recover", + "recovery", + "the", + "to" + ] + }, + { + "entry_id": 4, + "kind": "totp", + "label": "email-2fa", + "text": "email-2fa\nemail\nFastmail", + "tokens": [ + "2fa", + "email", + "fastmail" + ] + }, + { + "entry_id": 5, + "kind": "ssh", + "label": "prod-server", + "text": "prod-server\nproduction access\nops\nSHA256:abcdef", + "tokens": [ + "abcdef", + "access", + "ops", + "prod", + "production", + "server", + "sha256" + ] + }, + { + "entry_id": 6, + "kind": "nostr", + "label": "social", + "text": "social\nnpub1example", + "tokens": [ + "npub1example", + "social" + ] + }, + { + "entry_id": 9, + "kind": "password", + "label": "linked", + "text": "linked\ndepends_on\nsee recovery notes", + "tokens": [ + "depends_on", + "linked", + "notes", + "recovery", + "see" + ] + } + ] +} diff --git a/js/packages/test-vectors/src/index.ts b/js/packages/test-vectors/src/index.ts index b0d41f1d..35944bee 100644 --- a/js/packages/test-vectors/src/index.ts +++ b/js/packages/test-vectors/src/index.ts @@ -9,6 +9,7 @@ import manifestJson from "../fixtures/manifest.json"; import qrJson from "../fixtures/qr.json"; +import semanticJson from "../fixtures/semantic.json"; import bip39Json from "../fixtures/bip39_seeds.json"; import bip85Json from "../fixtures/bip85_entropy.json"; import passwordsV1Json from "../fixtures/passwords_v1.json"; @@ -315,3 +316,20 @@ export interface QrCase { } export const qrCases = (qrJson as { cases: QrCase[] }).cases; + +/** Semantic-index reference data, generated by the Python implementation. */ +export const semanticFixture = semanticJson as { + entries: Array>; + records: Array<{ + entry_id: number; + kind: string; + label: string; + text: string; + tokens: string[]; + }>; + queries: Record< + string, + Array<{ entry_id: number; kind: string; label: string; score: number; excerpt: string }> + >; + kind_filtered: Array<{ entry_id: number }>; +}; diff --git a/scripts/generate_ts_port_fixtures.py b/scripts/generate_ts_port_fixtures.py index 449bbfa3..d6c00ef8 100644 --- a/scripts/generate_ts_port_fixtures.py +++ b/scripts/generate_ts_port_fixtures.py @@ -1285,6 +1285,77 @@ def gen_qr() -> dict: } +def gen_semantic() -> dict: + """Reference records and rankings for the retrieval index. + + Two properties are being pinned. The rankings must match so a query + answered on one implementation is answered the same way on the other. And + the records must contain no secret: the file this produces is plaintext + beside an encrypted vault, so anything indexed is readable without the + master password. The key_value case below carries a value that must not + appear anywhere in the output. + """ + import tempfile + + from seedpass.core.semantic_index import SemanticIndex + + entries = [ + {"id": 0, "kind": "password", "label": "first-ever.example", + "username": "zero", "notes": "the very first entry a profile creates", + "tags": ["edge"]}, + {"id": 1, "kind": "password", "label": "bank.example", "username": "alice", + "url": "https://bank.example", "notes": "main current account", + "tags": ["money", "daily"]}, + {"id": 2, "kind": "key_value", "label": "deploy-token", "key": "DEPLOY_TOKEN", + "value": "SECRET-MUST-NOT-APPEAR", "notes": "ci pipeline", "tags": ["ops"]}, + {"id": 3, "kind": "document", "label": "recovery notes", + "content": "how to recover the bank account if locked out", "tags": ["docs"]}, + {"id": 4, "kind": "totp", "label": "email-2fa", "issuer": "Fastmail", + "notes": "", "tags": ["email"]}, + {"id": 5, "kind": "ssh", "label": "prod-server", "fingerprint": "SHA256:abcdef", + "notes": "production access", "tags": ["ops"]}, + {"id": 6, "kind": "nostr", "label": "social", "npub": "npub1example", + "notes": "", "tags": []}, + {"id": 7, "kind": "seed", "label": "child seed", "notes": "not an indexed kind"}, + {"id": 8, "kind": "document", "label": "", "content": "", "notes": "", "tags": []}, + {"id": 9, "kind": "password", "label": "linked", "notes": "", "tags": [], + "links": [{"relation": "depends_on", "note": "see recovery notes"}]}, + {"kind": "document", "label": "no id at all", "content": "skipped"}, + ] + queries = [ + "bank", + "bank account", + "ops production", + "email", + "recover locked", + "nothing matches here", + "DEPLOY_TOKEN", + "first entry", + ] + + with tempfile.TemporaryDirectory() as tmpdir: + index = SemanticIndex(Path(tmpdir)) + index.build(entries) + records = json.loads( + (Path(tmpdir) / "semantic_index" / "records.json").read_text( + encoding="utf-8" + ) + ) + results = {q: index.search(q, k=10) for q in queries} + kind_filtered = index.search("ops", k=10, kind="ssh") + + blob = json.dumps(records) + assert "SECRET-MUST-NOT-APPEAR" not in blob, "the index leaked a stored secret" + + return { + "description": "semantic index records and rankings", + "entries": entries, + "records": records, + "queries": results, + "kind_filtered": kind_filtered, + } + + def main() -> None: FIXTURES_DIR.mkdir(parents=True, exist_ok=True) @@ -1309,6 +1380,7 @@ def main() -> None: "password_kdf.json": gen_password_kdf(), "legacy_payloads.json": gen_legacy_payloads(), "qr.json": gen_qr(), + "semantic.json": gen_semantic(), } entries_fixture, vault_fixture = gen_entries_and_vault() files["entries_index.json"] = entries_fixture diff --git a/src/seedpass/core/semantic_index.py b/src/seedpass/core/semantic_index.py index cff601c5..4448674c 100644 --- a/src/seedpass/core/semantic_index.py +++ b/src/seedpass/core/semantic_index.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import os import re import time from dataclasses import dataclass @@ -26,7 +27,10 @@ class SemanticIndex: MANIFEST_FILENAME = "manifest.json" RECORDS_FILENAME = "records.json" SCHEMA_VERSION = 1 - MODEL_ID = "seedpass-token-overlap-v1" + #: Bumped from v1 when secret values stopped being indexed. An index + #: built by the old code still contains them, so `status()` reports the + #: stored id and a mismatch means "rebuild this, it holds secrets". + MODEL_ID = "seedpass-token-overlap-v2" ALLOWED_KINDS = { "document", @@ -85,6 +89,10 @@ class SemanticIndex: json.dumps(serializable, ensure_ascii=True, sort_keys=True, indent=2), encoding="utf-8", ) + # Labels, notes, usernames and URLs are still vault contents even with + # secret values excluded. Every other file in a profile is 0600; this + # one was created at the process umask (0664 on a default install). + os.chmod(self.records_path, 0o600) manifest = self._load_manifest() manifest.update( { @@ -157,8 +165,20 @@ class SemanticIndex: ) -> list[SemanticRecord]: records: list[SemanticRecord] = [] for entry in entries: - entry_id = int(entry.get("id", 0) or 0) - if entry_id <= 0: + # `id` 0 is a real entry -- it is the FIRST one any profile + # creates -- so it cannot double as "missing". The old test was + # `int(entry.get("id", 0) or 0) <= 0`, which silently dropped it + # along with genuinely absent ids, leaving every user's first + # entry permanently unfindable by search with nothing to indicate + # why. + raw_id = entry.get("id") + if raw_id is None: + continue + try: + entry_id = int(raw_id) + except (TypeError, ValueError): + continue + if entry_id < 0: continue kind = str(entry.get("kind") or entry.get("type") or "").strip().lower() if kind not in self.ALLOWED_KINDS: @@ -197,8 +217,15 @@ class SemanticIndex: parts.append(str(entry.get("username", "")).strip()) parts.append(str(entry.get("url", "")).strip()) elif kind == "key_value": + # The key NAME only. `value` is the stored secret itself, and it + # used to be indexed here -- which wrote it, in the clear, to + # semantic_index/records.json alongside a tokenized copy. Nobody + # searches for a secret they do not already know, so indexing it + # was pure downside: every other file in the profile is encrypted + # and 0600, and this one handed the secret to anything that could + # read the directory (a backup, a sync client, another user on a + # shared box) without the master password. parts.append(str(entry.get("key", "")).strip()) - parts.append(str(entry.get("value", "")).strip()) elif kind == "totp": parts.append(str(entry.get("issuer", "")).strip()) elif kind == "nostr": @@ -238,6 +265,7 @@ class SemanticIndex: json.dumps(manifest, ensure_ascii=True, sort_keys=True, indent=2), encoding="utf-8", ) + os.chmod(self.manifest_path, 0o600) def _load_records(self) -> list[dict[str, Any]]: if not self.records_path.exists(): diff --git a/src/tests/test_semantic_index.py b/src/tests/test_semantic_index.py index 6aedc675..25aef367 100644 --- a/src/tests/test_semantic_index.py +++ b/src/tests/test_semantic_index.py @@ -124,3 +124,101 @@ def test_semantic_index_service_build_and_status(tmp_path: Path) -> None: hybrid = service.search("agent", k=3, mode="hybrid") assert len(hybrid) >= 1 + + +def test_stored_secrets_are_never_written_to_the_index(tmp_path): + """The index is a plaintext file; secrets must not be in it. + + `_extract_text` used to append a key_value entry's `value` -- the stored + secret itself -- which `build()` then wrote to + semantic_index/records.json in the clear, alongside a tokenized copy that + leaked it just as thoroughly. Nobody searches for a secret they do not + already know, so indexing it was pure downside: it handed the value to + anything that could read the profile directory (a backup, a sync client, + another user on a shared machine) without the master password. + """ + from seedpass.core.semantic_index import SemanticIndex + + index = SemanticIndex(tmp_path) + index.build( + [ + { + "id": 1, + "kind": "key_value", + "label": "api-token", + "key": "DEPLOY_TOKEN", + "value": "SUPER-SECRET-VALUE-12345", + "notes": "production credentials", + "tags": ["ops"], + } + ] + ) + + raw = (tmp_path / "semantic_index" / "records.json").read_text(encoding="utf-8") + assert "SUPER-SECRET-VALUE-12345" not in raw + # Tokenized fragments leak it just as well as the whole string. + for fragment in ("super", "12345"): + assert fragment not in raw.lower() + + # The entry is still findable by everything that is not the secret. + assert index.search("token") + assert index.search("production") + assert "DEPLOY_TOKEN" in raw + + +def test_index_files_are_not_readable_by_other_users(tmp_path): + """Every other file in a profile is 0600; these were created at the umask.""" + from seedpass.core.semantic_index import SemanticIndex + + index = SemanticIndex(tmp_path) + index.set_enabled(True) + index.build([{"id": 1, "kind": "document", "label": "notes", "content": "hello"}]) + for name in ("records.json", "manifest.json"): + mode = (tmp_path / "semantic_index" / name).stat().st_mode & 0o777 + assert mode == 0o600, f"{name} is {oct(mode)}" + + +def test_model_id_marks_indexes_built_before_the_fix(tmp_path): + """A stale index still contains secrets, so it has to be distinguishable.""" + from seedpass.core.semantic_index import SemanticIndex + + index = SemanticIndex(tmp_path) + index.build([{"id": 1, "kind": "document", "label": "notes", "content": "hello"}]) + assert index.status()["model_id"] == "seedpass-token-overlap-v2" + + +def test_the_first_entry_a_profile_creates_is_searchable(tmp_path): + """Entry 0 is a real entry, not a missing one. + + The check was `int(entry.get("id", 0) or 0) <= 0`, which conflated "no + id" with "id 0" — and entry ids start at 0, so the first entry any user + ever created was silently absent from every search result, with nothing + to indicate why. + """ + from seedpass.core.semantic_index import SemanticIndex + + index = SemanticIndex(tmp_path) + index.build( + [ + {"id": 0, "kind": "password", "label": "bank.example", "notes": "first"}, + {"id": 1, "kind": "password", "label": "forum.example", "notes": "second"}, + ] + ) + assert [hit["entry_id"] for hit in index.search("bank")] == [0] + assert [hit["entry_id"] for hit in index.search("forum")] == [1] + + +def test_entries_without_a_usable_id_are_still_skipped(tmp_path): + """Which is what the old check was reaching for.""" + from seedpass.core.semantic_index import SemanticIndex + + index = SemanticIndex(tmp_path) + index.build( + [ + {"kind": "document", "label": "no id", "content": "text"}, + {"id": None, "kind": "document", "label": "null id", "content": "text"}, + {"id": "abc", "kind": "document", "label": "bad id", "content": "text"}, + {"id": -1, "kind": "document", "label": "negative", "content": "text"}, + ] + ) + assert index.search("text") == []