Notes front door: format-agnostic import into the graph pipeline

The same extractor→auditor→merger pipeline now runs over raw personal
notes in arbitrary formats (vision §1, DECIDED 2026-09-13):

- NotesSource records (data/graph/sources.json, deterministic id from
  path hash → idempotent re-imports) play the chatId role in mappings,
  checkpoints, and observation events; usage lands on the source record
  (appendSourceUsage, idempotent per startedAt, kept across upserts).
- mergeFindings is chat-store-free: callers pass MergeSource
  {messages, foldedAt} — sessions pass the chat record, notes pass
  messages:null (no quiz-event rebuild; a deterministic rebuild needs
  structured quiz tool parts).
- Second extractor prompt (NOTES_EXTRACTOR_INSTRUCTIONS) for general-
  purpose extraction; understanding judges the AUTHOR's grasp. Auditor
  wording input-agnostic. extract/audit share one generic runSubmitAgent.
- graphSnapshot.fileSources (srcId→filePath); notes sources render as
  static path entries in the detail card (FileTextIcon, never a link).
- One shared pipeline instance (exported from chat-handler): same
  single-flight registry, live status chips (label = file path), usage
  accounting, and true abort for both front doors.
- Transport: POST /api/graph/import (multipart, 3-wide worker pool) +
  Import notes… button in the graph view; scripts/import-notes.ts for
  batch runs (concurrency 4).
- real-data-extraction/ gitignored (raw import sources never committed).

355/355 tests; typecheck, lint, build green.

tests/extraction.test.ts

@@ -27,10 +27,12 @@ import type { UIMessage } from "ai";
 import type { PiEngine } from "../src/agent/models.ts";
 import type {
 	AuditResult,
+	ExtractionRunUsage,
 	GraphEdge,
 	GraphNode,
 	LedgerEvent,
 	NodeMapping,
+	NotesSource,
 	SessionCheckpoint,
 	SessionFindings,
 } from "../src/core/contracts.ts";
@@ -60,6 +62,7 @@ function memoryLedgers() {
 	const events: LedgerEvent[] = [];
 	const mappings: NodeMapping[] = [];
 	const checkpoints: SessionCheckpoint[] = [];
+	const sources = new Map<string, NotesSource>();
 	return {
 		appendNode: (n: GraphNode) => void nodes.push(n),
 		readNodes: () => nodes.map((n) => ({ ...n })),
@@ -71,6 +74,19 @@ function memoryLedgers() {
 		readMappings: () => mappings.map((m) => ({ ...m })),
 		appendCheckpoint: (c: SessionCheckpoint) => void checkpoints.push(c),
 		readCheckpoints: () => checkpoints.map((c) => ({ ...c })),
+		upsertSource: (s: NotesSource) => {
+			// Mirrors the real store: an upsert KEEPS the existing usage log.
+			const prev = sources.get(s.id);
+			sources.set(s.id, { ...s, extractions: [...(prev?.extractions ?? [])] });
+		},
+		readSources: () => [...sources.values()].map((s) => ({ ...s, extractions: [...(s.extractions ?? [])] })),
+		appendSourceUsage: (id: string, entry: ExtractionRunUsage) => {
+			const source = sources.get(id);
+			if (!source) return;
+			if (!source.extractions?.some((r) => r.startedAt === entry.startedAt)) {
+				source.extractions = [...(source.extractions ?? []), entry];
+			}
+		},
 	} satisfies GraphStore;
 }
 
@@ -513,3 +529,123 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
 		await assert.rejects(pipeline.runExtraction("chat_does_not_exist"), /Chat not found/);
 	});
 });
+
+// ── the notes front door (format-agnostic import) ────────────────────────────
+
+/** Raw personal notes — arbitrary format, no session, no message ids. */
+const NOTES_TEXT = `> [!quote] YOU
+> teach me vim
+
+> [!abstract] PI
+> Vim always starts in Normal mode — keys are commands, not text.`;
+
+/** What the notes-mode extractor submits: same shapes minus attribution,
+ *  and messages always empty (the file is the provenance). */
+const NOTES_FINDINGS = {
+	propositions: [
+		{
+			id: "p1",
+			name: "Vim normal mode",
+			statement: "Vim starts in Normal mode, where keys are commands.",
+			aliases: [],
+			messages: [],
+			evidence: ["Vim always starts in Normal mode — keys are commands, not text."],
+			understanding: "solid",
+		},
+	],
+	edges: [],
+} as const;
+
+const NOTES_VERDICT = {
+	propositions: [NOTES_FINDINGS.propositions[0]],
+	edges: [],
+	dropped: [],
+} as const;
+
+describe("the notes front door (importNotes — format-agnostic)", () => {
+	it("imports raw notes end to end: source record, file-scoped provenance, no quiz events", async () => {
+		const ledgers = memoryLedgers();
+		const seen: { kind?: string; label?: string }[] = [];
+		const pipeline = createExtractionPipeline({
+			chatStore,
+			graphStore: ledgers,
+			extractor: submitModel("submit_findings", NOTES_FINDINGS),
+			// During the auditor's call the registry must show the notes run.
+			auditor: submitModel("submit_review", NOTES_VERDICT, () => {
+				const run = extractionStatuses().find((r) => r.chatId.startsWith("src_"));
+				seen.push({ kind: run?.kind, label: run?.label });
+			}),
+		});
+		const stats = await pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
+
+		assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 });
+		// The run surfaced as a NOTES run labeled with the file path.
+		assert.deepEqual(seen, [{ kind: "notes", label: "/vault/learn vim.md" }]);
+		assert.equal(extractionStatuses().length, 0, "the registry clears when the run ends");
+
+		// The source record: deterministic src id, the file path, the usage log.
+		const source = ledgers.readSources()[0]!;
+		assert.match(source.id, /^src_/);
+		assert.equal(source.filePath, "/vault/learn vim.md");
+		assert.ok(Date.parse(source.importedAt));
+		assert.equal(source.extractions?.length, 1, "the run's usage persisted into the source record");
+		assert.equal(source.extractions![0]!.stats?.nodes, 1);
+		assert.ok(source.extractions![0]!.stages.extractor.steps.length > 0, "faux usage captured");
+
+		// File-scoped provenance: the mapping line carries the source id;
+		// the node carries NO message anchors.
+		const node = ledgers.readNodes()[0]!;
+		assert.equal(node.statement, "Vim starts in Normal mode, where keys are commands.");
+		assert.deepEqual(node.provenance, []);
+		assert.deepEqual(ledgers.readMappings(), [{ chatId: source.id, sourceId: "p1", graphNodeId: node.id }]);
+
+		// One observation event, zero quiz events — outcomes inside notes
+		// surface only through understanding observations (user-decided).
+		const events = ledgers.readEvents();
+		assert.deepEqual(
+			events.map((e) => [e.kind, e.chatId, e.ref, e.note]),
+			[["observation", source.id, "obs:p1", "solid"]],
+		);
+		// The checkpoint stamps the import.
+		assert.equal(ledgers.readCheckpoints()[0]!.chatId, source.id);
+	});
+
+	it("re-importing the same file is idempotent: same source identity, refs dedupe, usage history grows", async () => {
+		const ledgers = memoryLedgers();
+		const mk = () =>
+			createExtractionPipeline({
+				chatStore,
+				graphStore: ledgers,
+				extractor: submitModel("submit_findings", NOTES_FINDINGS),
+				auditor: submitModel("submit_review", NOTES_VERDICT),
+			});
+		const first = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
+		assert.deepEqual(first, { nodes: 1, edges: 0, events: 1 });
+
+		const second = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
+		assert.deepEqual(second, { nodes: 0, edges: 0, events: 0 }, "the same source id dedupes everything");
+
+		assert.equal(ledgers.readSources().length, 1, "one source identity — the path hashes to the same id");
+		assert.equal(ledgers.readSources()[0]!.extractions?.length, 2, "both runs billed their usage");
+		assert.equal(ledgers.readNodes().length, 1, "no duplicate node");
+	});
+
+	it("aborts like a chat run: true cancellation, nothing merged, partial usage in the source record", async () => {
+		const ledgers = memoryLedgers();
+		const pipeline = createExtractionPipeline({
+			chatStore,
+			graphStore: ledgers,
+			extractor: hangingModel(),
+			auditor: submitModel("submit_review", NOTES_VERDICT),
+		});
+		const runPromise = pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
+
+		const srcId = extractionStatuses().find((r) => r.chatId.startsWith("src_"))!.chatId;
+		assert.equal(abortExtraction(srcId), true);
+		assert.deepEqual(await runPromise, { nodes: 0, edges: 0, events: 0 });
+		assert.equal(ledgers.readNodes().length, 0, "nothing merged");
+		assert.equal(ledgers.readCheckpoints().length, 0, "nothing checkpointed");
+		assert.equal(ledgers.readSources()[0]!.extractions?.length, 1, "partial usage still persisted");
+		assert.equal(ledgers.readSources()[0]!.extractions![0]!.stats, undefined, "never merged → no stats");
+	});
+});

tests/graph.test.ts

@@ -23,14 +23,22 @@ import type { QuestionAnswerData } from "../src/core/question-core.ts";
 import { getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts";
 import { graphStore } from "../src/store/graph.ts";
 import { chatStore } from "../src/store/chats.ts";
-import { LEDGER_FILES, PATHS } from "../config.ts";
+import { LEDGER_FILES, PATHS, SOURCES_FILE } from "../config.ts";
 
 /** The wired fs adapters, exactly what src/server/stores.ts passes in production. */
 const graphDeps = { chatStore, graphStore } as const satisfies GraphDeps;
 
+/** The session merge source for a chat — what the pipeline passes: the live
+ *  transcript + its updatedAt checkpoint stamp. (Notes imports pass
+ *  messages: null — see the notes-merger test below.) */
+function sessionSource(chatId: string): { messages: UIMessage[]; foldedAt: string } {
+	const record = chatStore.getChat(chatId)!;
+	return { messages: record.messages as UIMessage[], foldedAt: record.updatedAt };
+}
+
 // ─── ledger isolation (backup-and-clear of data/graph/*.jsonl) ──────────────
 
-const LEDGER_PATHS = Object.values(LEDGER_FILES);
+const LEDGER_PATHS = [...Object.values(LEDGER_FILES), SOURCES_FILE];
 const backups: Array<{ file: string; bak: string }> = [];
 const createdChats: string[] = [];
 
@@ -233,7 +241,7 @@ function scopedGraph(chatIds: string[]) {
 
 describe("the merger (mergeFindings)", () => {
 	it("creates nodes from the verdicts, rebuilds quiz events deterministically, records observations", () => {
-		const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps);
+		const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id));
 		assert.deepEqual(stats, { nodes: 3, edges: 1, events: 6 });
 
 		const graph = scopedGraph([chat1.id]);
@@ -302,7 +310,7 @@ describe("the merger (mergeFindings)", () => {
 	});
 
 	it("is idempotent — re-merging the same verdicts appends nothing", () => {
-		const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps);
+		const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id));
 		assert.deepEqual(stats, { nodes: 0, edges: 0, events: 0 });
 		const graph = scopedGraph([chat1.id]);
 		assert.equal(graph.nodes.length, 3);
@@ -346,6 +354,7 @@ describe("the merger (mergeFindings)", () => {
 				dropped: [],
 			},
 			graphDeps,
+			sessionSource(chat2.id),
 		);
 		assert.deepEqual(stats, { nodes: 0, edges: 0, events: 1 });
 
@@ -363,6 +372,41 @@ describe("the merger (mergeFindings)", () => {
 		assert.deepEqual(m, { chatId: chat2.id, sourceId: "q1", graphNodeId: alphaId });
 	});
 
+	it("resolves imported-notes ids to their file paths in the snapshot (static provenance)", () => {
+		// The notes front door in miniature: a source record + a merge under
+		// its src id — the snapshot must label it with the file path (a
+		// static entry), never as a chat session.
+		graphStore.upsertSource({ id: "src_snapfixture", filePath: "/vault/learn vim.md", importedAt: "2026-09-13T00:00:00.000Z" });
+		mergeFindings(
+			"src_snapfixture",
+			{
+				propositions: [
+					{
+						id: "p1",
+						name: "Vim registers",
+						statement: "Vim registers map single keys to editing commands.",
+						aliases: [],
+						messages: [],
+						evidence: ["fixture evidence"],
+						understanding: "solid",
+					},
+				],
+				edges: [],
+				attribution: { quiz: {} },
+				dropped: [],
+			},
+			graphDeps,
+			{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
+		);
+		const snap = graphSnapshot(graphDeps);
+		const node = snap.nodes.find((n) => n.statement === "Vim registers map single keys to editing commands.")!;
+		assert.equal(snap.fileSources["src_snapfixture"], "/vault/learn vim.md");
+		assert.deepEqual(snap.nodeSessions[node.id], ["src_snapfixture"], "the file entry joins the node's sessions list");
+		assert.equal(snap.sessions["src_snapfixture"], undefined, "a notes source is never a chat session");
+		// The understanding observation joins under the source id like any session's.
+		assert.equal(snap.nodeObservations[node.id]!["src_snapfixture"], "solid");
+	});
+
 	it("derives sessions and per-session understanding observations in graphSnapshot", () => {
 		const snap = graphSnapshot(graphDeps);
 		const idByStatement = new Map(snap.nodes.map((n) => [n.statement, n.id]));
@@ -408,17 +452,54 @@ describe("the merger (mergeFindings)", () => {
 				dropped: [],
 			},
 			graphDeps,
+			sessionSource(chat3.id),
 		);
 		assert.equal(stats.nodes, 1, "created — the merger does not match strings");
 		assert.equal(getGraph(graphStore).nodes.filter((n) => n.statement === "Alpha intuition").length, 2);
 	});
 
-	it("returns zeros for unknown chats", () => {
-		assert.deepEqual(mergeFindings("chat_does_not_exist", fixtureAudit(), graphDeps), {
-			nodes: 0,
-			edges: 0,
-			events: 0,
+	it("merges the notes front door: source id as chatId, no quiz events, checkpoint at import time", () => {
+		// Raw personal notes carry no transcript — the merger gets no
+		// message parts, so NO quiz events are rebuilt; the understanding
+		// observation still lands, keyed to the source id.
+		const stats = mergeFindings(
+			"src_notesfixture",
+			{
+				propositions: [
+					{
+						id: "p1",
+						name: "Vim normal mode",
+						statement: "Vim starts in Normal mode, where keys are commands.",
+						aliases: [],
+						messages: [], // notes carry no message anchors — the file is the provenance
+						evidence: ["fixture evidence"],
+						understanding: "solid",
+					},
+				],
+				edges: [],
+				attribution: { quiz: {} }, // normalized by the notes pipeline — nothing to attribute
+				dropped: [],
+			},
+			graphDeps,
+			{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
+		);
+		assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 });
+
+		const node = getGraph(graphStore).nodes.find((n) => n.statement.startsWith("Vim starts"))!;
+		assert.deepEqual(node.provenance, [], "no message anchors — the mapping line carries the source");
+		assert.deepEqual(graphStore.readMappings().find((m) => m.chatId === "src_notesfixture"), {
+			chatId: "src_notesfixture",
+			sourceId: "p1",
+			graphNodeId: node.id,
 		});
+		// The observation event is file-scoped; no quiz events exist.
+		const events = graphStore.readEvents().filter((e) => e.chatId === "src_notesfixture");
+		assert.deepEqual(
+			events.map((e) => [e.kind, e.ref, e.note]),
+			[["observation", "obs:p1", "solid"]],
+		);
+		// The checkpoint stamps the import time.
+		assert.equal(graphStore.readCheckpoints().find((c) => c.chatId === "src_notesfixture")?.foldedAt, "2026-09-13T00:00:00.000Z");
 	});
 });
 
@@ -452,7 +533,7 @@ describe("the trigger helper (pendingQuizAnswers)", () => {
 		assert.equal(pendingQuizAnswers(chat.id, graphDeps), 1, "one answered quiz pending");
 		assert.equal(pendingQuizAnswers("chat_does_not_exist", graphDeps), 0);
 
-		mergeFindings(chat.id, fixtureAudit(), graphDeps);
+		mergeFindings(chat.id, fixtureAudit(), graphDeps, sessionSource(chat.id));
 		// call-s is unattributed here (fixtureAudit has no call-s attribution)
 		// — but it still becomes an event, so nothing stays pending.
 		assert.equal(pendingQuizAnswers(chat.id, graphDeps), 0, "extracted → nothing pending");

tests/usage.test.ts

@@ -10,7 +10,8 @@ import test from "node:test";
 import * as fs from "node:fs";
 
 import { chatStore } from "../src/store/chats.ts";
-import { PATHS } from "../config.ts";
+import { graphStore } from "../src/store/graph.ts";
+import { PATHS, SOURCES_FILE } from "../config.ts";
 import { callCost, contextWindow, jobCost, sessionCost, sumSteps, turnCost } from "../src/core/prices.ts";
 import type { CallUsage, ExtractionRunUsage, JobRecord, TurnUsage } from "../src/core/contracts.ts";
 
@@ -170,3 +171,40 @@ test("appendExtractionUsage: appends, idempotent per startedAt, not in list payl
 
 	fs.unlinkSync(`${PATHS.chats}/${record.id}.json`);
 });
+
+test("appendSourceUsage: appends to the source record, idempotent per startedAt, upsert keeps history", () => {
+	// The real fs graphStore — sources.json backed up and restored around
+	// this test (same discipline as the chat fixture files).
+	fs.mkdirSync(PATHS.graph, { recursive: true });
+	let bak: string | undefined;
+	if (fs.existsSync(SOURCES_FILE)) bak = `${SOURCES_FILE}.w5test.bak`;
+	if (bak) fs.renameSync(SOURCES_FILE, bak);
+	try {
+		const runEntry = (startedAt: number): ExtractionRunUsage => ({
+			startedAt,
+			at: "2026-09-13T00:00:00Z",
+			durationMs: 42_000,
+			stats: { nodes: 1, edges: 2, events: 3 },
+			stages: {
+				extractor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
+				auditor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
+			},
+		});
+		const id = "src_usagetest01";
+		graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-13T00:00:00Z" });
+		graphStore.appendSourceUsage(id, runEntry(1));
+		graphStore.appendSourceUsage(id, runEntry(1)); // same run appended twice → no-op
+		graphStore.appendSourceUsage(id, runEntry(2)); // a retried run is a NEW spend
+		assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2);
+
+		// An upsert (re-import) KEEPS the usage history.
+		graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-14T00:00:00Z" });
+		assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2);
+
+		// Unknown ids are a silent no-op (nothing to bill against).
+		graphStore.appendSourceUsage("src_nope", runEntry(1));
+	} finally {
+		fs.rmSync(SOURCES_FILE, { force: true });
+		if (bak) fs.renameSync(bak, SOURCES_FILE);
+	}
+});