Semantic topic clusters: local embeddings replace Louvain

Louvain only ever found the reasoning chains (the graph is disconnected
by nature — every connected component became its own cluster) and
edge-orphans could never join anything. Clusters are now RELATED TOPICS:
propositions grouped by meaning.

- Embedder (src/server/embeddings.ts): in-process transformers.js v4
  running EmbeddingGemma-300M q8 (MTEB 68.4, ~300MB one-time download),
  MRL-truncated to 512d, "title: none | text:" prefix on 100% of inputs,
  L2-normalized so cosine = dot. Model/dim mismatch voids the ledger
  (full re-embed). Injected at the composition root — the agent layer
  never imports the server layer (architecture law).
- Ledger (data/graph/embeddings.json): model + dim + vectors + the
  persisted cluster partition. Reclustering runs at the END of every
  embedding batch (post-merge hook, both front doors) — never per read
  (the agglomerative pass is ~1.5s at 338 nodes).
- Clustering (src/core/semantic-clusters.ts): hand-rolled average-linkage
  agglomerative over cosine (npm HDBSCAN is an abandoned alpha). Stragglers
  stay singletons by default; attachStragglers is the all-in toggle.
  Threshold 0.60 swept on the real graph: 26 topics, 74/90 edge-orphans
  land in real topics, 10 stragglers.
- Consumers (snapshot, settle, namer, replay) read the persisted
  partition — one cluster concept everywhere; core-survival label
  matching unchanged (20/26 topics kept labels).
- Louvain retired: detectCommunities + graphology* deps deleted
  (graphology-louvain had leaked into the client bundle since the
  clusters wave); scripts/bench-graph.ts (dead FA2 bench) removed.
- scripts/embed-graph.ts: backfill/cold-start (338 nodes in 150s
  including the one-time model download).

src/server/chat-handler.ts

@@ -4,6 +4,7 @@ import { createProberAgent, createTeacherAgent, type TeachingAgent } from "../ag
 import { piEngineFor } from "../agent/models.ts";
 import { createJobRunner } from "../agent/subagents/runner.ts";
 import { createExtractionPipeline } from "../agent/subagents/extraction.ts";
+import { embedNewNodes } from "./embeddings.ts";
 import type { ModelRole } from "../../config.ts";
 import { claimSteers, deliverJobResult, setJobRespawn, setTurnRunner } from "./jobs.ts";
 import { recordToPiMessages } from "./pi/record-adapter.ts";
@@ -40,7 +41,12 @@ const jobRunner = createJobRunner({
 /** The app's ONE extraction pipeline instance — both front doors share it:
  *  quiz-triggered session runs (turn-service) and notes imports
  *  (/api/graph/import) hit the same single-flight registry. */
-export const extraction = createExtractionPipeline(stores);
+export const extraction = createExtractionPipeline({
+	...stores,
+	// The REAL embedder (local transformer model): post-merge embedding
+	// batch — semantic clusters are computed and persisted here.
+	embedder: embedNewNodes,
+});
 
 // Wake-pipeline wiring (contracts.ts "THE ASYNC PROTOCOL" §4): the pipeline
 // fires its server-side teacher turns through THIS composition root, and

src/server/embeddings.ts

@@ -0,0 +1,125 @@
+/**
+ * THE EMBEDDER (DECIDED 2026-09-13): semantic topic clustering needs
+ * vectors, and vectors need a model running ON the user's machine — no
+ * external services, no API bills, no network dependency for re-embedding.
+ *
+ * Runtime: in-process transformers.js (@huggingface/transformers v4 over
+ * onnxruntime-node — native Apple Silicon acceleration). The model
+ * weights download ONCE to the local HF cache on first run (~300MB).
+ *
+ * Model: EmbeddingGemma-300M (q8) — best quality-per-megabyte small
+ * embedding model as of late 2025 (MTEB-en 68.36), built for on-device
+ * use, with MRL truncation so we store 512 of its 768 dims. Every input
+ * uses the model card's document format ("title: none | text: …"),
+ * consistently — mixing prefixed and unprefixed inputs degrades quality.
+ *
+ * Model swaps re-embed EVERYTHING: vectors from different models live in
+ * incompatible spaces. The ledger records model + dim; a mismatch marks
+ * the whole ledger stale and this module re-embeds from scratch (minutes
+ * at current scale).
+ */
+import { pipeline, type FeatureExtractionPipeline } from "@huggingface/transformers";
+import type { EmbeddingsLedger, GraphNode } from "../core/contracts.ts";
+import { semanticClusters, SEMANTIC_SIM_THRESHOLD } from "../core/semantic-clusters.ts";
+
+export const EMBEDDING_MODEL = "onnx-community/embeddinggemma-300m-ONNX";
+/** MRL truncation of the native 768 — negligible quality loss, half the
+ *  ledger size (2026-09-13 researcher sweep). */
+export const EMBEDDING_DIM = 512;
+/** Batch size bounds peak memory; propositions are tiny so 32 per call
+ *  keeps the pipeline warm without thrashing. */
+const EMBED_BATCH = 32;
+
+let pipelinePromise: Promise<FeatureExtractionPipeline> | null = null;
+
+function getPipeline(): Promise<FeatureExtractionPipeline> {
+	pipelinePromise ??= pipeline("feature-extraction", EMBEDDING_MODEL, { dtype: "q8" });
+	return pipelinePromise;
+}
+
+/** EmbeddingGemma's required document format (model card). */
+function embedInput(statement: string): string {
+	return `title: none | text: ${statement}`;
+}
+
+/** Truncate an L2-normalized vector to EMBEDDING_DIM (MRL: leading dims
+ *  carry most of the signal) and re-normalize so cosine = dot product. */
+function truncateNormalize(vec: readonly number[]): number[] {
+	const out = vec.slice(0, EMBEDDING_DIM);
+	let mag = 0;
+	for (const x of out) mag += x * x;
+	mag = Math.sqrt(mag);
+	return mag > 0 ? out.map((x) => x / mag) : out;
+}
+
+/**
+ * Embed every node the ledger is missing — the post-merge hook (new
+ * propositions only, typically a handful) AND the cold-start backfill
+ * (empty ledger = every node) in one function. Model/dim mismatch voids
+ * the ledger first (full re-embed). Also prunes vectors for nodes that
+ * no longer exist. Writes the whole ledger and returns counts for logs.
+ * Non-fatal by contract: the caller treats failure as "clustering runs
+ * without the new nodes until the next merge retries".
+ */
+export async function embedNewNodes(
+	graphStore: {
+		readNodes(): GraphNode[];
+		readEmbeddings(): EmbeddingsLedger;
+		writeEmbeddings(ledger: EmbeddingsLedger): void;
+	},
+): Promise<{ embedded: number; total: number }> {
+	const ledger = graphStore.readEmbeddings();
+	const stale = ledger.model !== EMBEDDING_MODEL || ledger.dim !== EMBEDDING_DIM;
+	const vectors: Record<string, number[]> = stale ? {} : { ...ledger.vectors };
+
+	// Node LINES may repeat an id — dedupe, first defines (getGraph rule).
+	const seen = new Set<string>();
+	const todo: GraphNode[] = [];
+	for (const node of graphStore.readNodes()) {
+		if (seen.has(node.id)) continue;
+		seen.add(node.id);
+		if (!vectors[node.id]) todo.push(node);
+	}
+
+	if (todo.length > 0) {
+		const extractor = await getPipeline();
+		for (let i = 0; i < todo.length; i += EMBED_BATCH) {
+			const batch = todo.slice(i, i + EMBED_BATCH);
+			const output = await extractor(batch.map((n) => embedInput(n.statement)), {
+				pooling: "mean",
+				normalize: true,
+			});
+			const rows = output.tolist() as number[][];
+			for (let j = 0; j < batch.length; j++) {
+				vectors[batch[j]!.id] = truncateNormalize(rows[j]!);
+			}
+		}
+	}
+
+	// Prune ghosts: vectors for node ids that no longer exist.
+	let pruned = 0;
+	for (const id of Object.keys(vectors)) {
+		if (!seen.has(id)) {
+			delete vectors[id];
+			pruned++;
+		}
+	}
+
+	// RECLUSTER AT THE END OF THE BATCH (user-decided 2026-09-13): the
+	// agglomerative pass is ~1.5s at current scale, so it runs HERE —
+	// once per embedding batch — and consumers (snapshot, settle, namer)
+	// read the persisted partition instead of recomputing per read.
+	const clusters = semanticClusters(vectors, { similarityThreshold: SEMANTIC_SIM_THRESHOLD });
+
+	const wrote = todo.length > 0 || pruned > 0 || stale || ledger.clusters.length !== clusters.length;
+	if (wrote) {
+		graphStore.writeEmbeddings({
+			model: EMBEDDING_MODEL,
+			dim: EMBEDDING_DIM,
+			vectors,
+			clusters,
+			updatedAt: new Date().toISOString(),
+		});
+	}
+	return { embedded: todo.length, total: Object.keys(vectors).length };
+}