Merge feat/graph-rendering: wave 5 — semantic clusters, regions renderer, persisted render tuning

This commit is contained in:
amosblomqvist
2026-09-14 05:20:45 +03:00
45 changed files with 7344 additions and 734 deletions
+2
View File
@@ -7,3 +7,5 @@ data/
.env.local
.memory/
SCOPE.md
# raw real-data import sources (user directive 2026-09-13: never commit)
real-data-extraction/
+52
View File
@@ -0,0 +1,52 @@
/**
* POST /api/graph/import — the format-agnostic front door's transport
* (vision §1, 2026-09-13): raw personal-notes files in, one extraction run
* per file through the SAME pipeline the sessions use. Multipart form data
* (field "files", any text format — the extractor is format-agnostic); the
* pipeline mints a deterministic source id per file path, persists the
* source record (the file's provenance + the run's usage), and runs
* extractor → auditor → merger over the raw text.
*
* Small concurrency cap: the model calls are the slow part, and a burst of
* 25 simultaneous runs would trip provider rate limits — a 3-wide worker
* pool keeps the run registry (the dock's status chips) and the ledgers
* healthy. The dock's chips + graph refetch track progress live through
* /api/graph/status, as with session runs.
*/
import { extraction } from "@/src/server";
import type { EmissionStats } from "@/src/core";
export const dynamic = "force-dynamic";
interface ImportResult {
filePath: string;
ok: boolean;
stats?: EmissionStats;
error?: string;
}
export async function POST(request: Request) {
const form = await request.formData();
const files = form.getAll("files").filter((f): f is File => f instanceof File);
if (files.length === 0) {
return Response.json({ results: [] as ImportResult[], error: "no files" }, { status: 400 });
}
const results: ImportResult[] = [];
const queue = [...files];
const workers = Array.from({ length: Math.min(3, files.length) }, async () => {
for (;;) {
const file = queue.shift();
if (!file) return;
try {
const text = await file.text();
const stats = await extraction.importNotes({ filePath: file.name, text });
results.push({ filePath: file.name, ok: true, stats });
} catch (err) {
results.push({ filePath: file.name, ok: false, error: err instanceof Error ? err.message : String(err) });
}
}
});
await Promise.all(workers);
return Response.json({ results });
}
+24
View File
@@ -0,0 +1,24 @@
/**
* PUT /api/graph/positions — the live-physics drag commit. The client
* anneals a drag to rest and sends the new positions; the server
* sanitizes (finite coords, known ids only), writes the ledger, and if
* the graph grew mid-drag runs one incremental settle to absorb the new
* nodes with the client's arrangement as anchors. See
* src/server/commit-positions.ts.
*
* PUT → 200 {positions: {...}} with whatever survived sanitizing
* (empty object = nothing written).
*/
import { commitPositions } from "@/src/server";
export const dynamic = "force-dynamic";
export async function PUT(request: Request) {
let body: unknown;
try {
body = await request.json();
} catch {
return Response.json({ error: "invalid JSON body" }, { status: 400 });
}
return Response.json({ positions: commitPositions(body) });
}
+31
View File
@@ -0,0 +1,31 @@
/**
* GET/PUT /api/graph/render-settings — server persistence for the graph
* view's rendering controls (node/edge/arrow/label sizes, the label-zoom
* threshold, the cluster-labels toggle). The file lives at
* data/render-settings.json (config.RENDER_SETTINGS_FILE).
*
* GET → 200 {settings} once saved, 200 null on first boot (client keeps
* in-memory defaults until the user changes something).
* PUT → 200 {settings} (the sanitized copy actually persisted — the
* client may echo it to stay honest with clamped values).
*
* Every body is passed through sanitizeRenderSettings (src/core) before
* touching disk: garbage clamps or falls back to defaults, never throws.
*/
import { readRenderSettings, writeRenderSettings } from "@/src/server";
export const dynamic = "force-dynamic";
export async function GET() {
return Response.json(readRenderSettings());
}
export async function PUT(request: Request) {
let body: unknown;
try {
body = await request.json();
} catch {
return Response.json({ error: "invalid JSON body" }, { status: 400 });
}
return Response.json(writeRenderSettings(body));
}
+3 -1
View File
@@ -12,7 +12,9 @@ export const dynamic = "force-dynamic";
export async function GET() {
const runs = extractionStatuses().map((run) => ({
chatId: run.chatId,
title: chatStore.getChat(run.chatId)?.title ?? run.chatId,
// The run's own label (the file path for notes imports) wins; chat
// runs fall back to the chat store's title.
title: run.label ?? chatStore.getChat(run.chatId)?.title ?? run.chatId,
phase: run.phase,
startedAt: run.startedAt,
// Live per-stage usage (the money channel): model + per-step CallUsage,
+19
View File
@@ -0,0 +1,19 @@
/**
* Cluster color identity — pure presentation math for the knowledge-graph
* regions. (The hull geometry that used to live here — convexHull,
* outwardNormal, offsetPolygon, traceRoundedPath — was RETIRED with the
* convex-hull renderer: regions are now a low-res world-space splat
* bitmap in graph-canvas.ts, which renders concave cluster shapes
* correctly and costs one blit per frame instead of 32 clusters × 4
* offset polygons. The file keeps the name for the barrel imports.)
*/
export type Pt = { x: number; y: number };
/** Stable hue per cluster id (content-hash): the region keeps its color
* across reshuffles because the id only changes when the members do. */
export function clusterHue(id: string): number {
let h = 0;
for (const ch of id) h = (h * 31 + ch.charCodeAt(0)) % 360;
return h;
}
File diff suppressed because it is too large Load Diff
+706 -472
View File
File diff suppressed because it is too large Load Diff
+28
View File
@@ -47,6 +47,9 @@ export const MODELS = {
extractor: { model: "z-ai/glm-5.3-flash", effort: "low" },
/** The auditor verifies evidence exists; keep it cheap. */
auditor: { model: "z-ai/glm-5.3-flash", effort: "low" },
/** Names detected communities — one singular call, given member names
* + statements. Purely a labeling flourish; keep it cheap. */
labeler: { model: "z-ai/glm-5.3-flash", effort: "low" },
} satisfies Record<string, ModelSpec>;
export type ModelRole = keyof typeof MODELS;
@@ -71,4 +74,29 @@ export const LEDGER_FILES = {
events: path.join(PATHS.graph, "events.jsonl"),
mappings: path.join(PATHS.graph, "mappings.jsonl"),
sessions: path.join(PATHS.graph, "sessions.jsonl"),
/** LLM-named community records (contracts/graph.ts ClusterLabeling). */
clusterLabels: path.join(PATHS.graph, "cluster-labels.jsonl"),
} as const;
/**
* Notes-import bookkeeping (NOT a ledger): one JSON map, src-id → NotesSource
* record (contracts/graph.ts) — the file path + import time + the run's usage
* log. Read-modify-write like a chat file, not append-only like the ledgers.
*/
export const SOURCES_FILE = path.join(PATHS.graph, "sources.json");
/** The layout ledger (read-modify-write like sources.json): nodeId →
* {x,y} resting positions from the incremental settle (core/layout.ts).
* Carries an updatedAt stamp for future live-physics conflict handling. */
export const POSITIONS_FILE = path.join(PATHS.graph, "positions.json");
/** The embedding ledger: one L2-normalized vector per node id, all from
* ONE model (swapping models re-embeds everything — vectors from
* different models live in incompatible spaces). Read-modify-write
* JSON like positions/sources. */
export const EMBEDDINGS_FILE = path.join(PATHS.graph, "embeddings.json");
/**
* Graph render settings (the settings-panel controls): one small JSON blob,
* read-modify-write. Shared shape/sanitizer: src/core/render-settings.ts.
*/
export const RENDER_SETTINGS_FILE = path.join(PATHS.graph, "render-settings.json");
+227
View File
@@ -0,0 +1,227 @@
# Graph rendering & cluster-first layout — research findings
*Compiled 2026-09-13 from four research passes (library fact-check ×2, prior-art ×2), all
claims source-verified by subagents against shipped package code (`npm pack` tarballs of
`sigma@3.0.3`, `@sigma/layer-webgl@3.0.0`), the sigma.js GitHub storybook source, and
primary docs. Speculation is marked where it appears. This document is the decision
basis for the rendering rebuild; nothing has been implemented yet.*
---
## 1. Requirements (as of this research)
Current symptoms to eliminate:
- Hull layers lag behind nodes during pan (desync).
- Settings changes shift hulls up-left until the next interaction (canvas-sizing race).
- GPU ~50–100% during pan/zoom (redundant full-canvas 2D repaints per camera event +
WebGL re-render per frame; the earlier `ctx.filter: blur` per frame caused coil whine).
- Stale/garbage boundary shapes at extreme zoom-out.
Feature set:
- Circles + directed arrows; precomputed layout (no runtime physics).
- Cluster regions: hull boundary around each community, dashed stroke, soft translucent
fill, stable color per region, region name label.
- Inverse zoom gates: node labels when zoomed IN, region labels when zoomed OUT,
cross-fading over a shared span.
- Pan, zoom-to-cursor, hover glow, click → detail card, edge-type filtering, camera
survives UI changes. 60fps, minimal GPU, ordinary laptop.
New direction (user, 2026-09-13):
- **Cluster-first placement**: layout depends on clustering — clusters placed as units,
nodes within a cluster close together (compact, natural hull shapes as a consequence).
- **Freeform experimentation**: a dev-only playground to try layouts/shapes/styles
against real and synthetic data; renderer must be dumb, data-driven, swappable.
Scale: 338 nodes / 211 edges today; low thousands expected (notes imports).
## 2. Verified facts about the current stack (sigma.js v3)
- **v3.0.3 ships a PUBLIC custom-layer API** — `createLayer` / `createCanvas` /
`createCanvasContext` / `killLayer` (typed in shipped `.d.ts`). Our current
`firstChild` DOM injection was improvising a worse version of a supported API.
- **Sigma sizes and DPR-scales ALL registered canvases** in its own `resize()` —
using the public API eliminates the canvas-sizing race by construction.
- **`afterRender` fires once per rendered frame** (rAF-debounced `scheduleRender()`).
Camera motion → full re-render each rAF. **Hover is cheap**: it only redraws the
`hoverNodes` WebGL layer + `edgeLabels` canvas via a separate rAF; no `afterRender`,
no full re-render. Sigma is strictly event-driven: no continuous loop, no rendering
when idle (verified: zero `setInterval`, all rAF sites are schedulers).
- **`renderer.graphToViewport()` is public and typed** — the sanctioned graph→pixel
path. `renderer.matrix` is **private**: building a `ctx.setTransform()` from it would
be guessing at internals; per-vertex `graphToViewport()` is the supported route.
- **The official cluster-label storybook example is exactly our pattern**:
custom DOM layer + `afterRender` + per-cluster `graphToViewport()` re-projection,
maintained in the sigma repo. Our minimal-change fix path is a demonstrated usage.
- Real v3 layer order (verified in source, NOT in docs):
`edges` (WebGL) → `edgeLabels` → `nodes` (WebGL) → `labels` → `hovers` →
`hoverNodes` (WebGL) → `mouse`.
### `@sigma/layer-webgl@3.0.0` (official background-layer package)
- `bindWebGLLayer` inserts a background WebGL layer (`beforeLayer: "edges"`), redrawn
on `afterRender` with the same camera matrix — fully in-loop, no desync.
- **`createContoursProgram` renders METABALL-style implicit fields, not hull polygons**
(verified from the fragment shader: fullscreen quad, summed radial falloffs,
analytic contour lines via `dFdx/dFdy`). Translucent fill: yes (arbitrary CSS color
with alpha). **Dashed stroke: NO** — solid anti-aliased line only; no dash parameter
exists. Per-community color: yes, but one program + one WebGL context per community,
and each runs a fullscreen fragment pass per frame (browsers cap ~16 WebGL contexts —
a problem for dozens of regions).
- **Decisive fact**: our exact visual (hull polygon + translucent fill + dashed
stroke) is **NOT achievable with shipped layer-webgl programs**. It would require a
custom `WebGLLayerProgram` (supported extension point, moderate effort, dashes DIY in
GLSL) — whereas Canvas 2D gives it for free with `ctx.setLineDash()`.
- v4-era programs (heatmap, density-splat, dashed/fill layer factories) are NOT in the
v3 package.
### sigma v4
- Real but **beta** (4.0.0-beta.5; docs site self-describes as alpha). Single `stage`
canvas + public `createLayer`, GPU picking with downsizing, SDF labels, depth
layers. Breaking migration. Not a production bet today.
### Pan GPU cost at our scale
- Structural facts verified (event-driven idle; full re-render per camera frame;
synchronous `gl.readPixels` picking per mousemove pre-#1546, since fixed).
- **Absolute GPU cost of panning 338–3k nodes in v3: UNVERIFIED** — no public
benchmark exists; nobody has measured it. If it matters, we measure it ourselves.
## 3. Verified facts about the candidates
| Candidate | Renderer | True hull fill+dash | Picking | Camera | Maintenance | License | Fit |
|---|---|---|---|---|---|---|---|
| sigma v3 + public 2D canvas layer | WebGL (+our 2D) | yes (our canvas, free) | GPU color-pick | built-in | active (3.0.3, 2026-04) | MIT | 7/10 |
| sigma v3 + layer-webgl contours | WebGL | **no** (metaballs, solid border) | — | built-in | active | MIT | 6/10 |
| sigma v4 + layers | WebGL, SDF | via custom program (beta risk) | GPU picking | built-in | **beta** | MIT | 6 (risky) |
| d3-zoom + Canvas 2D (hand-rolled loop) | Canvas 2D | yes, trivially | distance check | d3-zoom (verified math) | d3 very active | ISC | **8/10** |
| PixiJS v8 + pixi-viewport | WebGL/WebGPU/Canvas | yes (Graphics; dashes via shim `pixi-dashed-line`) | built-in events | pixi-viewport 6.0.3 | very active | MIT | 7/10 |
| cytoscape.js 3.34 | Canvas 2D | via bubblesets/layers extensions | built-in | built-in | active | MIT | 5/10 |
| vis-network 10.1 | Canvas 2D | **no** — clustering = supernodes (collapse, opposite of our regions) | built-in | built-in | sporadic | MIT | 2/10 |
| @cosmos.gl/graph (Cosmograph engine) | WebGL (luma.gl) | no (cluster = layout force only) | GPU | built-in | active | MIT | 3/10 |
Fact nuances we must not over-trust:
- "Canvas 2D is GPU-rasterized": **half-true**. Skia may raster on CPU or GPU depending
on browser/OS config. Either way the cost driver is **repaint count × pixel area
(DPR²)**, not "Skia = GPU".
- Canvas 2D at 60fps for hundreds–low-thousands of elements: **supported by practitioner
evidence** (cytoscape is fully canvas-based into the thousands; 10k animated canvas
elements ≈ 55fps in one 2025/26 benchmark). Pixel area matters more than node count
at our scale.
- Obsidian's graph view renderer: **UNVERIFIED** — no primary source. Do not cite.
- PixiJS exact bundle size: unmeasured. yFiles: commercial, unverified details.
## 4. Prior art for our specific features
### Region/hull visuals
- Nobody in the graph-tool world ships our region aesthetic: Gephi desktop, Gephi Lite
(the flagship sigma app — zero hull code in its tree), Obsidian — all partition by
**node coloring only**. The region feature is a differentiator, not a copied pattern.
- The aesthetic reference: ggforce `geom_mark_hull` (expand + round the hull, low-alpha
fill, integrated label placement).
- **Bostock's thick-stroke hull trick**: one path, wide stroke (e.g. 32px) +
`linejoin: round` = padded, rounded hull with zero polygon-offset math. Replaces our
offset-normal code outright.
- Named-region UX validation: Excalidraw frames / tldraw regions (dashed titled
container around content).
- Closest feature twin: **Cosmograph** — documents "cluster labels when zoomed out,
point labels when zoomed in" as a first-class use case (our exact inverse gate).
### Zoom-gated labels
- Consensus pattern = map-style per-layer zoom ranges (Mapbox/MapLibre
`minzoom`/`maxzoom` on symbol layers). State labels at far zoom, city labels at near
zoom is the standard cartographic solution; our inverse gates are the same mechanism
with opposite ranges. Sigma's own `labelRenderedSizeThreshold` is the zoom-in gate.
- Cross-fade + hysteresis over a shared span: no single canonical writeup (medium
confidence); nearest established practice is map label collision systems.
### The sync pattern (fixes our desync class structurally)
- Canonical and repeatedly documented: **one rAF loop with a dirty flag; input/DOM
events only mutate state.** All layers drawn in the same frame, under the same
camera transform — either same canvas (`setTransform` once, draw back-to-front) or
stacked canvases all redrawn by the same loop.
Sources: browser.engineering, MDN "Optimizing canvas", web.dev canvas performance.
### Picking
- Canvas 2D at our scale: nearest-node distance check per pointer event is the norm,
trivially fast (µs for 3k nodes; spatial grid only past ~10k).
## 5. Cluster-first layout (the new direction)
- **Two-level pattern is well-trodden**: place cluster anchors first (quotient graph —
one super-node per community, Gephi's "Groups by Partition" plugin is the desktop
prior art), layout nodes within each cluster, compose by translating local layouts
to cluster anchors. Bostock's Clustered Force Layout lineage; cosmos.gl's cluster
force verified from source (per-frame center-of-mass pull, or user-pinned positions —
cluster anchors as data independent of member layout).
- **graphology FA2 has NO cluster-aware settings** (full settings list read; gravity is
global). **LinLog mode** (`linLogMode: true`) is the documented "communities separate
tightly" switch — a one-afternoon A/B candidate before any custom pipeline.
- **No turnkey JS package does the composed two-level pipeline** — assemble from
graphology primitives (~150 lines of glue: quotient build, per-cluster FA2 with
circular init, compose transform, `noverlap` cleanup). The combination is our
synthesis from verified primitives.
- Deterministic given fixed init positions (no RNG dependency in FA2).
- **Server-side at extraction time is the better fit** for our architecture: positions
become data, rendering stays dumb. Browser worker fallback for live re-layout.
- Harness precedents: yFiles layout playground (polished), Observable/bl.ocks lineage
(notebook = graph + sliders + canvas), react-sigma layouts example. No canonical OSS
playground for our stack — copy the config-driven pattern (algorithm dropdown +
parameter sliders + seed input + live canvas).
## 6. Options (the decision)
**Option A — Stay on sigma v3; mount the hull canvas via the public
`createCanvas` API; redraw on `afterRender` via `graphToViewport()`.**
Supported, minimal-change, demonstrated by sigma's own cluster-label storybook. Kills
the desync, the sizing race, and the multi-event redraws (once per frame, sigma-sized).
Our exact visuals stay (dashes free in 2D). Keeps sigma's camera/labels/picking
machinery. Residual: full-canvas 2D repaint per interaction frame remains (raster cost,
mitigable with move-gating like sigma's own `hideLabelsOnMove`); WebGL + 2D raster
still both run per frame. Est: ~1 day.
**Option A′ — sigma + custom `WebGLLayerProgram` for hulls.** Supported extension
point, but dashes are DIY GLSL and we'd rebuild in shaders what Canvas 2D gives free.
No reason given A exists. Est: several days, for a worse iteration loop.
**Option B — d3-zoom + hand-rolled single Canvas 2D renderer.**
One canvas, one rAF + dirty flag, world-space geometry + one transform, all layers
back-to-front in the same frame. Lowest GPU/CPU, total control, perfectly matches the
new requirements: renderer is a dumb draw function over data (layout experiments can't
fight it), playground is trivial, cluster-first layout slots in as pure data. Costs
~600 lines + camera feel + picking + label gating owned by us. Est: ~1 focused day.
**Option C — PixiJS v8 + pixi-viewport.** Cleanest scene-graph answer (one ticker,
sync by construction), WebGPU headroom. But: dashes need a third-party shim, no graph
semantics, bundle, more machinery than Canvas 2D needs at our scale. Est: 1–2 days.
**Recommendation**: **Option B as the target** — it is the only option that structurally
matches all three requirement sets (bugs, cluster-first placement, freeform
experimentation), scored highest (8/10), and removes the vendor-internals class of
problems permanently. **Option A is a legitimate ~1-day stabilization** if we want the
current bugs dead immediately while deferring the rebuild; it reuses all our existing
visual code and is the officially sanctioned pattern, but it keeps us on sigma's
opinionated machinery and the experimentation direction will keep pushing against it.
**Layout work rides along independent of renderer choice**: first A/B
LinLog-only FA2 (one afternoon), then the two-level pipeline
(quotient → per-cluster FA2 → compose → noverlap) if needed, developed in the playground.
## 7. Facts we should NOT rely on
- "Obsidian's graph view uses renderer X" — unverified.
- "Sigma's documented canvas DOM order" — source-only, implementation detail, changed
in v4.
- "Canvas 2D is always GPU-rasterized" — false as stated.
- "Sigma v4 is ready" — beta.
- "Sigma pan cost at our scale is X%" — nobody has measured it publicly.
- "@cosmograph/cosmos is the maintained Cosmograph engine" — stale npm channel (Dec
2024); current engine is `@cosmos.gl/graph` v3.
- Any PixiJS bundle-size number — unmeasured.
## 8. Open items for the implementation phase
- Measure pan GPU/CPU ourselves (Chrome profiler) before/after whichever path is chosen.
- Playground design: `/playground` dev-only route, config-driven (layout algorithm
dropdown, parameter sliders, seed input, synthetic + real datasets), same renderer.
- Hull styling decision inside Option B: Bostock thick-stroke trick vs offset polygons
(geometry already unit-tested in `components/cluster-geometry.ts`).
- Label-gate hysteresis: small dead-band in log-zoom to avoid threshold flicker.
- Whether layout runs server-side at extraction (persisted as data) or in-browser.
+46 -2
View File
@@ -43,18 +43,60 @@ Every node records every **(chat, message) point** where it came up; every edge
Three stages, one pipeline — the sole graph writer:
- **Extractor (LLM) — session-only scope.** Sees ONLY the session transcript (tree-blind, with message ids). Reports freely, with verbatim evidence: propositions from anywhere in the session (one reasoning step each; prose counts as much as anything the teacher formalized), edges as the transcript articulates them, message-granular provenance, retroactive quiz attribution, and per proposition an **understanding level** — how strongly the learner accepted/understood it in this session, a CLOSED named-word scale (`solid` / `shaky` / `accepted` / `resisted` / `not yet engaged`; 2026-09-08: schema-enforced — exactly one word, no numbers, no justifications, which are dropped rather than stored). The levels are data for the future state model; their treatment is TBD.
- **Extractor (LLM) — single-source scope.** Sees ONLY the source text — a session transcript (tree-blind, with message ids) or raw notes (the front door's second prompt, above). Reports freely, with verbatim evidence: propositions from anywhere in the session (one reasoning step each; prose counts as much as anything the teacher formalized), edges as the transcript articulates them, message-granular provenance, retroactive quiz attribution, and per proposition an **understanding level** — how strongly the learner accepted/understood it in this session, a CLOSED named-word scale (`solid` / `shaky` / `accepted` / `resisted` / `not yet engaged`; 2026-09-08: schema-enforced — exactly one word, no numbers, no justifications, which are dropped rather than stored). The levels are data for the future state model; their treatment is TBD.
- **Auditor (LLM) — the global-graph scope.** Sees the extractor's output AND the existing graph. Two jobs: the faithfulness net (every candidate verified against verbatim evidence — unchanged), and the **identity judgment**: link each proposition to an existing node (recording the session's phrasing as an alias) or create it new. Conservative: duplicates over wrong merges.
- **Merger (deterministic).** Append-only ledger writes; quiz event kinds/refs rebuilt mechanically from quiz tool parts. Every judgment is an LLM's; everything mechanical stays code.
**Event ledger (amended):** quiz outcomes are rebuilt deterministically from quiz tool parts (correct / miss / don't-know, plus the learner's note). **There is no `taught` event** — no marking-things-done. The extractor's per-proposition understanding levels join the ledger as session-scoped observations (one closed-scale word per node per session). Identity is no longer exact-string matching — it is the auditor's judgment, with aliases as the phrasing record.
### The format-agnostic front door — notes imports (DECIDED 2026-09-13)
The same three-stage pipeline (extractor → auditor → merger) runs over **raw
personal notes in arbitrary formats** — an Obsidian vault, plain markdown,
anything textual. The extractor is format-blind by construction (it always
saw rendered text, never ChatRecord JSON), so the notes path needs only a
second extractor prompt (general-purpose extraction; the understanding word
judges the **author's** grasp from their own commentary) and a source
abstraction: each imported file becomes a `NotesSource` record (deterministic
id from the path hash — re-imports are idempotent), playing the `chatId` role
in mappings, checkpoints, and observation events.
Decided edges:
- **No session shaping.** Notes are never bent into teacher/learner
transcripts; quiz outcomes inside notes surface only via the extractor's
understanding observations (the deterministic quiz-event rebuild is a
session-pipeline feature — it needs structured quiz tool parts).
- **Static provenance.** Imported nodes show the file path in the sessions
list as a static entry — never a link. Understanding-assessment levels
apply to personal notes as to sessions.
- **Same single-flight registry and observability** as session runs: live
status, usage accounting per stage, true abort. One pipeline instance,
two front doors (quiz-triggered sessions; POST /api/graph/import).
### Understanding is derived (amended 2026-09-02)
Known vs. understood, event ledger as source of truth, derived state recomputed on read — carried over from the design doc in spirit. The state model itself is now an explicit design discussion: **node states should be session-independent and computed from the depth and strength of dependencies** (the `follows`/`supports` topology) plus quiz events and understanding notes. That redesign is pending ideation; the transitional five-class derivation (taught / known / missed / don't-know / candidate) was **removed** (2026-09-08) — the graph view renders state-neutral nodes until the state model is designed. The anchored calibration rubric stays parked until real data.
### Clusters — regions of the world model (DECIDED 2026-09-13, ideation)
When the graph fills up, the canvas needs a readable at-a-glance structure: **clusters** — objective regions of the world model (domains like "Lacanian psychoanalysis" or "photosynthesis"), discovered by **community detection** on the `follows`/`supports` edges. A community = nodes linked to each other far more than to the rest (modularity). Everything about them is **entirely inferred from the graph data** — the extraction/audit pipeline never knows clusters exist.
- **Algorithm: Louvain (modularity maximization), seeded for determinism.** In-stack via graphology's ecosystem, near-linear, scales far beyond our sizes. Determinism gives reproducibility (same graph → same clusters); the seed/graphology choice also keeps the upgrade path open: Louvain's dendrogram contains **hierarchy** — "math ⊃ differential calculus ⊃ chain rule" is one level down — deliberately NOT built now (multi-scale clusters stay parked). Dismissed: Leiden (better resolution guarantees, no JS port), Infomap (heuristic-strong, heavier, weaker JS support), Girvan–Newman (O(n³)), label propagation (unstable), spectral (needs k handed in).
- **Structure, not layout.** Clusters are a pure function of the *edges*, never of the visual layout. The force-directed layout already places structurally-close nodes together, so cluster hulls will look contiguous — but cluster identity must not inherit the layout's seed dependence and drift.
- **Derived, cached, auto-updated.** Clusters recompute when the ledger changes, cached against a graph version stamp — same doctrine as states and strengths: derived never stored, nothing hand-maintained.
- **Size-gated meaning.** Only communities ≥ ~3 nodes render as clusters (threshold tunable on real data) — a visible blob always means "a real domain"; lone nodes and tiny communities stay plain.
- **Naming: one cheap LLM call per new/changed cluster** — a single call (no agent loop) with the members' names + statements, producing a short domain label. Never part of extraction; a separate, cached seam.
- **Labeling persistence — the core-survival rule.** Each labeling record (append-only, like everything we persist) stores the label AND the member set it was produced with (the **core**). A recomputed community **keeps** its label while ≥ half its members are from that core; when under half are, it **relabels** with all current nodes (a new record is appended — old records stay as history: an audit trail of how the domains were named over time). Single successor: on splits/merges the strongest-overlap community inherits, the others get fresh labelings; a core with no ≥50% successor simply has no living heir. Cluster color keys to the labeling record — color survives exactly as long as the label does.
- **Rendering: glow blobs behind everything.** A dashed boundary encloses the cluster's hull; the fill is blurred, low-opacity, and sits *behind* the whole graph (edges and nodes float above it, so the blur naturally fogs the inter-cluster space). Per-cluster stable hue. **Cluster labels are inverse-gated** against the existing camera-ratio span: zoomed out = **domain view** (cluster labels on, node labels off), zoomed in = **proposition view** (node labels on, cluster labels faded) — one symmetric transition between the two views of one canvas.
- **Boundary geometry: computed at draw time, never stored** — a pure function of the member positions. This is the constraint that keeps future physics/moving nodes (likely not implemented soon) from invalidating anything: convex hull + padding + rounded corners is cheap per-frame; organic metaball hulls are the same family. Secondary until then.
- **Inter-cluster edges stay visible.** Bridges between domains are meaningful (Nietzsche's will-to-power *follows* Schopenhauer's will) — one of the most informative reads of the clustered view, not clutter.
- **Honest risk: sparsity.** Community detection wants density contrast, and the current real graph is thin — the 2k-node synthetic benchmark (open question 1) is the *primary* validation of clustering quality, not just of layout/render speed.
### Rendering (DECIDED — research report 2026-08-22)
**Render settings (2026-09-13):** four basic controls in the graph view — node size, edge width, arrowhead length, and **the zoom-level control** (the label-visibility threshold; 1× = fit zoom) — persisted per browser. The zoom control is deliberately THE single zoom knob: cluster labels gate inversely on it over the same fade span (implemented: node labels fade out over the span as cluster labels fade in — they never coexist). Settings changes rebuild only the renderer: layout positions and the camera never move. (Edge-length / spacing controls deliberately parked until a cluster-aware spacing design exists.)
- **Library: sigma.js v3 + graphology + graphology-layout-forceatlas2.** ~40 KB gzipped all-in — half of vis-network, a third of cytoscape, far below react-force-graph (~250 KB deps) and cosmos.gl (luma.gl stack).
- **Layout: synchronous run-then-freeze.** `forceAtlas2.assign(graph, { iterations: 100, settings: inferSettings(graph) })` runs once, writes x/y, and no animation loop ever starts — near-zero idle CPU (sigma only redraws on camera/graph events). <150 ms at hundreds of nodes; at ~2k enable `barnesHutOptimize`, then the bundled Web Worker, then server-cached positions — in that order, only if latency bites.
- **Labels at zoom:** sigma's `labelRenderedSizeThreshold` replicates Obsidian's "text fade threshold" exactly.
@@ -73,7 +115,7 @@ Every session starts with the learner's knowledge graph as base context for the
### Home page
The graph view replaces the current placeholder. v1: all nodes from all sessions, node color/shape by state class (the current five classes are transitional — see "Understanding is derived"), pan/zoom, labels at zoom. Clicking a node → the exact (session, message) points where it came up (message-granular provenance, deep-linked). Everything richer (frontier glow, rot map, blast radius) comes after the state model exists.
The graph view replaces the current placeholder. v1: all nodes from all sessions, pan/zoom, labels at zoom, state-neutral nodes (see "Understanding is derived"), and — once implemented — cluster glow-blobs with dashed hulls behind the graph, domain↔proposition as the two zoom-gated views (see "Clusters"). Clicking a node → the exact (session, message) points where it came up (message-granular provenance, deep-linked). Everything richer (frontier glow, rot map, blast radius) comes after the state model exists.
---
@@ -213,6 +255,8 @@ The teacher's context window and attention are precious — it has the lesson to
4. Home-graph v1 interactions beyond pan/zoom (node click → session list? — presumed cheap yes).
5. Grain mismatch frequency (`subsumes`-style relations) — per original doc, measured by data, not pre-solved.
6. Aggregation semantics for understanding (min / mean / two-level) — fit on real data, per original doc.
7. Cluster visual details — boundary aesthetic (convex vs. organic hull), label placement/typography, palette — decide during visual testing with real blobs (§1 Clusters; theory DECIDED 2026-09-13).
8. Cluster size threshold (~3) — tune on the synthetic benchmark + real data (§1 Clusters).
---
+7 -1
View File
@@ -4,7 +4,13 @@ const nextConfig: NextConfig = {
// Framework-free core lives in src/ (plain TS, imported by routes and CLI alike).
// The pi engine must stay external: its providers are runtime-loaded Node
// code (undici streams, catalog data) that breaks when bundled into routes.
serverExternalPackages: ["@earendil-works/pi-ai", "@earendil-works/pi-agent-core"],
// transformers.js (embeddings) is runtime-loaded ONNX — same reason as
// the pi engine: bundling onnxruntime-node breaks route builds.
serverExternalPackages: [
"@earendil-works/pi-ai",
"@earendil-works/pi-agent-core",
"@huggingface/transformers",
],
};
export default nextConfig;
+771 -72
View File
File diff suppressed because it is too large Load Diff
+4 -3
View File
@@ -16,6 +16,7 @@
"@ai-sdk/react": "^4.0.79",
"@earendil-works/pi-agent-core": "^0.85.1",
"@earendil-works/pi-ai": "^0.85.1",
"@huggingface/transformers": "^4.2.0",
"@mozilla/readability": "^0.6.0",
"@radix-ui/react-use-controllable-state": "^1.2.6",
"@rive-app/react-webgl2": "^4.32.1",
@@ -29,9 +30,10 @@
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
"cmdk": "^1.1.1",
"d3-force": "^3.0.0",
"d3-selection": "^3.0.0",
"d3-zoom": "^3.0.0",
"embla-carousel-react": "^8.6.0",
"graphology": "^0.26.0",
"graphology-layout-forceatlas2": "^0.10.1",
"katex": "^0.16.47",
"linkedom": "^0.18.13",
"lucide-react": "^1.33.0",
@@ -47,7 +49,6 @@
"react-dom": "^19.2.8",
"react-jsx-parser": "^2.4.1",
"remark-breaks": "^4.0.0",
"sigma": "^3.0.3",
"streamdown": "^2.5.0",
"tailwind-merge": "^3.6.0",
"tokenlens": "^1.3.1",
+2 -1
View File
@@ -8,13 +8,14 @@
import { chatStore } from "../src/store/chats.ts";
import { graphStore } from "../src/store/graph.ts";
import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts";
import { embedNewNodes } from "../src/server/embeddings.ts";
import { getGraph } from "../src/core/graph.ts";
async function main() {
const only = process.argv[2]; // optional: extract just one chat id
const chats = only ? chatStore.listChats().filter((c) => c.id === only) : chatStore.listChats();
console.log(`[backfill] ${chats.length} chats to extract`);
const pipeline = createExtractionPipeline({ chatStore, graphStore });
const pipeline = createExtractionPipeline({ chatStore, graphStore, embedder: embedNewNodes });
for (const chat of chats) {
const t0 = Date.now();
try {
-70
View File
@@ -1,70 +0,0 @@
/**
* Layout benchmark (W4; vision §1 "Rendering (DECIDED)" — "Pre-implementation
* validation: 2k-node synthetic benchmark to confirm layout time").
*
* npx tsx scripts/bench-graph.ts
*
* Two scales: the vision's target regime (hundreds of nodes, <150 ms budget
* with a synchronous run-then-freeze pass) and the stress case (2,000 nodes,
* ~4,000 edges), the latter timed with and without barnesHutOptimize.
*
* VERDICT (2026-08-22, Apple silicon, Node 24): hundreds of nodes are
* comfortably inside the budget (see the measured lines — ~25 ms at 100,
* ~90 ms at 300). At 2k nodes the naive O(n²) pass takes seconds, and
* barnesHutOptimize cuts it roughly in half — still over budget, which is
* exactly the vision's escalation ladder: Barnes-Hut is already switched on
* automatically above ~500 nodes in components/graph-view.tsx, and the
* bundled Web Worker / server-cached positions stay available as later
* rungs if real graphs ever get there. v1 learner graphs are dozens of
* nodes; the stress case is preparation, not expectation.
*/
import Graph from "graphology";
import forceAtlas2 from "graphology-layout-forceatlas2";
const ITERATIONS = 100;
/** A plausible learning graph: mostly-local chains plus long-range cross-links. */
function synthesize(nodeCount: number, edgeCount: number): Graph {
const graph = new Graph();
for (let i = 0; i < nodeCount; i++) {
graph.addNode(`n${i}`, {
label: `node ${i}`,
size: 6,
x: Math.random() * 100,
y: Math.random() * 100,
});
}
let added = 0;
for (let i = 0; added < edgeCount && i < edgeCount * 3; i++) {
const from = Math.floor(Math.random() * nodeCount);
const span = Math.random() < 0.8 ? 1 + Math.floor(Math.random() * 10) : Math.floor(Math.random() * nodeCount);
const to = (from + span) % nodeCount;
if (from === to || graph.hasEdge(`n${from}`, `n${to}`)) continue;
graph.addEdge(`n${from}`, `n${to}`, { type: "arrow", size: 1 });
added++;
}
return graph;
}
function bench(label: string, nodeCount: number, edgeCount: number, settings: Record<string, unknown>): void {
const graph = synthesize(nodeCount, edgeCount);
const start = performance.now();
forceAtlas2.assign(graph, {
iterations: ITERATIONS,
settings: { ...forceAtlas2.inferSettings(graph), ...settings },
});
const elapsed = performance.now() - start;
console.log(
`${label}: ${elapsed.toFixed(1)} ms (${graph.order} nodes, ${graph.size} edges, ${ITERATIONS} iterations)`,
);
}
console.log(`Target regime — synchronous run-then-freeze (budget: <150 ms):\n`);
bench(" 100 nodes, ~200 edges", 100, 200, {});
bench(" 300 nodes, ~600 edges", 300, 600, {});
bench(" 500 nodes, ~1000 edges", 500, 1000, {});
console.log(`\nStress case — 2k nodes (~4000 edges), the vision's escalation ladder:\n`);
bench(" naive repulsion (O(n²)) ", 2000, 4000, {});
bench(" barnesHutOptimize ", 2000, 4000, { barnesHutOptimize: true });
bench(" barnesHutOptimize (warmed JIT)", 2000, 4000, { barnesHutOptimize: true });
+23
View File
@@ -0,0 +1,23 @@
/**
* Embed the whole graph (backfill / cold start): one-time pass that
* fills the embedding ledger for every node — the same embedNewNodes the
* post-merge hook runs, just with an empty (or stale-model) ledger so
* every node is "new". First run downloads the model weights (~300MB)
* to the local HF cache; subsequent runs embed only what's missing.
*
* npx tsx scripts/embed-graph.ts
*/
import { graphStore } from "../src/store/graph.ts";
import { embedNewNodes, EMBEDDING_MODEL, EMBEDDING_DIM } from "../src/server/embeddings.ts";
async function main() {
const t0 = Date.now();
console.log(`[embed] model: ${EMBEDDING_MODEL} (dim ${EMBEDDING_DIM}, q8)`);
const { embedded, total } = await embedNewNodes(graphStore);
console.log(`[embed] embedded ${embedded} nodes → ledger holds ${total} vectors (${((Date.now() - t0) / 1000).toFixed(1)}s)`);
}
main().catch((err) => {
console.error("[embed] failed:", err);
process.exit(1);
});
+70
View File
@@ -0,0 +1,70 @@
/**
* Import raw notes files through the format-agnostic front door (vision §1,
* 2026-09-13) — every *.md / *.txt file in a directory becomes one
* extraction run (extractor → auditor → merger) over its raw text. Run
* from this worktree only — data never crosses trees:
*
* npx tsx --env-file=.env.local scripts/import-notes.ts <dir> [concurrency]
*
* Default dir: real-data-extraction/learn-sessions (the prototype
* learning-session exports — deliberately NOT imported as chats: they run
* through the notes path, raw). Runs in this process: the dev server's
* status chips do NOT see them (registry is per-process) — the dev server
* sees runs triggered through /api/graph/import.
*/
import * as fs from "node:fs";
import * as path from "node:path";
import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts";
import { embedNewNodes } from "../src/server/embeddings.ts";
import { clusterNamingSettled } from "../src/agent/subagents/cluster-namer.ts";
import { getGraph } from "../src/core/graph.ts";
import { stores } from "../src/server/stores.ts";
async function main() {
const dir = process.argv[2] ?? "real-data-extraction/learn-sessions";
const limit = Number(process.argv[3] ?? 4);
const files = fs
.readdirSync(dir)
.filter((f) => f.endsWith(".md") || f.endsWith(".markdown") || f.endsWith(".txt"))
.sort();
console.log(`[import-notes] ${files.length} note files from ${dir} (concurrency ${limit})`);
const pipeline = createExtractionPipeline({ ...stores, embedder: embedNewNodes });
const queue = [...files];
const totals = { nodes: 0, edges: 0, events: 0, failed: 0 };
const workers = Array.from({ length: Math.min(limit, files.length) }, async () => {
for (;;) {
const file = queue.shift();
if (!file) return;
const filePath = path.join(dir, file);
const t0 = Date.now();
try {
const stats = await pipeline.importNotes({ filePath, text: fs.readFileSync(filePath, "utf-8") });
totals.nodes += stats.nodes;
totals.edges += stats.edges;
totals.events += stats.events;
console.log(
`[import-notes] ${file}: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events in ${((Date.now() - t0) / 1000).toFixed(1)}s`,
);
} catch (err) {
totals.failed++;
console.error(`[import-notes] ${file} FAILED:`, err);
}
}
});
await Promise.all(workers);
// The post-merge cluster naming is fire-and-forget inside the pipeline;
// settle it before the process exits so labels actually land.
await clusterNamingSettled();
const graph = getGraph(stores.graphStore);
const labels = stores.graphStore.readClusterLabels().length;
console.log(
`[import-notes] done: +${totals.nodes} nodes, +${totals.edges} edges, +${totals.events} events, ${totals.failed} failed — ` +
`graph now ${graph.nodes.length} nodes, ${graph.edges.length} edges, ${graph.events.length} events, ` +
`${labels} named clusters`,
);
}
main();
+125
View File
@@ -0,0 +1,125 @@
/**
* Option A simulation: replay the CURRENT graph through the incremental
* layout as if it had arrived one imported-notes file at a time — the
* organic growth path the user described (one dependency structure /
* cluster at a time, orphans collecting on the rim).
*
* Reads ONLY the ledgers (nodes / edges / mappings / sources); writes ONLY
* data/graph/positions.json — no LLM calls, no extraction, no chat records.
*
* npx tsx scripts/replay-layout.ts [--depth=vertical|radial]
*
* --depth regenerates the ledger WITH inference-depth anchoring (the
* playground mode baked into the rest state); omit it for the free layout.
*/
import { graphStore } from "../src/store/graph.ts";
// --depth=vertical|radial: bake the playground depth mode into the ledger.
const depthArg = process.argv.find((a) => a.startsWith("--depth="));
const depthMode: "off" | "vertical" | "radial" =
depthArg === "--depth=vertical" ? "vertical" : depthArg === "--depth=radial" ? "radial" : "off";
import { incrementalLayout } from "../src/core/layout.ts";
import type { GraphNode, NodeMapping, Position } from "../src/core/contracts.ts";
function main() {
const t0 = Date.now();
// Unique nodes (lines may repeat an id — first defines).
const seen = new Set<string>();
const nodes: GraphNode[] = [];
for (const line of graphStore.readNodes()) {
if (seen.has(line.id)) continue;
seen.add(line.id);
nodes.push(line);
}
const edges = graphStore.readEdges();
// The semantic-topic source: the embedding ledger (run
// scripts/embed-graph.ts first if missing/stale).
const embeddings = graphStore.readEmbeddings();
if (Object.keys(embeddings.vectors).length === 0) {
console.warn("[replay] embedding ledger is empty — clusters will be absent (run scripts/embed-graph.ts)");
}
// Batches: each imported file, in real import order (sources.json
// importedAt). A node belongs to the file whose mapping first claims it
// (mapping line order = arrival); nodes no mapping covers (legacy) ride
// the final batch.
const sources = [...graphStore.readSources()].sort((a, b) => a.importedAt.localeCompare(b.importedAt));
const mappings: NodeMapping[] = graphStore.readMappings();
const batchOf = new Map<string, string>(); // nodeId → srcId
for (const m of mappings) {
if (!batchOf.has(m.graphNodeId)) batchOf.set(m.graphNodeId, m.chatId);
}
const delivered = new Set<string>();
let positions = new Map<string, Position>(Object.entries(graphStore.readPositions()));
if (positions.size > 0) {
console.log("[replay] discarding existing positions ledger — full replay from empty");
positions = new Map();
}
const byId = new Map(nodes.map((n) => [n.id, n]));
const unmappedFallback = sources[sources.length - 1]?.id ?? "";
console.log(`[replay] ${nodes.length} unique nodes, ${edges.length} edge lines, ${sources.length} sources`);
let maxOldShift = 0;
let shiftReport = "";
for (const source of sources) {
const batch = nodes.filter((n) => batchOf.get(n.id) === source.id);
const isLast = source.id === sources[sources.length - 1]!.id;
if (isLast) {
// Sweep in any never-mapped nodes with the last batch.
for (const n of nodes) {
if (!batchOf.has(n.id) && !delivered.has(n.id)) {
batch.push(n);
batchOf.set(n.id, unmappedFallback);
}
}
}
for (const n of batch) delivered.add(n.id);
// Edges activate once BOTH endpoints exist (a later file can claim
// one endpoint — the edge lights up then).
const activeEdges = edges.filter((e) => delivered.has(e.from) && delivered.has(e.to));
const before = new Map(positions);
const batchNodes = nodes.filter((n) => delivered.has(n.id));
positions = incrementalLayout(batchNodes, activeEdges, positions, {
// The same partition the real settle would see for this batch:
// the persisted semantic topics, filtered to delivered nodes
// (never recomputed — the agglomerative pass is ~1.5s at scale).
clusters: embeddings.clusters
.map((c) => ({ members: c.members.filter((id) => delivered.has(id)) }))
.filter((c) => c.members.length > 0),
...(depthMode === "off"
? {}
: {
depthAnchor: {
mode: depthMode,
strength: 0.8,
spacing: 60,
...(depthMode === "radial" ? { baseRadius: 220 } : {}),
},
}),
});
for (const [id, prev] of before) {
const now = positions.get(id)!;
const shift = Math.hypot(prev.x - now.x, prev.y - now.y);
if (shift > maxOldShift) {
maxOldShift = shift;
shiftReport = `${id} ${prev.x.toFixed(0)},${prev.y.toFixed(0)} -> ${now.x.toFixed(0)},${now.y.toFixed(0)} (batch ${batch.length} new, ${activeEdges.length} edges)`;
}
}
const label = source.filePath.split("/").pop() ?? source.id;
console.log(
`[replay] +${String(batch.length).padStart(3)} nodes ${String(activeEdges.length).padStart(4)} edges active (${label})`,
);
}
graphStore.writePositions(Object.fromEntries(positions));
console.log(
`[replay] done: ${positions.size} positions written in ${((Date.now() - t0) / 1000).toFixed(1)}s` +
` (max old-node shift within a batch: ${maxOldShift.toFixed(1)}: ${shiftReport})`,
);
}
main();
+8 -2
View File
@@ -66,7 +66,10 @@ export function toPiTool(def: LhTool<any, any>): AgentTool<any, any> {
name: def.name,
label: def.name,
description: def.description,
parameters: z.toJSONSchema(def.schema as z.ZodType) as never,
// io: "input" — the declaration describes what the MODEL must send,
// so .default() fields are optional in the schema (omission = the
// default); required-empty arrays are a known weak-model trap.
parameters: z.toJSONSchema(def.schema as z.ZodType, { io: "input" }) as never,
// Question tools opt into SEQUENTIAL execution: pi runs the whole
// batch in call order when any called tool opts in, so the
// tool-input-available chunks arrive in the model's call order — the
@@ -110,6 +113,9 @@ export function piToolDeclarations(registry: Record<string, LhTool<any, any>>) {
name,
label: name,
description: def.description,
parameters: z.toJSONSchema(def.schema as z.ZodType) as never,
// io: "input" — the declaration describes what the MODEL must send,
// so .default() fields are optional in the schema (omission = the
// default); required-empty arrays are a known weak-model trap.
parameters: z.toJSONSchema(def.schema as z.ZodType, { io: "input" }) as never,
}));
}
+198
View File
@@ -0,0 +1,198 @@
/**
* CLUSTER NAMING (vision §1, DECIDED 2026-09-13): the garnish layer over
* community detection. Detection is purely structural and deterministic
* (core/clusters.ts); naming is ONE singular cheap LLM call that labels
* every NEW/CHANGED community at once, given member names + statements.
*
* Core-survival: a community keeps an existing labeling while at least
* half its core survives in it (matchClusterLabelings); only communities
* with no surviving label AND at least MIN_CLUSTER_SIZE members are sent
* to the namer. Records are append-only and idempotent (content-hashed
* ids dedupe at the store seam), so a naming race or a re-run appends
* nothing new.
*
* Runs POST-EXTRACTION (user-decided 2026-09-13): both front doors call
* refreshClusterLabels after a successful merge. Failure is NON-FATAL —
* the graph and hulls render fine unlabeled; the next run retries.
*/
import { z } from "zod";
import { lhTool, toPiTools } from "../lh-tool.ts";
import type { PiEngine } from "../models.ts";
import { runSubagent } from "./pi-run.ts";
import type { GraphNode } from "../../core/contracts.ts";
import { getGraph, clusterContentId } from "../../core/graph.ts";
import { matchClusterLabelings, MIN_CLUSTER_SIZE, type Community } from "../../core/clusters.ts";
import type { GraphStore } from "../../core/ports.ts";
/** How many member propositions the namer sees per cluster — enough to
* recognize the domain without paying for the whole region. */
const MAX_MEMBERS_PER_CLUSTER = 12;
const CLUSTER_NAMER_INSTRUCTIONS = `You name communities of propositions in a personal knowledge graph.
You are given clusters. Each cluster is a set of related propositions
(theory fragments the author is learning) plus, for context, a sketch of
every OTHER cluster already named. For each UNNAMED cluster, mint ONE
short DOMAIN LABEL: 1-4 words naming the region of the world model those
propositions belong to together — like a section heading in a textbook.
Use the members' own vocabulary ("Differential forms", "Sales
psychology", "Server architecture"), not generic words ("Topics",
"Various concepts"). If the propositions are unrelated, that is fine:
name the best common denominator anyway — a reader should recognize the
region at a glance.
Your entire deliverable is ONE submit_labels call. Include EVERY
top-level field even when empty (labels: [] when nothing to name).`;
const labelsSchema = z.object({
labels: z
.array(
z.object({
/** The cluster key from the prompt ("c1", "c2", ...). */
key: z.string(),
label: z.string(),
}),
)
.default([]),
});
export interface UnnamedCluster {
key: string;
members: string[];
nodes: Pick<GraphNode, "name" | "statement">[];
}
/** One single-submit namer run (the same shape as every extraction stage):
* the whole deliverable is ONE zod-validated submit call. */
async function nameClusters(
engine: PiEngine | undefined,
signal: AbortSignal | undefined,
unnamed: UnnamedCluster[],
): Promise<{ key: string; label: string }[]> {
let submitted: { key: string; label: string }[] | undefined;
await runSubagent(
toPiTools({
submit_labels: lhTool({
name: "submit_labels",
description: "Submit the cluster labels.",
schema: labelsSchema,
execute: async (labels) => {
submitted = labels.labels;
return labels;
},
}),
}),
{
role: "labeler",
engine,
systemPrompt: CLUSTER_NAMER_INSTRUCTIONS,
prompt: JSON.stringify(
{
unnamedClusters: unnamed.map((c) => ({
key: c.key,
members: c.nodes.map((n) => ({ name: n.name, statement: n.statement })),
})),
},
null,
2,
),
maxTurns: 4,
abortSignal: signal,
shouldStop: (_message, toolResults) =>
toolResults.some((r) => r.toolName === "submit_labels" && !r.isError),
},
);
if (!submitted) throw new Error("the cluster namer never submitted labels");
return submitted;
}
export interface ClusterRefreshStats {
/** Communities the current graph divides into (all sizes). */
detected: number;
/** Communities that kept a label via core-survival. */
claimed: number;
/** Communities sent to the namer (unnamed, size >= MIN_CLUSTER_SIZE). */
named: number;
/** Labeling records actually appended (idempotent dedupe may shrink this). */
appended: number;
}
/**
* The post-extraction garnish: detect communities on the CURRENT graph,
* join persisted labelings (core-survival), and name what is new — one
* singular LLM call. Appends ClusterLabeling records; the NEXT snapshot
* read joins them (the payload is derived, never stored).
*
* Serialized: concurrent merges (parallel notes imports) chain here, so
* two runs never mint duplicate labels for the same community.
*/
async function doRefresh(deps: {
graphStore: GraphStore;
/** Test seam: a scripted engine (production uses the labeler role). */
engine?: PiEngine;
signal?: AbortSignal;
}): Promise<ClusterRefreshStats> {
const { graphStore, engine, signal } = deps;
const { nodes } = getGraph(graphStore);
// Semantic topics: the partition the embedder persisted in the
// embedding ledger (same source the snapshot renders — never
// recomputed here; the agglomerative pass is ~1.5s at scale).
const communities = graphStore.readEmbeddings().clusters;
const labelings = graphStore.readClusterLabels();
const claims = matchClusterLabelings(communities, labelings);
const byId = new Map(nodes.map((n) => [n.id, n]));
const claimed = claims.filter((c) => c !== null).length;
const unnamed = communities
.filter((c, i) => !claims[i] && c.members.length >= MIN_CLUSTER_SIZE)
.map((c: Community, i) => {
const nodesFor = c.members
.map((id) => byId.get(id))
.filter((n): n is GraphNode => n !== undefined)
.slice(0, MAX_MEMBERS_PER_CLUSTER)
.map((n) => ({ name: n.name, statement: n.statement }));
return { key: `c${i}`, members: c.members, nodes: nodesFor };
});
if (unnamed.length === 0) return { detected: communities.length, claimed, named: 0, appended: 0 };
const labels = await nameClusters(engine, signal, unnamed);
let appended = 0;
const seen = new Set<string>();
for (const { key, members } of unnamed) {
if (seen.has(key)) continue;
const label = labels.find((l) => l.key === key)?.label?.trim();
if (!label) continue;
seen.add(key);
graphStore.appendClusterLabel({
id: clusterContentId(members),
label,
core: members,
createdAt: new Date().toISOString(),
});
appended++;
}
return { detected: communities.length, claimed, named: unnamed.length, appended };
}
let namingTail: Promise<unknown> = Promise.resolve();
/**
* Fire the refresh, serialized behind any refresh already running (a
* parallel import's merge waits its turn; each run reads fresh ledgers).
* Waivable: call sites may fire-and-forget; scripts await the return.
*/
export function refreshClusterLabels(deps: {
graphStore: GraphStore;
engine?: PiEngine;
signal?: AbortSignal;
}): Promise<ClusterRefreshStats> {
const run = namingTail.then(() => doRefresh(deps));
namingTail = run.catch(() => {});
return run;
}
/** Await every refresh fired so far (scripts: don't exit mid-naming). */
export async function clusterNamingSettled(): Promise<void> {
await namingTail;
}
+389 -81
View File
@@ -1,7 +1,9 @@
import * as crypto from "node:crypto";
import type { UIMessage } from "ai";
import type { Usage } from "@earendil-works/pi-ai";
import { z } from "zod";
import { lhTool, toPiTools } from "../lh-tool.ts";
import type { ModelRole } from "../../../config.ts";
import type { PiEngine } from "../models.ts";
import { runSubagent } from "./pi-run.ts";
import type {
@@ -14,6 +16,9 @@ import type {
SessionFindings,
} from "../../core/contracts.ts";
import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts";
import type { GraphStore } from "../../core/ports.ts";
import { settleGraphLayout } from "../../core/layout-settle.ts";
import { refreshClusterLabels } from "./cluster-namer.ts";
/**
* The extraction pipeline (vision §1 construction, 2026-09-02 redesign) — the
@@ -42,6 +47,17 @@ import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../
* whatever the in-flight run missed). On failure nothing is checkpointed —
* the next trigger re-runs the whole thing.
*
* The FORMAT-AGNOSTIC FRONT DOOR (2026-09-13): importNotes() runs the SAME
* three stages over raw personal notes (arbitrary format — no chat, no
* session, no message ids). A notes-mode extractor prompt replaces the
* session one; the auditor and merger are shared (identity + deterministic
* appends don't care where text came from). Provenance is the file itself:
* source ids ("src_" + path hash) play the chatId role in mappings, events
* and checkpoints, and the source record (graphStore sources seam) carries
* the file path + the run's usage. No quiz events are ever built from
* notes — quiz outcomes inside the text surface only through the
* extractor's understanding observations (user-decided 2026-09-13).
*
* Observability (§5 baseline): every stage logs one line per step, its
* start, and its duration; the pipeline logs a failure line before
* rethrowing, so the server log reads start → stage timings → merged
@@ -154,7 +170,9 @@ const findingsSchema = z.object({
.describe("Every distinct proposition the session touched — true statements, misconception-free."),
edges: z
.array(edgeSchema)
.describe("Relations between propositions: follows (derivable) / supports (grounds without entailing)."),
.describe("Relations between propositions: follows (derivable) / supports (grounds without entailing). Pass [] when there are none — never omit the field.")
.default([]), // weak models omit required-but-empty arrays; omission means "none"
attribution: z
.object({
quiz: z
@@ -163,9 +181,51 @@ const findingsSchema = z.object({
"toolCallId of EVERY quiz line → the proposition id it was really testing. Attribute RETROACTIVELY — the whole transcript is visible; map each quiz to the proposition it tests (often articulated elsewhere in the session). null when genuinely unattributable.",
),
})
.describe("Quiz attribution — your only event judgment."),
.describe("Quiz attribution — your only event judgment. Pass { quiz: {} } when the session has no quiz lines — never omit the field.")
.default({ quiz: {} }), // weak models omit empty objects too
});
const NOTES_EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE source —
the user's own NOTES, in an ARBITRARY format (personal notes, an exported conversation, a document — anything) — and
report what those notes show the author engaging with, as structured findings for their global knowledge graph.
There is no session and no chat structure here: the source is raw text, the author's own material. Read it as such.
Report, with verbatim evidence quotes for everything:
NODE THEORY — the graph is a set of TRUE propositions about the world, full stop. A node is never anything about the
author or the source. The author's relationship to a proposition is expressed ENTIRELY by your understanding note; it
never leaks into the node itself. So no "the author understands X", no "mentioned in these notes": the node is just X.
A misconception is NOT a node of its own wrong claim — state the TRUE correction as the proposition.
1. PROPOSITIONS — every distinct reasoning-step-sized proposition the notes actually engage with:
- Each has a "name" — a VERY short label, a few words, a compact handle like a glossary headword ("Will to power",
"Divine command theory"). Not a sentence; it becomes the node's label in the graph, so keep it glanceable.
- And a "statement" — the self-contained TRUE proposition at full precision.
- Notes may explain things at length or quote material the author is studying: extract the propositions themselves,
one reasoning step each — never a topic blob, never a sub-step.
- "messages": ALWAYS an empty list — provenance is the file itself.
- "aliases": verbatim phrasings the notes used for the same proposition.
- "understanding": ONE word from the fixed scale — "solid", "shaky", "accepted", "resisted", or "not yet engaged" —
judging the AUTHOR's grasp of the proposition as the notes show it: their own commentary, self-assessments,
recorded quiz outcomes, the way they write about it. Strictly one word, no numbers, no justification (a
wrong-shaped note is rejected by the schema). When the notes give no signal, omit the field.
2. EDGES between propositions:
- "follows": to is DERIVABLE from from (understanding from makes understanding to achievable).
- "supports": from GROUNDS to without entailing it (helpful context, not a prerequisite).
- Only relations the notes actually articulate — with evidence.
There is no quiz attribution for notes — the source has no quiz events, and the schema has no attribution field.
Evidence discipline: every quote must appear VERBATIM in the source. The auditor rejects anything unbacked. When in
doubt, report; the auditor decides.
Include EVERY top-level field of the call even when empty — "edges": [] when there are no relations. NEVER omit
a field because it is empty.
Your entire deliverable is ONE submit_findings call.`;
const EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE session
transcript and report what was learned — as structured findings for the learner's global knowledge graph.
@@ -208,25 +268,35 @@ quiz outcomes carry that the learner believed the wrong version.
Evidence discipline: every quote must appear VERBATIM in the transcript view (a QUIZ line counts as the
transcript). The auditor rejects anything unbacked. When in doubt, report; the auditor decides.
Include EVERY top-level field of the call even when empty — "edges": [], "attribution": { quiz: {} } when there
are no quizzes. NEVER omit a field because it is empty.
Your entire deliverable is ONE submit_findings call.`;
/** One extractor run: the whole deliverable is ONE zod-validated submit
* call (a pi agent purely so the model can retry a schema-invalid call).
* Returns the submitted findings — captured from the execute seam, which
/** One single-submit agent run — the shared shape of EVERY LLM stage
* here (session + notes): the whole deliverable is ONE zod-validated
* submit call (a pi agent purely so the model can retry a schema-invalid
* call). The submitted value is captured from the execute seam, which
* only fires on a schema-valid call. */
async function runExtractor(
engine: PiEngine | undefined,
prompt: string,
signal: AbortSignal | undefined,
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void,
): Promise<SessionFindings> {
let submitted: SessionFindings | undefined;
async function runSubmitAgent<T>(opts: {
role: ModelRole;
toolName: string;
toolDescription: string;
schema: z.ZodType<T>;
systemPrompt: string;
prompt: string;
engine: PiEngine | undefined;
signal: AbortSignal | undefined;
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void;
neverSubmitted: string;
}): Promise<T> {
let submitted: T | undefined;
const result = await runSubagent(
toPiTools({
submit_findings: lhTool({
name: "submit_findings",
description: "Submit the session findings.",
schema: findingsSchema,
[opts.toolName]: lhTool({
name: opts.toolName,
description: opts.toolDescription,
schema: opts.schema,
execute: async (findings) => {
submitted = findings;
return findings;
@@ -234,28 +304,71 @@ async function runExtractor(
}),
}),
{
role: "extractor",
engine,
systemPrompt: EXTRACTOR_INSTRUCTIONS,
prompt,
role: opts.role,
engine: opts.engine,
systemPrompt: opts.systemPrompt,
prompt: opts.prompt,
maxTurns: 4,
abortSignal: signal,
abortSignal: opts.signal,
onAssistantMessage: (message, step) =>
onStep(
opts.onStep(
message.usage,
message.model,
step,
message.stopReason,
message.content.filter((c) => c.type === "toolCall").map((c) => (c as { name: string }).name).join(", ") || "(none)",
),
shouldStop: (_message, toolResults) => toolResults.some((r) => r.toolName === "submit_findings" && !r.isError),
shouldStop: (_message, toolResults) => toolResults.some((r) => r.toolName === opts.toolName && !r.isError),
},
);
void result;
if (!submitted) throw new Error("the extractor never submitted findings");
if (!submitted) throw new Error(opts.neverSubmitted);
return submitted;
}
/** The extractor over ONE session transcript (the chat front door). */
function runExtractor(
engine: PiEngine | undefined,
prompt: string,
signal: AbortSignal | undefined,
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void,
): Promise<SessionFindings> {
return runSubmitAgent({
role: "extractor",
toolName: "submit_findings",
toolDescription: "Submit the session findings.",
schema: findingsSchema,
systemPrompt: EXTRACTOR_INSTRUCTIONS,
prompt,
engine,
signal,
onStep,
neverSubmitted: "the extractor never submitted findings",
});
}
/** The extractor over ONE source of raw personal notes (the format-agnostic
* front door): same single-submit shape, notes-mode prompt + schema. */
function runNotesExtractor(
engine: PiEngine | undefined,
prompt: string,
signal: AbortSignal | undefined,
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void,
): Promise<NotesFindings> {
return runSubmitAgent({
role: "extractor",
toolName: "submit_findings",
toolDescription: "Submit the notes findings.",
schema: notesFindingsSchema,
systemPrompt: NOTES_EXTRACTOR_INSTRUCTIONS,
prompt,
engine,
signal,
onStep,
neverSubmitted: "the extractor never submitted findings",
});
}
// ─── the auditor ───────────────────────────────────────────────────────────
const auditSchema = findingsSchema.extend({
@@ -276,14 +389,25 @@ const auditSchema = findingsSchema.extend({
reason: z.string(),
}),
)
.describe("Everything you dropped, with the reason — the audit trail."),
.describe("Everything you dropped, with the reason — the audit trail. Pass [] when you dropped nothing — never omit the field.")
.default([]), // same weak-model tolerance as findings.edges
});
const AUDITOR_INSTRUCTIONS = `You are the audit stage of a learning application's extraction pipeline. You receive
ONE session transcript, a set of proposed findings (propositions, edges, quiz attribution), and the CURRENT
knowledge graph. Your jobs: the faithfulness net, and identity.
// The notes front door's shapes: everything session findings have, minus
// quiz attribution — raw notes have no quiz tool parts, so there is
// nothing to attribute (and no event to build: outcomes inside the text
// surface only through understanding observations, user-decided).
const notesFindingsSchema = findingsSchema.omit({ attribution: true });
const notesAuditSchema = auditSchema.omit({ attribution: true });
type NotesFindings = Omit<SessionFindings, "attribution">;
type NotesAuditResult = Omit<AuditResult, "attribution">;
FAITHFULNESS — verify every candidate against the transcript:
const AUDITOR_INSTRUCTIONS = `You are the audit stage of a learning application's extraction pipeline. You receive
ONE source — a session transcript, or the author's personal notes in an ARBITRARY format (the notes front door) — a
set of proposed findings (propositions, edges, and quiz attribution where the source has quiz lines), and the
CURRENT knowledge graph. Your jobs: the faithfulness net, and identity.
FAITHFULNESS — verify every candidate against the source text:
- PROPOSITIONS: every evidence quote must appear (near-)verbatim in the transcript view, and the statement
must be a faithful reading of it. The node theory applies: a statement must be a TRUE, self-contained,
world-facing fact — reword learner- or session-relative statements ("the learner understands X" → "X"), state
@@ -293,8 +417,10 @@ FAITHFULNESS — verify every candidate against the transcript:
- EDGES: the evidence must actually articulate the relation, in the direction claimed. Drop
direction-swapped or speculative edges.
- ATTRIBUTION: the QUIZ line's question must genuinely test the attributed proposition. Drop wrong
attributions to null, or re-point them to the proposition the question actually tests.
- UNDERSTANDING levels: verify against the quiz outcomes and the session's evidence; each note must be
attributions to null, or re-point them to the proposition the question actually tests. (Session
sources only — notes findings carry no attribution; there is nothing to verify.)
- UNDERSTANDING levels: verify against the quiz outcomes and the session's evidence — or, for notes,
the author's own words; each note must be
exactly one word from the fixed scale ("solid", "shaky", "accepted", "resisted", "not yet engaged") —
reword or drop anything else. No justifications.
@@ -311,7 +437,10 @@ Also drop propositions not referenced by anything else ONLY if they have no othe
well-evidenced proposition is still knowledge.
Never invent new candidates. Your entire deliverable is ONE submit_review call containing the verified findings
(the same shapes you received, with linkTo added) plus the dropped list.`;
(the same shapes you received, with linkTo added) plus the dropped list.
Include EVERY top-level field of the call even when empty — "edges": [], "dropped": []. NEVER omit a field
because it is empty.`;
/** The current graph, compact — the auditor's identity context. */
function graphView(nodes: GraphNode[]): string {
@@ -324,47 +453,47 @@ function graphView(nodes: GraphNode[]): string {
return lines.join("\n");
}
/** One auditor run — same single-submit shape as the extractor. */
async function runAuditor(
/** The auditor over session findings (the chat front door). */
function runAuditor(
engine: PiEngine | undefined,
prompt: string,
signal: AbortSignal | undefined,
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void,
): Promise<AuditResult> {
let submitted: AuditResult | undefined;
const result = await runSubagent(
toPiTools({
submit_review: lhTool({
name: "submit_review",
description: "Submit the verified findings subset (with identity verdicts) + the dropped list.",
schema: auditSchema,
execute: async (review) => {
submitted = review;
return review;
},
}),
}),
{
role: "auditor",
engine,
systemPrompt: AUDITOR_INSTRUCTIONS,
prompt,
maxTurns: 4,
abortSignal: signal,
onAssistantMessage: (message, step) =>
onStep(
message.usage,
message.model,
step,
message.stopReason,
message.content.filter((c) => c.type === "toolCall").map((c) => (c as { name: string }).name).join(", ") || "(none)",
),
shouldStop: (_message, toolResults) => toolResults.some((r) => r.toolName === "submit_review" && !r.isError),
},
);
void result;
if (!submitted) throw new Error("the auditor never submitted a review");
return submitted;
return runSubmitAgent({
role: "auditor",
toolName: "submit_review",
toolDescription: "Submit the verified findings subset (with identity verdicts) + the dropped list.",
schema: auditSchema,
systemPrompt: AUDITOR_INSTRUCTIONS,
prompt,
engine,
signal,
onStep,
neverSubmitted: "the auditor never submitted a review",
});
}
/** The auditor over notes findings (the format-agnostic front door): same
* faithfulness + identity job, notes-mode schema (no attribution). */
function runNotesAuditor(
engine: PiEngine | undefined,
prompt: string,
signal: AbortSignal | undefined,
onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void,
): Promise<NotesAuditResult> {
return runSubmitAgent({
role: "auditor",
toolName: "submit_review",
toolDescription: "Submit the verified findings subset (with identity verdicts) + the dropped list.",
schema: notesAuditSchema,
systemPrompt: AUDITOR_INSTRUCTIONS,
prompt,
engine,
signal,
onStep,
neverSubmitted: "the auditor never submitted a review",
});
}
// ─── the pipeline status registry ─────────────────────────────────────────────
@@ -376,6 +505,13 @@ export type ExtractionPhase = "extracting" | "auditing" | "merging";
* Exported for the status payload's consumer typing (the dock). */
export interface RunningExtraction {
chatId: string;
/** Which front door the run came through — label + usage-persistence
* depend on it (chats bill their chat record; notes bill the source
* record). */
kind: "chat" | "notes";
/** Display label (the file path for notes imports); the status route
* falls back to the chat store for chat runs. */
label?: string;
phase: ExtractionPhase;
startedAt: number;
/** Live per-stage usage, accumulated as steps bill (the dock's cost /
@@ -388,6 +524,10 @@ export interface RunningExtraction {
const runningExtractions = new Map<string, RunningExtraction>();
/** Last minted startedAt — kept monotonic so two runs starting in the same
* millisecond never collide on the usage idempotency key. */
let lastStartedAt = 0;
/** In-flight pipeline runs (oldest first) — the /api/graph/status payload. */
export function extractionStatuses(): RunningExtraction[] {
return [...runningExtractions.values()].sort((a, b) => a.startedAt - b.startedAt);
@@ -411,26 +551,51 @@ export interface ExtractionModels {
/** Test seam: a scripted engine per stage (production uses the roles). */
extractor?: PiEngine;
auditor?: PiEngine;
/** Test seam for the cluster namer (post-extraction garnish). */
labeler?: PiEngine;
}
export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {}
export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {
/** The post-merge embedding batch (semantic clusters run on it).
* INJECTED by the composition root (the agent layer never imports
* the server layer — architecture law): production passes the real
* local-model embedder (server/embeddings.ts embedNewNodes), tests
* fake it. Failure inside it is non-fatal by contract. */
embedder: (graphStore: GraphStore) => Promise<unknown>;
}
export interface NotesImportInput {
/** The source's file path — its identity (deterministic src id) AND its
* provenance label in the detail card. */
filePath: string;
/** The file's raw content, VERBATIM — any format. */
text: string;
}
export interface ExtractionPipeline {
/** Extract + audit + merge ONE chat. Single-flight per chat; resolves to
* the merger's append stats (or throws — no checkpoint on failure). */
runExtraction(chatId: string): Promise<EmissionStats>;
/** The format-agnostic front door (2026-09-13): run the same pipeline
* over ONE raw notes file. The source id is deterministic per file
* path, so re-importing the same file re-runs extraction against the
* same source identity (mappings + events dedupe by it). */
importNotes(input: NotesImportInput): Promise<EmissionStats>;
}
export function createExtractionPipeline(deps: ExtractionPipelineDeps): ExtractionPipeline {
const { chatStore, graphStore } = deps;
const { chatStore, graphStore, embedder: embed } = deps;
const inFlight = new Set<string>();
function setPhase(chatId: string, phase: ExtractionPhase): void {
function setPhase(chatId: string, phase: ExtractionPhase, kind: "chat" | "notes", label?: string): void {
const prev = runningExtractions.get(chatId);
if (!prev) lastStartedAt = Math.max(Date.now(), lastStartedAt + 1);
runningExtractions.set(chatId, {
chatId,
kind,
label: prev?.label ?? label, // phase changes never lose the label
phase,
startedAt: prev?.startedAt ?? Date.now(),
startedAt: prev?.startedAt ?? lastStartedAt,
usage: prev?.usage ?? {}, // phase changes never lose usage
controller: prev?.controller ?? new AbortController(),
});
@@ -525,7 +690,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
function persistRunUsage(chatId: string, durationMs: number, stats: EmissionStats | undefined): void {
const run = runningExtractions.get(chatId);
if (!run) return;
chatStore.appendExtractionUsage(chatId, {
const entry: ExtractionRunUsage = {
startedAt: run.startedAt,
at: new Date().toISOString(),
durationMs,
@@ -534,7 +699,9 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
extractor: run.usage.extractor ?? { model: "unknown", steps: [] },
auditor: run.usage.auditor ?? { model: "unknown", steps: [] },
},
});
};
if (run.kind === "notes") graphStore.appendSourceUsage(chatId, entry);
else chatStore.appendExtractionUsage(chatId, entry);
}
async function runExtraction(chatId: string): Promise<EmissionStats> {
@@ -543,19 +710,41 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
const t0 = performance.now();
try {
console.info(`[extraction ${chatId}] pipeline start`);
setPhase(chatId, "extracting");
setPhase(chatId, "extracting", "chat");
const findings = await extract(chatId);
setPhase(chatId, "auditing");
setPhase(chatId, "auditing", "chat");
const auditResult = await audit(chatId, findings);
setPhase(chatId, "merging");
// The merger re-reads the chat record itself — events are rebuilt
// from the transcript, never from stale copies.
const stats = mergeFindings(chatId, auditResult, { chatStore, graphStore });
setPhase(chatId, "merging", "chat");
// The merger merges from the CURRENT record — events are rebuilt
// from the live transcript, never from stale copies.
const record = chatStore.getChat(chatId);
if (!record) throw new Error(`Chat not found: ${chatId}`);
const stats = mergeFindings(chatId, auditResult, { chatStore, graphStore }, {
messages: record.messages as UIMessage[],
foldedAt: record.updatedAt,
});
// Semantic embeddings (DECIDED 2026-09-13): topics come from
// MEANING, so embed the new propositions BEFORE the settle and
// the namer look at clusters. Failure is non-fatal — clustering
// runs without the new nodes until the next merge retries.
await embed(graphStore).catch((err) => {
console.error(`[extraction ${chatId}] embedding failed (non-fatal):`, err);
});
// Option A: the layout ledger evolves with the merge (sync — the
// next snapshot fetch must already reflect the new nodes).
settleGraphLayout(graphStore);
persistRunUsage(chatId, Math.round(performance.now() - t0), stats);
console.info(
`[extraction ${chatId}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` +
`(${Math.round(performance.now() - t0)}ms total)`,
);
// Post-extraction garnish (DECIDED 2026-09-13): name new/changed
// clusters. Fire-and-forget — naming never fails the run.
refreshClusterLabels({
graphStore,
engine: deps.labeler,
signal: runningExtractions.get(chatId)?.controller.signal,
}).catch(() => {});
return stats;
} catch (err) {
persistRunUsage(chatId, Math.round(performance.now() - t0), undefined);
@@ -577,6 +766,125 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
}
}
return { runExtraction };
// ── the notes front door (format-agnostic: raw text in, same graph out) ──
/** The raw source view the notes stages see: the file path + the text
* VERBATIM. No message ids — provenance is the file itself. */
function notesView(input: { filePath: string; text: string }): string {
return `SOURCE (the author's personal notes): ${input.filePath}\n\n${input.text}`;
}
async function notesExtract(input: { id: string; filePath: string; text: string }): Promise<NotesFindings> {
const signal = runningExtractions.get(input.id)?.controller.signal;
signal?.throwIfAborted(); // an abort during a sync gap ends the run here
const view = notesView(input);
console.info(`[extraction ${input.id}] notes extractor starting (${view.split("\n").length} source lines)`);
const t0 = performance.now();
const findings = await runNotesExtractor(
deps.extractor,
view,
signal,
(usage, model, step, finishReason, tools) => {
recordStep(input.id, "extractor", callUsage(usage), model);
console.info(`[extraction] extractor step ${step}: ${finishReason}, tools: ${tools}`);
},
);
console.info(`[extraction ${input.id}] notes extractor done in ${Math.round(performance.now() - t0)}ms`);
return findings;
}
async function notesAudit(input: { id: string; filePath: string; text: string }, findings: NotesFindings): Promise<NotesAuditResult> {
const signal = runningExtractions.get(input.id)?.controller.signal;
signal?.throwIfAborted(); // an abort during the extract→audit gap ends here
const graph = getGraph(graphStore);
console.info(
`[extraction ${input.id}] notes auditor starting (${findings.propositions.length} propositions, ` +
`${findings.edges.length} edges, ${graph.nodes.length} graph nodes)`,
);
const t0 = performance.now();
const auditResult = await runNotesAuditor(
deps.auditor,
`Source:\n\n${notesView(input)}\n\nProposed findings:\n\n${JSON.stringify(findings, null, 2)}\n\n` +
`Current knowledge graph (${graph.nodes.length} nodes):\n\n${graphView(graph.nodes)}`,
signal,
(usage, model, step, finishReason, tools) => {
recordStep(input.id, "auditor", callUsage(usage), model);
console.info(`[extraction] auditor step ${step}: ${finishReason}, tools: ${tools}`);
},
);
if (auditResult.dropped.length > 0) {
console.info(
`[extraction ${input.id}] audit dropped ${auditResult.dropped.length}: ` +
auditResult.dropped.map((d) => `${d.ref} (${d.reason})`).join("; "),
);
}
console.info(
`[extraction ${input.id}] audit kept ${auditResult.propositions.length}/${findings.propositions.length} propositions, ` +
`${auditResult.propositions.filter((p) => p.linkTo !== undefined).length} linked to existing nodes`,
);
console.info(`[extraction ${input.id}] notes auditor done in ${Math.round(performance.now() - t0)}ms`);
return auditResult;
}
async function runNotesExtraction(input: { id: string; filePath: string; text: string; importedAt: string }): Promise<EmissionStats> {
const { id } = input;
if (inFlight.has(id)) return { nodes: 0, edges: 0, events: 0 };
inFlight.add(id);
const t0 = performance.now();
try {
console.info(`[extraction ${id}] notes pipeline start (${input.filePath})`);
setPhase(id, "extracting", "notes", input.filePath);
const findings = await notesExtract(input);
setPhase(id, "auditing", "notes", input.filePath);
const notesResult = await notesAudit(input, findings);
setPhase(id, "merging", "notes", input.filePath);
// Normalize to the merger's findings shape: notes carry no quiz
// attribution, and the merger scans no quiz parts for notes
// (messages: null) — nothing to attribute, nothing to build.
const auditResult: AuditResult = { ...notesResult, attribution: { quiz: {} } };
const stats = mergeFindings(id, auditResult, { chatStore, graphStore }, { messages: null, foldedAt: input.importedAt });
// Same post-merge chain as the chat path: embeddings first
// (topics from meaning), then the settle, then the namer.
await embed(graphStore).catch((err) => {
console.error(`[extraction ${id}] embedding failed (non-fatal):`, err);
});
// Option A: same post-merge settle as the chat path.
settleGraphLayout(graphStore);
persistRunUsage(id, Math.round(performance.now() - t0), stats);
console.info(
`[extraction ${id}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` +
`(${Math.round(performance.now() - t0)}ms total)`,
);
refreshClusterLabels({
graphStore,
engine: deps.labeler,
signal: runningExtractions.get(id)?.controller.signal,
}).catch(() => {});
return stats;
} catch (err) {
persistRunUsage(id, Math.round(performance.now() - t0), undefined);
if (err instanceof Error && err.name === "AbortError") {
console.info(`[extraction ${id}] aborted (${Math.round(performance.now() - t0)}ms) — nothing merged, nothing checkpointed`);
return { nodes: 0, edges: 0, events: 0 };
}
console.error(`[extraction ${id}] notes pipeline failed (${Math.round(performance.now() - t0)}ms):`, err);
throw err;
} finally {
inFlight.delete(id);
runningExtractions.delete(id); // done or failed — no longer running
}
}
/** Import ONE raw notes file: register the source (deterministic id per
* file path — re-import keeps the same identity and its usage history)
* and run the pipeline over it. */
async function importNotes(input: NotesImportInput): Promise<EmissionStats> {
const id = `src_${crypto.createHash("sha256").update(input.filePath).digest("hex").slice(0, 12)}`;
const importedAt = new Date().toISOString();
graphStore.upsertSource({ id, filePath: input.filePath, importedAt });
return runNotesExtraction({ id, ...input, importedAt });
}
return { runExtraction, importNotes };
}
+72
View File
@@ -0,0 +1,72 @@
/**
* CLUSTER LABEL MATCHING (vision §1). DETECTION moved to semantic topic
* clustering over embeddings (core/semantic-clusters.ts, DECIDED
* 2026-09-13) — Louvain over the edge structure is retired: it only ever
* found the reasoning chains (the graph is disconnected by nature), and
* edge-orphans could never join anything. What REMAINS here is the
* cluster contract + core-survival label matching, which is
* detection-agnostic: it consumes the same { members } shape semantic
* clustering emits.
*
* Labelings key to sorted MEMBER SETS, never to detection indices —
* a labeling record outlives any reshuffle of the partition.
*
* Browser-safe pure logic (no node builtins, no fs): the server runs
* matching for the snapshot payload, tests run it directly.
*/
import type { ClusterLabeling } from "./contracts.ts";
/** A detected community. Members are SORTED node ids — the member set IS
* the identity a labeling record can outlive a reshuffle with. */
export interface Community {
members: string[];
}
/** Communities below this size are not rendered, not named — presentation
* rule (user-decided 2026-09-13): a region of the world model needs at
* least three propositions to be worth a hull and a label. */
export const MIN_CLUSTER_SIZE = 3;
/**
* Claim labeling records for detected communities — CORE-SURVIVAL (DECIDED
* 2026-09-13): a community keeps a record's label while AT LEAST HALF of
* the record's core survives in it (overlap * 2 >= core.length; exactly
* half survives). Both sides are single-successor: one community holds at
* most one label, one record is claimed by at most one community — on a
* split, only the branch carrying the majority of the core keeps the name;
* on a merge, the best-overlapping record wins.
*
* Greedy assignment by overlap size (largest first; ties broken by newer
* record). Returns a claim (or null) per community, in input order —
* unclaimed communities are the namer's input. Pure: same inputs, same
* claims.
*/
export function matchClusterLabelings(
communities: readonly Community[],
labelings: readonly ClusterLabeling[],
): (ClusterLabeling | null)[] {
interface Pair {
community: number;
labeling: ClusterLabeling;
overlap: number;
}
const pairs: Pair[] = [];
communities.forEach((community, ci) => {
const members = new Set(community.members);
for (const labeling of labelings) {
const overlap = labeling.core.reduce((n, id) => n + (members.has(id) ? 1 : 0), 0);
if (overlap * 2 >= labeling.core.length && overlap > 0) {
pairs.push({ community: ci, labeling, overlap });
}
}
});
pairs.sort((a, b) => b.overlap - a.overlap || (a.labeling.createdAt < b.labeling.createdAt ? 1 : -1));
const claims: (ClusterLabeling | null)[] = communities.map(() => null);
const takenRecords = new Set<string>();
for (const pair of pairs) {
if (claims[pair.community] || takenRecords.has(pair.labeling.id)) continue;
claims[pair.community] = pair.labeling;
takenRecords.add(pair.labeling.id);
}
return claims;
}
+95
View File
@@ -20,6 +20,8 @@
/** Two relations only (DECIDED 2026-09-02): follows = b is derivable from a;
* supports = a grounds b without entailing it. No scalars, no encoded
* strength — strength is derived at read time from topology. */
import type { ExtractionRunUsage } from "./chat.ts";
export type EdgeType = "follows" | "supports";
/** What one extraction run merged into the graph ledgers — the merger's
@@ -37,10 +39,32 @@ export interface EmissionStats {
* edges record every (chat, message) where they came up, deep-linkable to
* the exact transcript point. */
export interface ProvenancePoint {
/** The chat this point anchors to — or a notes source id ("src_…"):
* imports carry no message anchors (provenance is the file itself,
* recorded through the mapping line + the source record). */
chatId: string;
messageId: string;
}
/** One imported notes file — the format-agnostic front door's bookkeeping
* (DECIDED 2026-09-13): personal notes in ARBITRARY formats run through the
* same extractor → auditor → merger pipeline, with the file itself as the
* provenance anchor. There is no chat: the detail card shows the file path
* as a STATIC entry (not a link), and understanding observations still
* apply (judged from the author's own voice in the notes). The run's usage
* log lives here, mirroring ChatRecord.extractions. Ids are "src_" + hash
* of the file path — deterministic, so re-importing the same file re-runs
* extraction against the same source id (mappings/events dedupe by it). */
export interface NotesSource {
id: string;
filePath: string;
/** ISO — when this file was imported (the checkpoint's foldedAt). */
importedAt: string;
/** The run's usage log — same append-only discipline as
* ChatRecord.extractions, idempotent per startedAt. */
extractions?: ExtractionRunUsage[];
}
export interface GraphNode {
/** Opaque stable id, store-assigned: "n_" + nanoid-like. */
id: string;
@@ -107,6 +131,8 @@ export interface LedgerEvent {
/** null = not yet attributed. */
nodeId: string | null;
kind: LedgerEventKind;
/** The chat that produced this evidence — or a notes source id
* ("src_…") for imported-notes observations. */
chatId: string;
at: string;
/** Idempotency key: the quiz part's toolCallId, or `obs:<sourceId>` for
@@ -244,11 +270,66 @@ export interface AuditResult {
}
/** The GET /api/graph payload — everything the home view's PURE renderer needs. */
/** cluster-labels.jsonl line — one LLM-named community (vision §1,
* DECIDED 2026-09-13). Append-only history: records are immutable, never
* edited or deleted; dead labelings remain as history. IDENTITY is the
* sorted member set ("core"), hashed into the id, so re-naming the same
* community dedupes and a labeling survives graph edits as long as at
* least half its core survives in some single community (core-survival;
* under half → the community is renamed, splits can keep only one
* successor). */
/** The embedding ledger shape (config.EMBEDDINGS_FILE): vectors for
* EVERY embedded node, all produced by one model at one dimensionality. */
export interface EmbeddingsLedger {
/** The embedding model id (e.g. the HF repo) — mismatch = stale
* vectors, the writer re-embeds everything. */
model: string;
/** Vector dimensionality (MRL-truncated on write). */
dim: number;
/** nodeId → L2-normalized vector (cosine similarity = dot product). */
vectors: Record<string, number[]>;
/** The topic partition over those vectors, computed by the SAME
* embedder run (the user-decided trigger: recluster at the end of
* every embedding batch, never per read — the agglomerative pass is
* ~1.5s at current scale). Consumers (snapshot, settle, namer) read
* this instead of recomputing. Empty when nothing is embedded. */
clusters: Array<{ members: string[] }>;
/** ISO timestamp of the last write. */
updatedAt: string;
}
export interface ClusterLabeling {
/** "clu_" + content hash of the sorted core. */
id: string;
/** The short domain label (1–4 words), e.g. "Differential forms". */
label: string;
/** The member node ids the label was minted for, sorted — the survival core. */
core: string[];
createdAt: string;
}
/** One rendered cluster in the snapshot payload (vision §1). Members are
* node ids; the renderer draws the hull from member positions at draw
* time and gates the label inversely on the node-label zoom control. */
export interface Cluster {
/** The labeling record id when claimed; unlabeled clusters still
* appear (hull, no name) with a member-hash id. */
id: string;
/** null until the naming call has minted/claimed a labeling. */
label: string | null;
/** Node ids, sorted. Hull + centroid are renderer concerns. */
members: string[];
}
export interface GraphSnapshot {
nodes: GraphNode[];
edges: GraphEdge[];
/** chatId → title, for every chat that touched the graph. */
sessions: Record<string, string>;
/** srcId → file path, for every IMPORTED NOTES file that touched the
* graph (the format-agnostic front door): a static provenance entry —
* the detail card shows the path as plain text, not a link. */
fileSources: Record<string, string>;
/** nodeId → chatIds that touched the node (click-through provenance). */
nodeSessions: Record<string, string[]>;
/** nodeId → chatId → that session's understanding observation (the
@@ -256,6 +337,20 @@ export interface GraphSnapshot {
* shown next to the session list — the confidence history). Multiple
* notes for one (node, chat) collapse to the latest. */
nodeObservations: Record<string, Record<string, string>>;
/** Structural communities (Louvain over follows+supports), size ≥ 3,
* joined with persisted LLM labelings. Purely structural — never
* depends on layout. */
clusters: Cluster[];
/** Resting layout positions per node id, when a positions ledger
* exists (core/layout.ts settle writes it). Absent on a fresh graph —
* the view then falls back to its own one-shot forceAtlas2 pass. */
positions?: Record<string, Position>;
}
/** A node's resting position in layout space (the positions ledger). */
export interface Position {
x: number;
y: number;
}
/** Derive a short chat title from the learner's first message (pure).
+85 -15
View File
@@ -38,10 +38,11 @@
* adapters in.
*/
import { randomBytes } from "node:crypto";
import { createHash, randomBytes } from "node:crypto";
import { isToolUIPart, type UIMessage } from "ai";
import type {
AuditResult,
Cluster,
EmissionStats,
GraphEdge,
GraphNode,
@@ -50,6 +51,7 @@ import type {
LedgerEvent,
LedgerEventKind,
} from "./contracts.ts";
import { matchClusterLabelings, MIN_CLUSTER_SIZE } from "./clusters.ts";
import type { QuestionAnswerData } from "./question-core.ts";
import type { ChatStore, GraphStore } from "./ports.ts";
@@ -64,6 +66,17 @@ function newGraphNodeId(): string {
return `n_${randomBytes(6).toString("hex")}`;
}
/** What one merge run merges FROM (the caller supplies it — the merger never
* reads the chat store): a session's transcript + its `updatedAt`
* checkpoint, or — for the format-agnostic notes front door (2026-09-13) —
* `messages: null`: raw notes carry no tool parts, so no quiz events are
* rebuilt (quiz outcomes inside notes surface only through the extractor's
* understanding observations), and the checkpoint stamps the import. */
export interface MergeSource {
messages: UIMessage[] | null;
foldedAt: string;
}
// ─── transcript scanning (shared by the merger + trigger) ─────────────────────
/** A flattened, order-preserving view of every tool part in a transcript. */
@@ -126,11 +139,15 @@ const samePoint = (a: ProvenancePoint, b: ProvenancePoint): boolean =>
* parts, nodeId from the extractor's attribution through this run's
* mapping) plus one observation event per proposition carrying an
* understanding note (ref `obs:<sourceId>`). No taught events, ever.
*
* `chatId` is the run's source key: a real chat, or a notes source id
* ("src_…") for the format-agnostic import path — same ledgers, same
* idempotency, one graph. The `source` parameter carries the transcript
* being merged (null messages = raw notes: no quiz events) and the
* checkpoint stamp; the merger itself is chat-store-free.
*/
export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDeps): EmissionStats {
const { graphStore: ledgers, chatStore } = deps;
const record = chatStore.getChat(chatId);
if (!record) return { nodes: 0, edges: 0, events: 0 };
export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDeps, source: MergeSource): EmissionStats {
const { graphStore: ledgers } = deps;
const stats: EmissionStats = { nodes: 0, edges: 0, events: 0 };
// 1. Resolution: this run's propositions → graph node ids. Existing
@@ -214,7 +231,7 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep
for (const event of ledgers.readEvents()) {
if (event.chatId === chatId && event.ref) seenRefs.add(event.ref);
}
for (const part of toolParts(record.messages)) {
for (const part of source.messages ? toolParts(source.messages) : []) {
if (part.name !== "quiz" || part.state !== "output-available") continue;
const event = quizEvent(part, chatId, audit, mapping, now);
if (event?.ref !== undefined && !seenRefs.has(event.ref)) {
@@ -241,8 +258,9 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep
stats.events++;
}
// 4. Checkpoint: this chat is extracted through its current updatedAt.
ledgers.appendCheckpoint({ chatId, foldedAt: record.updatedAt });
// 4. Checkpoint: this source is extracted through its foldedAt (the
// chat's updatedAt, or the import time for notes).
ledgers.appendCheckpoint({ chatId, foldedAt: source.foldedAt });
return stats;
}
@@ -314,6 +332,16 @@ export function pendingQuizAnswers(chatId: string, deps: GraphDeps): number {
// ─── derivation (reads only — everything below is recomputable) ─────────────
/** The content identity of a community: "clu_" + sha256 of the sorted
* member ids (12 hex). Deterministic — the same member set always mints
* the same id — so re-naming races and idempotent re-runs dedupe at the
* store seam (appendClusterLabel). Lives here (server-only core), not in
* clusters.ts, because clusters.ts is barrel-exported and must stay
* browser-safe (no node:crypto). */
export function clusterContentId(members: readonly string[]): string {
return "clu_" + createHash("sha256").update([...members].sort().join("\n")).digest("hex").slice(0, 12);
}
export interface DerivedGraph {
nodes: GraphNode[];
edges: GraphEdge[];
@@ -423,20 +451,28 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot {
if (event.nodeId !== null) touch(event.nodeId, event.chatId);
}
// Titles for every chat the graph can point back at.
const chatIds = new Set<string>();
// Labels for every source the graph can point back at: real chats
// (title, deep-linkable) and imported notes files (path, static — the
// format-agnostic front door's provenance).
const sourceIds = new Set<string>();
for (const chats of nodeChats.values()) {
for (const id of chats) chatIds.add(id);
for (const id of chats) sourceIds.add(id);
}
const sessions: Record<string, string> = {};
for (const id of chatIds) {
const fileSources: Record<string, string> = {};
for (const id of sourceIds) {
const chat = deps.chatStore.getChat(id);
if (chat) sessions[id] = chat.title;
if (chat) {
sessions[id] = chat.title;
continue;
}
const source = deps.graphStore.readSources().find((s) => s.id === id);
if (source) fileSources[id] = source.filePath;
}
const nodeSessions: Record<string, string[]> = {};
for (const [nodeId, chats] of nodeChats) {
const ids = [...chats].filter((id) => sessions[id] !== undefined);
const ids = [...chats].filter((id) => sessions[id] !== undefined || fileSources[id] !== undefined);
if (ids.length > 0) nodeSessions[nodeId] = ids;
}
@@ -461,7 +497,41 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot {
(nodeObservations[nodeId] ??= {})[chatId] = note;
}
return { nodes, edges, sessions, nodeSessions, nodeObservations };
// Semantic topic communities (DECIDED 2026-09-13, replacing structural
// Louvain): average-linkage clustering over embeddings — related
// topics, not reasoning chains. Edge-orphans join topics by meaning;
// stragglers stay loose (filtered at the presentation threshold
// below). The partition is computed ONCE per embedding batch by the
// embedder (the agglomerative pass is ~1.5s at current scale) and
// PERSISTED in the embedding ledger — reads just load it. Nodes whose
// embedding hasn't landed yet simply sit unclustered until the next
// read after the embedder runs.
const communities = deps.graphStore.readEmbeddings().clusters;
const claims = matchClusterLabelings(communities, deps.graphStore.readClusterLabels());
const clusters: Cluster[] = communities
.map((community, index) => ({ community, claim: claims[index] }))
.filter(({ community }) => community.members.length >= MIN_CLUSTER_SIZE)
.map(({ community, claim }) => ({
id: claim?.id ?? clusterContentId(community.members),
label: claim?.label ?? null,
members: community.members,
}));
// Resting layout (Option A): the positions ledger, when the settle has
// ever run. Absent on a fresh graph — the view falls back to its own
// one-shot pass until the first merge writes the ledger.
const positions = deps.graphStore.readPositions();
return {
nodes,
edges,
sessions,
fileSources,
nodeSessions,
nodeObservations,
clusters,
...(Object.keys(positions).length > 0 ? { positions } : {}),
};
}
// Re-export: the contract owns the shape; legacy importers still reach it.
+7
View File
@@ -17,8 +17,15 @@
export * from "./contracts.ts";
export * from "./ports.ts";
export * from "./render-settings.ts";
export * from "./question-core.ts";
// clusters.ts (Louvain) is deliberately NOT re-exported: nothing in the
// shell runs community detection — the snapshot carries the partition
// from the server. Keeping it out keeps the heavy clustering deps out of the
// client bundle (verified 2026-09-13: chunks contained it while this
// line existed). Same logic as layout-settle.ts below.
export * from "./layout.ts";
export * from "./plan-tree.ts";
export * from "./phases.ts";
export * from "./pending.ts";
+44
View File
@@ -0,0 +1,44 @@
import type { EmbeddingsLedger, GraphEdge, GraphNode } from "./contracts.ts";
import { incrementalLayout } from "./layout.ts";
import type { Position } from "./contracts.ts";
/**
* The merge-time settle: recompute the resting layout from the CURRENT
* graph + the positions ledger, and replace the ledger. Called by the
* extraction pipeline after every merge (the sole writer) and by the
* positions-commit endpoint when the graph grew under a live-physics
* commit (client positions become the anchor set). Synchronous — it must
* finish before the next /api/graph fetch reflects the change.
*
* SPLIT FROM layout.ts deliberately (2026-09-13): layout.ts is in the
* browser barrel, and the settle needs the Louvain partition for the
* cluster centroid pull — the heavy deps must not ship in the client
* bundle. Server code imports this module DIRECTLY; it is never
* re-exported through src/core/index.ts.
*/
export function settleGraphLayout(graphStore: {
readNodes(): GraphNode[];
readEdges(): GraphEdge[];
readPositions(): Record<string, Position>;
writePositions(positions: Record<string, Position>): void;
readEmbeddings(): EmbeddingsLedger;
}): void {
const nodes = graphStore.readNodes();
if (nodes.length === 0) return;
// Node LINES may repeat an id (link-appends) — dedupe, first defines.
const seen = new Set<string>();
const unique = nodes.filter((n) => {
if (seen.has(n.id)) return false;
seen.add(n.id);
return true;
});
const edges = graphStore.readEdges();
const previous = new Map(Object.entries(graphStore.readPositions()));
// The cluster partition for the centroid pull: the SAME persisted
// semantic topics the snapshot renders (computed by the embedder
// once per batch, stored in the embedding ledger) — one cluster
// concept everywhere.
const communities = graphStore.readEmbeddings().clusters;
const next = incrementalLayout(unique, edges, previous, { clusters: communities });
graphStore.writePositions(Object.fromEntries(next));
}
+629
View File
@@ -0,0 +1,629 @@
/**
* The layout engine — d3-force, everywhere (DECIDED 2026-09-13).
*
* One engine owns the equilibrium: the merge-time incremental settle AND
* the live drag physics run the SAME force set (links as springs, charge
* repulsion, weak gravity), so the positions ledger and the transient
* simulation agree on what "resting" means. (FA2 was replaced after a
* research round: it has no annealing/pinning/settle model and mixing it
* with a different live engine visibly re-flows the graph on every drag.)
*
* Layout EVOLVES instead of reshuffling (Option A): each settle warm-starts
* from the positions ledger and spring-ties every previously-positioned
* node toward its anchor (a per-node anchor force). New nodes seed near
* their positioned neighbors; orphans seed on the rim OUTSIDE the core and
* are anchored there, so they read as "not yet connected knowledge".
*
* Design constraints (kept deliberately, for live physics):
* - PURE: no fs, no server types — this module runs server-side (the
* merge settle) and in the browser (the live simulation, via the core
* barrel). A pinned node is just fx/fy; an anchored node is a spring.
* - DETERMINISTIC: seeds derive from node id; d3-force ticks are
* deterministic — same inputs, same layout.
*/
import {
forceSimulation,
forceLink,
forceManyBody,
forceX,
forceY,
type Simulation,
type SimulationLinkDatum,
type SimulationNodeDatum,
} from "d3-force";
import type { GraphEdge, GraphNode, Position } from "./contracts.ts";
import { seededRandom } from "./question-core.ts";
/** A node the d3 simulation mutates (x/y/vx/vy; fx/fy = pinned). */
export interface LayoutNode extends SimulationNodeDatum {
id: string;
}
/** A link as d3 sees it (id strings resolve to node refs on init). */
type SimLink = SimulationLinkDatum<LayoutNode>;
/** How the inference rank is expressed spatially (playground, 2026-09-13):
* the graph's `follows`/`supports` edges encode derivation (premise →
* conclusion), so depth-0 nodes (no incoming edges — the fundamentals)
* can anchor a shared baseline and every derivation level rises from it.
* `vertical`: levels are horizontal rows, arrows point up.
* `radial`: levels are concentric rings, foundations at the center, the
* outer boundary IS the edge of the user's knowledge.
* `off`: today's free layout. */
export type DepthAnchorMode = "off" | "vertical" | "radial";
export interface DepthAnchor {
mode: DepthAnchorMode;
/** Anchor pull toward the level (same units as clusterPull). */
strength: number;
/** Distance between consecutive inference levels (world units). */
spacing: number;
/** Baseline point: vertical mode uses .y as the depth-0 line; radial
* mode uses the full point as the ring center. Default (0,0). */
center?: { x: number; y: number };
/** Inner ring radius (radial only; depth-0 nodes sit at this ring). */
baseRadius?: number;
}
/** Inference depth per node: the longest chain of directed edges from a
* base node. CYCLE-SAFE (2026-09-13, user requirement — the DAG property
* is typical, not guaranteed): Tarjan SCC condensation collapses each
* strongly connected component to one super-node, ranks the (always
* acyclic) condensation, and every cycle member shares its component's
* depth — mutual support reads as equal footing. Iterative Tarjan (no
* recursion depth limits), O(V+E), deterministic. */
export function inferenceDepths(
nodes: ReadonlyArray<Pick<GraphNode, "id">>,
edges: ReadonlyArray<LayoutEdge>,
): Map<string, number> {
const valid = new Set(nodes.map((n) => n.id));
const adj = new Map<string, string[]>();
for (const e of edges) {
if (e.from === e.to || !valid.has(e.from) || !valid.has(e.to)) continue;
let list = adj.get(e.from);
if (!list) adj.set(e.from, (list = []));
list.push(e.to);
}
// Iterative Tarjan SCC.
const num = new Map<string, number>();
const low = new Map<string, number>();
const onStack = new Set<string>();
const stack: string[] = [];
const comp = new Map<string, number>();
let counter = 0;
let compCount = 0;
for (const start of valid) {
if (num.has(start)) continue;
num.set(start, counter);
low.set(start, counter);
counter++;
stack.push(start);
onStack.add(start);
const work: [string, number][] = [[start, 0]];
while (work.length > 0) {
const frame = work[work.length - 1]!;
const v = frame[0];
const outs = adj.get(v) ?? [];
if (frame[1] < outs.length) {
const i = frame[1];
frame[1] = i + 1;
const w = outs[i]!;
if (!num.has(w)) {
num.set(w, counter);
low.set(w, counter);
counter++;
stack.push(w);
onStack.add(w);
work.push([w, 0]);
} else if (onStack.has(w)) {
low.set(v, Math.min(low.get(v)!, num.get(w)!));
}
} else {
work.pop();
if (low.get(v) === num.get(v)) {
let w: string;
do {
w = stack.pop()!;
onStack.delete(w);
comp.set(w, compCount);
} while (w !== v);
compCount++;
}
if (work.length > 0) {
const parent = work[work.length - 1]![0];
low.set(parent, Math.min(low.get(parent)!, low.get(v)!));
}
}
}
}
// Condensation predecessors, then longest path. Tarjan emits sinks
// first, so descending comp index = topological order (sources first).
const preds: Set<number>[] = Array.from({ length: compCount }, () => new Set<number>());
for (const e of edges) {
const a = comp.get(e.from);
const b = comp.get(e.to);
if (a == null || b == null || a === b) continue;
preds[b]!.add(a);
}
const depth = new Array<number>(compCount).fill(0);
for (let c = compCount - 1; c >= 0; c--) {
let d = 0;
for (const p of preds[c]!) d = Math.max(d, depth[p]! + 1);
depth[c] = d;
}
const out = new Map<string, number>();
for (const n of nodes) out.set(n.id, depth[comp.get(n.id)!] ?? 0);
return out;
}
export interface SimOptions {
/** Link spring rest length (world units). */
linkDistance?: number;
/** Link spring stiffness MULTIPLIER (1 = d3's default
* 1/min(degree) stiffness). */
linkStrength?: number;
/** Many-body repulsion strength (negative). */
chargeStrength?: number;
/** Center-of-mass gravity: pulls loose components inward. Weak —
* anchors, not gravity, hold the evolved layout's frame. */
gravity?: { x: number; y: number; strength: number };
/** Alpha decay per tick — higher = snappier settle. The merge settle
* uses d3's default 0.0228 (~300 ticks); the live sim passes ~0.05. */
alphaDecay?: number;
/** Velocity damping per tick (d3 default 0.4). Higher = more friction. */
velocityDecay?: number;
/** Inference-depth anchoring (see DepthAnchor). Off by default. */
depthAnchor?: DepthAnchor;
/** Cluster centroid pull (the "virtual node" without the node):
* members of each cluster accelerate toward their own cluster's
* centroid. Two O(n) passes per tick — cheaper than a real virtual
* node (which would need one extra link spring per member plus a
* maintained position). Strength 0 or absent = force off.
* Callers supply the partition (Louvain output shape). */
clusterPull?: {
clusters: ReadonlyArray<{ members: ReadonlyArray<string> }>;
strength: number;
};
}
/** Link input: edges are undirected for layout purposes. */
export interface LayoutEdge {
from: string;
to: string;
}
export const LINK_DISTANCE = 40;
export const CHARGE_STRENGTH = -120;
/** Per-tick anchor re-blend: fraction of an anchored node's displacement
* from its previous position that SURVIVES each tick. 0.85 = gentle
* evolution where forces are strong, stillness where they are not —
* the exact FA2-era constant. Position-space (alpha-independent), so it
* cannot be overpowered by d3's long-range link springs the way a force-
* space anchor can (forceLink is a Hooke spring: strength*(d-L)*alpha). */
const ANCHOR_RETENTION = 0.85;
/** Gravity strengths. d3 manyBody repulsion falls off as 1/d, so two
* 20-node blobs at distance d push with ~n1*n2*|charge|/d; cluster
* spacing settles where that equals g*d*n/2 — g=0.03 lands clusters
* ~400 world units apart (the FA2-era density), and the anchor re-blend
* (position-space, alpha-independent) keeps gravity's transient sag
* from ever moving an anchored node at rest. */
const BOOTSTRAP_GRAVITY = 0.2;
/** Exported: the live sim reuses this so its rest state matches the
* server settle's equilibrium (one engine, one rest). */
export const INCREMENTAL_GRAVITY = 0.12;
/** Exported: default cluster-centroid pull strength (tuned by replay —
* compact enough to read as blobs, weak enough that cluster shape is
* still the edge network's). */
export const CLUSTER_PULL_STRENGTH = 0.15;
/**
* The one force set. Returns a STOPPED simulation — callers drive it
* manually (tick-per-frame live, or a bounded tick loop when settling).
* Deterministic; mutates `nodes` in place (d3 semantics: impure by design,
* zero per-tick allocation).
*/
export function createGraphSimulation(
nodes: LayoutNode[],
edges: ReadonlyArray<LayoutEdge>,
options: SimOptions = {},
): Simulation<LayoutNode, SimLink> {
// Link stiffness: d3's default is 1/min(deg) (degree-scaled springs).
// linkStrength multiplies it, so 1 = unchanged default behavior.
const degree = new Map<string, number>();
for (const e of edges) {
degree.set(e.from, (degree.get(e.from) ?? 0) + 1);
degree.set(e.to, (degree.get(e.to) ?? 0) + 1);
}
const linkStrength = options.linkStrength ?? 1;
const sim = forceSimulation<LayoutNode, SimLink>(nodes)
.force(
"link",
forceLink<LayoutNode, SimLink>(
edges.map((e) => ({ source: e.from, target: e.to })),
)
.id((d) => d.id)
.distance(options.linkDistance ?? LINK_DISTANCE)
.strength(
(l) =>
(linkStrength * 1) /
Math.min(degree.get((l.source as LayoutNode).id) ?? 1, degree.get((l.target as LayoutNode).id) ?? 1),
),
)
.force("charge", forceManyBody<LayoutNode>().strength(options.chargeStrength ?? CHARGE_STRENGTH))
.alphaDecay(options.alphaDecay ?? 0.0228)
.velocityDecay(options.velocityDecay ?? 0.4)
.stop();
// Gravity, mode-projected: in vertical mode the depth force OWNS the
// y axis, so gravity goes x-only (it would fight the level lines); in
// radial mode a center pull is purely radial and would fight the
// rings, so it is dropped entirely (rings already bound the radius).
// Depth mode projections (DECIDED 2026-09-13), with the ORPHAN RULE
// (user-decided, same day): a node with no edges at all has no
// dependency structure — depth is a property of inference chains, so
// stamping unconnected propositions onto the baseline / base ring
// assigns them meaning they don't have. Orphans are exempt from the
// depth force and keep the REGULAR view's forces instead (full center
// gravity + full cluster pull, unprojected).
const mode = options.depthAnchor?.mode ?? "off";
let orphan: Uint8Array | null = null;
if (mode !== "off") {
const connected = new Set<string>();
const valid = new Set(nodes.map((n) => n.id));
for (const e of edges) {
// Self-loops and dangling endpoints are not a structure (same
// guards as inferenceDepths).
if (e.from === e.to || !valid.has(e.from) || !valid.has(e.to)) continue;
connected.add(e.from);
connected.add(e.to);
}
orphan = new Uint8Array(nodes.length);
nodes.forEach((n, i) => {
if (!connected.has(n.id)) orphan![i] = 1;
});
}
const g = options.gravity;
if (g && mode === "off") {
sim.force("x", forceX<LayoutNode>(g.x).strength(g.strength));
sim.force("y", forceY<LayoutNode>(g.y).strength(g.strength));
} else if (g) {
// Mode-projected gravity, EXACTLY d3's forceX/forceY math (restoring
// force × strength × alpha): structured nodes get x-only in vertical
// mode (the depth force owns y) and none in radial mode (the rings
// own the radius); ORPHANS get the full regular-view pull in every
// mode.
const orphanMask = orphan!;
sim.force("gravity", (alpha: number) => {
for (let i = 0; i < nodes.length; i++) {
const n = nodes[i]!;
if (orphanMask[i]) {
n.vx = (n.vx ?? 0) + (g.x - (n.x ?? 0)) * g.strength * alpha;
n.vy = (n.vy ?? 0) + (g.y - (n.y ?? 0)) * g.strength * alpha;
} else if (mode === "vertical") {
n.vx = (n.vx ?? 0) + (g.x - (n.x ?? 0)) * g.strength * alpha;
}
// radial + structured: nothing — the rings own the radius.
}
});
}
// Cluster centroid pull: the membership index is resolved ONCE (nodes
// don't change clusters mid-sim); each tick is two allocation-free
// O(n) passes (sum, then pull). Singletons (<2 members) skip themselves.
const cp = options.clusterPull;
if (cp && cp.strength > 0 && cp.clusters.length > 0) {
const clusterIdx = new Map<string, number>();
cp.clusters.forEach((c, i) => {
for (const id of c.members) clusterIdx.set(id, i);
});
const nodeCluster = nodes.map((n) => clusterIdx.get(n.id) ?? -1);
const k = cp.clusters.length;
const sumX = new Float64Array(k);
const sumY = new Float64Array(k);
const cnt = new Float64Array(k);
const strength = cp.strength;
// The cluster centroid pull also respects the depth mode: vertical
// → x-only (the depth force owns y); radial → tangential only
// (the radial component would drag members off their rings; the
// tangential component still tightens each cluster's angular
// spread, which is all clustering needs to do there).
const ringCx = options.depthAnchor?.center?.x ?? 0;
const ringCy = options.depthAnchor?.center?.y ?? 0;
sim.force("clusterPull", (alpha: number) => {
sumX.fill(0);
sumY.fill(0);
cnt.fill(0);
for (let i = 0; i < nodes.length; i++) {
const ci = nodeCluster[i]!;
if (ci < 0) continue;
const n = nodes[i]!;
sumX[ci] = sumX[ci]! + (n.x ?? 0);
sumY[ci] = sumY[ci]! + (n.y ?? 0);
cnt[ci] = cnt[ci]! + 1;
}
for (let i = 0; i < nodes.length; i++) {
const ci = nodeCluster[i]!;
if (ci < 0 || (cnt[ci] ?? 0) < 2) continue;
const n = nodes[i]!;
const px = n.x ?? 0;
const py = n.y ?? 0;
const ux = sumX[ci]! / cnt[ci]! - px;
const uy = sumY[ci]! / cnt[ci]! - py;
if (orphan && orphan[i]) {
// Orphans: full regular-view pull (see the ORPHAN RULE above).
n.vx = (n.vx ?? 0) + ux * strength * alpha;
n.vy = (n.vy ?? 0) + uy * strength * alpha;
} else if (mode === "vertical") {
n.vx = (n.vx ?? 0) + ux * strength * alpha;
} else if (mode === "radial") {
const rx = px - ringCx;
const ry = py - ringCy;
const r = Math.hypot(rx, ry);
if (r > 1e-9) {
// Tangential component: u − (u·r̂)r̂
const dot = (ux * rx + uy * ry) / (r * r);
n.vx = (n.vx ?? 0) + (ux - dot * rx) * strength * alpha;
n.vy = (n.vy ?? 0) + (uy - dot * ry) * strength * alpha;
}
} else {
n.vx = (n.vx ?? 0) + ux * strength * alpha;
n.vy = (n.vy ?? 0) + uy * strength * alpha;
}
}
});
}
// ── Inference-depth anchoring ────────────────────────────────────────
// The rank is COMPUTED (not simulated): per-node target scalar derived
// from inferenceDepths, then one O(n) pull pass per tick. Soft knob —
// high strength approaches pinned rows/rings while low strength leaves
// organic sag.
const da = options.depthAnchor;
if (da && da.mode !== "off" && da.strength > 0) {
const depths = inferenceDepths(nodes, edges);
const strength = da.strength;
const spacing = da.spacing;
const c = da.center ?? { x: 0, y: 0 };
const orphanMask = orphan; // ORPHAN RULE: depth skips unstructured nodes
if (da.mode === "vertical") {
const baseY = c.y;
sim.force("depth", (alpha: number) => {
for (let i = 0; i < nodes.length; i++) {
if (orphanMask?.[i]) continue;
const n = nodes[i]!;
const targetY = baseY - (depths.get(n.id) ?? 0) * spacing;
n.vy = (n.vy ?? 0) + (targetY - (n.y ?? 0)) * strength * alpha;
}
});
} else {
const baseR = da.baseRadius ?? 0;
const cx0 = c.x;
const cy0 = c.y;
sim.force("depth", (alpha: number) => {
for (let i = 0; i < nodes.length; i++) {
if (orphanMask?.[i]) continue;
const n = nodes[i]!;
const dx = (n.x ?? 0) - cx0;
const dy = (n.y ?? 0) - cy0;
// Degenerate direction at the exact ring center: nudge
// outward on a deterministic axis.
const r = Math.hypot(dx, dy);
const ux = r > 1e-9 ? dx / r : 1;
const uy = r > 1e-9 ? dy / r : 0;
const target = baseR + (depths.get(n.id) ?? 0) * spacing;
const mag = (target - r) * strength * alpha;
n.vx = (n.vx ?? 0) + ux * mag;
n.vy = (n.vy ?? 0) + uy * mag;
}
});
}
}
return sim;
}
export interface IncrementalLayoutOptions {
/** Manual ticks for the settle (default 300 — d3's anneal to alphaMin). */
ticks?: number;
/** Per-tick anchor re-blend (default 0.85; 1 = fully pinned). */
anchorRetention?: number;
/** Center-of-mass gravity strength override (tuning/A-B experiments;
* defaults to the internal bootstrap/incremental constants). */
gravityStrength?: number;
/** Cluster partition (Louvain output shape). Supplied by callers so
* this module stays louvain-free (the browser bundle must not carry
* embedding ledger; absent = no cluster pull. */
clusters?: ReadonlyArray<{ members: ReadonlyArray<string> }>;
/** Cluster centroid pull strength (default CLUSTER_PULL_STRENGTH). */
clusterPullStrength?: number;
/** Inference-depth anchoring (see DepthAnchor). `center` defaults to
* the anchored core's centroid. */
depthAnchor?: DepthAnchor;
}
/**
* One incremental settle. `nodes`/`edges` describe the CURRENT graph;
* `previous` is the positions ledger (empty on the very first settle —
* then this degrades to a seeded-scatter bootstrap). Returns positions
* for EVERY node in `nodes`.
*/
export function incrementalLayout(
nodes: ReadonlyArray<Pick<GraphNode, "id">>,
edges: ReadonlyArray<LayoutEdge>,
previous: ReadonlyMap<string, Position>,
options: IncrementalLayoutOptions = {},
): Map<string, Position> {
const ticks = options.ticks ?? 300;
const retention = options.anchorRetention ?? ANCHOR_RETENTION;
// Anchors: the warm-start set (the previous rest state).
const anchors = new Map<string, Position>();
for (const node of nodes) {
const prev = previous.get(node.id);
if (prev) anchors.set(node.id, { x: prev.x, y: prev.y });
}
// The anchored core's bounding box + centroid, for rim seeds.
let minX = Infinity;
let maxX = -Infinity;
let minY = Infinity;
let maxY = -Infinity;
let cx = 0;
let cy = 0;
for (const a of anchors.values()) {
minX = Math.min(minX, a.x);
maxX = Math.max(maxX, a.x);
minY = Math.min(minY, a.y);
maxY = Math.max(maxY, a.y);
cx += a.x;
cy += a.y;
}
const hasCore = anchors.size > 0;
if (hasCore) {
cx /= anchors.size;
cy /= anchors.size;
}
const coreRadius = hasCore ? Math.max(maxX - minX, maxY - minY) / 2 : 1;
// Adjacency for neighbor seeding (undirected: edges include assertions).
const adjacency = new Map<string, string[]>();
const linkTo = (a: string, b: string): void => {
if (a === b) return;
let list = adjacency.get(a);
if (!list) adjacency.set(a, (list = []));
list.push(b);
};
for (const e of edges) {
linkTo(e.from, e.to);
linkTo(e.to, e.from);
}
// The rim radius derives from the CONNECTED core's mean radius — orphans
// never inflate it (a max-span-based radius feeds back: each batch's
// rim pushes the next batch's rim farther out, and the layout runs
// away). 1.7x matches the density the FA2-era settle produced
// (orphan mean r ~338 vs connected ~194).
let rimRadius = 0;
let connectedAnchored = 0;
for (const [id, a] of anchors) {
if ((adjacency.get(id)?.length ?? 0) === 0) continue;
connectedAnchored++;
rimRadius += Math.hypot(a.x - cx, a.y - cy);
}
if (connectedAnchored > 0) rimRadius = (rimRadius / connectedAnchored) * 1.7;
else if (hasCore) rimRadius = coreRadius * 1.15; // no connected anchors yet: bbox fallback
rimRadius += 4;
// Seed every node.
const simNodes: LayoutNode[] = nodes.map((node) => {
const rand = seededRandom(node.id);
const anchor = anchors.get(node.id);
let x: number;
let y: number;
if (anchor) {
// Warm start: resume from the previous rest position.
x = anchor.x;
y = anchor.y;
} else {
// New node: seed near its positioned neighbors (the cluster it
// arrived with), else on the rim outside the core, else scatter.
let sx = 0;
let sy = 0;
let found = 0;
for (const nbr of adjacency.get(node.id) ?? []) {
const na = anchors.get(nbr);
if (na) {
sx += na.x;
sy += na.y;
found++;
}
}
if (found > 0) {
x = sx / found + (rand() - 0.5) * 8;
y = sy / found + (rand() - 0.5) * 8;
} else if (hasCore) {
const angle = rand() * Math.PI * 2;
const radius = rimRadius + rand() * 8;
x = cx + Math.cos(angle) * radius;
y = cy + Math.sin(angle) * radius;
// An orphan's intended rest spot IS the rim seed: anchor it
// there so gravity cannot sink it into the core. (A later
// batch linking it to the graph pulls it in naturally —
// but then it also has a previous position, and its anchor
// becomes that previous position, exactly like any node.)
anchors.set(node.id, { x, y });
} else {
// Bootstrap: the classic seeded scatter.
x = (rand() - 0.5) * 100;
y = (rand() - 0.5) * 100;
}
}
// d3 is fine with coincident seeds (no division), but nudge exact
// overlaps of a brand-new node onto its only neighbor anyway.
if (x === 0 && y === 0) x = 0.5;
return { id: node.id, x, y };
});
const sim = createGraphSimulation(simNodes, edges, {
gravity: {
x: cx,
y: cy,
strength: options.gravityStrength ?? (hasCore ? INCREMENTAL_GRAVITY : BOOTSTRAP_GRAVITY),
},
clusterPull: options.clusters
? { clusters: options.clusters, strength: options.clusterPullStrength ?? CLUSTER_PULL_STRENGTH }
: undefined,
depthAnchor:
options.depthAnchor && options.depthAnchor.mode !== "off"
? { ...options.depthAnchor, center: options.depthAnchor.center ?? { x: cx, y: cy } }
: options.depthAnchor,
});
const dMode = options.depthAnchor?.mode ?? "off";
const dCenter = options.depthAnchor?.center ?? { x: cx, y: cy };
const byId = new Map(simNodes.map((n) => [n.id, n]));
for (let i = 0; i < ticks; i++) {
sim.tick();
// Anchor re-blend (position space, alpha-independent — see the
// ANCHOR_RETENTION note). Pinned nodes (fx/fy) are left to d3.
// DEPTH-MODE PROJECTION: the re-blend preserves the user's
// arrangement in the FREE dimension only — x in vertical mode, the
// angle in radial mode — because the depth force owns the
// constrained dimension (a node's level/ring is meaning, not
// memory). Off-mode: full 2D blend as before.
if (hasCore && retention < 1) {
for (const [id, a] of anchors) {
const n = byId.get(id);
if (!n || n.fx != null || n.fy != null) continue;
const x = n.x ?? 0;
const y = n.y ?? 0;
if (dMode === "vertical") {
n.x = a.x + (x - a.x) * retention;
} else if (dMode === "radial") {
const rx = x - dCenter.x;
const ry = y - dCenter.y;
const r = Math.hypot(rx, ry);
const ax = a.x - dCenter.x;
const ay = a.y - dCenter.y;
const ar = Math.hypot(ax, ay);
if (r > 1e-9 && ar > 1e-9) {
const theta = Math.atan2(ry, rx);
const thetaA = Math.atan2(ay, ax);
const blended = thetaA + Math.atan2(Math.sin(theta - thetaA), Math.cos(theta - thetaA)) * retention;
n.x = dCenter.x + r * Math.cos(blended);
n.y = dCenter.y + r * Math.sin(blended);
}
} else {
n.x = a.x + (x - a.x) * retention;
n.y = a.y + (y - a.y) * retention;
}
}
}
}
const out = new Map<string, Position>();
for (const n of simNodes) out.set(n.id, { x: n.x ?? 0, y: n.y ?? 0 });
return out;
}
+32
View File
@@ -22,8 +22,12 @@ import type {
ExtractionRunUsage,
NodeMapping,
PlanTree,
Position,
SessionCheckpoint,
TurnUsage,
NotesSource,
ClusterLabeling,
EmbeddingsLedger,
} from "./contracts.ts";
// ─── ChatStore — the session record's home (data/chats) ─────────────────────
@@ -80,6 +84,34 @@ export interface GraphStore {
/** Append one emission checkpoint. */
appendCheckpoint(checkpoint: SessionCheckpoint): void;
readCheckpoints(): SessionCheckpoint[];
/** Append one cluster labeling — idempotent by id (the id hashes the
* sorted core, so identical member sets dedupe). */
appendClusterLabel(labeling: ClusterLabeling): void;
readClusterLabels(): ClusterLabeling[];
/** Upsert one imported-notes source record (config.SOURCES_FILE — a
* JSON map, not a ledger; an upsert keeps the existing usage log). */
upsertSource(source: NotesSource): void;
/** Every notes source record, by id order-insensitive. */
readSources(): NotesSource[];
/** Append one run's usage to a source record — idempotent per
* startedAt, mirroring ChatRecord.extractions. */
appendSourceUsage(id: string, entry: ExtractionRunUsage): void;
/** The layout ledger (config.POSITIONS_FILE): nodeId → resting {x,y}.
* WRITTEN ONLY by the incremental settle after a merge (core/layout.ts
* settleGraphLayout) — read-only for everything else. */
readPositions(): Record<string, Position>;
/** Replace the whole ledger (the settle is a full-graph computation). */
writePositions(positions: Record<string, Position>): void;
/** The embedding ledger (config.EMBEDDINGS_FILE). WRITTEN ONLY by
* the embedder (server/embeddings.ts) after a merge — read-only for
* clustering consumers (snapshot, settle, namer). */
readEmbeddings(): EmbeddingsLedger;
/** Replace the whole ledger (embedding is a full-batch computation). */
writeEmbeddings(ledger: EmbeddingsLedger): void;
}
// ─── JobStore — background sub-agent jobs (data/jobs) ─────────────────────────
+280
View File
@@ -0,0 +1,280 @@
/**
* Graph render settings: the shape, the defaults, the sanitizer, and the
* slider mapping (log-scaled, default-centered) — shared by the client (components/graph-canvas.ts,
* graph-view.tsx), the server store (src/store/render-settings.ts, which
* persists them to data/render-settings.json), and the API route.
*
* This module is pure: no DOM, no d3, no node APIs — safe everywhere.
*
* Settings are server-persisted (render-settings.json); the localStorage era
* is over. Whatever arrives from disk or the network is passed through
* sanitizeRenderSettings before use — never trusted raw.
*
* The labelZoom semantics: the sigma-compatible "camera ratio" threshold
* (1 = fit zoom, SMALLER = zoomed in). Node labels show at ratio <= labelZoom
* and fade over LABELS_FADE_SPAN; cluster region labels fade in inversely.
* Persisted values keep these units across the renderer swap.
*/
import {
CHARGE_STRENGTH,
CLUSTER_PULL_STRENGTH,
INCREMENTAL_GRAVITY,
LINK_DISTANCE,
type DepthAnchorMode,
} from "./layout.ts";
export interface GraphRenderSettings {
nodeSize: number;
edgeWidth: number;
arrowLength: number;
/** Camera-ratio threshold for node labels (see module doc). */
labelZoom: number;
/** Node + hover label font size, px at fit zoom (scales with camera). */
labelSize: number;
/** Toggle for the cluster region labels (hulls always draw). */
clusterLabels: boolean;
/** Region rendering (the blurred cluster tints). Server-persisted
* since 2026-09-13 (user request) — was session-only dev state. */
regions: RegionTuning;
/** Live-sim physics + inference-depth anchoring. Server-persisted
* since 2026-09-13 (user request) — was session-only dev state. */
physics: PhysicsTuning;
}
const clamp = (v: number, min: number, max: number) => Math.min(Math.max(v, min), max);
/**
* Coerce arbitrary JSON (server file, network body, localStorage leftovers)
* into valid settings: missing keys fall back to defaults, out-of-range
* values clamp, wrong types reset. Never throws.
*/
export function sanitizeRenderSettings(raw: unknown): GraphRenderSettings {
const r = (raw ?? {}) as Partial<GraphRenderSettings>;
const num = (v: unknown, fallback: number, min: number, max: number) => {
const n = typeof v === "number" && Number.isFinite(v) ? v : fallback;
return clamp(n, min, max);
};
return {
nodeSize: num(r.nodeSize, DEFAULT_RENDER_SETTINGS.nodeSize, NODE_SIZE_RANGE.min, NODE_SIZE_RANGE.max),
edgeWidth: num(r.edgeWidth, DEFAULT_RENDER_SETTINGS.edgeWidth, EDGE_WIDTH_RANGE.min, EDGE_WIDTH_RANGE.max),
arrowLength: num(r.arrowLength, DEFAULT_RENDER_SETTINGS.arrowLength, ARROW_LENGTH_RANGE.min, ARROW_LENGTH_RANGE.max),
// Range matches the log slider's span [LABEL_ZOOM_MIN, LABEL_ZOOM_MAX].
labelZoom: num(r.labelZoom, DEFAULT_RENDER_SETTINGS.labelZoom, LABEL_ZOOM_MIN, LABEL_ZOOM_MAX),
labelSize: num(r.labelSize, DEFAULT_RENDER_SETTINGS.labelSize, LABEL_SIZE_RANGE.min, LABEL_SIZE_RANGE.max),
clusterLabels:
typeof r.clusterLabels === "boolean" ? r.clusterLabels : DEFAULT_RENDER_SETTINGS.clusterLabels,
// Nested sections (2026-09-13): old files lack them → defaults.
regions: sanitizeRegionTuning(r.regions),
physics: sanitizePhysicsTuning(r.physics),
};
}
// ─── Region + physics tuning (moved from the renderer, 2026-09-13) ─────────────
//
// The renderer's region/physics controls became USER-FACING persisted
// settings: one shape + one sanitizer in core, consumed by the settings
// store (this file's schema), the panel, and the renderer's
// setRegionTuning/setPhysicsTuning. Every clamp below is a SWEEP-VALIDATED
// safe range (2026-09-13, real graph, renderer's exact sim): each field at
// both range ends AND all four range corners settles cleanly — no NaN, no
// explosion (linkStrength ≥ 8 explodes the layout, 5 wobbles; cap 4),
// no non-settle (velocityDecay 0 never damps; floor 0.1), no 3x sprawl
// (gravity 0 / charge -1000 excluded).
/** Region rendering knobs (the blurred cluster tints). */
export interface RegionTuning {
/** Dilation half-width of the round-cap edge strokes, screen px at
* fit zoom (the expansion of the cluster skeleton). */
dilation: number;
/** Box-blur radius in bitmap px (3 passes ≈ Gaussian). Fixed set:
* see REGION_BLUR_OPTIONS (dropdown in the panel — non-integer
* radii broke the rendering, so the control is integer-only). */
blur: number;
/** Max bitmap dimension (cell = worldSpan / this). Fixed set:
* see REGION_BITMAP_OPTIONS (dropdown in the panel). */
bitmap: number;
/** Region alpha, applied ONCE after the blur (dilate at full
* opacity → blur → scale). Constant interior, soft edges; overlaps
* of any count never stack (DECIDED 2026-09-14 — per-stroke alpha
* accumulated through source-over and double-counted orphans). */
edgeAlpha: number;
}
export const REGION_BITMAP_OPTIONS = [128, 192, 256, 384, 512] as const;
export const REGION_BLUR_OPTIONS = [2, 4, 6, 8, 10] as const;
export const DEFAULT_REGION_TUNING: RegionTuning = {
dilation: 28,
blur: 6,
bitmap: 384,
edgeAlpha: 0.35,
};
/** Nearest member of a fixed option set (dropdowns have no free values). */
function snapToOption(value: number, options: readonly number[]): number {
return options.reduce((best, o) => (Math.abs(o - value) < Math.abs(best - value) ? o : best), options[0] as number);
}
/** Clamp/validate a region tuning. `bitmap` and `blur` snap to the nearest
* allowed option (they're fixed sets — dropdowns, not free numbers). */
export function sanitizeRegionTuning(input: Partial<RegionTuning> | null | undefined): RegionTuning {
const d = DEFAULT_REGION_TUNING;
const num = (v: unknown, fallback: number): number =>
typeof v === "number" && Number.isFinite(v) ? v : fallback;
const bitmap = num(input?.bitmap, d.bitmap);
return {
dilation: Math.min(Math.max(num(input?.dilation, d.dilation), REGION_DILATION_RANGE.min), REGION_DILATION_RANGE.max),
blur: snapToOption(num(input?.blur, d.blur), REGION_BLUR_OPTIONS),
bitmap: snapToOption(bitmap, REGION_BITMAP_OPTIONS),
edgeAlpha: Math.min(Math.max(num(input?.edgeAlpha, d.edgeAlpha), REGION_EDGE_ALPHA_RANGE.min), 1),
};
}
/** Live-sim physics controls (user-facing since 2026-09-13): every value
* feeds createGraphSimulation — the SAME engine the server settle runs —
* so tuning the feel changes exactly what a drag relaxes toward. */
export interface PhysicsTuning {
/** Node repulsion (d3 charge, negative = push apart). */
chargeStrength: number;
/** Edge attraction: link-spring stiffness multiplier (1 = d3 default). */
linkStrength: number;
/** Link spring rest length (world units). */
linkDistance: number;
/** Central pull toward the graph centroid. */
gravity: number;
/** Per-cluster pull toward the cluster centroid. 0 = off. */
clusterPull: number;
/** Per-tick velocity damping (d3 default 0.4). */
velocityDecay: number;
/** Inference-depth anchoring mode (see DepthAnchor in core). */
depthMode: DepthAnchorMode;
/** Depth-anchor pull strength. */
depthStrength: number;
/** Spacing between inference levels (world units). */
depthSpacing: number;
/** Inner ring radius, radial mode (world units). */
depthBaseRadius: number;
}
export const DEFAULT_PHYSICS_TUNING: PhysicsTuning = {
chargeStrength: CHARGE_STRENGTH,
linkStrength: 1,
linkDistance: LINK_DISTANCE,
gravity: INCREMENTAL_GRAVITY,
clusterPull: CLUSTER_PULL_STRENGTH,
velocityDecay: 0.4,
depthMode: "off",
// 0.8 by sweep on the real graph: 215/216 arrows read upward with
// organic sag (mean |dev| from level ~34u at spacing 60); 1.5 gives
// crisp rows (dev ~18) if the promoted default wants them.
depthStrength: 0.8,
depthSpacing: 60,
depthBaseRadius: 220,
};
/** Clamp anything into a usable tuning (sliders/JSON can be empty/NaN). */
export function sanitizePhysicsTuning(input: Partial<PhysicsTuning> | null | undefined): PhysicsTuning {
const d = DEFAULT_PHYSICS_TUNING;
const num = (v: unknown, fallback: number): number =>
typeof v === "number" && Number.isFinite(v) ? v : fallback;
return {
chargeStrength: Math.min(Math.max(num(input?.chargeStrength, d.chargeStrength), CHARGE_RANGE.min), CHARGE_RANGE.max),
linkStrength: Math.min(Math.max(num(input?.linkStrength, d.linkStrength), LINK_STRENGTH_RANGE.min), LINK_STRENGTH_RANGE.max),
linkDistance: Math.min(Math.max(num(input?.linkDistance, d.linkDistance), LINK_DISTANCE_RANGE.min), LINK_DISTANCE_RANGE.max),
gravity: Math.min(Math.max(num(input?.gravity, d.gravity), GRAVITY_RANGE.min), GRAVITY_RANGE.max),
clusterPull: Math.min(Math.max(num(input?.clusterPull, d.clusterPull), 0), CLUSTER_PULL_RANGE.max),
velocityDecay: Math.min(Math.max(num(input?.velocityDecay, d.velocityDecay), VELOCITY_DECAY_RANGE.min), VELOCITY_DECAY_RANGE.max),
depthMode: input?.depthMode === "vertical" || input?.depthMode === "radial" ? input.depthMode : "off",
depthStrength: Math.min(Math.max(num(input?.depthStrength, d.depthStrength), DEPTH_STRENGTH_RANGE.min), DEPTH_STRENGTH_RANGE.max),
depthSpacing: Math.min(Math.max(num(input?.depthSpacing, d.depthSpacing), DEPTH_SPACING_RANGE.min), DEPTH_SPACING_RANGE.max),
depthBaseRadius: Math.min(Math.max(num(input?.depthBaseRadius, d.depthBaseRadius), DEPTH_BASE_RADIUS_RANGE.min), DEPTH_BASE_RADIUS_RANGE.max),
};
}
// Sizes are screen px AT FIT ZOOM (the world-space renderer scales them
// with the camera, so a label reads at labelSize x k/kFit on screen).
// Defaults were re-tuned for world-scaling on 2026-09-13: the old values
// were calibrated in the screen-fixed era, so everything rendered ~2x
// too big once sizes started scaling with the graph.
export const DEFAULT_RENDER_SETTINGS: GraphRenderSettings = {
nodeSize: 2,
edgeWidth: 1,
arrowLength: 2,
labelZoom: 0.5,
labelSize: 2,
clusterLabels: true,
regions: DEFAULT_REGION_TUNING,
physics: DEFAULT_PHYSICS_TUNING,
};
// ─── The slider mapping ────────────────────────────────────────────────────────
//
// EVERY slider is log-scaled over its setting's range with the DEFAULT at
// position 50 (dead center), per the user's spec: centered defaults, coarse
// steps (the UI steps by 5 → 21 stops), no fine-grained linear tracks.
// Ranges are chosen so each default is the geometric mean of its ends.
export const NODE_SIZE_RANGE = { min: 0.5, max: 8 } as const;
export const EDGE_WIDTH_RANGE = { min: 0.25, max: 4 } as const;
export const ARROW_LENGTH_RANGE = { min: 0.5, max: 8 } as const;
export const LABEL_SIZE_RANGE = { min: 0.5, max: 8 } as const;
export const LABEL_ZOOM_MIN = 0.0625; // slider far left: labels only very deep in
export const LABEL_ZOOM_MAX = 4; // slider far right: labels visible at fit zoom
// ─── Region + physics slider ranges ──────────────────────────────────────────
//
// Same laws as the size sliders: log-scaled, DEFAULT AT 50 (each default is
// the geometric mean of its ends), coarse 5-step tracks. Signed/zeroed
// fields get special treatment:
// chargeStrength is NEGATIVE — the slider maps |charge| and displays "-x";
// clusterPull has an OFF zone — position 0 is exactly 0 (see
// toOffSliderPosition), the log range starts one step in. Blur used to
// have one too; it's a dropdown since 2026-09-13 (non-integers broke).
export const REGION_DILATION_RANGE = { min: 7, max: 112 } as const; // gm = 28
export const REGION_EDGE_ALPHA_RANGE = { min: 0.12, max: 1 } as const; // gm ≈ 0.35
export const CHARGE_RANGE = { min: -480, max: -30 } as const; // |gm| = 120
export const LINK_STRENGTH_RANGE = { min: 0.25, max: 4 } as const; // gm = 1
export const LINK_DISTANCE_RANGE = { min: 10, max: 160 } as const; // gm = 40
export const GRAVITY_RANGE = { min: 0.03, max: 0.48 } as const; // gm = 0.12
export const CLUSTER_PULL_RANGE = { min: 0.0225, max: 1 } as const; // gm = 0.15; 0 = off
export const VELOCITY_DECAY_RANGE = { min: 0.18, max: 0.9 } as const; // gm ≈ 0.40
export const DEPTH_STRENGTH_RANGE = { min: 0.2, max: 3.2 } as const; // gm = 0.8
export const DEPTH_SPACING_RANGE = { min: 15, max: 240 } as const; // gm = 60
export const DEPTH_BASE_RADIUS_RANGE = { min: 55, max: 880 } as const; // gm = 220
/** Off-aware slider mapping (position 0 → value 0 = OFF; the log range
* starts one step in). Inverse of fromOffSliderPosition. */
export function toOffSliderPosition(value: number, min: number, max: number): number {
if (value <= 0) return 0;
return toSliderPosition(value, min, max);
}
/** Off-aware slider mapping: position 0 → 0 (off), else log-scaled. */
export function fromOffSliderPosition(position: number, min: number, max: number): number {
if (position <= 0) return 0;
return fromSliderPosition(position, min, max);
}
/** Setting value → slider position [0, 100] (log scale, default at 50). */
export function toSliderPosition(value: number, min: number, max: number): number {
const v = clamp(value, min, max);
const logSpan = Math.log(max / min);
return (100 * Math.log(v / min)) / logSpan;
}
/** Slider position [0, 100] → setting value (inverse of toSliderPosition). */
export function fromSliderPosition(position: number, min: number, max: number): number {
const p = clamp(position, 0, 100) / 100;
const logSpan = Math.log(max / min);
return min * Math.exp(p * logSpan);
}
/** Display units for labelZoom: normalized so the default (0.5) reads as 1. */
export function labelZoomDisplay(labelZoom: number): string {
const v = Math.round((labelZoom / DEFAULT_RENDER_SETTINGS.labelZoom) * 100) / 100;
return `${v}×`;
}
+141
View File
@@ -0,0 +1,141 @@
// Semantic topic clustering — the REPLACEMENT for Louvain community
// detection (DECIDED 2026-09-13): structural clusters were just the
// reasoning chains (the graph is disconnected by nature — every connected
// component became its own "cluster"), and edge-orphans could never join
// anything. Clusters are now RELATED TOPICS: nodes whose proposition
// MEANING is close, computed over the embedding ledger
// (config.EMBEDDINGS_FILE). This pulls orphans into topics by meaning
// and groups small chains with the large chains they discuss.
//
// Algorithm: threshold AGGLOMERATIVE (average linkage over cosine
// similarity). No JS HDBSCAN is trustworthy (the npm package is an
// abandoned alpha), and at proposition scale (~100s-1000s of short texts)
// brute-force pairwise cosine is sub-second — so the honest ~60-line
// hand-roll wins. "Stragglers" (nodes that fit nothing) fall out
// naturally: they simply never merge, and stay singletons.
/** The default similarity threshold, SWEPT ON THE REAL GRAPH
* (2026-09-13): 0.60 → 26 topics of ~20 propositions, 74 of the 90
* edge-orphans land in real topics, stragglers 10. Higher (0.65-0.68)
* gives tighter micro-topics with far more stragglers; lower (0.55)
* collapses into ~13 mega-topics. */
export const SEMANTIC_SIM_THRESHOLD = 0.6;
/** Cosine similarity of two vectors (defensively normalizes inputs;
* ledger vectors are already L2-normalized, so this is a dot product). */
export function cosineSimilarity(a: readonly number[], b: readonly number[]): number {
let dot = 0;
let ma = 0;
let mb = 0;
for (let i = 0; i < a.length; i++) {
dot += a[i]! * b[i]!;
ma += a[i]! * a[i]!;
mb += b[i]! * b[i]!;
}
if (ma === 0 || mb === 0) return 0;
return dot / (Math.sqrt(ma) * Math.sqrt(mb));
}
export interface SemanticClusterOptions {
/** Merge while the best cluster pair's average-linkage similarity is
* >= this (default SEMANTIC_SIM_THRESHOLD). */
similarityThreshold?: number;
/** After the cut, force every singleton into its nearest topic
* ("all-in" mode) instead of leaving stragglers loose. Default
* false: some propositions genuinely fit nothing. */
attachStragglers?: boolean;
}
/** Average-linkage agglomerative clustering over embedding vectors.
* Pure and deterministic (stable Map iteration, greedy by best
* similarity, first-found wins ties). Returns clusters size-desc — the
* same { members } shape Louvain produced, so label matching and the
* renderer contract are unchanged. An empty/missing ledger returns []
* (nothing embedded yet: no clusters, every node loose — the snapshot
* degrades gracefully until the first embed). */
export function semanticClusters(
vectors: Readonly<Record<string, readonly number[]>>,
options: SemanticClusterOptions = {},
): Array<{ members: string[] }> {
const ids = Object.keys(vectors);
if (ids.length === 0) return [];
const simThreshold = options.similarityThreshold ?? SEMANTIC_SIM_THRESHOLD;
const attach = options.attachStragglers ?? false;
// Sum vectors per cluster: average-linkage similarity between clusters
// A and B is dot(sumA, sumB) / (|A|·|B|) — the exact mean of pairwise
// dots — so the merge loop never revisits the raw O(n²) matrix.
const sums = new Map<number, number[]>();
const members = new Map<number, string[]>();
ids.forEach((id, i) => {
sums.set(i, [...vectors[id]!]);
members.set(i, [id]);
});
// Greedy merge: repeatedly join the pair with the highest
// average-linkage similarity until it drops below the threshold.
// O(k²) pair scan per merge — trivial at proposition scale.
for (;;) {
let bestA = -1;
let bestB = -1;
let bestSim = -Infinity;
for (const [a, sumA] of sums) {
for (const [b, sumB] of sums) {
if (a >= b) continue;
let dot = 0;
for (let i = 0; i < sumA.length; i++) dot += sumA[i]! * sumB[i]!;
const sim = dot / (members.get(a)!.length * members.get(b)!.length);
if (sim > bestSim) {
bestSim = sim;
bestA = a;
bestB = b;
}
}
}
if (bestA < 0 || bestSim < simThreshold) break;
// Merge B into A (deterministic direction: lower cluster index).
const sumA = sums.get(bestA)!;
const sumB = sums.get(bestB)!;
for (let i = 0; i < sumA.length; i++) sumA[i] = sumA[i]! + sumB[i]!;
members.get(bestA)!.push(...members.get(bestB)!);
sums.delete(bestB);
members.delete(bestB);
}
// Optional all-in pass: singletons join their nearest topic regardless
// of threshold — exactly the "forced membership" mode. The difference
// from stragglers mode is this one pass. Singleton-vs-cluster
// average-linkage similarity is dot(sumS, sumK) / |K|.
if (attach) {
const singletons = [...members.entries()].filter(([, m]) => m.length === 1).map(([k]) => k);
for (const s of singletons) {
if (!members.has(s)) continue; // absorbed by an earlier attach
let bestTarget = -1;
let bestSim = -Infinity;
for (const [k, m] of members) {
if (k === s || m.length < 2) continue;
let dot = 0;
const sumS = sums.get(s)!;
const sumK = sums.get(k)!;
for (let i = 0; i < sumS.length; i++) dot += sumS[i]! * sumK[i]!;
const sim = dot / m.length;
if (sim > bestSim) {
bestSim = sim;
bestTarget = k;
}
}
if (bestTarget >= 0) {
const sumT = sums.get(bestTarget)!;
const sumS = sums.get(s)!;
for (let i = 0; i < sumT.length; i++) sumT[i] = sumT[i]! + sumS[i]!;
members.get(bestTarget)!.push(...members.get(s)!);
sums.delete(s);
members.delete(s);
}
}
}
return [...members.values()]
.map((m) => ({ members: [...m].sort() }))
.sort((a, b) => b.members.length - a.members.length || (a.members[0] ?? "").localeCompare(b.members[0] ?? ""));
}
+10 -1
View File
@@ -4,6 +4,7 @@ import { createProberAgent, createTeacherAgent, type TeachingAgent } from "../ag
import { piEngineFor } from "../agent/models.ts";
import { createJobRunner } from "../agent/subagents/runner.ts";
import { createExtractionPipeline } from "../agent/subagents/extraction.ts";
import { embedNewNodes } from "./embeddings.ts";
import type { ModelRole } from "../../config.ts";
import { claimSteers, deliverJobResult, setJobRespawn, setTurnRunner } from "./jobs.ts";
import { recordToPiMessages } from "./pi/record-adapter.ts";
@@ -37,7 +38,15 @@ const jobRunner = createJobRunner({
jobStore: stores.jobStore,
deliver: deliverJobResult,
});
const extraction = createExtractionPipeline(stores);
/** The app's ONE extraction pipeline instance — both front doors share it:
* quiz-triggered session runs (turn-service) and notes imports
* (/api/graph/import) hit the same single-flight registry. */
export const extraction = createExtractionPipeline({
...stores,
// The REAL embedder (local transformer model): post-merge embedding
// batch — semantic clusters are computed and persisted here.
embedder: embedNewNodes,
});
// Wake-pipeline wiring (contracts.ts "THE ASYNC PROTOCOL" §4): the pipeline
// fires its server-side teacher turns through THIS composition root, and
+57
View File
@@ -0,0 +1,57 @@
/**
* Server-side commit for the live physics drag (PUT /api/graph/positions).
* The client's renderer anneals a drag to rest and PUTs the new rest
* state; the server trusts it for the nodes it knows about and, if the
* graph grew while the user was dragging (a merge landed), runs ONE
* incremental settle (settleGraphLayout) so the fresh nodes are absorbed
* into the frame the user just arranged — client positions act as
* anchors, never as a clobber.
*
* Every body is sanitized: garbage coordinates clamp or drop, unknown
* ids are discarded, nothing throws.
*/
import type { Position } from "../core/contracts.ts";
import { getGraph } from "../core/graph.ts";
import { settleGraphLayout } from "../core/layout-settle.ts";
import { stores } from "./stores.ts";
/** Max |x| or |y| accepted — NaN/Infinity clamp here, absurd too. */
const MAX_COORD = 1e6;
/**
* Accepts the raw PUT body (a Record of id → {x, y}) and returns the
* sanitized positions actually persisted (empty object = nothing written,
* e.g. a body with no known ids — the caller reports but never fails).
*/
export function commitPositions(input: unknown): Record<string, Position> {
if (typeof input !== "object" || input === null) return {};
const raw = input as Record<string, unknown>;
// Only ids the graph actually has (first line wins, like getGraph).
const known = new Map<string, boolean>();
for (const line of stores.graphStore.readNodes()) {
if (!known.has(line.id)) known.set(line.id, true);
}
const positions: Record<string, Position> = {};
for (const [id, value] of Object.entries(raw)) {
if (!known.has(id)) continue;
if (typeof value !== "object" || value === null) continue;
const { x, y } = value as Record<string, unknown>;
if (typeof x !== "number" || typeof y !== "number" || !Number.isFinite(x) || !Number.isFinite(y))
continue;
positions[id] = {
x: Math.min(Math.max(x, -MAX_COORD), MAX_COORD),
y: Math.min(Math.max(y, -MAX_COORD), MAX_COORD),
};
}
if (Object.keys(positions).length === 0) return {};
stores.graphStore.writePositions(positions);
// The graph grew past what the client knew (a merge landed mid-drag):
// absorb the new nodes with the client's rest state as anchors.
const unique = getGraph(stores.graphStore).nodes.length;
if (unique > Object.keys(positions).length) settleGraphLayout(stores.graphStore);
return positions;
}
+125
View File
@@ -0,0 +1,125 @@
/**
* THE EMBEDDER (DECIDED 2026-09-13): semantic topic clustering needs
* vectors, and vectors need a model running ON the user's machine — no
* external services, no API bills, no network dependency for re-embedding.
*
* Runtime: in-process transformers.js (@huggingface/transformers v4 over
* onnxruntime-node — native Apple Silicon acceleration). The model
* weights download ONCE to the local HF cache on first run (~300MB).
*
* Model: EmbeddingGemma-300M (q8) — best quality-per-megabyte small
* embedding model as of late 2025 (MTEB-en 68.36), built for on-device
* use, with MRL truncation so we store 512 of its 768 dims. Every input
* uses the model card's document format ("title: none | text: …"),
* consistently — mixing prefixed and unprefixed inputs degrades quality.
*
* Model swaps re-embed EVERYTHING: vectors from different models live in
* incompatible spaces. The ledger records model + dim; a mismatch marks
* the whole ledger stale and this module re-embeds from scratch (minutes
* at current scale).
*/
import { pipeline, type FeatureExtractionPipeline } from "@huggingface/transformers";
import type { EmbeddingsLedger, GraphNode } from "../core/contracts.ts";
import { semanticClusters, SEMANTIC_SIM_THRESHOLD } from "../core/semantic-clusters.ts";
export const EMBEDDING_MODEL = "onnx-community/embeddinggemma-300m-ONNX";
/** MRL truncation of the native 768 — negligible quality loss, half the
* ledger size (2026-09-13 researcher sweep). */
export const EMBEDDING_DIM = 512;
/** Batch size bounds peak memory; propositions are tiny so 32 per call
* keeps the pipeline warm without thrashing. */
const EMBED_BATCH = 32;
let pipelinePromise: Promise<FeatureExtractionPipeline> | null = null;
function getPipeline(): Promise<FeatureExtractionPipeline> {
pipelinePromise ??= pipeline("feature-extraction", EMBEDDING_MODEL, { dtype: "q8" });
return pipelinePromise;
}
/** EmbeddingGemma's required document format (model card). */
function embedInput(statement: string): string {
return `title: none | text: ${statement}`;
}
/** Truncate an L2-normalized vector to EMBEDDING_DIM (MRL: leading dims
* carry most of the signal) and re-normalize so cosine = dot product. */
function truncateNormalize(vec: readonly number[]): number[] {
const out = vec.slice(0, EMBEDDING_DIM);
let mag = 0;
for (const x of out) mag += x * x;
mag = Math.sqrt(mag);
return mag > 0 ? out.map((x) => x / mag) : out;
}
/**
* Embed every node the ledger is missing — the post-merge hook (new
* propositions only, typically a handful) AND the cold-start backfill
* (empty ledger = every node) in one function. Model/dim mismatch voids
* the ledger first (full re-embed). Also prunes vectors for nodes that
* no longer exist. Writes the whole ledger and returns counts for logs.
* Non-fatal by contract: the caller treats failure as "clustering runs
* without the new nodes until the next merge retries".
*/
export async function embedNewNodes(
graphStore: {
readNodes(): GraphNode[];
readEmbeddings(): EmbeddingsLedger;
writeEmbeddings(ledger: EmbeddingsLedger): void;
},
): Promise<{ embedded: number; total: number }> {
const ledger = graphStore.readEmbeddings();
const stale = ledger.model !== EMBEDDING_MODEL || ledger.dim !== EMBEDDING_DIM;
const vectors: Record<string, number[]> = stale ? {} : { ...ledger.vectors };
// Node LINES may repeat an id — dedupe, first defines (getGraph rule).
const seen = new Set<string>();
const todo: GraphNode[] = [];
for (const node of graphStore.readNodes()) {
if (seen.has(node.id)) continue;
seen.add(node.id);
if (!vectors[node.id]) todo.push(node);
}
if (todo.length > 0) {
const extractor = await getPipeline();
for (let i = 0; i < todo.length; i += EMBED_BATCH) {
const batch = todo.slice(i, i + EMBED_BATCH);
const output = await extractor(batch.map((n) => embedInput(n.statement)), {
pooling: "mean",
normalize: true,
});
const rows = output.tolist() as number[][];
for (let j = 0; j < batch.length; j++) {
vectors[batch[j]!.id] = truncateNormalize(rows[j]!);
}
}
}
// Prune ghosts: vectors for node ids that no longer exist.
let pruned = 0;
for (const id of Object.keys(vectors)) {
if (!seen.has(id)) {
delete vectors[id];
pruned++;
}
}
// RECLUSTER AT THE END OF THE BATCH (user-decided 2026-09-13): the
// agglomerative pass is ~1.5s at current scale, so it runs HERE —
// once per embedding batch — and consumers (snapshot, settle, namer)
// read the persisted partition instead of recomputing per read.
const clusters = semanticClusters(vectors, { similarityThreshold: SEMANTIC_SIM_THRESHOLD });
const wrote = todo.length > 0 || pruned > 0 || stale || ledger.clusters.length !== clusters.length;
if (wrote) {
graphStore.writeEmbeddings({
model: EMBEDDING_MODEL,
dim: EMBEDDING_DIM,
vectors,
clusters,
updatedAt: new Date().toISOString(),
});
}
return { embedded: todo.length, total: Object.keys(vectors).length };
}
+2
View File
@@ -7,8 +7,10 @@
export * from "./chat-handler.ts";
export * from "./chats.ts";
export * from "./graph.ts";
export * from "./commit-positions.ts";
export * from "./jobs.ts";
export * from "./stores.ts";
export { readRenderSettings, writeRenderSettings } from "../store/render-settings.ts";
export * from "./turn-liveness.ts";
export * from "./observability.ts";
export * from "./model-catalog.ts";
+185 -1
View File
@@ -15,6 +15,10 @@
* (chatId, sourceId) → graphNodeId
* sessions.jsonl SessionCheckpoint— emission checkpoints (foldedAt)
*
* Plus config.SOURCES_FILE (sources.json): the imported-notes bookkeeping
* (NOT a ledger) — one JSON map, src-id → NotesSource, read-modify-write
* like a chat file. The format-agnostic front door (2026-09-13).
*
* Implements core/ports.ts GraphStore. data/graph/* is exclusively this
* adapter's. Reads are strict against the contracts.ts shapes (the 2026-09-02
* graph redesign): corrupt lines are skipped, never guessed at — the ledgers
@@ -25,13 +29,18 @@
import * as fs from "node:fs";
import * as path from "node:path";
import { LEDGER_FILES } from "../../config.ts";
import { EMBEDDINGS_FILE, LEDGER_FILES, POSITIONS_FILE, SOURCES_FILE } from "../../config.ts";
import type {
ClusterLabeling,
EmbeddingsLedger,
GraphEdge,
GraphNode,
ExtractionRunUsage,
LedgerEvent,
LedgerEventKind,
NotesSource,
NodeMapping,
Position,
ProvenancePoint,
SessionCheckpoint,
} from "../core/contracts.ts";
@@ -80,6 +89,14 @@ function readValid<T>(file: string, isValid: (v: unknown) => v is T): T[] {
const isProvenancePoint = (v: unknown): v is ProvenancePoint =>
isRecord(v) && isString(v.chatId) && isString(v.messageId);
const isClusterLabeling = (v: unknown): v is ClusterLabeling =>
isRecord(v) &&
isString(v.id) &&
isString(v.label) &&
Array.isArray(v.core) &&
v.core.every(isString) &&
isString(v.createdAt);
const isGraphNode = (v: unknown): v is GraphNode =>
isRecord(v) &&
isString(v.id) &&
@@ -157,4 +174,171 @@ export const graphStore: GraphStore = {
readCheckpoints(): SessionCheckpoint[] {
return readValid(LEDGER_FILES.sessions, isCheckpoint);
},
// ── cluster labelings (config.LEDGER_FILES.clusterLabels) ──
/** Idempotent by id: the id is a hash of the sorted core, so re-appending
* the same member set (re-naming races, idempotent re-runs) is a no-op. */
appendClusterLabel(labeling: ClusterLabeling): void {
if (this.readClusterLabels().some((l) => l.id === labeling.id)) return;
appendLine(LEDGER_FILES.clusterLabels, labeling);
},
readClusterLabels(): ClusterLabeling[] {
return readValid(LEDGER_FILES.clusterLabels, isClusterLabeling);
},
// ── imported-notes bookkeeping (config.SOURCES_FILE — a JSON map) ──
/** Upsert: keeps the existing record's usage log when re-importing. */
upsertSource(source: NotesSource): void {
const all = readSourceMap();
const prev = all[source.id];
all[source.id] = prev ? { ...source, extractions: prev.extractions } : source;
writeSourceMap(all);
},
readSources(): NotesSource[] {
return Object.values(readSourceMap());
},
/** Idempotent per startedAt — same discipline as chats.appendExtractionUsage. */
appendSourceUsage(id: string, entry: ExtractionRunUsage): void {
const all = readSourceMap();
const source = all[id];
if (!source) return; // no such source — nothing to bill against
source.extractions = source.extractions?.some((r) => r.startedAt === entry.startedAt)
? source.extractions
: [...(source.extractions ?? []), entry];
writeSourceMap(all);
},
// ── the layout ledger (config.POSITIONS_FILE — a JSON map) ──
readPositions(): Record<string, Position> {
return readPositionMap();
},
writePositions(positions: Record<string, Position>): void {
writePositionMap(positions);
},
readEmbeddings(): EmbeddingsLedger {
return readEmbeddingsLedger();
},
writeEmbeddings(ledger: EmbeddingsLedger): void {
writeEmbeddingsLedger(ledger);
},
};
/** positions.json — { positions, updatedAt }. Read-modify-write, NOT
* append-only: the settle is a full-graph computation, so it replaces the
* whole map. The updatedAt stamp exists for future live-physics conflict
* handling (a browser commit must be able to detect "the graph changed
* underneath me"); reads just return the map. */
function readPositionMap(): Record<string, Position> {
let raw: string;
try {
raw = fs.readFileSync(POSITIONS_FILE, "utf-8");
} catch {
return {}; // no ledger yet — nothing has ever settled
}
try {
const parsed: unknown = JSON.parse(raw);
if (!isRecord(parsed) || !isRecord(parsed.positions)) return {};
const out: Record<string, Position> = {};
for (const [id, value] of Object.entries(parsed.positions)) {
if (isPosition(value)) out[id] = { x: value.x, y: value.y };
}
return out;
} catch {
return {};
}
}
function writePositionMap(positions: Record<string, Position>): void {
fs.mkdirSync(path.dirname(POSITIONS_FILE), { recursive: true });
fs.writeFileSync(
POSITIONS_FILE,
JSON.stringify({ positions, updatedAt: new Date().toISOString() }, null, "\t"),
"utf-8",
);
}
/** embeddings.json — the whole EmbeddingsLedger at once. Missing/corrupt
* file reads as an EMPTY ledger (model "") so consumers see "nothing
* embedded yet" rather than an error; the embedder backfills. */
function readEmbeddingsLedger(): EmbeddingsLedger {
let raw: string;
try {
raw = fs.readFileSync(EMBEDDINGS_FILE, "utf-8");
} catch {
return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" };
}
try {
const parsed: unknown = JSON.parse(raw);
if (!isRecord(parsed)) return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" };
const model = typeof parsed.model === "string" ? parsed.model : "";
const dim = typeof parsed.dim === "number" && Number.isFinite(parsed.dim) ? parsed.dim : 0;
const vectors: Record<string, number[]> = {};
if (isRecord(parsed.vectors)) {
for (const [id, value] of Object.entries(parsed.vectors)) {
if (
Array.isArray(value) &&
value.length > 0 &&
value.every((x) => typeof x === "number" && Number.isFinite(x))
) {
vectors[id] = value as number[];
}
}
}
const clusters: Array<{ members: string[] }> = [];
if (Array.isArray(parsed.clusters)) {
for (const c of parsed.clusters) {
if (
isRecord(c) &&
Array.isArray(c.members) &&
c.members.length > 0 &&
c.members.every((m) => typeof m === "string")
) {
clusters.push({ members: [...(c.members as string[])] });
}
}
}
return { model, dim, vectors, clusters, updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : "" };
} catch {
return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" };
}
}
function writeEmbeddingsLedger(ledger: EmbeddingsLedger): void {
fs.mkdirSync(path.dirname(EMBEDDINGS_FILE), { recursive: true });
fs.writeFileSync(EMBEDDINGS_FILE, JSON.stringify(ledger, null, "\t"), "utf-8");
}
const isPosition = (v: unknown): v is Position =>
isRecord(v) && typeof v.x === "number" && Number.isFinite(v.x) && typeof v.y === "number" && Number.isFinite(v.y);
/** sources.json — { id: NotesSource }. Read-modify-write, NOT append-only:
* the usage log lives inside the record (like a chat's JSON file). */
function readSourceMap(): Record<string, NotesSource> {
let raw: string;
try {
raw = fs.readFileSync(SOURCES_FILE, "utf-8");
} catch {
return {};
}
try {
const parsed: unknown = JSON.parse(raw);
if (!isRecord(parsed)) return {};
const out: Record<string, NotesSource> = {};
for (const [id, value] of Object.entries(parsed)) {
if (isNotesSource(value)) out[id] = value;
}
return out;
} catch {
return {};
}
}
function writeSourceMap(map: Record<string, NotesSource>): void {
fs.mkdirSync(path.dirname(SOURCES_FILE), { recursive: true });
fs.writeFileSync(SOURCES_FILE, JSON.stringify(map, null, "\t"), "utf-8");
}
const isNotesSource = (v: unknown): v is NotesSource =>
isRecord(v) && isString(v.id) && isString(v.filePath) && isString(v.importedAt);
+40
View File
@@ -0,0 +1,40 @@
/**
* Server-side persistence for the graph render settings (the settings-panel
* controls): data/render-settings.json — read-modify-write, NOT a ledger.
*
* The shape, defaults, and sanitizer live in src/core/render-settings.ts
* (pure, shared with the client); this module only moves bytes. Whatever
* comes off disk is sanitized by the caller on load — and we sanitize on
* write too, so the file can never hold an out-of-range value that would
* silently shift a slider.
*/
import fs from "node:fs";
import path from "node:path";
import { RENDER_SETTINGS_FILE } from "../../config.ts";
import { DEFAULT_RENDER_SETTINGS, sanitizeRenderSettings, type GraphRenderSettings } from "../core/render-settings.ts";
/** The persisted settings, or null when no file exists yet (first boot). */
export function readRenderSettings(): GraphRenderSettings | null {
let raw: string;
try {
raw = fs.readFileSync(RENDER_SETTINGS_FILE, "utf-8");
} catch {
return null; // never booted / unreadable → caller uses defaults
}
try {
return sanitizeRenderSettings(JSON.parse(raw));
} catch {
return null; // corrupt → defaults, never crash the view
}
}
/** Persist settings (sanitized). Returns what was written. */
export function writeRenderSettings(raw: unknown): GraphRenderSettings {
const settings = sanitizeRenderSettings(raw);
fs.mkdirSync(path.dirname(RENDER_SETTINGS_FILE), { recursive: true });
fs.writeFileSync(RENDER_SETTINGS_FILE, JSON.stringify(settings, null, "\t"), "utf-8");
return settings;
}
export { DEFAULT_RENDER_SETTINGS };
+85
View File
@@ -0,0 +1,85 @@
/**
* Community detection tests (vision §1, DECIDED 2026-09-13) — pure
* structural Louvain over follows+supports, deterministic via seed.
* Assertions are on SHAPE and MEMBER SETS, never community indices
* (indices reshuffle; member sets are what labelings key to).
*/
import { strict as assert } from "node:assert";
import { describe, test } from "node:test";
import { matchClusterLabelings, MIN_CLUSTER_SIZE, type Community } from "../src/core/clusters.ts";
import { clusterHue } from "../components/cluster-geometry.ts";
import type { ClusterLabeling } from "../src/core/contracts.ts";
const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${prefix}${i}`);
/** A clique on the given ids. */
const clique = (nodeIds: string[]) =>
nodeIds.flatMap((a, i) => nodeIds.slice(i + 1).map((b) => ({ from: a, to: b })));
const labeling = (id: string, core: string[], createdAt = "2026-09-01T00:00:00.000Z"): ClusterLabeling => ({
id,
label: `label-${id}`,
core,
createdAt,
});
describe("matchClusterLabelings", () => {
test("claims a labeling while at least half its core survives in the community", () => {
const community: Community = { members: ["a", "b", "c", "d", "e"] };
const records = [labeling("clu_1", ["a", "b", "x", "y"])]; // 2 of 4 survive — exactly half
const [claim] = matchClusterLabelings([community], records);
assert.equal(claim?.id, "clu_1");
// One fewer survivor: under half → relabel.
const [dead] = matchClusterLabelings([{ members: ["a", "c", "d", "e", "f"] }], records);
assert.equal(dead, null);
});
test("single successor: a split core keeps its label in only one community", () => {
const records = [labeling("clu_1", ["a", "b", "c", "d"])];
const left: Community = { members: ["a", "b", "z"] }; // 2 of 4 — half
const right: Community = { members: ["c", "d", "w"] }; // 2 of 4 — half
// Tie: larger overlap wins; equal → both eligible, greedy takes first come.
const claims = matchClusterLabelings([left, right], records);
const claimed = claims.filter((c) => c !== null);
assert.equal(claimed.length, 1);
assert.equal(claimed[0]?.id, "clu_1");
});
test("greedy: the best-overlapping community wins the record", () => {
const records = [labeling("clu_1", ["a", "b", "c", "d"])];
const weak: Community = { members: ["a", "b", "q", "r", "s"] }; // 2/4
const strong: Community = { members: ["a", "b", "c", "x", "y"] }; // 3/4
const claims = matchClusterLabelings([weak, strong], records);
assert.equal(claims[0], null);
assert.equal(claims[1]?.id, "clu_1");
});
test("one record per community; newest record breaks ties", () => {
const oldR = labeling("clu_old", ["a", "b"], "2026-09-01T00:00:00.000Z");
const newR = labeling("clu_new", ["a", "b"], "2026-09-02T00:00:00.000Z");
const community: Community = { members: ["a", "b", "c"] };
const [claim] = matchClusterLabelings([community], [oldR, newR]);
assert.equal(claim?.id, "clu_new");
});
test("no labelings: every community unclaimed; empty communities array ok", () => {
assert.deepEqual(matchClusterLabelings([{ members: ["a", "b"] }], []), [null]);
assert.deepEqual(matchClusterLabelings([], [labeling("clu_1", ["a"])]), []);
});
test("MIN_CLUSTER_SIZE is 3 (the render/naming threshold)", () => {
assert.equal(MIN_CLUSTER_SIZE, 3);
});
});
describe("clusterHue", () => {
test("is stable per id and spreads", () => {
// Same id → same hue, always; different ids → usually different hues.
assert.equal(clusterHue("cluster-a"), clusterHue("cluster-a"));
const hues = new Set<number>();
for (let i = 0; i < 40; i++) hues.add(clusterHue(`cluster-${i}`));
assert.ok(hues.size > 20, `expected spread, got ${hues.size} distinct hues for 40 ids`);
for (const h of hues) assert.ok(h >= 0 && h < 360);
});
});
+278
View File
@@ -27,15 +27,20 @@ import type { UIMessage } from "ai";
import type { PiEngine } from "../src/agent/models.ts";
import type {
AuditResult,
ClusterLabeling,
ExtractionRunUsage,
GraphEdge,
GraphNode,
LedgerEvent,
NodeMapping,
NotesSource,
SessionCheckpoint,
SessionFindings,
} from "../src/core/contracts.ts";
import type { GraphStore } from "../src/core/ports.ts";
import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts";
import { clusterNamingSettled, refreshClusterLabels } from "../src/agent/subagents/cluster-namer.ts";
import { semanticClusters } from "../src/core/semantic-clusters.ts";
import { chatStore } from "../src/store/chats.ts";
import { PATHS } from "../config.ts";
@@ -60,6 +65,16 @@ function memoryLedgers() {
const events: LedgerEvent[] = [];
const mappings: NodeMapping[] = [];
const checkpoints: SessionCheckpoint[] = [];
const sources = new Map<string, NotesSource>();
const clusterLabels: ClusterLabeling[] = [];
const positions = new Map<string, { x: number; y: number }>();
const embeddings = {
model: "",
dim: 0,
updatedAt: "",
vectors: new Map<string, number[]>(),
clusters: new Array<Array<string>>(),
};
return {
appendNode: (n: GraphNode) => void nodes.push(n),
readNodes: () => nodes.map((n) => ({ ...n })),
@@ -71,6 +86,40 @@ function memoryLedgers() {
readMappings: () => mappings.map((m) => ({ ...m })),
appendCheckpoint: (c: SessionCheckpoint) => void checkpoints.push(c),
readCheckpoints: () => checkpoints.map((c) => ({ ...c })),
upsertSource: (s: NotesSource) => {
// Mirrors the real store: an upsert KEEPS the existing usage log.
const prev = sources.get(s.id);
sources.set(s.id, { ...s, extractions: [...(prev?.extractions ?? [])] });
},
readSources: () => [...sources.values()].map((s) => ({ ...s, extractions: [...(s.extractions ?? [])] })),
appendSourceUsage: (id: string, entry: ExtractionRunUsage) => {
const source = sources.get(id);
if (!source) return;
if (!source.extractions?.some((r) => r.startedAt === entry.startedAt)) {
source.extractions = [...(source.extractions ?? []), entry];
}
},
// Cluster labelings — mirrors the real store: append dedupes by id.
appendClusterLabel: (l: ClusterLabeling) => {
if (!clusterLabels.some((x) => x.id === l.id)) clusterLabels.push(l);
},
readClusterLabels: () => clusterLabels.map((l) => ({ ...l, core: [...l.core] })),
// The layout ledger — settleGraphLayout writes it after each merge.
readPositions: () => Object.fromEntries([...positions].map(([id, p]) => [id, { ...p }])),
writePositions: (next) => {
positions.clear();
for (const [id, p] of Object.entries(next)) positions.set(id, { x: p.x, y: p.y });
},
// The embedding ledger — the embedder writes it after a merge.
readEmbeddings: () => ({ model: embeddings.model, dim: embeddings.dim, vectors: Object.fromEntries([...embeddings.vectors].map(([id, v]) => [id, [...v]])), clusters: [...embeddings.clusters.map((c) => ({ members: [...c] }))], updatedAt: embeddings.updatedAt }),
writeEmbeddings: (next) => {
embeddings.model = next.model;
embeddings.dim = next.dim;
embeddings.updatedAt = next.updatedAt;
embeddings.vectors.clear();
for (const [id, v] of Object.entries(next.vectors)) embeddings.vectors.set(id, [...v]);
embeddings.clusters = next.clusters.map((c) => [...c.members]);
},
} satisfies GraphStore;
}
@@ -288,8 +337,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
embedder: fakeEmbedder,
extractor: submitModel("submit_findings", FINDINGS),
auditor: submitModel("submit_review", verdict),
labeler: submitModel("submit_labels", { labels: [] }),
});
const stats = await pipeline.runExtraction(chatId);
@@ -328,8 +379,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
embedder: fakeEmbedder,
extractor: submitModel("submit_findings", FINDINGS),
auditor: silentModel(), // never submits the review
labeler: submitModel("submit_labels", { labels: [] }),
});
await assert.rejects(pipeline.runExtraction(chatId), /the auditor never submitted a review/);
assert.equal(ledgers.readNodes().length, 0);
@@ -338,8 +391,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
const failingExtractor = createExtractionPipeline({
chatStore,
graphStore: ledgers,
embedder: fakeEmbedder,
extractor: silentModel(),
auditor: submitModel("submit_review", { ...FINDINGS, dropped: [] }),
labeler: submitModel("submit_labels", { labels: [] }),
});
await assert.rejects(failingExtractor.runExtraction(chatId), /the extractor never submitted findings/);
assert.equal(ledgers.readCheckpoints().length, 0);
@@ -368,10 +423,12 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
embedder: fakeEmbedder,
// Each stage's first generate is the observation point: the
// registry must already carry this chat at the right phase.
extractor: submitModel("submit_findings", FINDINGS, () => seen.push(phaseOf())),
auditor: submitModel("submit_review", verdict, () => seen.push(phaseOf())),
labeler: submitModel("submit_labels", { labels: [] }),
});
await pipeline.runExtraction(chatId);
assert.deepEqual(seen, ["extracting", "auditing"]);
@@ -400,12 +457,14 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
embedder: fakeEmbedder,
// During the auditor's generate, the registry must already carry
// the extractor's fully accumulated live usage (2 steps × 10/5).
extractor: submitModel("submit_findings", FINDINGS),
auditor: submitModel("submit_review", verdict, () =>
usageSeen.push(extractionStatuses().find((r) => r.chatId === chatId)?.usage.extractor?.steps.at(-1)?.inputTokens),
),
labeler: submitModel("submit_labels", { labels: [] }),
});
await pipeline.runExtraction(chatId);
@@ -440,8 +499,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
embedder: fakeEmbedder,
extractor: hangingModel(), // hangs until aborted — no step finishes
auditor: submitModel("submit_review", { propositions: [], edges: [], attribution: { quiz: {} }, dropped: [] }),
labeler: submitModel("submit_labels", { labels: [] }),
});
const runPromise = pipeline.runExtraction(chatId);
@@ -490,8 +551,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
embedder: fakeEmbedder,
extractor: gatedExtractor,
auditor: silentModel(),
labeler: submitModel("submit_labels", { labels: [] }),
});
const first = pipeline.runExtraction(chatId);
@@ -507,9 +570,224 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
embedder: fakeEmbedder,
extractor: silentModel(),
auditor: silentModel(),
labeler: submitModel("submit_labels", { labels: [] }),
});
await assert.rejects(pipeline.runExtraction("chat_does_not_exist"), /Chat not found/);
});
});
// ── the notes front door (format-agnostic import) ────────────────────────────
/** Raw personal notes — arbitrary format, no session, no message ids. */
const NOTES_TEXT = `> [!quote] YOU
> teach me vim
> [!abstract] PI
> Vim always starts in Normal mode — keys are commands, not text.`;
/** What the notes-mode extractor submits: same shapes minus attribution,
* and messages always empty (the file is the provenance). */
const NOTES_FINDINGS = {
propositions: [
{
id: "p1",
name: "Vim normal mode",
statement: "Vim starts in Normal mode, where keys are commands.",
aliases: [],
messages: [],
evidence: ["Vim always starts in Normal mode — keys are commands, not text."],
understanding: "solid",
},
],
edges: [],
} as const;
const NOTES_VERDICT = {
propositions: [NOTES_FINDINGS.propositions[0]],
edges: [],
dropped: [],
} as const;
const CLUSTER_FINDINGS = {
propositions: [
{ id: "p1", name: "Modal editing", statement: "Vim edits text through modal command modes.", aliases: [], messages: [], evidence: ["Normal mode keys are commands."], understanding: "solid" },
{ id: "p2", name: "Verb-object grammar", statement: "Vim commands compose as verb and object.", aliases: [], messages: [], evidence: ["d then motion deletes."], understanding: "solid" },
{ id: "p3", name: "Operator-pending", statement: "Vim enters operator-pending mode between verb and object.", aliases: [], messages: [], evidence: ["After d, a motion is expected."], understanding: "solid" },
],
edges: [
{ from: "p1", to: "p2", type: "follows", messages: [], evidence: ["e"] },
{ from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] },
],
} as const;
const CLUSTER_VERDICT = {
propositions: CLUSTER_FINDINGS.propositions,
edges: CLUSTER_FINDINGS.edges,
dropped: [],
} as const;
/** Test seam for the embedder: fakes the post-merge embedding batch —
* one identical unit vector per node → semanticClusters merges them all
* into ONE topic — then persists the partition like the real embedder. */
const fakeEmbedder = async (store: GraphStore) => {
const seen = new Set<string>();
const ids: string[] = [];
for (const n of store.readNodes()) {
if (seen.has(n.id)) continue;
seen.add(n.id);
ids.push(n.id);
}
const vectors = Object.fromEntries(ids.map((id) => [id, [1, 0, 0]]));
store.writeEmbeddings({
model: "test/embedder",
dim: 3,
vectors,
clusters: semanticClusters(vectors),
updatedAt: "2026-09-13T00:00:00.000Z",
});
};
describe("cluster naming (the post-extraction garnish)", () => {
it("names a new community after a notes merge — one record, claimed on the next refresh", async () => {
const ledgers = memoryLedgers();
const namerCalls: number[] = [];
const labeler = submitModel("submit_labels", { labels: [{ key: "c0", label: "Vim modal grammar" }] }, () => namerCalls.push(1));
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
embedder: fakeEmbedder,
extractor: submitModel("submit_findings", CLUSTER_FINDINGS),
auditor: submitModel("submit_review", CLUSTER_VERDICT),
labeler,
});
const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes, any format" });
assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 });
// The refresh is fire-and-forget inside the pipeline — settle it.
await clusterNamingSettled();
assert.equal(namerCalls.length, 1, "ONE singular naming call");
const records = ledgers.readClusterLabels();
assert.equal(records.length, 1);
assert.equal(records[0]!.label, "Vim modal grammar");
assert.equal(records[0]!.core.length, 3);
assert.match(records[0]!.id, /^clu_/);
// Second refresh: core-survival claims it — nothing new to name.
const again = await refreshClusterLabels({ graphStore: ledgers, engine: labeler });
assert.equal(again.named, 0);
assert.equal(again.appended, 0);
assert.ok(again.claimed >= 1);
assert.equal(ledgers.readClusterLabels().length, 1, "no duplicate records");
});
it("naming failure never fails the run", async () => {
const ledgers = memoryLedgers();
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
embedder: fakeEmbedder,
extractor: submitModel("submit_findings", CLUSTER_FINDINGS),
auditor: submitModel("submit_review", CLUSTER_VERDICT),
labeler: silentModel(), // never submits — the namer throws
});
const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes" });
assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 }, "the merge itself succeeded");
await clusterNamingSettled();
assert.equal(ledgers.readClusterLabels().length, 0, "nothing named, nothing appended");
});
});
describe("the notes front door (importNotes — format-agnostic)", () => {
it("imports raw notes end to end: source record, file-scoped provenance, no quiz events", async () => {
const ledgers = memoryLedgers();
const seen: { kind?: string; label?: string }[] = [];
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
embedder: fakeEmbedder,
extractor: submitModel("submit_findings", NOTES_FINDINGS),
// During the auditor's call the registry must show the notes run.
auditor: submitModel("submit_review", NOTES_VERDICT, () => {
const run = extractionStatuses().find((r) => r.chatId.startsWith("src_"));
seen.push({ kind: run?.kind, label: run?.label });
}),
labeler: submitModel("submit_labels", { labels: [] }),
});
const stats = await pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 });
// The run surfaced as a NOTES run labeled with the file path.
assert.deepEqual(seen, [{ kind: "notes", label: "/vault/learn vim.md" }]);
assert.equal(extractionStatuses().length, 0, "the registry clears when the run ends");
// The source record: deterministic src id, the file path, the usage log.
const source = ledgers.readSources()[0]!;
assert.match(source.id, /^src_/);
assert.equal(source.filePath, "/vault/learn vim.md");
assert.ok(Date.parse(source.importedAt));
assert.equal(source.extractions?.length, 1, "the run's usage persisted into the source record");
assert.equal(source.extractions![0]!.stats?.nodes, 1);
assert.ok(source.extractions![0]!.stages.extractor.steps.length > 0, "faux usage captured");
// File-scoped provenance: the mapping line carries the source id;
// the node carries NO message anchors.
const node = ledgers.readNodes()[0]!;
assert.equal(node.statement, "Vim starts in Normal mode, where keys are commands.");
assert.deepEqual(node.provenance, []);
assert.deepEqual(ledgers.readMappings(), [{ chatId: source.id, sourceId: "p1", graphNodeId: node.id }]);
// One observation event, zero quiz events — outcomes inside notes
// surface only through understanding observations (user-decided).
const events = ledgers.readEvents();
assert.deepEqual(
events.map((e) => [e.kind, e.chatId, e.ref, e.note]),
[["observation", source.id, "obs:p1", "solid"]],
);
// The checkpoint stamps the import.
assert.equal(ledgers.readCheckpoints()[0]!.chatId, source.id);
});
it("re-importing the same file is idempotent: same source identity, refs dedupe, usage history grows", async () => {
const ledgers = memoryLedgers();
const mk = () =>
createExtractionPipeline({
chatStore,
graphStore: ledgers,
embedder: fakeEmbedder,
extractor: submitModel("submit_findings", NOTES_FINDINGS),
auditor: submitModel("submit_review", NOTES_VERDICT),
labeler: submitModel("submit_labels", { labels: [] }),
});
const first = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
assert.deepEqual(first, { nodes: 1, edges: 0, events: 1 });
const second = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
assert.deepEqual(second, { nodes: 0, edges: 0, events: 0 }, "the same source id dedupes everything");
assert.equal(ledgers.readSources().length, 1, "one source identity — the path hashes to the same id");
assert.equal(ledgers.readSources()[0]!.extractions?.length, 2, "both runs billed their usage");
assert.equal(ledgers.readNodes().length, 1, "no duplicate node");
});
it("aborts like a chat run: true cancellation, nothing merged, partial usage in the source record", async () => {
const ledgers = memoryLedgers();
const pipeline = createExtractionPipeline({
chatStore,
graphStore: ledgers,
embedder: fakeEmbedder,
extractor: hangingModel(),
auditor: submitModel("submit_review", NOTES_VERDICT),
labeler: submitModel("submit_labels", { labels: [] }),
});
const runPromise = pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
const srcId = extractionStatuses().find((r) => r.chatId.startsWith("src_"))!.chatId;
assert.equal(abortExtraction(srcId), true);
assert.deepEqual(await runPromise, { nodes: 0, edges: 0, events: 0 });
assert.equal(ledgers.readNodes().length, 0, "nothing merged");
assert.equal(ledgers.readCheckpoints().length, 0, "nothing checkpointed");
assert.equal(ledgers.readSources()[0]!.extractions?.length, 1, "partial usage still persisted");
assert.equal(ledgers.readSources()[0]!.extractions![0]!.stats, undefined, "never merged → no stats");
});
});
+227
View File
@@ -0,0 +1,227 @@
/**
* Pure-helper tests for the hand-rolled canvas renderer
* (components/graph-canvas.ts). The renderer class itself is DOM-bound
* (canvas, rAF, ResizeObserver) and is verified live in the browser; the
* math it runs on every frame must be exact, so it is pinned here.
*
* The camera-ratio convention is the load-bearing contract with the
* persisted settings: sigma's ratio (1 = fit, SMALLER = zoomed in) is kept
* verbatim so stored labelZoom values survive the renderer swap.
*/
import assert from "node:assert/strict";
import { describe, it as test } from "node:test";
import {
arrowGeometry,
boxBlurRound,
fitTransform,
labelAlpha,
pickNearest,
zoomRatioOf,
LABELS_FADE_SPAN,
} from "../components/graph-canvas.ts";
import {
sanitizePhysicsTuning,
sanitizeRegionTuning,
DEFAULT_PHYSICS_TUNING,
DEFAULT_REGION_TUNING,
REGION_BITMAP_OPTIONS,
} from "../src/core/render-settings.ts";
describe("zoomRatioOf (the sigma-compatible camera ratio)", () => {
test("1 at fit zoom, smaller when zoomed in, larger when zoomed out", () => {
assert.equal(zoomRatioOf(2, 2), 1); // at fit
assert.equal(zoomRatioOf(2, 4), 0.5); // zoomed in 2x -> ratio halves
assert.equal(zoomRatioOf(2, 1), 2); // zoomed out 2x -> ratio doubles
});
});
describe("labelAlpha (the single zoom gate)", () => {
test("fully visible at/below the threshold, faded out after the span", () => {
assert.equal(labelAlpha(0.5, 0.5), 1); // at the threshold
assert.equal(labelAlpha(0.25, 0.5), 1); // zoomed further in: stays 1
assert.equal(labelAlpha(0.75, 0.5), 0.5); // mid-fade
assert.equal(labelAlpha(1.0, 0.5), 0); // one span out: gone
assert.equal(labelAlpha(2.0, 0.5), 0); // far out: clamped
});
test("node labels and cluster labels cross-fade over the exact same span", () => {
// The renderer draws cluster labels at 1 - labelAlpha; the two must
// always sum to 1 within the fade and never coexist at full opacity.
for (const ratio of [0, 0.25, 0.5, 0.6, 0.75, 0.9, 1, 2, 4]) {
const node = labelAlpha(ratio, 0.5);
const cluster = 1 - node;
assert.ok(node + cluster > 0.999 && node + cluster < 1.001, `sum at ratio ${ratio}`);
assert.ok(node <= 1 && cluster <= 1, `clamped at ratio ${ratio}`);
}
});
test("the span constant is unchanged from the sigma era (stored settings depend on it)", () => {
assert.equal(LABELS_FADE_SPAN, 0.5);
});
});
describe("fitTransform", () => {
test("centers the bbox and respects padding", () => {
// A 100x50 graph in a 500x300 viewport, 40px padding:
// available 420x220 -> k = min(4.2, 4.4) = 4.2
const t = fitTransform({ minX: -50, maxX: 50, minY: -25, maxY: 25 }, 500, 300, 40);
assert.ok(Math.abs(t.k - 4.2) < 1e-9, `k=${t.k}`);
assert.ok(Math.abs(t.x - 250) < 1e-9, `x=${t.x}`); // centered
assert.ok(Math.abs(t.y - 150) < 1e-9, `y=${t.y}`);
// World center maps to viewport center:
assert.ok(Math.abs(0 * t.k + t.x - 250) < 1e-9);
});
test("never returns a non-positive scale for degenerate bboxes", () => {
const t = fitTransform({ minX: 5, maxX: 5, minY: 5, maxY: 5 }, 100, 100, 10);
assert.ok(t.k > 0 && Number.isFinite(t.k));
});
});
describe("arrowGeometry", () => {
test("the tip lands exactly on the target disc edge", () => {
const g = arrowGeometry({ x: 0, y: 0 }, { x: 100, y: 0 }, 6, 6, 5);
assert.ok(g, "non-degenerate edge");
assert.ok(Math.abs(g!.tip.x - 94) < 1e-9 && Math.abs(g!.tip.y) < 1e-9);
});
test("the shaft starts outside the source disc", () => {
const g = arrowGeometry({ x: 0, y: 0 }, { x: 100, y: 0 }, 6, 6, 5)!;
assert.ok(Math.abs(g.shaftFrom.x - 6) < 1e-9 && Math.abs(g.shaftFrom.y) < 1e-9);
// Shaft ends at the head base; head is drawn from base to tip.
assert.ok(Math.abs(g.shaftTo.x - 89) < 1e-9);
});
test("the head triangle is symmetric around the shaft axis (0.8x wideness)", () => {
const g = arrowGeometry({ x: 0, y: 0 }, { x: 100, y: 0 }, 6, 6, 5)!;
// halfW = 0.4 x headLength = 2; both base corners share x, mirrored y.
assert.ok(Math.abs(Math.abs(g.left.y) - 2) < 1e-9 && Math.abs(Math.abs(g.right.y) - 2) < 1e-9);
assert.ok(Math.abs(g.left.y + g.right.y) < 1e-9); // mirror
assert.ok(Math.abs(g.left.x - g.right.x) < 1e-9);
});
test("works at arbitrary angles (diagonal stays normalized)", () => {
const a = { x: 0, y: 0 };
const b = { x: 60, y: 80 }; // len 100, dir (0.6, 0.8)
const g = arrowGeometry(a, b, 10, 10, 8)!;
assert.ok(Math.abs(g.tip.x - (60 - 0.6 * 10)) < 1e-9 && Math.abs(g.tip.y - (80 - 0.8 * 10)) < 1e-9);
});
test("null for coincident endpoints (never a NaN arrow)", () => {
assert.equal(arrowGeometry({ x: 5, y: 5 }, { x: 5, y: 5 }, 6, 6, 5), null);
});
});
describe("pickNearest (distance-check picking)", () => {
const nodes = [
{ id: "a", x: 10, y: 10 },
{ id: "b", x: 50, y: 50 },
{ id: "c", x: 90, y: 10 },
];
test("picks the nearest node within the pad", () => {
assert.equal(pickNearest(nodes, 12, 12, 16)?.id, "a");
assert.equal(pickNearest(nodes, 48, 52, 16)?.id, "b");
assert.equal(pickNearest(nodes, 88, 8, 16)?.id, "c");
});
test("the picking pad extends the hitbox beyond the disc", () => {
// 20px away, only reachable because the pad covers it.
assert.equal(pickNearest(nodes, 28, 18, 25)?.id, "a");
// Beyond the pad: nothing.
assert.equal(pickNearest(nodes, 30, 40, 3), null);
});
test("ties resolve deterministically (first scanned wins)", () => {
const pair = [
{ id: "first", x: 0, y: 0 },
{ id: "second", x: 10, y: 0 },
];
assert.equal(pickNearest(pair, 5, 0, 10)?.id, "first");
});
test("empty node list picks nothing", () => {
assert.equal(pickNearest([], 0, 0, 100), null);
});
});
describe("sanitizeRegionTuning (dev number inputs)", () => {
test("defaults survive untouched; partial input keeps the rest", () => {
const t = sanitizeRegionTuning(undefined);
assert.deepEqual(t, DEFAULT_REGION_TUNING);
const half = sanitizeRegionTuning({ blur: 7.2 });
assert.equal(half.blur, 8); // snaps into the fixed set
assert.equal(half.dilation, DEFAULT_REGION_TUNING.dilation);
});
test("garbage clamps: NaN falls back, out-of-range clamps", () => {
const t = sanitizeRegionTuning({ dilation: Number.NaN, blur: -5, bitmap: 99999, edgeAlpha: 42 });
assert.equal(t.dilation, DEFAULT_REGION_TUNING.dilation); // NaN -> default
assert.equal(t.blur, 2); // snaps into the fixed set
assert.equal(t.bitmap, REGION_BITMAP_OPTIONS[REGION_BITMAP_OPTIONS.length - 1]); // 99999 snaps to the largest option
assert.equal(t.edgeAlpha, 1);
});
test("empty string / null / undefined values fall back per-field", () => {
const t = sanitizeRegionTuning({ edgeAlpha: Number.NaN });
assert.equal(t.edgeAlpha, DEFAULT_REGION_TUNING.edgeAlpha);
});
});
describe("boxBlurRound (premultiplied region blur)", () => {
test("spreads a single opaque px into its neighborhood and conserves sum", () => {
// 5x5 RGBA, one white opaque pixel at (2,2)
const w = 5, h = 5;
const a = new Float32Array(w * h * 4);
const center = (2 * w + 2) * 4;
a[center] = 255; a[center + 1] = 255; a[center + 2] = 255; a[center + 3] = 255;
const tmp = new Float32Array(a.length);
const b = new Float32Array(a.length);
boxBlurRound(a, tmp, b, w, h, 1);
// Alpha conservation: total alpha unchanged.
let sumA = 0;
for (let i = 3; i < b.length; i += 4) sumA += b[i]!;
assert.ok(Math.abs(sumA - 255) < 1e-3, `alpha sum ${sumA}`);
// The center dimmed, the 8 neighbors picked some up.
assert.ok(b[center + 3]! < 255 && b[center + 3]! > 0);
const up = ((1 * w) + 2) * 4 + 3;
assert.ok(b[up]! > 0, "neighbor got alpha");
// Far corner (4,4) untouched.
assert.equal(b[(4 * 5 + 4) * 4 + 3], 0);
});
test("radius 0 is a plain copy", () => {
const a = new Float32Array([1, 2, 3, 4, 5, 6, 7, 8]);
const tmp = new Float32Array(8);
const b = new Float32Array(8);
boxBlurRound(a, tmp, b, 1, 2, 0);
assert.deepEqual([...b], [1, 2, 3, 4, 5, 6, 7, 8]);
});
});
describe("sanitizePhysicsTuning (physics controls)", () => {
test("defaults survive untouched; partial input keeps the rest", () => {
assert.deepEqual(sanitizePhysicsTuning(undefined), DEFAULT_PHYSICS_TUNING);
const half = sanitizePhysicsTuning({ clusterPull: 0.6 });
assert.equal(half.clusterPull, 0.6);
assert.equal(half.chargeStrength, DEFAULT_PHYSICS_TUNING.chargeStrength);
});
test("garbage clamps: NaN falls back, out-of-range clamps", () => {
const t = sanitizePhysicsTuning({
chargeStrength: Number.NaN,
linkStrength: 99,
linkDistance: -5,
gravity: 7,
clusterPull: -1,
velocityDecay: 3,
});
assert.equal(t.chargeStrength, DEFAULT_PHYSICS_TUNING.chargeStrength);
assert.equal(t.linkStrength, 4); // sweep-validated cap (8 explodes, 5 wobbles)
assert.equal(t.linkDistance, 10);
assert.equal(t.gravity, 0.48);
assert.equal(t.clusterPull, 0);
assert.equal(t.velocityDecay, 0.9);
});
});
+153 -11
View File
@@ -20,17 +20,27 @@ import { after, describe, it } from "node:test";
import type { UIMessage } from "ai";
import type { AuditResult } from "../src/core/contracts.ts";
import type { QuestionAnswerData } from "../src/core/question-core.ts";
import { getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts";
import { clusterContentId, getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts";
import { graphStore } from "../src/store/graph.ts";
import { chatStore } from "../src/store/chats.ts";
import { LEDGER_FILES, PATHS } from "../config.ts";
import { LEDGER_FILES, PATHS, SOURCES_FILE, EMBEDDINGS_FILE } from "../config.ts";
/** The wired fs adapters, exactly what src/server/stores.ts passes in production. */
const graphDeps = { chatStore, graphStore } as const satisfies GraphDeps;
/** The session merge source for a chat — what the pipeline passes: the live
* transcript + its updatedAt checkpoint stamp. (Notes imports pass
* messages: null — see the notes-merger test below.) */
function sessionSource(chatId: string): { messages: UIMessage[]; foldedAt: string } {
const record = chatStore.getChat(chatId)!;
return { messages: record.messages as UIMessage[], foldedAt: record.updatedAt };
}
// ─── ledger isolation (backup-and-clear of data/graph/*.jsonl) ──────────────
const LEDGER_PATHS = Object.values(LEDGER_FILES);
// EMBEDDINGS_FILE too: the semantic-cluster source (read by graphSnapshot)
// — isolated like the rest so this suite never touches the real vectors.
const LEDGER_PATHS = [...Object.values(LEDGER_FILES), SOURCES_FILE, EMBEDDINGS_FILE];
const backups: Array<{ file: string; bak: string }> = [];
const createdChats: string[] = [];
@@ -233,7 +243,7 @@ function scopedGraph(chatIds: string[]) {
describe("the merger (mergeFindings)", () => {
it("creates nodes from the verdicts, rebuilds quiz events deterministically, records observations", () => {
const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps);
const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id));
assert.deepEqual(stats, { nodes: 3, edges: 1, events: 6 });
const graph = scopedGraph([chat1.id]);
@@ -302,7 +312,7 @@ describe("the merger (mergeFindings)", () => {
});
it("is idempotent — re-merging the same verdicts appends nothing", () => {
const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps);
const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id));
assert.deepEqual(stats, { nodes: 0, edges: 0, events: 0 });
const graph = scopedGraph([chat1.id]);
assert.equal(graph.nodes.length, 3);
@@ -346,6 +356,7 @@ describe("the merger (mergeFindings)", () => {
dropped: [],
},
graphDeps,
sessionSource(chat2.id),
);
assert.deepEqual(stats, { nodes: 0, edges: 0, events: 1 });
@@ -363,6 +374,100 @@ describe("the merger (mergeFindings)", () => {
assert.deepEqual(m, { chatId: chat2.id, sourceId: "q1", graphNodeId: alphaId });
});
it("resolves imported-notes ids to their file paths in the snapshot (static provenance)", () => {
// The notes front door in miniature: a source record + a merge under
// its src id — the snapshot must label it with the file path (a
// static entry), never as a chat session.
graphStore.upsertSource({ id: "src_snapfixture", filePath: "/vault/learn vim.md", importedAt: "2026-09-13T00:00:00.000Z" });
mergeFindings(
"src_snapfixture",
{
propositions: [
{
id: "p1",
name: "Vim registers",
statement: "Vim registers map single keys to editing commands.",
aliases: [],
messages: [],
evidence: ["fixture evidence"],
understanding: "solid",
},
],
edges: [],
attribution: { quiz: {} },
dropped: [],
},
graphDeps,
{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
);
const snap = graphSnapshot(graphDeps);
const node = snap.nodes.find((n) => n.statement === "Vim registers map single keys to editing commands.")!;
assert.equal(snap.fileSources["src_snapfixture"], "/vault/learn vim.md");
assert.deepEqual(snap.nodeSessions[node.id], ["src_snapfixture"], "the file entry joins the node's sessions list");
assert.equal(snap.sessions["src_snapfixture"], undefined, "a notes source is never a chat session");
// The understanding observation joins under the source id like any session's.
assert.equal(snap.nodeObservations[node.id]!["src_snapfixture"], "solid");
});
it("derives semantic clusters in graphSnapshot, joined with persisted labelings", () => {
// Three propositions; the cluster comes from the EMBEDDING ledger
// (semantic topics, DECIDED 2026-09-13) — the embedder persists the
// partition, the snapshot just reads it. Here we play the embedder.
const tri = ["Alpha.", "Beta.", "Gamma cluster."] as const;
const merge = mergeFindings(
"src_clusterfixture",
{
propositions: tri.map((statement, i) => ({
id: `p${i + 1}`,
name: `Cluster node ${i + 1}`,
statement,
aliases: [],
messages: [],
evidence: ["fixture evidence"],
})),
edges: [
{ from: "p1", to: "p2", type: "follows", messages: [], evidence: ["e"] },
{ from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] },
{ from: "p1", to: "p3", type: "follows", messages: [], evidence: ["e"] },
],
attribution: { quiz: {} },
dropped: [],
},
graphDeps,
{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
);
assert.equal(merge.nodes, 3);
// The embedder's contract: recluster at the end of the batch and
// PERSIST the partition in the embeddings ledger.
const ids = tri.map((s) => getGraph(graphStore).nodes.find((n) => n.statement === s)!.id);
graphStore.writeEmbeddings({
model: "test/embedder",
dim: 2,
vectors: {},
clusters: [{ members: ids }],
updatedAt: "2026-09-13T00:00:00.000Z",
});
const snap = graphSnapshot(graphDeps);
// Presentation threshold: every cluster is >= 3 members.
assert.ok(snap.clusters.every((c) => c.members.length >= 3));
// Our trio is exactly one cluster, unlabeled until the namer runs.
const mine = snap.clusters.find((c) => c.members.length === 3 && tri.every((s) => c.members.includes(snap.nodes.find((n) => n.statement === s)!.id)));
assert.ok(mine, "the 3-clique is a cluster");
assert.equal(mine!.label, null);
assert.match(mine!.id, /^clu_/);
// Mint the labeling the namer would append — the next read joins it.
graphStore.appendClusterLabel({
id: clusterContentId(mine!.members),
label: "Test region",
core: mine!.members,
createdAt: "2026-09-13T00:00:00.000Z",
});
const snap2 = graphSnapshot(graphDeps);
const claimed = snap2.clusters.find((c) => c.members.length === 3 && tri.every((s) => c.members.includes(snap2.nodes.find((n) => n.statement === s)!.id)))!;
assert.equal(claimed.label, "Test region");
assert.equal(claimed.id, clusterContentId(mine!.members), "the claimed cluster carries the record's id");
});
it("derives sessions and per-session understanding observations in graphSnapshot", () => {
const snap = graphSnapshot(graphDeps);
const idByStatement = new Map(snap.nodes.map((n) => [n.statement, n.id]));
@@ -408,17 +513,54 @@ describe("the merger (mergeFindings)", () => {
dropped: [],
},
graphDeps,
sessionSource(chat3.id),
);
assert.equal(stats.nodes, 1, "created — the merger does not match strings");
assert.equal(getGraph(graphStore).nodes.filter((n) => n.statement === "Alpha intuition").length, 2);
});
it("returns zeros for unknown chats", () => {
assert.deepEqual(mergeFindings("chat_does_not_exist", fixtureAudit(), graphDeps), {
nodes: 0,
edges: 0,
events: 0,
it("merges the notes front door: source id as chatId, no quiz events, checkpoint at import time", () => {
// Raw personal notes carry no transcript — the merger gets no
// message parts, so NO quiz events are rebuilt; the understanding
// observation still lands, keyed to the source id.
const stats = mergeFindings(
"src_notesfixture",
{
propositions: [
{
id: "p1",
name: "Vim normal mode",
statement: "Vim starts in Normal mode, where keys are commands.",
aliases: [],
messages: [], // notes carry no message anchors — the file is the provenance
evidence: ["fixture evidence"],
understanding: "solid",
},
],
edges: [],
attribution: { quiz: {} }, // normalized by the notes pipeline — nothing to attribute
dropped: [],
},
graphDeps,
{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
);
assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 });
const node = getGraph(graphStore).nodes.find((n) => n.statement.startsWith("Vim starts"))!;
assert.deepEqual(node.provenance, [], "no message anchors — the mapping line carries the source");
assert.deepEqual(graphStore.readMappings().find((m) => m.chatId === "src_notesfixture"), {
chatId: "src_notesfixture",
sourceId: "p1",
graphNodeId: node.id,
});
// The observation event is file-scoped; no quiz events exist.
const events = graphStore.readEvents().filter((e) => e.chatId === "src_notesfixture");
assert.deepEqual(
events.map((e) => [e.kind, e.ref, e.note]),
[["observation", "obs:p1", "solid"]],
);
// The checkpoint stamps the import time.
assert.equal(graphStore.readCheckpoints().find((c) => c.chatId === "src_notesfixture")?.foldedAt, "2026-09-13T00:00:00.000Z");
});
});
@@ -452,7 +594,7 @@ describe("the trigger helper (pendingQuizAnswers)", () => {
assert.equal(pendingQuizAnswers(chat.id, graphDeps), 1, "one answered quiz pending");
assert.equal(pendingQuizAnswers("chat_does_not_exist", graphDeps), 0);
mergeFindings(chat.id, fixtureAudit(), graphDeps);
mergeFindings(chat.id, fixtureAudit(), graphDeps, sessionSource(chat.id));
// call-s is unattributed here (fixtureAudit has no call-s attribution)
// — but it still becomes an event, so nothing stays pending.
assert.equal(pendingQuizAnswers(chat.id, graphDeps), 0, "extracted → nothing pending");
+315
View File
@@ -0,0 +1,315 @@
/**
* Option A — incremental anchored force layout (core/layout.ts). The
* properties the user asked for: the layout EVOLVES (old placements stay
* recognizable), new nodes land near their cluster, orphans collect on
* the rim, and everything is deterministic.
*/
import assert from "node:assert/strict";
import { describe, it } from "node:test";
import { createGraphSimulation, incrementalLayout, inferenceDepths } from "../src/core/layout.ts";
import { settleGraphLayout } from "../src/core/layout-settle.ts";
import type { GraphNode, GraphEdge, Position } from "../src/core/contracts.ts";
const node = (id: string): GraphNode => ({
id,
name: id,
statement: id,
aliases: [],
createdAt: "2026-09-13T00:00:00.000Z",
provenance: [],
});
const edge = (from: string, to: string): GraphEdge => ({ from, to, type: "supports", provenance: [] });
const mean = (positions: Map<string, Position>, ids: string[]): { x: number; y: number } => ({
x: ids.reduce((s, id) => s + positions.get(id)!.x, 0) / ids.length,
y: ids.reduce((s, id) => s + positions.get(id)!.y, 0) / ids.length,
});
describe("incrementalLayout", () => {
it("bootstraps deterministically from an empty ledger", () => {
const nodes = ["a", "b", "c", "d"].map(node);
const edges = [edge("a", "b"), edge("b", "c"), edge("c", "d")];
const first = incrementalLayout(nodes, edges, new Map());
const second = incrementalLayout(nodes, edges, new Map());
assert.equal(first.size, 4);
for (const [id, p] of first) {
assert.ok(Number.isFinite(p.x) && Number.isFinite(p.y), `finite coords for ${id}`);
assert.deepEqual(p, second.get(id), `deterministic position for ${id}`);
}
});
it("rest state is stable: a re-settle with nothing new barely moves nodes", () => {
const nodes = ["a", "b", "c", "d", "e"].map(node);
const edges = [edge("a", "b"), edge("b", "c"), edge("c", "d"), edge("d", "e"), edge("a", "e")];
const rest = incrementalLayout(nodes, edges, new Map());
const span =
Math.max(...[...rest.values()].map((p) => p.x)) - Math.min(...[...rest.values()].map((p) => p.x));
const again = incrementalLayout(nodes, edges, rest);
let maxShift = 0;
for (const [id, p] of rest) {
const q = again.get(id)!;
maxShift = Math.max(maxShift, Math.hypot(p.x - q.x, p.y - q.y));
}
assert.ok(maxShift < span * 0.1, `re-settle drift ${maxShift.toFixed(2)} should be < 10% of span ${span.toFixed(2)}`);
});
it("evolves: an anchored node stays near its anchor while a new neighbor joins it", () => {
const core = ["a", "b", "c"].map(node);
const coreEdges = [edge("a", "b"), edge("b", "c")];
const previous = incrementalLayout(core, coreEdges, new Map());
const anchor = previous.get("b")!;
const dist = Math.hypot;
const nearby = ["d", "e"].map(node);
const next = incrementalLayout(
[...core, ...nearby],
[...coreEdges, edge("b", "d"), edge("d", "e")],
previous,
);
// The anchor keeps its neighborhood (bounded drift), and the new
// nodes landed near it rather than being flung elsewhere.
const coreSpanBefore =
Math.max(...[...previous.values()].map((p) => p.x)) - Math.min(...[...previous.values()].map((p) => p.x));
assert.ok(
dist(next.get("b")!.x - anchor.x, next.get("b")!.y - anchor.y) < coreSpanBefore,
"anchored node b stays near its previous position",
);
const centroid = mean(next, ["b", "d", "e"]);
assert.ok(
dist(next.get("d")!.x - centroid.x, next.get("d")!.y - centroid.y) < coreSpanBefore * 2,
"new neighbor d landed near the cluster it arrived with",
);
});
it("rim: an orphan ends up outside the anchored core", () => {
const core = ["a", "b", "c", "d", "e"].map(node);
const coreEdges = [
edge("a", "b"),
edge("b", "c"),
edge("c", "d"),
edge("d", "e"),
edge("a", "e"),
edge("b", "d"),
];
const previous = incrementalLayout(core, coreEdges, new Map());
const next = incrementalLayout([...core, node("orphan")], coreEdges, previous);
const c = mean(previous, ["a", "b", "c", "d", "e"]);
const coreMax = Math.max(
...[...next.entries()].filter(([id]) => id !== "orphan").map(([, p]) => Math.hypot(p.x - c.x, p.y - c.y)),
);
const orphanR = Math.hypot(next.get("orphan")!.x - c.x, next.get("orphan")!.y - c.y);
assert.ok(orphanR > coreMax, `orphan at r=${orphanR.toFixed(1)} should sit outside core max r=${coreMax.toFixed(1)}`);
});
});
describe("settleGraphLayout", () => {
it("dedupes repeated node lines, runs, and writes the whole ledger", () => {
const written: Record<string, Position>[] = [];
const lines = [node("a"), node("b"), node("a")]; // "a" appended twice
const store = {
readNodes: () => lines.map((l) => ({ ...l })),
readEdges: () => [edge("a", "b")],
readPositions: () => ({} as Record<string, Position>),
writePositions: (p: Record<string, Position>) => void written.push(p),
readEmbeddings: () => ({ model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }),
};
settleGraphLayout(store);
assert.equal(written.length, 1);
assert.deepEqual(Object.keys(written[0]!).sort(), ["a", "b"], "one position per UNIQUE node");
});
it("does nothing on an empty graph", () => {
const written: Record<string, Position>[] = [];
settleGraphLayout({
readNodes: () => [],
readEdges: () => [],
readPositions: () => ({} as Record<string, Position>),
writePositions: (p) => void written.push(p),
readEmbeddings: () => ({ model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }),
});
assert.equal(written.length, 0);
});
});
describe("clusterPull (the centroid force)", () => {
it("compacts each cluster toward its OWN centroid; strength 0 = off", () => {
// Two loose chains, far apart, edges only WITHIN each chain.
const mk = (p: string, baseX: number) =>
Array.from({ length: 8 }, (_, i) => ({ id: `${p}${i}`, x: baseX + i * 40, y: i * 15 }));
const a = mk("a", -400);
const b = mk("b", 400);
const nodes = [...a, ...b];
const edges = [
...a.slice(0, -1).map((n, i) => ({ from: n.id, to: a[i + 1]!.id })),
...b.slice(0, -1).map((n, i) => ({ from: n.id, to: b[i + 1]!.id })),
];
const clusters = [{ members: a.map((n) => n.id) }, { members: b.map((n) => n.id) }];
const spreadOf = (ns: { id: string; x: number; y: number }[]): number => {
let sum = 0;
for (const group of [ns.filter((n) => n.id[0] === "a"), ns.filter((n) => n.id[0] === "b")]) {
const cx = group.reduce((s, n) => s + n.x, 0) / group.length;
const cy = group.reduce((s, n) => s + n.y, 0) / group.length;
sum += group.reduce((s, n) => s + Math.hypot(n.x - cx, n.y - cy), 0) / group.length;
}
return sum;
};
const run = (strength: number) => {
const sim = createGraphSimulation(
nodes.map((n) => ({ ...n })),
edges,
{ clusterPull: { clusters, strength } },
);
for (let i = 0; i < 150; i++) sim.tick();
return spreadOf(sim.nodes() as unknown as { id: string; x: number; y: number }[]);
};
const pulled = run(0.15);
const unpulled = run(0);
// The force compacts (well under the unpulled baseline)...
assert.ok(pulled < unpulled * 0.8, `pulled spread ${pulled.toFixed(1)} should be well under ${unpulled.toFixed(1)}`);
// ...and both groups survive as separate blobs (no merge).
assert.ok(pulled > 0, "spread is finite/positive");
});
});
describe("inferenceDepths (SCC-safe ranking)", () => {
const nodes = (ids: string[]) => ids.map((id) => ({ id }));
it("ranks a chain, a diamond takes the longest path, orphans are depth 0", () => {
// chain a->b->c; diamond a->d, b->d, c->d (d reachable at 1 and 2 and 3)
const ns = nodes(["a", "b", "c", "d", "loner"]);
const edges = [
{ from: "a", to: "b" },
{ from: "b", to: "c" },
{ from: "a", to: "d" },
{ from: "b", to: "d" },
{ from: "c", to: "d" },
];
const d = inferenceDepths(ns, edges);
assert.equal(d.get("a"), 0);
assert.equal(d.get("b"), 1);
assert.equal(d.get("c"), 2);
assert.equal(d.get("d"), 3); // longest path, not shortest
assert.equal(d.get("loner"), 0);
});
it("cycles share a depth (mutual support = equal footing)", () => {
// x <-> y (a cycle), fed by s; z follows x.
const ns = nodes(["s", "x", "y", "z"]);
const edges = [
{ from: "s", to: "x" },
{ from: "x", to: "y" },
{ from: "y", to: "x" },
{ from: "x", to: "z" },
];
const d = inferenceDepths(ns, edges);
assert.equal(d.get("s"), 0);
assert.equal(d.get("x"), d.get("y"), "cycle members share a depth");
assert.equal(d.get("x"), 1);
assert.equal(d.get("z"), 2);
// Deterministic across calls.
assert.deepEqual([...inferenceDepths(ns, edges).entries()], [...d.entries()]);
});
});
describe("depthAnchor force", () => {
const nodes = (ids: string[]) => ids.map((id) => ({ id, x: 0, y: 0 }));
it("vertical: the premise lands below its conclusion", () => {
const ns = nodes(["base", "derived"]);
const edges = [{ from: "base", to: "derived" }];
const sim = createGraphSimulation(ns, edges, {
depthAnchor: { mode: "vertical", strength: 1.2, spacing: 100, center: { x: 0, y: 0 } },
});
for (let i = 0; i < 300; i++) sim.tick();
const byId = new Map(sim.nodes().map((n) => [n.id, n]));
const baseY = byId.get("base")!.y!;
const derivedY = byId.get("derived")!.y!;
assert.ok(baseY > derivedY, `base (${baseY.toFixed(0)}) should sit BELOW derived (${derivedY.toFixed(0)})`);
assert.ok(
baseY - derivedY > 60,
`level separation should be substantial (got ${(baseY - derivedY).toFixed(0)})`,
);
});
it("radial: inference depth becomes radius", () => {
const ns = nodes(["base", "mid", "top"]);
const edges = [
{ from: "base", to: "mid" },
{ from: "mid", to: "top" },
];
const sim = createGraphSimulation(ns, edges, {
depthAnchor: { mode: "radial", strength: 1.2, spacing: 80, baseRadius: 100, center: { x: 0, y: 0 } },
});
for (let i = 0; i < 300; i++) sim.tick();
const r = (id: string) => {
const n = sim.nodes().find((x) => x.id === id)!;
return Math.hypot(n.x!, n.y!);
};
assert.ok(r("base") < r("mid"), `base ring (${r("base").toFixed(0)}) inside mid (${r("mid").toFixed(0)})`);
assert.ok(r("mid") < r("top"), `mid ring (${r("mid").toFixed(0)}) inside top (${r("top").toFixed(0)})`);
});
it("settle with depthAnchor: old nodes keep x memory but snap to their level", () => {
// Node "old" was previously placed at y=300 (off-level). With
// vertical anchoring a re-settle must move it to the baseline
// while preserving (roughly) its x arrangement.
const previous = new Map([["old", { x: 250, y: 300 }], ["peer", { x: 260, y: 290 }]]);
const next = incrementalLayout(
[{ id: "old" }, { id: "peer" }, { id: "child" }],
[
{ from: "old", to: "child" },
{ from: "peer", to: "child" },
],
previous,
{ depthAnchor: { mode: "vertical", strength: 0.6, spacing: 80 } },
);
const old = next.get("old")!;
// The baseline defaults to the anchor centroid: (300+290)/2 = 295.
assert.ok(Math.abs(old.y - 295) < 20, `old node snaps to the baseline (y=${old.y.toFixed(1)})`);
assert.ok(old.x > 150, `x memory preserved (x=${old.x.toFixed(1)})`);
const child = next.get("child")!;
assert.ok(child.y < old.y - 40, `child rises above its premises (child=${child.y.toFixed(1)}, old=${old.y.toFixed(1)})`);
});
it("orphans are exempt from depth anchoring — regular forces govern them (ORPHAN RULE)", () => {
// A premise→conclusion chain plus an unconnected loner. Gravity's
// center is FAR from the baseline, so the two are distinguishable:
// a depth-stamped orphan would sit on the baseline; an exempt one
// follows gravity instead.
const ns = [
{ id: "base", x: 0, y: 0 },
{ id: "derived", x: 10, y: 0 },
{ id: "loner", x: 0, y: 0 },
];
const sim = createGraphSimulation(ns, [{ from: "base", to: "derived" }], {
gravity: { x: 0, y: 500, strength: 0.1 },
depthAnchor: { mode: "vertical", strength: 1.5, spacing: 120, center: { x: 0, y: 0 } },
});
for (let i = 0; i < 400; i++) sim.tick();
const byId = new Map(sim.nodes().map((n) => [n.id, n]));
assert.ok(Math.abs(byId.get("base")!.y!) < 40, "structured base sits on the baseline");
assert.ok(byId.get("derived")!.y! < -60, "derived rises a level");
const lonerY = byId.get("loner")!.y!;
assert.ok(lonerY > 250, `the orphan follows gravity, not the baseline (y=${lonerY.toFixed(0)})`);
assert.ok(Math.abs(lonerY - 500) < 120, `the orphan converges on gravity's center (y=${lonerY.toFixed(0)})`);
});
it("radial: orphans are exempt — they gather at the center, not on the base ring", () => {
const ns = [
{ id: "base", x: 0, y: 0 },
{ id: "mid", x: 10, y: 0 },
{ id: "loner", x: 300, y: 300 },
];
const sim = createGraphSimulation(ns, [{ from: "base", to: "mid" }], {
gravity: { x: 0, y: 0, strength: 0.1 },
depthAnchor: { mode: "radial", strength: 1.5, spacing: 80, baseRadius: 150, center: { x: 0, y: 0 } },
});
for (let i = 0; i < 400; i++) sim.tick();
const r = (id: string) => {
const n = sim.nodes().find((x) => x.id === id)!;
return Math.hypot(n.x!, n.y!);
};
assert.ok(Math.abs(r("base") - 150) < 40, `base sits on the base ring (r=${r("base").toFixed(0)})`);
assert.ok(r("loner") < 60, `the orphan follows gravity to the center, not the ring (r=${r("loner").toFixed(0)})`);
});
});
+217
View File
@@ -0,0 +1,217 @@
/**
* Tests for src/core/render-settings.ts — the shared settings contract
* (shape, defaults, sanitizer, label-zoom slider mapping). Pure module,
* pure tests: whatever JSON arrives from disk or the network goes through
* sanitizeRenderSettings and must come out valid, never throw.
*/
import assert from "node:assert/strict";
import { describe, it as test } from "node:test";
import {
ARROW_LENGTH_RANGE,
CHARGE_RANGE,
CLUSTER_PULL_RANGE,
DEFAULT_RENDER_SETTINGS,
DEPTH_BASE_RADIUS_RANGE,
DEPTH_SPACING_RANGE,
DEPTH_STRENGTH_RANGE,
EDGE_WIDTH_RANGE,
GRAVITY_RANGE,
LABEL_SIZE_RANGE,
LABEL_ZOOM_MAX,
LABEL_ZOOM_MIN,
LINK_DISTANCE_RANGE,
LINK_STRENGTH_RANGE,
NODE_SIZE_RANGE,
REGION_BLUR_OPTIONS,
sanitizeRegionTuning,
REGION_DILATION_RANGE,
REGION_EDGE_ALPHA_RANGE,
VELOCITY_DECAY_RANGE,
DEFAULT_PHYSICS_TUNING,
DEFAULT_REGION_TUNING,
fromOffSliderPosition,
fromSliderPosition,
labelZoomDisplay,
sanitizeRenderSettings,
toOffSliderPosition,
toSliderPosition,
} from "../src/core/render-settings.ts";
describe("sanitizeRenderSettings", () => {
test("garbage in → full defaults out", () => {
assert.deepEqual(sanitizeRenderSettings(null), DEFAULT_RENDER_SETTINGS);
assert.deepEqual(sanitizeRenderSettings(undefined), DEFAULT_RENDER_SETTINGS);
assert.deepEqual(sanitizeRenderSettings("junk"), DEFAULT_RENDER_SETTINGS);
assert.deepEqual(sanitizeRenderSettings(42), DEFAULT_RENDER_SETTINGS);
});
test("partial input → defaults fill the gaps", () => {
const s = sanitizeRenderSettings({ nodeSize: 6 });
assert.equal(s.nodeSize, 6);
assert.equal(s.edgeWidth, DEFAULT_RENDER_SETTINGS.edgeWidth);
assert.equal(s.labelSize, DEFAULT_RENDER_SETTINGS.labelSize);
assert.equal(s.labelZoom, DEFAULT_RENDER_SETTINGS.labelZoom);
assert.equal(s.clusterLabels, DEFAULT_RENDER_SETTINGS.clusterLabels);
});
test("out-of-range values clamp; wrong types reset", () => {
const s = sanitizeRenderSettings({
nodeSize: 999,
edgeWidth: 0,
arrowLength: -5,
labelZoom: 100,
labelSize: "big" as unknown as number,
clusterLabels: "yes" as unknown as boolean,
});
assert.equal(s.nodeSize, NODE_SIZE_RANGE.max); // clamped to max
assert.equal(s.edgeWidth, EDGE_WIDTH_RANGE.min); // clamped to min
assert.equal(s.arrowLength, ARROW_LENGTH_RANGE.min);
assert.equal(s.labelZoom, LABEL_ZOOM_MAX);
assert.equal(s.labelSize, DEFAULT_RENDER_SETTINGS.labelSize); // NaN-safe reset
assert.equal(s.clusterLabels, DEFAULT_RENDER_SETTINGS.clusterLabels);
});
test("the low ends accept small sizes (log-slider floors)", () => {
const s = sanitizeRenderSettings({ nodeSize: 0.5, edgeWidth: 0.25, arrowLength: 0.5, labelSize: 0.5 });
assert.deepEqual(
{ nodeSize: s.nodeSize, edgeWidth: s.edgeWidth, arrowLength: s.arrowLength, labelSize: s.labelSize },
{ nodeSize: 0.5, edgeWidth: 0.25, arrowLength: 0.5, labelSize: 0.5 },
);
});
test("a fully valid object passes through unchanged", () => {
const valid = {
nodeSize: 4,
edgeWidth: 2,
arrowLength: 3,
labelZoom: 0.25,
labelSize: 4,
clusterLabels: false,
regions: DEFAULT_REGION_TUNING,
physics: DEFAULT_PHYSICS_TUNING,
};
assert.deepEqual(sanitizeRenderSettings(valid), valid);
});
test("the pre-2026-09-13 stored shape (flat, no regions/physics) still loads", () => {
// data/render-settings.json on disk predates the nested sections —
// those loads must get defaults, not crash, not shift anything.
const s = sanitizeRenderSettings({ nodeSize: 4, edgeWidth: 1, arrowLength: 5, labelZoom: 0.5, clusterLabels: true });
assert.deepEqual(s.regions, DEFAULT_REGION_TUNING);
assert.deepEqual(s.physics, DEFAULT_PHYSICS_TUNING);
assert.equal(s.nodeSize, 4);
});
test("nested sections sanitize: garbage clamps inside them without touching the rest", () => {
const s = sanitizeRenderSettings({
nodeSize: 3,
regions: { blur: -5, bitmap: 999, edgeAlpha: 9 },
physics: { linkStrength: 99, gravity: 7, depthMode: "sideways" },
});
assert.equal(s.nodeSize, 3);
assert.equal(s.regions.blur, 2); // snaps into the fixed set {2,4,6,8,10}
assert.equal(s.regions.bitmap, 512); // snaps to the nearest fixed option
assert.equal(s.regions.edgeAlpha, 1);
assert.equal(s.regions.dilation, DEFAULT_REGION_TUNING.dilation);
assert.equal(s.physics.linkStrength, 4); // sweep-validated cap
assert.equal(s.physics.gravity, 0.48);
assert.equal(s.physics.depthMode, "off"); // unknown mode → off
assert.equal(s.physics.chargeStrength, DEFAULT_PHYSICS_TUNING.chargeStrength);
});
test("the pre-wave stored shape (no labelSize) still loads", () => {
// localStorage-era records lacked labelSize — must not shift anything.
const s = sanitizeRenderSettings({ nodeSize: 4, edgeWidth: 1, arrowLength: 5, labelZoom: 0.5, clusterLabels: true });
assert.equal(s.labelSize, DEFAULT_RENDER_SETTINGS.labelSize);
assert.equal(s.nodeSize, 4);
});
});
describe("slider mapping (log scale, default-centered)", () => {
const RANGES: [string, number, number, number][] = [
["nodeSize", NODE_SIZE_RANGE.min, NODE_SIZE_RANGE.max, DEFAULT_RENDER_SETTINGS.nodeSize],
["edgeWidth", EDGE_WIDTH_RANGE.min, EDGE_WIDTH_RANGE.max, DEFAULT_RENDER_SETTINGS.edgeWidth],
["arrowLength", ARROW_LENGTH_RANGE.min, ARROW_LENGTH_RANGE.max, DEFAULT_RENDER_SETTINGS.arrowLength],
["labelSize", LABEL_SIZE_RANGE.min, LABEL_SIZE_RANGE.max, DEFAULT_RENDER_SETTINGS.labelSize],
["labelZoom", LABEL_ZOOM_MIN, LABEL_ZOOM_MAX, DEFAULT_RENDER_SETTINGS.labelZoom],
];
test("every default sits dead-center at position 50", () => {
for (const [name, min, max, dflt] of RANGES)
assert.equal(Math.round(toSliderPosition(dflt, min, max)), 50, name);
});
test("round-trips both directions across the whole span", () => {
for (const [name, min, max] of RANGES) {
for (let p = 0; p <= 100; p += 5) {
const v = fromSliderPosition(p, min, max);
assert.ok(Math.abs(toSliderPosition(v, min, max) - p) < 1e-9, `${name} @ ${p}`);
}
}
});
test("slider extremes map to the range ends and clamp beyond", () => {
for (const [name, min, max] of RANGES) {
assert.equal(fromSliderPosition(0, min, max), min, name);
assert.ok(Math.abs(fromSliderPosition(100, min, max) - max) < 1e-9, name); // exp/log float noise
assert.equal(toSliderPosition(min * 0.1, min, max), 0, name); // clamps
assert.equal(toSliderPosition(max * 10, min, max), 100, name);
}
});
test("display units normalize the default to 1", () => {
assert.equal(labelZoomDisplay(0.5), "1×");
assert.equal(labelZoomDisplay(0.25), "0.5×");
assert.equal(labelZoomDisplay(4), "8×");
assert.equal(labelZoomDisplay(0.0625), "0.13×"); // rounded to 2 decimals
});
test("every region/physics default sits dead-center at position 50", () => {
// The same law for the 2026-09-13 sliders: default = geometric mean
// of the range ends (repulsion maps |charge|; friction tolerates
// the sub-0.5° drift).
const pairs: [number, number, number][] = [
[REGION_DILATION_RANGE.min, REGION_DILATION_RANGE.max, DEFAULT_REGION_TUNING.dilation],
[REGION_EDGE_ALPHA_RANGE.min, REGION_EDGE_ALPHA_RANGE.max, DEFAULT_REGION_TUNING.edgeAlpha],
// |charge| in [30, 480] (the range stores negative ends).
[-CHARGE_RANGE.max, -CHARGE_RANGE.min, -DEFAULT_PHYSICS_TUNING.chargeStrength],
[LINK_STRENGTH_RANGE.min, LINK_STRENGTH_RANGE.max, DEFAULT_PHYSICS_TUNING.linkStrength],
[LINK_DISTANCE_RANGE.min, LINK_DISTANCE_RANGE.max, DEFAULT_PHYSICS_TUNING.linkDistance],
[GRAVITY_RANGE.min, GRAVITY_RANGE.max, DEFAULT_PHYSICS_TUNING.gravity],
[VELOCITY_DECAY_RANGE.min, VELOCITY_DECAY_RANGE.max, DEFAULT_PHYSICS_TUNING.velocityDecay],
[DEPTH_STRENGTH_RANGE.min, DEPTH_STRENGTH_RANGE.max, DEFAULT_PHYSICS_TUNING.depthStrength],
[DEPTH_SPACING_RANGE.min, DEPTH_SPACING_RANGE.max, DEFAULT_PHYSICS_TUNING.depthSpacing],
[DEPTH_BASE_RADIUS_RANGE.min, DEPTH_BASE_RADIUS_RANGE.max, DEFAULT_PHYSICS_TUNING.depthBaseRadius],
];
for (const [min, max, d] of pairs) {
assert.ok(Math.abs(toSliderPosition(d, min, max) - 50) < 0.6, `default ${d} in [${min}, ${max}]`);
}
// Off-zone fields: the default sits at 50 INSIDE the on-part.
assert.ok(Math.abs(toOffSliderPosition(DEFAULT_PHYSICS_TUNING.clusterPull, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max) - 50) < 0.6);
});
test("off-slider mapping: position 0 ↔ value 0; the log part round-trips", () => {
assert.equal(toOffSliderPosition(0, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max), 0);
assert.equal(fromOffSliderPosition(0, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max), 0);
const v = fromOffSliderPosition(toOffSliderPosition(0.15, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max), CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max);
assert.ok(Math.abs(v - 0.15) < 1e-9);
// One step in (position 5) sits just above the log floor:
// min * (max/min)^(5/100), NOT the floor itself (that's position 0,
// the off zone).
const stepIn = CLUSTER_PULL_RANGE.min * (CLUSTER_PULL_RANGE.max / CLUSTER_PULL_RANGE.min) ** (5 / 100);
assert.ok(Math.abs(fromOffSliderPosition(5, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max) - stepIn) < 1e-9);
});
test("blur is a fixed set: the default is 6, non-members snap to the nearest", () => {
// Non-integer radii broke the rendering, so the control became a
// dropdown — the sanitizer enforces the set for ANY input.
assert.equal(DEFAULT_REGION_TUNING.blur, 6);
assert.equal(sanitizeRegionTuning({ blur: 3.7 }).blur, 4);
assert.equal(sanitizeRegionTuning({ blur: 7.2 }).blur, 8);
assert.equal(sanitizeRegionTuning({ blur: -5 }).blur, 2);
assert.equal(sanitizeRegionTuning({ blur: 99 }).blur, 10);
// Stored blur 0 (the old off zone) snaps back into the set.
assert.equal(sanitizeRegionTuning({ blur: 0 }).blur, 2);
});
});
+118
View File
@@ -0,0 +1,118 @@
/**
* Semantic topic clustering (the Louvain REPLACEMENT, DECIDED
* 2026-09-13): average-linkage agglomerative clustering over the
* embedding ledger — related TOPICS, not reasoning chains. Pure and
* deterministic, so these tests are exact.
*/
import { strict as assert } from "node:assert";
import { describe, test } from "node:test";
import { semanticClusters, SEMANTIC_SIM_THRESHOLD, cosineSimilarity } from "../src/core/semantic-clusters.ts";
/** Two clearly-related one-hot-ish vectors (high cosine similarity). */
function near(vec: number[], target: number[], amount = 0.2): number[] {
let dot = 0;
for (let i = 0; i < vec.length; i++) dot += vec[i]! * target[i]!;
const mixed = vec.map((x, i) => x + amount * target[i]!);
let mag = 0;
for (const x of mixed) mag += x * x;
mag = Math.sqrt(mag);
return mixed.map((x) => x / mag);
}
function axis(dim: number, k: number): number[] {
return Array.from({ length: dim }, (_, i) => (i === k ? 1 : 0));
}
describe("cosineSimilarity", () => {
test("identical unit vectors → 1; orthogonal → 0; defensive normalization", () => {
const a = axis(4, 0);
const b = axis(4, 1);
assert.ok(cosineSimilarity(a, a) > 0.999);
assert.ok(Math.abs(cosineSimilarity(a, b)) < 1e-9);
// Unnormalized twins still match.
assert.ok(cosineSimilarity([3, 0], [5, 0]) > 0.999);
});
});
describe("semanticClusters", () => {
test("empty ledger: zero clusters (nothing embedded yet)", () => {
assert.deepEqual(semanticClusters({}), []);
});
test("related vectors merge; distant ones stay separate (threshold)", () => {
const dim = 8;
const topicA = axis(dim, 0);
const topicB = axis(dim, 1);
const vectors = {
a1: near(topicA, topicB, 0.1),
a2: near(topicA, topicB, 0.15),
a3: near(topicA, topicB, 0.05),
b1: near(topicB, topicA, 0.1),
b2: near(topicB, topicA, 0.12),
};
const clusters = semanticClusters(vectors);
// Both topics form; average-linkage similarity within each is well
// above the default threshold, across is well below.
const big = clusters.filter((c) => c.members.length >= 2);
assert.equal(big.length, 2);
assert.deepEqual(
big.map((c) => c.members.sort()).sort((x, y) => x[0]!.localeCompare(y[0]!)),
[["a1", "a2", "a3"], ["b1", "b2"]],
);
});
test("threshold is the granularity knob: higher splits, lower merges", () => {
const dim = 6;
const vectors = {
x: near(axis(dim, 0), axis(dim, 1), 0.3),
y: near(axis(dim, 1), axis(dim, 0), 0.3),
};
// Similarity ~0.72: merges below that, splits above.
const sim = cosineSimilarity(vectors.x!, vectors.y!);
assert.equal(semanticClusters(vectors, { similarityThreshold: sim - 0.01 }).length, 1);
assert.equal(semanticClusters(vectors, { similarityThreshold: sim + 0.01 }).length, 2);
});
test("stragglers stay singletons by default; attachStragglers forces membership", () => {
const dim = 6;
const topic = axis(dim, 0);
const vectors = {
t1: near(topic, axis(dim, 1), 0.1),
t2: near(topic, axis(dim, 2), 0.1),
t3: near(topic, axis(dim, 3), 0.1),
loner: axis(dim, 5), // orthogonal: fits nothing
};
const loose = semanticClusters(vectors);
const looseLoner = loose.find((c) => c.members.includes("loner"));
assert.equal(looseLoner?.members.length, 1, "default: loner is a straggler");
assert.equal(loose.find((c) => c.members.length >= 3)?.members.length, 3);
const allIn = semanticClusters(vectors, { attachStragglers: true });
assert.equal(allIn.length, 1, "attach: the loner joins the nearest topic");
assert.ok(allIn[0]!.members.includes("loner"));
});
test("members sorted; clusters ordered by size then id (deterministic)", () => {
const dim = 4;
const vectors = {
solo: axis(dim, 3),
p1: near(axis(dim, 0), axis(dim, 1), 0.05),
p2: near(axis(dim, 0), axis(dim, 2), 0.05),
q1: near(axis(dim, 1), axis(dim, 0), 0.05),
q2: near(axis(dim, 1), axis(dim, 2), 0.05),
};
const a = semanticClusters(vectors);
const b = semanticClusters(vectors);
assert.deepEqual(a, b, "same input → identical partition");
for (let i = 1; i < a.length; i++) {
assert.ok(a[i - 1]!.members.length >= a[i]!.members.length, "size-descending");
}
for (const c of a) {
assert.deepEqual(c.members, [...c.members].sort(), "members sorted");
}
});
test("default threshold is the swept real-graph value", () => {
assert.equal(SEMANTIC_SIM_THRESHOLD, 0.6);
});
});
+39 -1
View File
@@ -10,7 +10,8 @@ import test from "node:test";
import * as fs from "node:fs";
import { chatStore } from "../src/store/chats.ts";
import { PATHS } from "../config.ts";
import { graphStore } from "../src/store/graph.ts";
import { PATHS, SOURCES_FILE } from "../config.ts";
import { callCost, contextWindow, jobCost, sessionCost, sumSteps, turnCost } from "../src/core/prices.ts";
import type { CallUsage, ExtractionRunUsage, JobRecord, TurnUsage } from "../src/core/contracts.ts";
@@ -170,3 +171,40 @@ test("appendExtractionUsage: appends, idempotent per startedAt, not in list payl
fs.unlinkSync(`${PATHS.chats}/${record.id}.json`);
});
test("appendSourceUsage: appends to the source record, idempotent per startedAt, upsert keeps history", () => {
// The real fs graphStore — sources.json backed up and restored around
// this test (same discipline as the chat fixture files).
fs.mkdirSync(PATHS.graph, { recursive: true });
let bak: string | undefined;
if (fs.existsSync(SOURCES_FILE)) bak = `${SOURCES_FILE}.w5test.bak`;
if (bak) fs.renameSync(SOURCES_FILE, bak);
try {
const runEntry = (startedAt: number): ExtractionRunUsage => ({
startedAt,
at: "2026-09-13T00:00:00Z",
durationMs: 42_000,
stats: { nodes: 1, edges: 2, events: 3 },
stages: {
extractor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
auditor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
},
});
const id = "src_usagetest01";
graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-13T00:00:00Z" });
graphStore.appendSourceUsage(id, runEntry(1));
graphStore.appendSourceUsage(id, runEntry(1)); // same run appended twice → no-op
graphStore.appendSourceUsage(id, runEntry(2)); // a retried run is a NEW spend
assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2);
// An upsert (re-import) KEEPS the usage history.
graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-14T00:00:00Z" });
assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2);
// Unknown ids are a silent no-op (nothing to bill against).
graphStore.appendSourceUsage("src_nope", runEntry(1));
} finally {
fs.rmSync(SOURCES_FILE, { force: true });
if (bak) fs.renameSync(bak, SOURCES_FILE);
}
});