diff --git a/.gitignore b/.gitignore index 1143e7e..d1030a5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ data/ .env.local .memory/ SCOPE.md +# raw real-data import sources (user directive 2026-09-13: never commit) +real-data-extraction/ diff --git a/app/api/graph/import/route.ts b/app/api/graph/import/route.ts new file mode 100644 index 0000000..7c44afd --- /dev/null +++ b/app/api/graph/import/route.ts @@ -0,0 +1,52 @@ +/** + * POST /api/graph/import — the format-agnostic front door's transport + * (vision §1, 2026-09-13): raw personal-notes files in, one extraction run + * per file through the SAME pipeline the sessions use. Multipart form data + * (field "files", any text format — the extractor is format-agnostic); the + * pipeline mints a deterministic source id per file path, persists the + * source record (the file's provenance + the run's usage), and runs + * extractor → auditor → merger over the raw text. + * + * Small concurrency cap: the model calls are the slow part, and a burst of + * 25 simultaneous runs would trip provider rate limits — a 3-wide worker + * pool keeps the run registry (the dock's status chips) and the ledgers + * healthy. The dock's chips + graph refetch track progress live through + * /api/graph/status, as with session runs. + */ +import { extraction } from "@/src/server"; +import type { EmissionStats } from "@/src/core"; + +export const dynamic = "force-dynamic"; + +interface ImportResult { + filePath: string; + ok: boolean; + stats?: EmissionStats; + error?: string; +} + +export async function POST(request: Request) { + const form = await request.formData(); + const files = form.getAll("files").filter((f): f is File => f instanceof File); + if (files.length === 0) { + return Response.json({ results: [] as ImportResult[], error: "no files" }, { status: 400 }); + } + + const results: ImportResult[] = []; + const queue = [...files]; + const workers = Array.from({ length: Math.min(3, files.length) }, async () => { + for (;;) { + const file = queue.shift(); + if (!file) return; + try { + const text = await file.text(); + const stats = await extraction.importNotes({ filePath: file.name, text }); + results.push({ filePath: file.name, ok: true, stats }); + } catch (err) { + results.push({ filePath: file.name, ok: false, error: err instanceof Error ? err.message : String(err) }); + } + } + }); + await Promise.all(workers); + return Response.json({ results }); +} \ No newline at end of file diff --git a/app/api/graph/positions/route.ts b/app/api/graph/positions/route.ts new file mode 100644 index 0000000..da0d280 --- /dev/null +++ b/app/api/graph/positions/route.ts @@ -0,0 +1,24 @@ +/** + * PUT /api/graph/positions — the live-physics drag commit. The client + * anneals a drag to rest and sends the new positions; the server + * sanitizes (finite coords, known ids only), writes the ledger, and if + * the graph grew mid-drag runs one incremental settle to absorb the new + * nodes with the client's arrangement as anchors. See + * src/server/commit-positions.ts. + * + * PUT → 200 {positions: {...}} with whatever survived sanitizing + * (empty object = nothing written). + */ +import { commitPositions } from "@/src/server"; + +export const dynamic = "force-dynamic"; + +export async function PUT(request: Request) { + let body: unknown; + try { + body = await request.json(); + } catch { + return Response.json({ error: "invalid JSON body" }, { status: 400 }); + } + return Response.json({ positions: commitPositions(body) }); +} diff --git a/app/api/graph/render-settings/route.ts b/app/api/graph/render-settings/route.ts new file mode 100644 index 0000000..bc82f1b --- /dev/null +++ b/app/api/graph/render-settings/route.ts @@ -0,0 +1,31 @@ +/** + * GET/PUT /api/graph/render-settings — server persistence for the graph + * view's rendering controls (node/edge/arrow/label sizes, the label-zoom + * threshold, the cluster-labels toggle). The file lives at + * data/render-settings.json (config.RENDER_SETTINGS_FILE). + * + * GET → 200 {settings} once saved, 200 null on first boot (client keeps + * in-memory defaults until the user changes something). + * PUT → 200 {settings} (the sanitized copy actually persisted — the + * client may echo it to stay honest with clamped values). + * + * Every body is passed through sanitizeRenderSettings (src/core) before + * touching disk: garbage clamps or falls back to defaults, never throws. + */ +import { readRenderSettings, writeRenderSettings } from "@/src/server"; + +export const dynamic = "force-dynamic"; + +export async function GET() { + return Response.json(readRenderSettings()); +} + +export async function PUT(request: Request) { + let body: unknown; + try { + body = await request.json(); + } catch { + return Response.json({ error: "invalid JSON body" }, { status: 400 }); + } + return Response.json(writeRenderSettings(body)); +} diff --git a/app/api/graph/status/route.ts b/app/api/graph/status/route.ts index 0ba43a6..e4a3866 100644 --- a/app/api/graph/status/route.ts +++ b/app/api/graph/status/route.ts @@ -12,7 +12,9 @@ export const dynamic = "force-dynamic"; export async function GET() { const runs = extractionStatuses().map((run) => ({ chatId: run.chatId, - title: chatStore.getChat(run.chatId)?.title ?? run.chatId, + // The run's own label (the file path for notes imports) wins; chat + // runs fall back to the chat store's title. + title: run.label ?? chatStore.getChat(run.chatId)?.title ?? run.chatId, phase: run.phase, startedAt: run.startedAt, // Live per-stage usage (the money channel): model + per-step CallUsage, diff --git a/components/cluster-geometry.ts b/components/cluster-geometry.ts new file mode 100644 index 0000000..a81bd83 --- /dev/null +++ b/components/cluster-geometry.ts @@ -0,0 +1,19 @@ +/** + * Cluster color identity — pure presentation math for the knowledge-graph + * regions. (The hull geometry that used to live here — convexHull, + * outwardNormal, offsetPolygon, traceRoundedPath — was RETIRED with the + * convex-hull renderer: regions are now a low-res world-space splat + * bitmap in graph-canvas.ts, which renders concave cluster shapes + * correctly and costs one blit per frame instead of 32 clusters × 4 + * offset polygons. The file keeps the name for the barrel imports.) + */ + +export type Pt = { x: number; y: number }; + +/** Stable hue per cluster id (content-hash): the region keeps its color + * across reshuffles because the id only changes when the members do. */ +export function clusterHue(id: string): number { + let h = 0; + for (const ch of id) h = (h * 31 + ch.charCodeAt(0)) % 360; + return h; +} diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts new file mode 100644 index 0000000..91950d8 --- /dev/null +++ b/components/graph-canvas.ts @@ -0,0 +1,1373 @@ +"use client"; + +/** + * The hand-rolled single-canvas graph renderer (decided 2026-09-13 after + * the research pass — docs/research/graph-rendering.md, Option B). + * + * Replaces sigma.js. Why: sigma gave us WebGL node/edge rendering, + * picking, and a camera, but our cluster regions had to live on foreign + * canvases injected next to its private DOM stack, drawn from a separate + * event loop — so hulls lagged behind nodes during pan, raced sigma's + * canvas sizing on renderer rebuilds, and every camera event re-rasterized + * the full hull canvas (GPU spikes + audible coil whine). At our scale + * (338 → low thousands of nodes) one Canvas 2D is comfortably inside + * 60fps territory, and a single loop makes the desync class of bugs + * structurally impossible. + * + * Architecture (the canonical pattern — browser.engineering, MDN + * "Optimizing canvas", web.dev canvas performance): + * - The renderer is a DUMB DRAW FUNCTION over plain data: nodes with + * world-space positions, edges, clusters, colors, settings. Layout + * (forceAtlas2) and filtering happen OUTSIDE, in the React component; + * this module never computes a position. + * - ONE canvas, ONE requestAnimationFrame loop with a dirty flag. Input + * and DOM events only mutate state and mark dirty; the loop draws at + * most once per frame with every layer under the same camera — + * hull fills → hull strokes → edges → nodes → region labels → node + * labels → hover. Nothing can ever lag anything else. + * - Camera: d3-zoom supplies the verified pan / zoom-to-cursor / pinch + * math (screen = world·k + [x,y]). The camera lives in the renderer + * instance, so data refetches and settings changes never move it. + * - Picking: nearest-node distance check (µs at our scale), with a + * PICKING_PAD so the whole hover halo is hit-testable. + * + * Zoom-gated labels (the Obsidian text-fade feel): sigma's camera "ratio" + * convention is kept for the settings' persisted semantics — ratio 1 = + * fit zoom, SMALLER = zoomed in (ratio = kFit / k). Node labels are + * fully visible at ratio ≤ labelZoom and fade out across LABELS_FADE_SPAN + * of ratio; cluster labels fade in over the exact same span (sums to 1) + * so the two never coexist. All labels gate together — no per-node + * popping. + * + * The hover glow is unchanged in look: a soft halo in the node's own + * color at a constant radius, opacity cross-faded over ~180ms, with the + * name brightening on the SAME factor. Only opacity animates; the only + * per-frame work when idle is a dirty-flag check (no rAF is scheduled at + * all until something changes). + * + * Unit tests cover the pure helpers (tests/graph-canvas.test.ts); the + * class itself is DOM-bound and exercised live. Module scope stays + * DOM-free so the file imports cleanly under Node. + */ + +import { select } from "d3-selection"; +import { zoom, zoomIdentity, type ZoomBehavior, type ZoomTransform } from "d3-zoom"; +import type { Simulation, SimulationLinkDatum } from "d3-force"; +import { clusterHue, type Pt } from "./cluster-geometry.ts"; +import { + CHARGE_STRENGTH, + CLUSTER_PULL_STRENGTH, + createGraphSimulation, + DEFAULT_PHYSICS_TUNING, + DEFAULT_REGION_TUNING, + sanitizePhysicsTuning, + sanitizeRegionTuning, + type PhysicsTuning, + type RegionTuning, + INCREMENTAL_GRAVITY, + LINK_DISTANCE, + type DepthAnchorMode, + type GraphRenderSettings, + type LayoutNode, + type Position, +} from "@/src/core"; + +export type { GraphRenderSettings }; + +// ─── Public data contract ────────────────────────────────────────────────── + +/** One node to draw: world-space position + the short glanceable label. */ +export interface CanvasNode { + id: string; + x: number; + y: number; + label: string; +} + +/** One directed edge (follows | supports); arrow points prerequisite → dependent. */ +export interface CanvasEdge { + from: string; + to: string; +} + +/** One cluster region: member node ids + the LLM-coined label (or null). */ +export interface CanvasCluster { + id: string; + label: string | null; + members: string[]; +} + +/** Theme colors, resolved to #rrggbb by the caller (canvas wants hex, + * and the renderer must not read CSS itself). */ +export interface CanvasColors { + node: string; + edge: string; + label: string; + hoverText: string; +} + +export interface CanvasData { + nodes: CanvasNode[]; + edges: CanvasEdge[]; + clusters: CanvasCluster[]; + colors: CanvasColors; + /** Body font family (labels must match the app typography). */ + fontFamily: string; +} + +export interface RendererCallbacks { + /** A node was clicked (screen position included for card anchoring). */ + onSelect: (nodeId: string, screenPos: Pt) => void; + /** The empty stage was clicked. */ + onStageClick: () => void; + /** Any camera move (pan/zoom) — the caller dismisses anchored UI. */ + onCameraMove: () => void; + /** The live sim settled after a drag: commit the new rest state + * (the caller PUTs it to the positions ledger). */ + onCommit: (positions: Record) => void; +} + +// ─── Constants (ported verbatim from the sigma view) ──────────────────────── + +const LABEL_SIZE = 12; // px — the default label size (settings.labelSize scales it) +const LABEL_WEIGHT = "500"; +const LABEL_MAX_WIDTH = 170; // px — wraps long names instead of a wide strip +const LABEL_GAP = 2; // px between node edge and first label line (was 6 — user: way too far) +const LABEL_LINE_HEIGHT = 1.3; +/** Zoom span (camera-ratio units) over which labels fade together. */ +export const LABELS_FADE_SPAN = 0.5; + +const GLOW_SCALE = 1.7; // halo outer radius = node radius x this (proportional, like blurring a copy of the disc) +const GLOW_ALPHA = 0.3; // halo opacity at the disc edge, fading outward +const HOVER_FADE_MS = 180; // hover halo opacity fade length +const ZOOM_EASE_MS = 45; // wheel-zoom glide: exponential-ease time constant (~250ms per notch) + +/** Extra pickable radius (px) beyond the disc — the whole halo is + * hit-testable (the old flicker fix, minus the picking shader). */ +const PICKING_PAD = 10; + +/** Click vs drag threshold: pointer travel under this many px is a click. */ +const CLICK_SLOP = 4; +/** Live-sim tuning — see the class doc comment. */ +const LIVE_ALPHA_DECAY = 0.05; // snappier anneal than the settle's 0.0228 +const DRAG_ALPHA_TARGET = 0.3; // d3-drag's own reheat constant +const SIM_SETTLE_ALPHA = 0.005; // below this, the sim counts as settled +const COMMIT_DEBOUNCE_MS = 250; // settle → ledger PUT delay (new drags cancel) + +/** Cluster region styling — a low-res world-space SPLAT bitmap (the + * density-field technique, Canvas-2D edition): every member node paints + * a soft radial-gradient disc and every intra-cluster edge a round-cap + * stroke; upscaling the bitmap with bilinear smoothing IS the blur. + * Replaces the convex-hull fills + borders: concave cluster shapes are + * finally correct (the hull bridged every gap with straight lines — + * the jagged close-up look), there is no border to lag behind, and a + * live-physics frame pays one tiny blit instead of 32 clusters × 4 + * offset polygons re-rasterized at DPR 2 (the old GPU coil-whine). */ +const REGION_EVERY = 3; // live-sim frames: rebuild every Nth, blit the rest +const REGION_LABEL_SIZE = 14; + + +/** One horizontal+vertical box-blur round over PREMULTIPLIED RGBA + * (Float32, stride 4, length w*h*4). Premultiplying first is what keeps + * the blur halo-free: blurring non-premultiplied RGB over varying alpha + * bleeds solid color into transparent space. Pure: DOM-free, tested. */ +export function boxBlurRound( + src: Float32Array, + tmp: Float32Array, + dst: Float32Array, + w: number, + h: number, + radius: number, +): void { + if (radius <= 0) { + dst.set(src); + return; + } + const div = radius * 2 + 1; + // Horizontal: src → tmp (clamped window, running sum). + for (let y = 0; y < h; y++) { + const row = y * w * 4; + let sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; + for (let x = -radius; x <= radius; x++) { + const i = row + Math.min(Math.max(x, 0), w - 1) * 4; + sum0 += src[i]!; + sum1 += src[i + 1]!; + sum2 += src[i + 2]!; + sum3 += src[i + 3]!; + } + for (let x = 0; x < w; x++) { + const o = row + x * 4; + tmp[o] = sum0 / div; + tmp[o + 1] = sum1 / div; + tmp[o + 2] = sum2 / div; + tmp[o + 3] = sum3 / div; + const addI = row + Math.min(x + radius + 1, w - 1) * 4; + const subI = row + Math.max(x - radius, 0) * 4; + sum0 += src[addI]! - src[subI]!; + sum1 += src[addI + 1]! - src[subI + 1]!; + sum2 += src[addI + 2]! - src[subI + 2]!; + sum3 += src[addI + 3]! - src[subI + 3]!; + } + } + // Vertical: tmp → dst. + for (let x = 0; x < w; x++) { + const col = x * 4; + let sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0; + for (let y = -radius; y <= radius; y++) { + const i = col + Math.min(Math.max(y, 0), h - 1) * w * 4; + sum0 += tmp[i]!; + sum1 += tmp[i + 1]!; + sum2 += tmp[i + 2]!; + sum3 += tmp[i + 3]!; + } + for (let y = 0; y < h; y++) { + const o = col + y * w * 4; + dst[o] = sum0 / div; + dst[o + 1] = sum1 / div; + dst[o + 2] = sum2 / div; + dst[o + 3] = sum3 / div; + const addI = col + Math.min(y + radius + 1, h - 1) * w * 4; + const subI = col + Math.max(y - radius, 0) * w * 4; + sum0 += tmp[addI]! - tmp[subI]!; + sum1 += tmp[addI + 1]! - tmp[subI + 1]!; + sum2 += tmp[addI + 2]! - tmp[subI + 2]!; + sum3 += tmp[addI + 3]! - tmp[subI + 3]!; + } + } +} + +/** All "px" constants above are screen px AT FIT ZOOM. The renderer scales + * them into world units (÷ kFit) once per geometry build; camera zoom then + * scales nodes/labels/arrowheads with the graph (Obsidian-style), while + * edge widths and hull borders stay viewer-relative hairlines (÷ k). */ + +const clamp01 = (v: number) => Math.min(Math.max(v, 0), 1); + +// ─── Pure helpers (unit-tested) ───────────────────────────────────────────── + +/** + * The sigma-compatible "camera ratio": 1 at fit zoom, SMALLER when zoomed + * in (ratio = kFit / k). The persisted labelZoom setting is denominated in + * these units, so the gate semantics survive the renderer swap unchanged. + */ +export function zoomRatioOf(kFit: number, k: number): number { + return kFit / k; +} + +/** + * Node-label opacity at a given camera ratio: 1 at/below the labelZoom + * threshold, fading to 0 across LABELS_FADE_SPAN of ratio. Cluster labels + * use `1 - this` over the same span — the two cross-fade and never + * coexist at full opacity. + */ +export function labelAlpha(ratio: number, labelZoom: number, span = LABELS_FADE_SPAN): number { + return clamp01(1 - (ratio - labelZoom) / span); +} + +/** The camera transform that frames `bbox` in a w×h viewport with padding. */ +export function fitTransform( + bbox: { minX: number; maxX: number; minY: number; maxY: number }, + w: number, + h: number, + padding: number, +): { x: number; y: number; k: number } { + const bw = Math.max(bbox.maxX - bbox.minX, 1e-6); + const bh = Math.max(bbox.maxY - bbox.minY, 1e-6); + const k = Math.min((w - 2 * padding) / bw, (h - 2 * padding) / bh); + const cx = (bbox.minX + bbox.maxX) / 2; + const cy = (bbox.minY + bbox.maxY) / 2; + return { x: w / 2 - cx * k, y: h / 2 - cy * k, k }; +} + +/** The arrowhead geometry for a directed edge a→b: shaft endpoints + * (clear of both node discs) and the head triangle (tip on the target + * disc edge, wideness = 0.8 × head length — the sigma arrow shape). */ +export function arrowGeometry( + a: Pt, + b: Pt, + fromRadius: number, + toRadius: number, + headLength: number, +): { shaftFrom: Pt; shaftTo: Pt; tip: Pt; left: Pt; right: Pt } | null { + const dx = b.x - a.x; + const dy = b.y - a.y; + const len = Math.hypot(dx, dy); + if (len < 1e-6) return null; // degenerate (coincident) — skip + const ux = dx / len; + const uy = dy / len; + const tip = { x: b.x - ux * toRadius, y: b.y - uy * toRadius }; + const base = { x: tip.x - ux * headLength, y: tip.y - uy * headLength }; + const halfW = 0.4 * headLength; + return { + shaftFrom: { x: a.x + ux * fromRadius, y: a.y + uy * fromRadius }, + shaftTo: base, + tip, + left: { x: base.x - uy * halfW, y: base.y + ux * halfW }, + right: { x: base.x + uy * halfW, y: base.y - ux * halfW }, + }; +} + +/** Nearest node within maxDist of a screen point (linear scan — µs at + * our scale; a spatial grid only earns its keep past ~10k nodes). */ +export function pickNearest( + nodes: T[], + px: number, + py: number, + maxDist: number, +): T | null { + let best: T | null = null; + let bestD = maxDist * maxDist; // compare squared distances + for (const n of nodes) { + const d = (n.x - px) ** 2 + (n.y - py) ** 2; + if (d < bestD) { + bestD = d; + best = n; + } + } + return best; +} + +// ─── Canvas text helpers (ported verbatim) ───────────────────────────────── + +/** "#rrggbb" + alpha → "rgba(...)". */ +function rgbaFromHex(hex: string, alpha: number): string { + const r = Number.parseInt(hex.slice(1, 3), 16); + const g = Number.parseInt(hex.slice(3, 5), 16); + const b = Number.parseInt(hex.slice(5, 7), 16); + return `rgba(${r}, ${g}, ${b}, ${alpha})`; +} + +/** Word-wrap for canvas text. Never truncates: overlong single words are + * hard-broken by character so no line ever exceeds maxWidth. */ +function wrapLabel(context: CanvasRenderingContext2D, text: string, maxWidth: number): string[] { + const lines: string[] = []; + for (const word of text.split(/\s+/).filter(Boolean)) { + const last = lines.length > 0 ? lines[lines.length - 1]! : null; + if (last !== null) { + const joined = `${last} ${word}`; + if (context.measureText(joined).width <= maxWidth) { + lines[lines.length - 1] = joined; + continue; + } + } + let chunk = ""; + for (const char of word) { + if (chunk && context.measureText(`${chunk}${char}`).width > maxWidth) { + lines.push(chunk); + chunk = char; + } else { + chunk += char; + } + } + if (chunk) lines.push(chunk); + } + return lines; +} + +/** World-unit sizes for the label pass, derived once per geometry build. */ +interface WorldSizes { + nodeR: number; // node disc radius, world units + glowOuter: number; // hover halo outer radius, world units + labelSize: number; // node label font, world units + labelGap: number; // node edge → first label line, world units +} + +/** The pre-wrapped node name, centered below the node in WORLD units — + * identical geometry at rest and on hover, so hover only changes the + * color, nothing jumps. Lines are wrapped once per geometry build, never + * per frame. */ +function drawNameLines( + context: CanvasRenderingContext2D, + x: number, + y: number, + lines: string[], + world: WorldSizes, + color: string, +): void { + context.fillStyle = color; + context.textAlign = "center"; + context.textBaseline = "top"; + const firstTop = y + world.nodeR + world.labelGap; + for (let i = 0; i < lines.length; i++) { + context.fillText(lines[i]!, x, firstTop + i * world.labelSize * LABEL_LINE_HEIGHT); + } +} + +// ─── The renderer ─────────────────────────────────────────────────────────── + +export class GraphCanvasRenderer { + private canvas: HTMLCanvasElement; + private ctx: CanvasRenderingContext2D; + private callbacks: RendererCallbacks; + private zoomBehavior: ZoomBehavior; + private transform: ZoomTransform = zoomIdentity; + /** Set once the first fit happens; the camera then survives setData. */ + private hasFit = false; + /** The k that frames the whole graph — the label-gate's "1×". */ + private kFit = 1; + /** World-space bounds of the current nodes — fit() and the resize-time + * kFit refresh share it (computed once per setData, not per resize). */ + private worldBBox: { minX: number; maxX: number; minY: number; maxY: number } | null = null; + + private data: CanvasData | null = null; + /** id -> node for the current data (avoids O(n^2) scans). */ + private nodeById = new Map(); + private settings: GraphRenderSettings | null = null; + private visible: ReadonlySet | null = null; + + // ── Live physics ── + // One d3-force sim shared with the server settle (createGraphSimulation + // in @/src/core): dragging pins fx/fy and reheats (alphaTarget), release + // un-pins and the sim anneals; once alpha drops below SIM_SETTLE_ALPHA + // the new rest state is debounced-committed via onCommit. The sim mutates + // data.nodes IN PLACE (they ARE the sim nodes — zero copying), so every + // tick just bumps geoVersion and the draw loop rebuilds geometry. + private sim: Simulation> | null = null; + private simById = new Map(); + private simActive = false; + private dragId: string | null = null; + private dragOrigin: Pt | null = null; + private simMoved = false; // any real movement since the last commit + private commitTimer: ReturnType | null = null; + + private frameId: number | null = null; + private dirty = true; + /** True while tick() runs: markDirty inside the loop (d3's zoom event + * fires from advanceZoom) must NOT schedule — the loop schedules + * itself below. Un-guarded, each glide frame leaked an extra queued + * tick and draws multiplied per displayed frame (20fps zoom bug). */ + private inTick = false; + + // Hover transition: fade 0 = resting, 1 = fully hovered; the animating + // flag keeps the loop scheduled until the fade settles (then rAF stops). + private hoverId: string | null = null; + /** The node the glow is drawn on. OUTLIVES hoverId on hover-out so the + * fade has something to draw (hoverId nulls instantly; without this + * the glow popped off in one frame instead of fading). Cleared when + * the outgoing fade completes. */ + private hoverNode: CanvasNode | null = null; + private hoverFade = 0; + private hoverFrom = 0; + private hoverTarget = 0; + private hoverStart = 0; + // Smooth wheel zoom: wheel events accumulate a target scale (cursor- + // anchored); each tick eases the camera toward it instead of jumping. + private zoomTarget: { k: number; x: number; y: number } | null = null; + /** Clock for frame-rate-independent easing (updated every tick). */ + private lastTickAt = 0; + private animating = false; + + private resizeObserver: ResizeObserver; + private downPos: Pt | null = null; + + // ── Cached world geometry ───────────────────────────────────────────── + // Everything above is drawn per frame; everything below is derived from + // data+visibility+settings+kFit and rebuilt ONLY when one of those + // changes (geoVersion). Camera moves just re-rasterize the same paths. + private geoVersion = 0; // bump = geometry inputs changed + private builtVersion = -1; // version the cache was built at + private nodesPath: Path2D | null = null; + private shaftsPath: Path2D = new Path2D(); + private headsPath: Path2D = new Path2D(); + private regionCanvas: HTMLCanvasElement | null = null; + /** World rect the bitmap covers (drawImage target coords). */ + private regionRect: { minX: number; minY: number; w: number; h: number } | null = null; + /** Region labels: cluster centroid + hue, computed at splat build. */ + private regionMeta: { label: string | null; cx: number; cy: number; hue: number }[] = []; + private regionTuning: RegionTuning = DEFAULT_REGION_TUNING; + private physicsTuning: PhysicsTuning = DEFAULT_PHYSICS_TUNING; + /** Gravity center for the live sim (graph bbox centroid at setData). */ + private simCenter: Pt = { x: 0, y: 0 }; + private regionBuiltVersion = -1; + private frameCount = 0; // draw counter for the live-sim splat throttle + private labelLines = new Map(); + private worldLabelFont = ""; + private world: WorldSizes = { nodeR: 0, glowOuter: 0, labelSize: 0, labelGap: 0 }; + + constructor(canvas: HTMLCanvasElement, callbacks: RendererCallbacks) { + this.canvas = canvas; + const ctx = canvas.getContext("2d"); + if (!ctx) throw new Error("2d context unavailable"); + this.ctx = ctx; + this.callbacks = callbacks; + + // d3-zoom owns pan / wheel zoom-to-cursor / touch pinch; every + // transform change just marks dirty (draws happen in OUR loop). + this.zoomBehavior = zoom() + .scaleExtent([0.02, 40]) + .on("zoom", ({ transform }) => { + this.transform = transform; + this.markDirty(); + this.callbacks.onCameraMove(); + }); + select(canvas) + .call(this.zoomBehavior) + // No surprise dblclick-zoom (sigma had none). + .on("dblclick.zoom", null) + // Wheel is OURS now: d3's instant per-notch jump felt janky; the + // eased glide below replaces it (pan and pinch stay d3's). + .on("wheel.zoom", null); + + canvas.addEventListener("wheel", this.onWheel, { passive: false }); + canvas.addEventListener("pointermove", this.onPointerMove); + canvas.addEventListener("pointerdown", this.onPointerDown); + canvas.addEventListener("pointerup", this.onPointerUp); + canvas.addEventListener("pointerleave", this.onPointerLeave); + + this.resizeObserver = new ResizeObserver(() => { + this.syncSize(); + if (!this.hasFit) this.fit(); // first real size → fit once + else if (this.worldBBox) { + // Keep the label-gate's "1×" honest: kFit is viewport-bound, + // so a resize changes it even though the camera persists. + const w = this.canvas.clientWidth; + const h = this.canvas.clientHeight; + if (w >= 10 && h >= 10) { + this.kFit = fitTransform(this.worldBBox, w, h, 40).k; + this.geoVersion++; // resize redefines the fit-zoom scale + } + } + // Redraw SYNCHRONOUSLY: syncSize may have reallocated the backing + // store (which clears the canvas), and during live window resizes + // rAF is starved by relayout — a deferred draw would leave the + // canvas blank for the whole drag (the resize-blanking bug). + this.draw(); + }); + this.resizeObserver.observe(canvas); + this.syncSize(); + } + + // ── Public API ── + + setData(data: CanvasData): void { + this.data = data; + this.nodeById = new Map(data.nodes.map((n) => [n.id, n])); + let minX = Infinity; + let minY = Infinity; + let maxX = -Infinity; + let maxY = -Infinity; + for (const n of data.nodes) { + if (n.x < minX) minX = n.x; + if (n.y < minY) minY = n.y; + if (n.x > maxX) maxX = n.x; + if (n.y > maxY) maxY = n.y; + } + this.worldBBox = { minX, maxX, minY, maxY }; + if (!this.hasFit && data.nodes.length > 0) this.fit(); + // Rebuild the live sim around the fresh nodes. New data means the + // server spoke (merge or refetch); any pending commit is moot and + // the graph is at rest until the next drag. + if (this.commitTimer) { + clearTimeout(this.commitTimer); + this.commitTimer = null; + } + this.simActive = false; + this.dragId = null; + this.simMoved = false; + this.simCenter = { x: (minX + maxX) / 2, y: (minY + maxY) / 2 }; + this.buildSim(); + this.geoVersion++; // new data → rebuild world geometry + this.markDirty(); + } + + setSettings(settings: GraphRenderSettings): void { + this.settings = settings; + this.geoVersion++; // sizes/wrap width changed → rebuild geometry + this.markDirty(); + } + + /** Dev tuning for region rendering (session-only, never persisted): + * invalid numbers fall back to defaults — the number inputs can be + * emptied while typing. Forces a region bitmap rebuild. */ + setRegionTuning(input: Partial): void { + this.regionTuning = sanitizeRegionTuning(input); + this.regionBuiltVersion = -2; // never equals geoVersion → rebuild + this.markDirty(); + } + + /** Live-sim physics controls: rebuilds the sim from the CURRENT + * node positions (same node objects — x/y and any drag pin fx/fy + * survive; velocities restart clean, which reads as "re-relax with + * the new feel"). A mid-settle change re-anneals, which is the + * honest response to new physics. */ + setPhysicsTuning(input: Partial): void { + this.physicsTuning = sanitizePhysicsTuning(input); + this.buildSim(); + // Reheat: a tuning change re-anneals the layout from its current + // state, so mode toggles (off/vertical/radial) visibly RE-FORM the + // graph instead of waiting for the next drag. Mid-drag is safe — + // buildSim preserved the pinned node's fx/fy. + if (this.sim) { + this.sim.alpha(1); + this.simActive = true; + } + this.markDirty(); + } + + setVisible(visible: ReadonlySet | null): void { + this.visible = visible; + this.geoVersion++; // member visibility changes hulls + paths + this.markDirty(); + } + + /** Frame the whole graph (padding included). Keeps the camera on later + * setData calls — call fit() only for an explicit re-frame. */ + fit(): void { + if (!this.data || this.data.nodes.length === 0 || !this.worldBBox) return; + const w = this.canvas.clientWidth; + const h = this.canvas.clientHeight; + if (w < 10 || h < 10) return; // layout not ready yet; the ResizeObserver retries + const t = fitTransform(this.worldBBox, w, h, 40); + this.kFit = t.k; + this.hasFit = true; + this.geoVersion++; // kFit defines the world-unit scale + select(this.canvas).call( + this.zoomBehavior.transform, + zoomIdentity.translate(t.x, t.y).scale(t.k), + ); + } + + destroy(): void { + if (this.frameId !== null) cancelAnimationFrame(this.frameId); + if (this.commitTimer) clearTimeout(this.commitTimer); + this.sim = null; + this.simActive = false; + this.dragId = null; + this.resizeObserver.disconnect(); + select(this.canvas).on(".zoom", null); + this.canvas.removeEventListener("wheel", this.onWheel); + this.canvas.removeEventListener("pointermove", this.onPointerMove); + this.canvas.removeEventListener("pointerdown", this.onPointerDown); + this.canvas.removeEventListener("pointerup", this.onPointerUp); + this.canvas.removeEventListener("pointerleave", this.onPointerLeave); + this.canvas.style.cursor = ""; + } + + // ── Event handlers (state mutation only — never draw) ── + + /** Wheel: accumulate the zoom target; the camera GLIDES toward it in the + * tick loop (cursor-anchored). Sensitivity formula is d3-zoom's own. */ + private onWheel = (e: WheelEvent) => { + e.preventDefault(); + const delta = -e.deltaY * (e.deltaMode === 1 ? 0.05 : e.deltaMode ? 1 : 0.002); + const { x, y } = this.eventPos(e); + const current = this.zoomTarget?.k ?? this.transform.k; + // Must stay inside the zoomBehavior's scaleExtent [0.02, 40]. + const k = Math.min(Math.max(current * Math.exp(delta), 0.02), 40); + this.zoomTarget = { k, x, y }; + // Fresh easing clock so the glide starts gentle, never half-done. + this.lastTickAt = performance.now(); + this.markDirty(); + }; + + private onPointerMove = (e: PointerEvent) => { + const pos = this.eventPos(e); + if (this.dragId) { + const n = this.simById.get(this.dragId); + if (n) { + const moved = + this.dragOrigin !== null && + Math.hypot(pos.x - this.dragOrigin.x, pos.y - this.dragOrigin.y) > CLICK_SLOP; + if (moved) { + // First real movement: reheat and glue the node to the cursor. + if (this.commitTimer) { + clearTimeout(this.commitTimer); + this.commitTimer = null; + } + this.sim?.alphaTarget(DRAG_ALPHA_TARGET); + this.simActive = true; + this.markDirty(); + } + const { x: tx, y: ty, k } = this.transform; + n.fx = (pos.x - tx) / k; + n.fy = (pos.y - ty) / k; + } + return; + } + const node = this.pickAt(pos.x, pos.y); + if ((node?.id ?? null) !== this.hoverId) this.setHover(node?.id ?? null); + this.canvas.style.cursor = node ? "pointer" : ""; + }; + + private onPointerDown = (e: PointerEvent) => { + const pos = this.eventPos(e); + const node = this.pickAt(pos.x, pos.y); + if (node && this.sim) { + // Node press: pin the node (at ITS OWN position — the cursor + // may be inside the pick pad, off-center). preventDefault kills + // the compatibility mousedown, so d3-zoom never starts a pan. + // Physics only reheats once the pointer exceeds CLICK_SLOP — + // a plain click stays perfectly still. + e.preventDefault(); + try { + this.canvas.setPointerCapture(e.pointerId); + } catch { + /* capture is best-effort */ + } + this.dragId = node.id; + this.dragOrigin = pos; + const n = this.simById.get(node.id); + if (n) { + n.fx = n.x; + n.fy = n.y; + } + this.canvas.style.cursor = "grabbing"; + this.markDirty(); + return; + } + this.downPos = pos; + this.zoomTarget = null; // a pan takes over — cancel any zoom glide + }; + + private onPointerUp = (e: PointerEvent) => { + // Node press ending: click (no slop) → select; drag → release-and-relax. + if (this.dragId) { + const id = this.dragId; + this.dragId = null; + this.dragOrigin = null; + try { + this.canvas.releasePointerCapture(e.pointerId); + } catch { + /* already released */ + } + this.canvas.style.cursor = ""; + const n = this.simById.get(id); + if (n) { + n.fx = null; + n.fy = null; + } + if (this.simActive) { + // Real drag: the sim anneals to the new rest state; advanceSim + // detects the settle and schedules the commit. + this.sim?.alphaTarget(0); + this.markDirty(); + } else { + // Never heated: a plain click on the node. + this.callbacks.onSelect(id, this.eventPos(e)); + } + return; + } + if (!this.downPos) return; + const pos = this.eventPos(e); + const moved = Math.hypot(pos.x - this.downPos.x, pos.y - this.downPos.y); + this.downPos = null; + if (moved > CLICK_SLOP) return; // that was a drag (pan), not a click + const node = this.pickAt(pos.x, pos.y); + if (node) this.callbacks.onSelect(node.id, pos); + else this.callbacks.onStageClick(); + }; + + private onPointerLeave = () => { + if (this.dragId) return; // pointer capture keeps the drag alive + this.downPos = null; + this.setHover(null); + }; + + // ── Internals ── + + private eventPos(e: { clientX: number; clientY: number }): Pt { + const rect = this.canvas.getBoundingClientRect(); + return { x: e.clientX - rect.left, y: e.clientY - rect.top }; + } + + private pickAt(px: number, py: number): CanvasNode | null { + if (!this.data || !this.settings) return null; + // Screen positions, filtered to visible nodes. + const { x: tx, y: ty, k } = this.transform; + // Discs scale with the camera now: their screen radius is + // nodeSize × k/kFit; the halo pad stays a constant screen pad. + const maxDist = this.settings.nodeSize * (k / this.kFit) + PICKING_PAD; + const projected: { id: string; x: number; y: number }[] = []; + for (const n of this.data.nodes) { + if (this.visible && !this.visible.has(n.id)) continue; + projected.push({ id: n.id, x: n.x * k + tx, y: n.y * k + ty }); + } + const hit = pickNearest(projected, px, py, maxDist); + if (!hit) return null; + return this.nodeById.get(hit.id) ?? null; + } + + private setHover(nodeId: string | null): void { + if (nodeId === this.hoverId && this.hoverTarget === (nodeId ? 1 : 0)) return; + this.hoverId = nodeId; + // Hover-out deliberately KEEPS hoverNode: the glow must fade out + // from the last hovered node, not vanish with the id. + if (nodeId) this.hoverNode = this.nodeById.get(nodeId) ?? null; + // Rebase the animation from the CURRENT fade: rapid hover toggles + // glide instead of snapping (ported from the sigma-era behavior). + this.hoverFrom = this.hoverFade; + this.hoverTarget = nodeId ? 1 : 0; + this.hoverStart = performance.now(); + this.animating = true; + this.markDirty(); + } + + private markDirty(): void { + this.dirty = true; + if (this.frameId === null && !this.inTick) { + this.frameId = requestAnimationFrame(() => { + this.frameId = null; + this.tick(); + }); + } + } + + private tick(): void { + this.inTick = true; + try { + const now = performance.now(); + // Clamp the dt: a long idle gap must not teleport the ease forward. + const dt = Math.min(Math.max(now - this.lastTickAt, 8), 100); + this.lastTickAt = now; + const hoverAnim = this.advanceHover(); + const zoomAnim = this.advanceZoom(dt); + const simAnim = this.advanceSim(); + this.draw(); + // Keep frames coming ONLY while an animation is in motion (hover + // fade, zoom glide, or live physics); an idle graph schedules no + // rAF at all. + this.animating = hoverAnim || zoomAnim || simAnim; + } finally { + this.inTick = false; + } + if (this.animating) { + this.frameId = requestAnimationFrame(() => { + this.frameId = null; + this.tick(); + }); + } + } + + /** One live-sim step; returns true while physics is still in motion. + * Runs only while simActive (drag reheat or post-release anneal). */ + private advanceSim(): boolean { + if (!this.sim || !this.simActive) return false; + this.sim.tick(); + // Movement check: did any node actually move (a click-pin that + // never reheated leaves everything exactly at rest)? + for (const n of this.sim.nodes()) { + if (n.vx || n.vy) { + this.simMoved = true; + break; + } + } + this.geoVersion++; // positions changed → geometry rebuild in draw() + // Settled (drag released long ago, alpha annealed away)? + if (!this.dragId && this.sim.alpha() < SIM_SETTLE_ALPHA) { + this.simActive = false; + this.scheduleCommit(); + return false; + } + return true; + } + + /** Debounced rest-state commit. A new drag or setData cancels it. */ + private scheduleCommit(): void { + if (this.commitTimer) clearTimeout(this.commitTimer); + this.commitTimer = setTimeout(() => { + this.commitTimer = null; + if (!this.data || !this.simMoved) return; + this.simMoved = false; + const positions: Record = {}; + for (const n of this.data.nodes) positions[n.id] = { x: n.x, y: n.y }; + this.callbacks.onCommit(positions); + }, COMMIT_DEBOUNCE_MS); + } + + /** Hover fade step; returns true while still moving. */ + private advanceHover(): boolean { + if (this.hoverFade === this.hoverTarget) return false; + const t = Math.min((performance.now() - this.hoverStart) / HOVER_FADE_MS, 1); + this.hoverFade = this.hoverFrom + (this.hoverTarget - this.hoverFrom) * t; + // Outgoing fade finished — release the glow node. + if (this.hoverTarget === 0 && this.hoverFade === 0) this.hoverNode = null; + return t < 1; + } + + /** Zoom glide step: ease k toward the wheel target, keeping the world + * point under the cursor pinned. Frame-rate independent (exponential + * approach, time constant ZOOM_EASE_MS). Returns true while gliding. */ + private advanceZoom(dt: number): boolean { + const target = this.zoomTarget; + if (!target) return false; + const t = this.transform; + const blend = 1 - Math.exp(-dt / ZOOM_EASE_MS); + let k = t.k + (target.k - t.k) * blend; + let done = false; + if (Math.abs(target.k - k) / target.k < 0.002) { + k = target.k; + done = true; + } + const scale = k / t.k; + const x = target.x - (target.x - t.x) * scale; + const y = target.y - (target.y - t.y) * scale; + if (done) this.zoomTarget = null; + // Applied through d3-zoom so constraints + the zoom event (markDirty, + // onCameraMove) fire exactly as with any other camera change. + select(this.canvas).call( + this.zoomBehavior.transform, + zoomIdentity.translate(x, y).scale(k), + ); + return !done; + } + + /** Resize the backing store to the CSS box × devicePixelRatio; all + * drawing then happens in CSS pixels through the DPR transform. */ + private syncSize(): void { + const w = this.canvas.clientWidth; + const h = this.canvas.clientHeight; + const dpr = window.devicePixelRatio || 1; + const bw = Math.max(1, Math.round(w * dpr)); + const bh = Math.max(1, Math.round(h * dpr)); + if (this.canvas.width !== bw || this.canvas.height !== bh) { + this.canvas.width = bw; + this.canvas.height = bh; + } + this.ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + } + + /** Derive ALL world geometry from data + visibility + settings + kFit. + * Runs only when geoVersion moves — never per frame, never per camera + * change. Zoom/pan pay zero geometry math, only rasterization. */ + private buildGeometry(): void { + this.builtVersion = this.geoVersion; + const data = this.data; + const settings = this.settings; + if (!data || !settings) return; + + // Screen px at fit zoom → world units. + const s = 1 / this.kFit; + const nodeR = settings.nodeSize * s; + this.world = { + nodeR, + glowOuter: nodeR * GLOW_SCALE, + labelSize: settings.labelSize * s, + labelGap: LABEL_GAP * s, + }; + this.worldLabelFont = `${LABEL_WEIGHT} ${settings.labelSize * s}px ${data.fontFamily}`; + const arrowLen = settings.arrowLength * s; + const visible = this.visible; + const isVisible = (id: string) => !visible || visible.has(id); + + // Node discs: one Path2D, one fill() per frame. + const nodesPath = new Path2D(); + for (const n of data.nodes) { + if (!isVisible(n.id)) continue; + nodesPath.moveTo(n.x + nodeR, n.y); + nodesPath.arc(n.x, n.y, nodeR, 0, Math.PI * 2); + } + this.nodesPath = nodesPath; + + // Edge shafts + arrowheads: tip lands on the target disc edge. + const shafts = new Path2D(); + const heads = new Path2D(); + for (const e of data.edges) { + if (!isVisible(e.from) || !isVisible(e.to)) continue; + const a = this.nodeById.get(e.from); + const b = this.nodeById.get(e.to); + if (!a || !b) continue; + const g = arrowGeometry(a, b, nodeR, nodeR, arrowLen); + if (!g) continue; + shafts.moveTo(g.shaftFrom.x, g.shaftFrom.y); + shafts.lineTo(g.shaftTo.x, g.shaftTo.y); + heads.moveTo(g.tip.x, g.tip.y); + heads.lineTo(g.left.x, g.left.y); + heads.lineTo(g.right.x, g.right.y); + heads.closePath(); + } + this.shaftsPath = shafts; + this.headsPath = heads; + + // Cluster regions are NOT built here: they rasterize into their own + // low-res bitmap (see buildRegions), on their own rebuild schedule + // (live-sim frames skip 2 of 3 rebuilds and blit the last bitmap). + + // Label wrapping: measureText once per node per geometry build, + // never per frame. Wrap width keeps the bigger-label/bigger-wrap + // behavior from the sigma era. + this.labelLines = new Map(); + const ctx = this.ctx; + const prevFont = ctx.font; + ctx.font = this.worldLabelFont; + const maxW = LABEL_MAX_WIDTH * (settings.labelSize / LABEL_SIZE) * s; + for (const n of data.nodes) { + if (isVisible(n.id)) this.labelLines.set(n.id, wrapLabel(ctx, n.label, maxW)); + } + ctx.font = prevFont; + } + + /** (Re)create the live sim from data.nodes at their current + * positions + the current physics tuning. Called by setData and + * setPhysicsTuning. */ + private buildSim(): void { + const data = this.data; + if (!data || data.nodes.length === 0) { + this.sim = null; + this.simById = new Map(); + return; + } + const t = this.physicsTuning; + this.sim = createGraphSimulation(data.nodes, data.edges, { + gravity: { x: this.simCenter.x, y: this.simCenter.y, strength: t.gravity }, + alphaDecay: LIVE_ALPHA_DECAY, + velocityDecay: t.velocityDecay, + chargeStrength: t.chargeStrength, + linkStrength: t.linkStrength, + linkDistance: t.linkDistance, + clusterPull: + data.clusters.length > 0 ? { clusters: data.clusters, strength: t.clusterPull } : undefined, + depthAnchor: + t.depthMode === "off" + ? undefined + : { + mode: t.depthMode, + strength: t.depthStrength, + spacing: t.depthSpacing, + baseRadius: t.depthBaseRadius, + center: this.simCenter, + }, + }); + this.simById = new Map(data.nodes.map((n) => [n.id, n as LayoutNode])); + } + + /** Rasterize cluster regions into the low-res world-space bitmap. + * Runs on data/visibility/settings changes at rest, and every + * REGION_EVERY-th frame while the live sim moves nodes (the in-between + * frames blit the previous bitmap — one sim step of staleness at + * 120fps is invisible). Positions come straight from data.nodes. */ + private buildRegions(): void { + const data = this.data; + if (!data || data.clusters.length === 0) { + this.regionCanvas = null; + this.regionRect = null; + this.regionMeta = []; + return; + } + const visible = this.visible; + const isVisible = (id: string): boolean => !visible || visible.has(id); + + const tuning = this.regionTuning; + const s = 1 / this.kFit; + + // World rect over VISIBLE cluster members (regions cover members + // and their intra-cluster edges — nothing else). + let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity; + for (const c of data.clusters) { + for (const id of c.members) { + if (!isVisible(id)) continue; + const n = this.nodeById.get(id); + if (!n) continue; + if (n.x < minX) minX = n.x; + if (n.x > maxX) maxX = n.x; + if (n.y < minY) minY = n.y; + if (n.y > maxY) maxY = n.y; + } + } + if (minX > maxX) { + this.regionCanvas = null; + this.regionRect = null; + this.regionMeta = []; + return; + } + + // Bitmap sizing: longest world span maps to `bitmap` px. The rect + // is padded by the stroke half-width (dilation) plus the blur + // reach so nothing clips at the bitmap edge. + // + // SCREEN-ANCHORED RADII (user report 2026-09-14: a tight search + // collapsed the blur). The stroke width is anchored to kFit (screen + // px at fit zoom — the camera persists, searches don't refit), but + // the blur radius was anchored to `cell`, which shrinks with the + // FILTERED span: close-together matches → tiny cell → blur reach + // (blur·cell·kFit) collapsed on screen while the stroke stayed put. + // The blur now uses the stroke's anchoring: its world reach is + // blur · cellRef, cellRef being the FULL-graph cell (worldBBox — + // the same bbox kFit comes from). Span-independent, and with + // nothing filtered cell == cellRef, so the look is unchanged. + const dilationW = tuning.dilation * s; + const bbox = this.worldBBox; + const cellRef = bbox + ? Math.max(bbox.maxX - bbox.minX, bbox.maxY - bbox.minY) / tuning.bitmap + : 0; + const blurW = tuning.blur * cellRef; + const padWorld = dilationW + blurW * 2; + // Cell floor: a single visible node (span 0) zeroed the cell and + // blew the bitmap dims to Infinity (getImageData TypeError). The + // floor is the padded rect's own reach, so dims stay finite (long + // side ≤ bitmap · (1 + span/2·pad)) however tight the search. + const spanNow = Math.max(maxX - minX, maxY - minY); + const cell = Math.max(spanNow, padWorld * 2) / tuning.bitmap; + minX -= padWorld; + minY -= padWorld; + maxX += padWorld; + maxY += padWorld; + const spanX = maxX - minX; + const spanY = maxY - minY; + const bw = Math.max(1, Math.ceil(spanX / cell)); + const bh = Math.max(1, Math.ceil(spanY / cell)); + let rc = this.regionCanvas; + if (!rc) { + rc = document.createElement("canvas"); + this.regionCanvas = rc; + } + if (rc.width !== bw || rc.height !== bh) { + rc.width = bw; + rc.height = bh; // also clears + } + // willReadFrequently: the blur step getImageData/putImageData every + // rebuild — Chrome keeps this canvas on the CPU instead of GPU- + // shuttling it. + const rctx = rc.getContext("2d", { willReadFrequently: true }); + if (!rctx) return; + rctx.clearRect(0, 0, bw, bh); + rctx.lineCap = "round"; + rctx.lineWidth = (dilationW * 2) / cell; + + // Which cluster is a node in (for intra-cluster edges)? + const clusterOf = new Map(); + data.clusters.forEach((c, i) => { + for (const id of c.members) clusterOf.set(id, i); + }); + + // The region IS the dilated skeleton: intra-cluster edges as + // round-cap strokes (Minkowski dilation). NO blanket per-node discs + // — they double-counted where nodes sit on edges (user-identified + // hot spots, when clusters were edge-based). + this.regionMeta = []; + data.clusters.forEach((cluster, ci) => { + const hue = clusterHue(cluster.id); + // Full-opacity strokes: alpha is applied ONCE at the very end + // (dilate at 1 → blur → scale by edgeAlpha). Baking per-stroke + // alpha in here made source-over stacking ACCUMULATE it — an + // orphan disc overlapping anything read as a darker double- + // counted spot (user report 2026-09-14). At alpha 1 the alpha + // channel is a pure coverage indicator: overlaps of any count + // contribute exactly 1.0. + rctx.strokeStyle = `hsla(${hue}, 65%, 60%, 1)`; + rctx.beginPath(); + let members = 0; + let sumX = 0; + let sumY = 0; + // Endpoints actually covered by a stroke's round caps. + const touched = new Set(); + for (const e of data.edges) { + if (clusterOf.get(e.from) !== ci) continue; + if (clusterOf.get(e.to) !== ci) continue; + // Search filter: stroke only fully-visible edges. Invisible-node + // segments still stroked into the (visible-only) bitmap cut + // diagonal hue streaks across other clusters' regions (user- + // reported 2026-09-14) — rect, discs, and anchors already filter. + if (!isVisible(e.from) || !isVisible(e.to)) continue; + const a = this.nodeById.get(e.from); + const b = this.nodeById.get(e.to); + if (!a || !b) continue; + rctx.moveTo((a.x - minX) / cell, (a.y - minY) / cell); + rctx.lineTo((b.x - minX) / cell, (b.y - minY) / cell); + touched.add(e.from); + touched.add(e.to); + } + rctx.stroke(); + + // Member discs for the SEMANTIC era (DECIDED 2026-09-13): clusters + // are now topics by meaning, and many members (orphans, isolated + // micro-topics) share NO intra-cluster edge with anyone — edge + // strokes alone left them completely uncolored. A disc (same + // radius as a round cap = dilation) covers exactly the members no + // stroke reached: every member is painted by its edges' caps OR + // one disc, never both — the no-double-counting rule survives. + rctx.fillStyle = `hsla(${hue}, 65%, 60%, 1)`; + const discR = dilationW / cell; + for (const id of cluster.members) { + if (touched.has(id) || !isVisible(id)) continue; + const n = this.nodeById.get(id); + if (!n) continue; + rctx.beginPath(); + rctx.arc((n.x - minX) / cell, (n.y - minY) / cell, discR, 0, Math.PI * 2); + rctx.fill(); + } + + // Label anchor: member centroid (world coords). + for (const id of cluster.members) { + if (!isVisible(id)) continue; + const n = this.nodeById.get(id); + if (!n) continue; + members++; + sumX += n.x; + sumY += n.y; + } + if (members > 0) { + this.regionMeta.push({ label: cluster.label || null, cx: sumX / members, cy: sumY / members, hue }); + } + }); + + // THE blur + alpha (in that order, by design — user-decided + // 2026-09-14): 3 box-blur rounds on the premultiplied bitmap, + // THEN scale the whole field by edgeAlpha once. Bilinear + // upscaling alone is no blur at all (tent ramps — the JPEG look); + // this makes the field genuinely smooth before the blit. The + // radius is blurW in bitmap px (screen-anchored — see sizing + // above), an integer because boxBlurRound indexes with the loop + // counter. Radius 0 (blur off) still runs this path — boxBlurRound + // copies — so the end-alpha applies uniformly with or without + // blur. + { + const blurPx = tuning.blur > 0 ? Math.max(1, Math.round(blurW / cell)) : 0; + const img = rctx.getImageData(0, 0, bw, bh); + const n = bw * bh * 4; + const a = new Float32Array(n); + for (let i = 0; i < n; i += 4) { + const alpha = img.data[i + 3]! / 255; + a[i] = img.data[i]! * alpha; + a[i + 1] = img.data[i + 1]! * alpha; + a[i + 2] = img.data[i + 2]! * alpha; + a[i + 3] = img.data[i + 3]!; + } + const b = new Float32Array(n); + const tmp = new Float32Array(n); // scratch inside each round only + boxBlurRound(a, tmp, b, bw, bh, blurPx); // a → b + boxBlurRound(b, tmp, a, bw, bh, blurPx); // b → a + boxBlurRound(a, tmp, b, bw, bh, blurPx); // a → b (result) + // Unpremultiply back into pixels; scale the alpha by edgeAlpha + // HERE — the one, final application. Result: constant interior + // alpha everywhere covered, tapering only across the blur edges. + for (let i = 0; i < n; i += 4) { + const alpha = b[i + 3]!; + if (alpha <= 0) { + img.data[i] = 0; + img.data[i + 1] = 0; + img.data[i + 2] = 0; + img.data[i + 3] = 0; + } else { + // Un-premultiply: pm = c * a/255, so c = pm / a * 255. + img.data[i] = Math.round((b[i]! / alpha) * 255); + img.data[i + 1] = Math.round((b[i + 1]! / alpha) * 255); + img.data[i + 2] = Math.round((b[i + 2]! / alpha) * 255); + img.data[i + 3] = Math.round(alpha * tuning.edgeAlpha); + } + } + rctx.putImageData(img, 0, 0); + } + this.regionRect = { minX, minY, w: spanX, h: spanY }; + } + + /** THE draw function: every layer, one frame, one camera. All drawing + * happens in WORLD coordinates under the camera transform; nothing is + * recomputed per frame except layer styling. */ + private draw(): void { + this.dirty = false; + const data = this.data; + const settings = this.settings; + if (!data || !settings) return; + const w = this.canvas.clientWidth; + const h = this.canvas.clientHeight; + const dpr = window.devicePixelRatio || 1; + const ctx = this.ctx; + + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + ctx.clearRect(0, 0, w, h); + + this.frameCount++; + // Regions: rebuild on stale geometry at rest; during the live sim, + // every REGION_EVERY-th frame (others blit the last bitmap). + if ( + this.regionBuiltVersion !== this.geoVersion && + (!this.simActive || this.frameCount % REGION_EVERY === 0) + ) { + this.buildRegions(); + this.regionBuiltVersion = this.geoVersion; + } + if (this.builtVersion !== this.geoVersion) this.buildGeometry(); + if (!this.nodesPath) return; + + const { x: tx, y: ty, k } = this.transform; + // THE camera: one transform, then everything is world coordinates. + ctx.setTransform(dpr * k, 0, 0, dpr * k, dpr * tx, dpr * ty); + + const ratio = zoomRatioOf(this.kFit, k); + const nodeAlpha = labelAlpha(ratio, settings.labelZoom); + const regionAlpha = 1 - nodeAlpha; + const { nodeR, glowOuter } = this.world; + + // 1. Cluster regions: one tiny world-space bitmap blit. The upscale + // blur is the region softness; adjacent cluster tints blend + // source-over where they overlap (decided: multiply is invisible + // on the dark background). + if (this.regionCanvas && this.regionRect) { + const { minX, minY, w, h } = this.regionRect; + ctx.imageSmoothingEnabled = true; + ctx.drawImage(this.regionCanvas, minX, minY, w, h); + } + + // 3. Edges: shafts + arrowheads from cached paths; width constant on + // screen (÷ k), geometry in world units (scales with the graph). + if (settings.edgeWidth > 0) { + ctx.strokeStyle = data.colors.edge; + ctx.fillStyle = data.colors.edge; + ctx.lineWidth = settings.edgeWidth / k; + ctx.lineCap = "round"; + ctx.stroke(this.shaftsPath); + ctx.fill(this.headsPath); + } + + // 4. Nodes: one cached Path2D, one fill. + ctx.fillStyle = data.colors.node; + ctx.fill(this.nodesPath); + + // 5. Cluster region labels — inverse-gated on THE zoom control. + // Viewer-relative by design (UI, not graph content): drawn at the + // world centroid with a font of REGION_LABEL_SIZE / k, which is a + // constant 18px on screen at any zoom. + if (settings.clusterLabels && regionAlpha > 0.01) { + ctx.save(); + ctx.globalAlpha = regionAlpha; + ctx.font = `${LABEL_WEIGHT} ${REGION_LABEL_SIZE / k}px ${data.fontFamily}`; + ctx.textAlign = "center"; + ctx.textBaseline = "middle"; + for (const rm of this.regionMeta) { + if (!rm.label) continue; + ctx.fillStyle = `hsla(${rm.hue}, 55%, 78%, 0.85)`; + ctx.fillText(rm.label, rm.cx, rm.cy); + } + ctx.restore(); + } + + // 6. Node labels — one global gate, faded together. World-scaled + // with the graph (they grow as you zoom in, Obsidian-style). + if (nodeAlpha > 0.01) { + ctx.save(); + ctx.globalAlpha = nodeAlpha; + ctx.font = this.worldLabelFont; + for (const [id, lines] of this.labelLines) { + const n = this.nodeById.get(id); + if (!n) continue; + drawNameLines(ctx, n.x, n.y, lines, this.world, data.colors.label); + } + ctx.restore(); + } + + // 7. Hover glow (topmost): halo in the node's color + the brightened + // name, both on the SAME fade factor. World units like everything + // else. Shows even when labels are zoomed out (hover is its own + // affordance, not a label). + if (this.hoverNode && this.hoverFade > 0.02) { + const n = this.visible && !this.visible.has(this.hoverNode.id) + ? null // filtered out by search — no glow on hidden nodes + : this.hoverNode; + if (n) { + const glow = ctx.createRadialGradient(n.x, n.y, nodeR, n.x, n.y, glowOuter); + glow.addColorStop(0, rgbaFromHex(data.colors.node, GLOW_ALPHA * this.hoverFade)); + glow.addColorStop(1, rgbaFromHex(data.colors.node, 0)); + ctx.fillStyle = glow; + ctx.beginPath(); + ctx.arc(n.x, n.y, glowOuter, 0, Math.PI * 2); + ctx.arc(n.x, n.y, nodeR, 0, Math.PI * 2, true); + ctx.fill("evenodd"); + ctx.save(); + ctx.globalAlpha = this.hoverFade; + ctx.font = this.worldLabelFont; + drawNameLines(ctx, n.x, n.y, this.labelLines.get(n.id) ?? [], this.world, data.colors.hoverText); + ctx.restore(); + } + } + } +} diff --git a/components/graph-view.tsx b/components/graph-view.tsx index 017664e..9ce650e 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -3,74 +3,113 @@ /** * The home knowledge-graph view (W4; vision §1 "Rendering (DECIDED)"). * - * sigma.js v3 + graphology + forceAtlas2, run-then-freeze: ONE synchronous - * forceAtlas2.assign (100 iterations, inferSettings) inside a useEffect - * writes x/y. No layout animation ever starts; the ONLY animation is the - * short hover fade (enter/leave transitions), strictly bounded in time - * and cancelled on unmount — idle CPU stays near zero. Physics is - * placement only: pan/zoom, no dragging, no editing. + * ARCHITECTURE (2026-09-13, decided after the research pass — + * docs/research/graph-rendering.md, Option B): sigma.js is GONE. The + * renderer is now a hand-rolled single Canvas 2D (./graph-canvas.ts) — + * a DUMB DRAW FUNCTION over plain data with ONE requestAnimationFrame + * loop and a dirty flag. Everything that made the sigma era fragile is + * structurally impossible now: hulls cannot lag nodes (one loop, one + * camera, one frame), settings changes cannot shift anything (the + * camera lives in the renderer and no rebuild ever happens), and the + * per-camera-event full-canvas repaints (GPU spikes + coil whine) are + * gone (at most one draw per frame, and NO frame at all while idle). * - * Labels render the node `name` — the very short, glanceable label the - * extractor coins (a glossary-headword-style handle); the full-precision - * `statement` lives on the detail card. Custom canvas drawers replace - * sigma's defaults: - * - defaultDrawNodeLabel: plain wrapped text, centered below the node, - * word-wrapped at a max line width — no truncation, no one-line sprawl. - * - defaultDrawNodeHover: NO background box (sigma's default is a - * hardcoded white one). Instead the hovered node gets a soft glow - * halo in the node's color at a constant radius, flush with the - * disc edge, and its name brightens — the "this is clickable, - * click for the full fact" affordance. Entering/leaving hover fades - * ONE transition factor shared by halo and text alike, so the - * halo's opacity and the text's muted→bright cross-fade stay in - * sync (no growth, no pulse; interpolating from the current value - * means rapid hover toggles glide instead of snapping). Hover - * always shows the name, even when labels are zoomed out. - * Node hit-testing is the disc PLUS a pad: sigma picks nodes from a - * half-resolution framebuffer (no v3 setting enlarges hitboxes), and the - * glow halo is bigger than the stock hitbox — touching the halo - * de-hovered the node, which oscillated (the hover flicker). A - * NodeCircleProgram subclass (PICKING_VERTEX_SHADER below) adds - * PICKING_PAD_PX of pickable radius under #ifdef PICKING_MODE only, per - * the maintainer-endorsed v3 pattern (#1493); the visible disc stays - * pixel-identical, and the whole halo becomes hoverable/clickable. - * Labels are zoom-gated GLOBALLY with a fade: all names appear at once - * when the camera is at fit zoom or closer (ratio <= 1), then fade out - * together across a short zoom span before label rendering switches off — - * no per-node size gating, so labels never pop in one by one. (The Obsidian - * text-fade feel, applied to the whole graph.) + * Layout fallback (ledger absent): ONE synchronous incrementalLayout + * pass (d3-force, core/layout.ts) inside a useEffect writes world-space + * x/y, then the positions are handed to the renderer as data. No layout + * animation ever starts; the only animation is the + * short hover fade, strictly bounded in time — idle CPU is zero (the + * loop schedules no rAF until something actually changes). * - * The search panel (top-right overlay): a search box matching name or - * statement text — the transitional node-state classes are gone (2026-09-08; - * nodes render uniformly until the state model ideation lands), so there - * are no state legend rows or toggles anymore. Filtering flows through - * sigma's official - * nodeReducer/edgeReducer settings (`hidden`), so rendering, hover, - * picking, and labels all respect it; edges drop when either endpoint is - * hidden, and an open detail card closes if its node is filtered out. - * Clicking a node opens the detail card anchored just below the node's - * viewport position (flipping above near the bottom edge); any camera move - * (pan/zoom) dismisses it. The `requires` edges render as arrows pointing - * prerequisite → dependent. + * The component's job is everything AROUND drawing: fetch + refetch the + * snapshot, run the layout, resolve theme colors to hex, filter via the + * search box, own the settings panel / detail card / import button / + * pipeline chips, and FEED the renderer (data, settings, visibility). + * The renderer's contract stays presentation-only — which is exactly + * what the upcoming playground and cluster-first layout work needs: a + * renderer that draws whatever data it is given, never questioning it. + * + * Visual semantics, all preserved verbatim from the sigma era: + * - Node labels (the short extractor-coined names) wrap under the + * node; one GLOBAL zoom gate fades them together (no per-node pop). + * - Hover: a soft glow halo in the node's own color at a constant + * radius, opacity cross-faded ~180ms with the name brightening on + * the same factor; the whole halo is hit-testable (picking pad). + * - Cluster hulls (vision §1): soft layered translucent fills + a + * dashed boundary per cluster, stable hue from the cluster id; the + * region label at the centroid fades in over the exact zoom span + * node labels fade out (THE single zoom control) — and is + * toggleable in the settings panel (hulls always draw). + * - The persisted labelZoom setting keeps sigma's camera-ratio + * convention (1 = fit zoom, SMALLER = zoomed in): labels until + * zoom 0.5x means labels appear once zoomed in past 2x fit. + * - Click a node -> the detail card anchored just below it (flipping + * above near the bottom edge); any camera move dismisses it. + * Clicking the empty stage closes it. Search filtering hides nodes + * (and their edges) from rendering, picking, hover, AND labels; + * an open card closes if its node is filtered out. + * + * Rendering settings (2026-09-13): the gear in the search panel reveals + * five sliders (node size, edge width, arrowhead length, label size, the + * zoom threshold) plus the cluster-labels checkbox. SERVER-persisted + * (data/render-settings.json via /api/graph/render-settings): loaded + * once after mount, saved debounced on every user change — one source + * of truth shared by every browser. The label-zoom slider is log-scaled + * (its useful band 0.25–0.5 sits mid-slider, not crammed at the left) + * and displays normalized units where the default 0.5 reads as "1×". + * Settings apply by just redrawing — no rebuild, no re-layout, and the + * camera never moves. */ -import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; +import { useDeferredValue, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import Link from "next/link"; -import { Loader2Icon, SearchIcon, WaypointsIcon } from "lucide-react"; -import Graph from "graphology"; -import forceAtlas2 from "graphology-layout-forceatlas2"; -import type { NodeHoverDrawingFunction, NodeLabelDrawingFunction } from "sigma/rendering"; -import { seededRandom, type GraphNode, type GraphSnapshot } from "@/src/core"; +import { FileTextIcon, Loader2Icon, SearchIcon, SettingsIcon, UploadIcon, WaypointsIcon } from "lucide-react"; +import { incrementalLayout, type GraphNode, type GraphSnapshot } from "@/src/core"; +import { GraphCanvasRenderer, type GraphRenderSettings } from "./graph-canvas.ts"; +import { + ARROW_LENGTH_RANGE, + CHARGE_RANGE, + CLUSTER_PULL_RANGE, + DEPTH_BASE_RADIUS_RANGE, + DEPTH_SPACING_RANGE, + DEPTH_STRENGTH_RANGE, + DEFAULT_RENDER_SETTINGS, + EDGE_WIDTH_RANGE, + GRAVITY_RANGE, + LABEL_SIZE_RANGE, + LABEL_ZOOM_MAX, + LABEL_ZOOM_MIN, + LINK_DISTANCE_RANGE, + LINK_STRENGTH_RANGE, + NODE_SIZE_RANGE, + REGION_BITMAP_OPTIONS, + REGION_BLUR_OPTIONS, + REGION_DILATION_RANGE, + REGION_EDGE_ALPHA_RANGE, + VELOCITY_DECAY_RANGE, + fromOffSliderPosition, + fromSliderPosition, + labelZoomDisplay, + sanitizeRenderSettings, + toOffSliderPosition, + toSliderPosition, +} from "@/src/core"; -/** The uniform node look (no state classes — WebGL wants hex, not oklch): - * the accent color + the old "known" size, until the state model lands. */ +/** The uniform node look (no state classes — canvas wants hex, not oklch): + * the accent color, until the state model lands. */ const NODE_COLOR_TOKEN = "--primary"; -const NODE_SIZE = 8; + +// ---- Render settings (the settings-panel controls) ------------------------ +// +// The shape, defaults, sanitizer, and slider mappings live in +// src/core/render-settings.ts (shared with the server store + tests). + +type RenderSettings = GraphRenderSettings; /** * Pure filter predicate for the search panel: a node is visible when the * query is empty or matches its name or statement (case-insensitive). - * Exported for tests; the sigma reducers just consume its verdict. + * Exported for tests; the visibility feed just consumes its verdict. */ export function isNodeVisible(node: Pick, query: string): boolean { const q = query.trim().toLowerCase(); @@ -78,225 +117,7 @@ export function isNodeVisible(node: Pick, query return node.name.toLowerCase().includes(q) || node.statement.toLowerCase().includes(q); } -// ---- Label drawing (canvas) --------------------------------------------- -// -// sigma's defaults draw the label on ONE line to the right of the node with -// a white box on hover. We replace both: names wrap under the node at a max -// line width, muted at rest; on hover the node pulses a soft glow in its -// own state color and the name brightens (no background box). - -const LABEL_MAX_WIDTH = 170; // px — wraps long names instead of a wide strip -const LABEL_GAP = 6; // px between node edge and first label line -const LABEL_LINE_HEIGHT = 1.3; -/** Camera ratio at/below which ALL labels appear at once (1 = fit zoom). */ -const LABELS_CAMERA_RATIO = 1; -/** Zoom span (camera-ratio units) over which labels fade out together. */ -const LABELS_FADE_SPAN = 0.5; - -// Hover glow: a soft halo ring in the node's own color at a CONSTANT -// radius, starting flush at the disc edge (no gap — a gap reads as a -// false outline/border between node and glow). Only the halo's OPACITY -// animates (hover enter/leave); it never grows and never pulses. -const GLOW_PAD = 9; // px the halo extends beyond the node disc (constant) -const GLOW_ALPHA = 0.45; // halo opacity at the disc edge, fading outward -const HOVER_FADE_MS = 180; // hover halo opacity fade length - -// ---- Padded picking (WebGL) ---------------------------------------------- -// -// sigma picks nodes by reading a single pixel from a picking framebuffer -// rendered at HALF the CSS resolution: small nodes get tiny, ragged hit -// edges, and our glow halo (GLOW_PAD beyond the disc) is NOT pickable, so -// moving the cursor from the disc onto its own halo de-hovered it — an -// oscillation loop (the hover flicker). sigma v3 has no setting for hit -// padding; the maintainer-endorsed pattern (#1493, fixed officially in v4 -// as `nodePickingPadding`) is a node program whose vertex shader enlarges -// the disc ONLY in the picking shader pass. - -/** Extra radius (px) added to each node's PICKABLE disc (picking layer - * only — the visible disc is untouched). >= GLOW_PAD so the entire hover - * halo is hit-testable. */ -const PICKING_PAD_PX = 10; - -/** - * The node-circle vertex shader, verbatim from sigma 3.0.3 - * (dist/index-fad77a13.esm.js, SHADER_SOURCE$5) with ONE change: under - * `#ifdef PICKING_MODE` (sigma compiles this source twice — visible render - * vs picking readback — so the visible pass never takes this branch) the - * size gains PICKING_PAD_PX of radius, which is the same on-screen unit as - * `data.size` (the px radius the canvas drawers use for the halo). - */ -const PICKING_VERTEX_SHADER = /* glsl */ ` -attribute vec4 a_id; -attribute vec4 a_color; -attribute vec2 a_position; -attribute float a_size; -attribute float a_angle; - -uniform mat3 u_matrix; -uniform float u_sizeRatio; -uniform float u_correctionRatio; - -varying vec4 v_color; -varying vec2 v_diffVector; -varying float v_radius; -varying float v_border; - -const float bias = 255.0 / 254.0; - -void main() { - float pickedSize = a_size; -#ifdef PICKING_MODE - // Picking pass only: enlarge the hitbox by PICKING_PAD_PX of radius. - pickedSize += ${PICKING_PAD_PX.toFixed(1)}; -#endif - float size = pickedSize * u_correctionRatio / u_sizeRatio * 4.0; - vec2 diffVector = size * vec2(cos(a_angle), sin(a_angle)); - vec2 position = a_position + diffVector; - gl_Position = vec4( - (u_matrix * vec3(position, 1)).xy, - 0, - 1 - ); - - v_diffVector = diffVector; - v_radius = size / 2.0; - -#ifdef PICKING_MODE - // For picking mode, we use the ID as the color: - v_color = a_id; -#else - // For normal mode, we use the color: - v_color = a_color; -#endif - - v_color.a *= bias; -} -`; - -/** "#rrggbb" + alpha → "rgba(...)" (tokenHex only ever emits #rrggbb). */ -function rgbaFromHex(hex: string, alpha: number): string { - const r = Number.parseInt(hex.slice(1, 3), 16); - const g = Number.parseInt(hex.slice(3, 5), 16); - const b = Number.parseInt(hex.slice(5, 7), 16); - return `rgba(${r}, ${g}, ${b}, ${alpha})`; -} - -/** Word-wrap for canvas text. Never truncates: overlong single words are - * hard-broken by character so no line ever exceeds maxWidth. */ -function wrapLabel(context: CanvasRenderingContext2D, text: string, maxWidth: number): string[] { - const lines: string[] = []; - for (const word of text.split(/\s+/).filter(Boolean)) { - const last = lines.length > 0 ? lines[lines.length - 1]! : null; - if (last !== null) { - const joined = `${last} ${word}`; - if (context.measureText(joined).width <= maxWidth) { - lines[lines.length - 1] = joined; - continue; - } - } - let chunk = ""; - for (const char of word) { - if (chunk && context.measureText(`${chunk}${char}`).width > maxWidth) { - lines.push(chunk); - chunk = char; - } else { - chunk += char; - } - } - if (chunk) lines.push(chunk); - } - return lines; -} - -/** The wrapped node name (the short label), centered below the node — used by both drawers - * with identical geometry so hover only changes the color, nothing jumps. */ -function drawName( - context: CanvasRenderingContext2D, - data: { label?: string | null; x: number; y: number; size: number }, - settings: { labelSize: number; labelWeight: string; labelFont: string }, - color: string, -): void { - if (!data.label) return; - const size = settings.labelSize; - context.save(); - context.font = `${settings.labelWeight} ${size}px ${settings.labelFont}`; - const lines = wrapLabel(context, data.label, LABEL_MAX_WIDTH); - context.fillStyle = color; - context.textAlign = "center"; - context.textBaseline = "top"; - const firstTop = data.y + data.size + LABEL_GAP; - for (let i = 0; i < lines.length; i++) { - context.fillText(lines[i]!, data.x, firstTop + i * size * LABEL_LINE_HEIGHT); - } - context.restore(); -} - -/** The hovered-state paint at a given transition factor (0 = resting, - * 1 = fully hovered): a glow halo in the node's own state color at a - * CONSTANT radius (opacity fades in/out; the halo never grows) starting - * flush at the disc edge (the evenodd punch-out keeps the disc itself - * unpainted). The name is drawn at partial alpha in the bright hover - * color — the muted resting label on the layer beneath shows through, - * so text and halo cross-fade on the SAME factor and stay in sync. */ -function paintHover( - context: CanvasRenderingContext2D, - data: { label?: string | null; x: number; y: number; size: number; color: string }, - settings: { labelSize: number; labelWeight: string; labelFont: string }, - fade: number, - hoverTextColor: string, -): void { - context.save(); - if (fade > 0.02) { - const inner = data.size; - const outer = data.size + GLOW_PAD; - const glow = context.createRadialGradient(data.x, data.y, inner, data.x, data.y, outer); - glow.addColorStop(0, rgbaFromHex(data.color, GLOW_ALPHA * fade)); - glow.addColorStop(1, rgbaFromHex(data.color, 0)); - context.fillStyle = glow; - context.beginPath(); - context.arc(data.x, data.y, outer, 0, Math.PI * 2); - context.arc(data.x, data.y, inner, 0, Math.PI * 2, true); - context.fill("evenodd"); - } - // Same fade factor as the halo: text and glow rise together. The - // bright name at partial alpha over the muted resting label beneath is - // a smooth muted→bright cross-fade rather than an instant color swap. - context.globalAlpha = fade; - drawName(context, data, settings, hoverTextColor); - context.restore(); -} - -/** Mutable hover-transition state shared between the drawers and the - * effect's enter/leave animation. */ -interface HoverState { - fade: number; - /** Set on every sigma-driven hover paint; the leave animation draws - * through it directly, since sigma stops calling drawHover once the - * mouse has left the node. */ - repaint: ((fade: number) => void) | null; -} - -/** Make the pair of canvas label drawers bound to the theme's colors. */ -function makeLabelDrawers(labelColor: string, hoverTextColor: string, hover: HoverState) { - const drawLabel: NodeLabelDrawingFunction = (context, data, settings) => { - drawName(context, data, settings, labelColor); - }; - - const drawHover: NodeHoverDrawingFunction = (context, data, settings) => { - hover.repaint = (fade) => { - context.save(); - context.setTransform(1, 0, 0, 1, 0, 0); - context.clearRect(0, 0, context.canvas.width, context.canvas.height); - context.restore(); - if (fade > 0) paintHover(context, data, settings, fade, hoverTextColor); - }; - paintHover(context, data, settings, hover.fade, hoverTextColor); - }; - - return { drawLabel, drawHover }; -} - -// ---- Color helpers (WebGL/canvas want hex, not oklch) -------------------- +// ---- Color helpers (canvas wants hex, not oklch) --------------------------- /** Resolve a CSS variable (oklch in this app) to #rrggbb via the canvas. */ function cssColorToHex(css: string): string { @@ -322,27 +143,139 @@ function tokenHex(cssVar: string): string { interface Selection { node: GraphNode; - chatIds: string[]; - /** Node position in container pixels (from graphToViewport at click time). */ + /** The sources that touched this node: chat ids (deep-linkable sessions) + * and notes-source ids ("src_…" — imported files, static entries). */ + sourceIds: string[]; + /** Node position in container pixels (at click time; camera moves + * dismiss the card, so it never needs re-anchoring). */ anchor: { x: number; y: number }; } +/** One labeled range row for the rendering-settings panel. */ +/** Compact size display for the log sliders: "2", "1.6", "0.75". */ +function sizeDisplay(v: number): string { + return `${v >= 1 ? Math.round(v * 10) / 10 : Math.round(v * 100) / 100}`; +} + +function SettingSlider({ + label, + min, + max, + step, + value, + display, + onChange, +}: { + label: string; + min: number; + max: number; + step: number; + value: number; + /** Value display override (e.g. "1.5×" for zoom) — defaults to the raw number. */ + display?: string; + onChange: (v: number) => void; +}) { + return ( + + ); +} + export function GraphView() { const containerRef = useRef(null); + const canvasRef = useRef(null); const cardRef = useRef(null); const [snapshot, setSnapshot] = useState(null); const [selected, setSelected] = useState(null); - // Search panel state: the query text. - const [query, setQuery] = useState(""); - // The sigma instance and its graphology graph live inside the mount - // effect; refs expose them to the filters effect, and `rendererEpoch` - // re-triggers it when a new renderer is constructed. - const rendererRef = useRef(null); - const graphRef = useRef(null); - const [rendererEpoch, setRendererEpoch] = useState(0); + // The latest snapshot for the renderer's click callbacks (they are + // constructed once, so they read through this ref, never a closure). + const snapshotRef = useRef(null); + snapshotRef.current = snapshot; - // The id set that survives the current search — consumed by the sigma - // reducers and the "no matches" hint. + // Search panel state: the query text + the settings section toggle. + const [query, setQuery] = useState(""); + const [settingsOpen, setSettingsOpen] = useState(false); + // Render settings: SERVER-persisted (data/render-settings.json via + // /api/graph/render-settings) — sizes AND the region/physics sections + // (user-facing since 2026-09-13; the dev-session era is over). Loaded + // once after mount; every user change saves back, debounced. Defaults + // render until the load resolves — a fraction of a second at worst. + const [settings, setSettings] = useState(DEFAULT_RENDER_SETTINGS); + // The two nested sections, unpacked for the renderer feed effects + // below (same names the old session state had). + const regionTuning = settings.regions; + const physicsTuning = settings.physics; + const settingsRef = useRef(settings); + settingsRef.current = settings; + // True once the user has moved a control this mount: the server copy + // must never clobber a (possibly newer) local change. + const userTouched = useRef(false); + // True once the initial GET settled — saves are held back until then, + // so merely mounting never PUTs defaults over the server copy. + const serverLoaded = useRef(false); + const savePending = useRef(false); + const changeSettings = (update: (cur: RenderSettings) => RenderSettings) => { + userTouched.current = true; + setSettings(update); + }; + const saveToServer = (s: RenderSettings) => { + fetch("/api/graph/render-settings", { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify(s), + }).catch((err) => console.error("[graph] settings save failed:", err)); + }; + useEffect(() => { + fetch("/api/graph/render-settings") + .then((res) => (res.ok ? (res.json() as Promise) : null)) + .then((raw) => { + // Apply only if the user hasn't already moved a slider — + // if they outran the network, their change wins. + if (raw !== null && !userTouched.current) setSettings(sanitizeRenderSettings(raw)); + }) + .catch(() => { + // server unreachable → in-memory defaults keep working + }) + .finally(() => { + serverLoaded.current = true; + // A change made during the fetch was held back — send now. + if (savePending.current) { + savePending.current = false; + saveToServer(settingsRef.current); + } + }); + }, []); + useEffect(() => { + if (!userTouched.current) return; // mount-run / server-apply: no save + if (!serverLoaded.current) { + savePending.current = true; // hold until the GET settles + return; + } + const id = setTimeout(() => saveToServer(settings), 400); + return () => clearTimeout(id); + }, [settings]); + + // The renderer instance lives across snapshot refetches and settings + // changes (its camera is never reset by either); `posRef` carries the + // frozen layout out of the layout effect. + const rendererRef = useRef(null); + const posRef = useRef | null>(null); + + // The id set that survives the current search — consumed by the + // visibility feed and the "no matches" hint. const visibleIds = useMemo(() => { const set = new Set(); if (snapshot) @@ -368,6 +301,26 @@ export function GraphView() { }; }, [refreshTick]); + // Notes import (the format-agnostic front door): file picker → POST + // /api/graph/import; the runs surface in the status chips below, and the + // graph refetches when they finish (the runs-empty transition). + const fileInputRef = useRef(null); + const [importing, setImporting] = useState(false); + const onImportFiles = (files: FileList | null) => { + if (!files || files.length === 0) return; + setImporting(true); + const form = new FormData(); + for (const file of Array.from(files)) form.append("files", file); + fetch("/api/graph/import", { method: "POST", body: form }) + .then(async (res) => { + const body = (await res.json()) as { results: { filePath: string; ok: boolean }[] }; + const failed = body.results.filter((r) => !r.ok); + if (failed.length > 0) console.error("notes import failed for:", failed); + }) + .catch((err) => console.error("import failed:", err)) + .finally(() => setImporting(false)); + }; + // Background pipeline status: poll /api/graph/status; when a run // finishes (some -> none), refetch the graph. const [pipelineRuns, setPipelineRuns] = useState<{ chatId: string; title: string; phase: string; startedAt: number }[]>([]); @@ -392,191 +345,135 @@ export function GraphView() { }; }, []); - // Build the graphology graph, lay it out once, mount sigma. + // Layout (Option A): the positions ledger (written by the server's + // incremental settle after every merge) IS the layout — the renderer + // receives it as frozen data. The local one-shot forceAtlas2 pass runs + // ONLY as a fallback when the ledger is absent (a graph never merged + // through the pipeline, e.g. tests or a fresh install). useEffect(() => { if (!snapshot || snapshot.nodes.length === 0) return; - const container = containerRef.current; - if (!container) return; - - const edgeColor = tokenHex("--ring"); - const labelColor = tokenHex("--muted-foreground"); - const hoverTextColor = tokenHex("--foreground"); - const nodeColor = tokenHex(NODE_COLOR_TOKEN); - // Mutable hover-transition state: 0 = resting, 1 = fully hovered. - const hover: HoverState = { fade: 0, repaint: null }; - const { drawLabel, drawHover } = makeLabelDrawers(labelColor, hoverTextColor, hover); - - const graph = new Graph(); - for (const node of snapshot.nodes) { - // Seeded start positions (FA2 requires non-zero x/y) keep the - // frozen layout stable for a given node set across remounts. - const rand = seededRandom(node.id); - graph.addNode(node.id, { - label: node.name, - size: NODE_SIZE, - color: nodeColor, - x: (rand() - 0.5) * 100, - y: (rand() - 0.5) * 100, - }); - } - for (const edge of snapshot.edges) { - // Snapshot edges are already merged by (from,to,type); the guard - // is just belt-and-braces for the simple graphology graph. - if (!graph.hasEdge(edge.from, edge.to)) { - graph.addEdge(edge.from, edge.to, { type: "arrow", color: edgeColor, size: 1 }); + // The ledger path: every positioned node uses its resting position; + // nodes the ledger doesn't know yet (raced mid-settle) fall through + // to seeded scatter below. + const ledger = snapshot.positions; + if (ledger) { + const positions = new Map(); + for (const node of snapshot.nodes) { + const p = ledger[node.id]; + if (p) positions.set(node.id, { x: p.x, y: p.y }); + } + if (positions.size > 0) { + posRef.current = positions; + return () => { + posRef.current = null; + }; } } - - // Run-then-freeze: one synchronous pass, writes x/y, then done. - // Barnes-Hut comes on automatically above ~500 nodes (the vision's - // first escalation rung — see scripts/bench-graph.ts for the numbers). - const settings = { - ...forceAtlas2.inferSettings(graph), - ...(graph.order > 500 ? { barnesHutOptimize: true } : {}), - }; - forceAtlas2.assign(graph, { iterations: 100, settings }); - - let killed = false; - let renderer: import("sigma").default | null = null; - // rAF id of the hover fade transition — the ONLY animation (see the - // file header). Lives out here so the effect cleanup can cancel it. - let hoverAnimId: number | null = null; - // Both sigma and its rendering package touch WebGL globals at module - // scope, so both load dynamically (SSR-safe; the effect is browser-only). - void Promise.all([import("sigma"), import("sigma/rendering")]).then( - ([{ default: Sigma }, { NodeCircleProgram }]) => { - if (killed) return; - // Padded picking (see PICKING_VERTEX_SHADER): the visible disc is - // pixel-identical to the stock circle program; the PICKABLE disc - // gains PICKING_PAD_PX of radius, so the whole hover halo is - // hit-testable and small nodes are comfortable to hover/click. - // (Maintainer-endorsed pattern for sigma v3; v4 ships this as - // the `nodePickingPadding` setting.) - const PaddedNodeCircleProgram = class extends NodeCircleProgram { - getDefinition() { - return { - ...super.getDefinition(), - VERTEX_SHADER_SOURCE: PICKING_VERTEX_SHADER, - }; - } - }; - renderer = new Sigma(graph, container, { - nodeProgramClasses: { circle: PaddedNodeCircleProgram }, - defaultEdgeType: "arrow", - // All labels appear/disappear together: no per-node size gating - // (threshold 0) and NO sigma label-grid declutter (density → - // ∞) — clustered nodes otherwise lose their cells' label - // quota and pop in late, un-faded, mid-zoom. The camera-ratio - // gate below is the ONLY label culling: every label appears at - // the same zoom with the same fade. - labelRenderedSizeThreshold: 0, - labelDensity: Number.MAX_SAFE_INTEGER, - // Match the app typography (Geist via the body's stack). - labelFont: getComputedStyle(document.body).fontFamily, - labelWeight: "500", - labelSize: 12, - labelColor: { color: labelColor }, - renderEdgeLabels: false, - defaultDrawNodeLabel: drawLabel, - defaultDrawNodeHover: drawHover, - }); - graphRef.current = graph; - rendererRef.current = renderer; - setRendererEpoch((e) => e + 1); - renderer.on("clickNode", ({ node }) => { - const graphNode = snapshot.nodes.find((n) => n.id === node); - if (!graphNode) return; - const pos = graph.getNodeAttributes(node); - const anchor = renderer?.graphToViewport({ x: pos.x ?? 0, y: pos.y ?? 0 }); - setSelected({ - node: graphNode, - chatIds: snapshot.nodeSessions[node] ?? [], - anchor: anchor ?? { x: 0, y: 0 }, - }); - }); - // Clicking the empty stage closes the node card. - renderer.on("clickStage", () => setSelected(null)); - // One global label gate with a fade: names are fully visible at fit - // zoom or closer, then fade out together across LABELS_FADE_SPAN of - // zoom (the labels canvas layer's opacity), and only once fully - // faded does label rendering switch off. Any pan/zoom also detaches - // the card from its node — close it. - const labelsCanvas = renderer.getCanvases().labels; - let labelsRendering: boolean | null = null; - const syncRenderLabels = (ratio: number) => { - // 0 at fit zoom → 1 at the end of the fade span - const fade = (ratio - LABELS_CAMERA_RATIO) / LABELS_FADE_SPAN; - const alpha = Math.min(Math.max(1 - fade, 0), 1); - const shouldRender = alpha > 0; - if (shouldRender !== labelsRendering) { - labelsRendering = shouldRender; - renderer?.setSetting("renderLabels", shouldRender); - } - if (labelsCanvas) labelsCanvas.style.opacity = alpha.toFixed(3); - }; - syncRenderLabels(renderer.getCamera().getState().ratio); - renderer.getCamera().on("updated", (state) => { - syncRenderLabels(state.ratio); - setSelected(null); - }); - // The hover fade: entering schedules renders (sigma re-invokes - // drawHover with the new fade); leaving draws through the captured - // repainter, since sigma stops calling drawHover once unhovered. - // Both directions are bounded (~HOVER_FADE_MS) and cancel each - // other, so idle CPU returns to zero after every transition. - const animateHoverTo = (target: number) => { - if (hoverAnimId !== null) { - cancelAnimationFrame(hoverAnimId); - hoverAnimId = null; - } - const from = hover.fade; - if (from === target) { - if (target === 0) hover.repaint?.(0); - return; - } - const start = performance.now(); - const tick = (now: number) => { - const t = Math.min((now - start) / HOVER_FADE_MS, 1); - hover.fade = from + (target - from) * t; - if (target === 0) hover.repaint?.(hover.fade); - else renderer?.scheduleRefresh(); - hoverAnimId = t < 1 ? requestAnimationFrame(tick) : null; - }; - hoverAnimId = requestAnimationFrame(tick); - }; - renderer.on("enterNode", () => animateHoverTo(1)); - renderer.on("leaveNode", () => animateHoverTo(0)); - renderer.on("leaveStage", () => animateHoverTo(0)); - }); - + // Bootstrap fallback: the ledger is absent (fresh graph) — run + // the SAME engine the server settle uses (d3-force via + // incrementalLayout, seeded deterministic) and freeze the result. + // Deterministic for a given node set across remounts. + posRef.current = incrementalLayout(snapshot.nodes, snapshot.edges, new Map()); return () => { - killed = true; - if (hoverAnimId !== null) cancelAnimationFrame(hoverAnimId); - rendererRef.current = null; - graphRef.current = null; - renderer?.kill(); + posRef.current = null; }; }, [snapshot]); - // Filtering through sigma's official reducers: hidden nodes drop out of - // rendering, picking, hover, AND labels; edges drop when either endpoint - // is hidden. Re-runs whenever filters change or a renderer is (re)built - // (epoch), so the reducers always close over current values. + // Construct the renderer once the canvas exists; destroy when it goes + // away (the empty-graph placeholder replaces it). The callbacks are + // created once and read through refs, so they never go stale. + const isEmpty = snapshot === null || snapshot.nodes.length === 0; + useEffect(() => { + if (isEmpty) return; + const canvas = canvasRef.current; + if (!canvas) return; + const renderer = new GraphCanvasRenderer(canvas, { + onSelect: (nodeId, pos) => { + const snap = snapshotRef.current; + const node = snap?.nodes.find((n) => n.id === nodeId); + if (!snap || !node) return; + setSelected({ + node, + sourceIds: snap.nodeSessions[node.id] ?? [], + anchor: pos, + }); + }, + onStageClick: () => setSelected(null), + // Any camera move detaches the card from its node — close it. + onCameraMove: () => setSelected(null), + // Live physics settled: persist the new rest state. Fire-and- + // forget — the next snapshot fetch carries the ledger anyway. + onCommit: (positions) => { + void fetch("/api/graph/positions", { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(positions), + }).catch((err) => console.error("[graph] positions commit failed", err)); + }, + }); + rendererRef.current = renderer; + return () => { + rendererRef.current = null; + renderer.destroy(); + }; + }, [isEmpty]); + + // Feed #1 — data: positions + edges + clusters + resolved colors. Runs + // after the layout effect (same commit, declaration order) and after + // the construct effect, so the renderer exists and positions are fresh. useEffect(() => { const renderer = rendererRef.current; - const graph = graphRef.current; - if (!renderer || !graph) return; - renderer.setSetting("nodeReducer", (key, attrs) => - visibleIds.has(key) ? attrs : { ...attrs, hidden: true }, - ); - renderer.setSetting("edgeReducer", (key, attrs) => { - const source = graph.source(key); - const target = graph.target(key); - return visibleIds.has(source) && visibleIds.has(target) ? attrs : { ...attrs, hidden: true }; + const positions = posRef.current; + if (!renderer || !snapshot || snapshot.nodes.length === 0 || !positions) return; + renderer.setData({ + nodes: snapshot.nodes.map((n) => ({ + id: n.id, + x: positions.get(n.id)?.x ?? 0, + y: positions.get(n.id)?.y ?? 0, + label: n.name, + })), + edges: snapshot.edges.map((e) => ({ from: e.from, to: e.to })), + clusters: snapshot.clusters.map((c) => ({ id: c.id, label: c.label, members: c.members })), + colors: { + node: tokenHex(NODE_COLOR_TOKEN), + edge: tokenHex("--ring"), + label: tokenHex("--muted-foreground"), + hoverText: tokenHex("--foreground"), + }, + // Match the app typography (Geist via the body's stack). + fontFamily: getComputedStyle(document.body).fontFamily, }); + }, [snapshot, isEmpty]); + + // Feed #2 — settings: just marks the frame dirty. No rebuild, no + // re-layout, no camera touch (the whole point of the new renderer). + const deferredSettings = useDeferredValue(settings); + useEffect(() => { + rendererRef.current?.setSettings(deferredSettings); + }, [deferredSettings, isEmpty]); + + // Feed #3 — visibility: hidden nodes drop out of rendering, picking, + // hover, AND labels; edges drop when either endpoint is hidden. + useEffect(() => { + rendererRef.current?.setVisible(query.trim() ? visibleIds : null); // The open detail card cannot outlive its node's visibility. setSelected((sel) => (sel && visibleIds.has(sel.node.id) ? sel : null)); - }, [rendererEpoch, visibleIds]); + }, [visibleIds, query]); + + // Feed #4 — region tuning. `isEmpty` in the deps (same as Feed #2): + // the settings fetch resolves BEFORE the snapshot creates the renderer, + // so the load-time run hits a null renderer and would never re-fire — + // the renderer would silently keep the default tuning forever. + useEffect(() => { + rendererRef.current?.setRegionTuning(regionTuning); + }, [regionTuning, isEmpty]); + + // Feed #5 — physics tuning: rebuilds the live sim in place (drag pins + // survive; a mid-settle change re-anneals, the honest response). Same + // isEmpty dep as Feed #4 — see there. + useEffect(() => { + rendererRef.current?.setPhysicsTuning(physicsTuning); + }, [physicsTuning, isEmpty]); // Position the detail card: below the clicked node, flipped above near // the bottom edge, clamped horizontally. Imperative (ref + layout effect, @@ -602,8 +499,6 @@ export function GraphView() { card.style.visibility = "visible"; }, [selected]); - const isEmpty = snapshot === null || snapshot.nodes.length === 0; - return (
{isEmpty ? ( @@ -619,7 +514,11 @@ export function GraphView() {
) : ( -
+
+ {/* THE single canvas: every layer (hulls, edges, nodes, both + label systems, hover) draws into it, once per frame. */} + +
)} {/* Legend + filters: search box and per-state toggle rows. */} @@ -632,14 +531,331 @@ export function GraphView() { onChange={(e) => setQuery(e.target.value)} placeholder="Search nodes…" spellCheck={false} - className="h-8 w-full rounded-lg border bg-background pl-8 pr-2 text-xs outline-none placeholder:text-muted-foreground focus:border-ring" + className="h-8 w-full rounded-lg border bg-background pl-8 pr-8 text-xs outline-none placeholder:text-muted-foreground focus:border-ring" /> +
+ {settingsOpen && ( +
+ {/* All sliders: log-scaled 0-100, step 5, so every + DEFAULT sits at 50 (src/core/render-settings.ts). */} + + changeSettings((cur) => ({ ...cur, nodeSize: fromSliderPosition(p, NODE_SIZE_RANGE.min, NODE_SIZE_RANGE.max) }))} + /> + + changeSettings((cur) => ({ ...cur, edgeWidth: fromSliderPosition(p, EDGE_WIDTH_RANGE.min, EDGE_WIDTH_RANGE.max) }))} + /> + + changeSettings((cur) => ({ ...cur, arrowLength: fromSliderPosition(p, ARROW_LENGTH_RANGE.min, ARROW_LENGTH_RANGE.max) }))} + /> + + changeSettings((cur) => ({ ...cur, labelSize: fromSliderPosition(p, LABEL_SIZE_RANGE.min, LABEL_SIZE_RANGE.max) }))} + /> + {/* Displayed units normalize the default (0.5) to "1×". */} + + changeSettings((cur) => ({ ...cur, labelZoom: fromSliderPosition(p, LABEL_ZOOM_MIN, LABEL_ZOOM_MAX) }))} + /> + + {/* Regions: the blurred cluster tints. Sliders + a fixed-set + dropdown for the bitmap resolution (it's a discrete + choice, not a free number). Server-persisted like the + sizes; every range end was sweep-validated on the real + graph. */} +
+ + Regions + + + changeSettings((cur) => ({ + ...cur, + regions: { ...cur.regions, dilation: fromSliderPosition(p, REGION_DILATION_RANGE.min, REGION_DILATION_RANGE.max) }, + }))} + /> + + + changeSettings((cur) => ({ + ...cur, + regions: { ...cur.regions, edgeAlpha: fromSliderPosition(p, REGION_EDGE_ALPHA_RANGE.min, REGION_EDGE_ALPHA_RANGE.max) }, + }))} + /> + +
+ {/* Physics: the live sim's forces. Every value feeds the SAME + engine the server settle runs (createGraphSimulation); + changing one rebuilds the sim from the current positions — + drag mid-settle re-anneals, pins survive. Sliders over + sweep-validated ranges (src/core/render-settings.ts): + range ends and all four corners settle cleanly — no NaN, + no explosion, no non-settle. Repulsion is displayed as + the negative it is; cluster pull's far left = off. */} +
+ + Physics + + + + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, depthStrength: fromSliderPosition(p, DEPTH_STRENGTH_RANGE.min, DEPTH_STRENGTH_RANGE.max) }, + }))} + /> + + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, depthSpacing: fromSliderPosition(p, DEPTH_SPACING_RANGE.min, DEPTH_SPACING_RANGE.max) }, + }))} + /> + + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, depthBaseRadius: fromSliderPosition(p, DEPTH_BASE_RADIUS_RANGE.min, DEPTH_BASE_RADIUS_RANGE.max) }, + }))} + /> + + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, chargeStrength: -fromSliderPosition(p, -CHARGE_RANGE.max, -CHARGE_RANGE.min) }, + }))} + /> + + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, linkStrength: fromSliderPosition(p, LINK_STRENGTH_RANGE.min, LINK_STRENGTH_RANGE.max) }, + }))} + /> + + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, linkDistance: fromSliderPosition(p, LINK_DISTANCE_RANGE.min, LINK_DISTANCE_RANGE.max) }, + }))} + /> + + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, gravity: fromSliderPosition(p, GRAVITY_RANGE.min, GRAVITY_RANGE.max) }, + }))} + /> + + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, clusterPull: fromOffSliderPosition(p, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max) }, + }))} + /> + + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, velocityDecay: fromSliderPosition(p, VELOCITY_DECAY_RANGE.min, VELOCITY_DECAY_RANGE.max) }, + }))} + /> +
+
+ )} {snapshot !== null && visibleIds.size === 0 && query.trim() !== "" && (
No matching nodes.
)} +
+ { + onImportFiles(e.target.files); + e.target.value = ""; // allow re-picking the same file + }} + /> + +
)} @@ -670,20 +886,38 @@ export function GraphView() { >
{selected.node.name}
{selected.node.statement}
- {selected.chatIds.length > 0 ? ( + {selected.sourceIds.length > 0 ? (
Sessions
- {selected.chatIds.map((chatId) => { - // That session's understanding observation — the + {selected.sourceIds.map((sourceId) => { + // That source's understanding observation — the // extractor's named-word note (confidence history). - const note = snapshot?.nodeObservations?.[selected.node.id]?.[chatId]; + const note = snapshot?.nodeObservations?.[selected.node.id]?.[sourceId]; + // An imported-notes file: a STATIC entry — the path, + // plain text, never a link (no session to open). + if (snapshot?.fileSources?.[sourceId] !== undefined) { + return ( +
+ + + {snapshot.fileSources[sourceId]} + + {note && ( + + {note} + + )} +
+ ); + } + // A real session: deep-linkable provenance. return ( -
+
- {snapshot?.sessions[chatId] ?? chatId} + {snapshot?.sessions[sourceId] ?? sourceId} {/* The note is a fixed-scale word (solid/shaky/...) — the session name is the main thing; the chip is bounded so diff --git a/config.ts b/config.ts index 3dbfba7..8162991 100644 --- a/config.ts +++ b/config.ts @@ -47,6 +47,9 @@ export const MODELS = { extractor: { model: "z-ai/glm-5.3-flash", effort: "low" }, /** The auditor verifies evidence exists; keep it cheap. */ auditor: { model: "z-ai/glm-5.3-flash", effort: "low" }, + /** Names detected communities — one singular call, given member names + * + statements. Purely a labeling flourish; keep it cheap. */ + labeler: { model: "z-ai/glm-5.3-flash", effort: "low" }, } satisfies Record; export type ModelRole = keyof typeof MODELS; @@ -71,4 +74,29 @@ export const LEDGER_FILES = { events: path.join(PATHS.graph, "events.jsonl"), mappings: path.join(PATHS.graph, "mappings.jsonl"), sessions: path.join(PATHS.graph, "sessions.jsonl"), + /** LLM-named community records (contracts/graph.ts ClusterLabeling). */ + clusterLabels: path.join(PATHS.graph, "cluster-labels.jsonl"), } as const; + +/** + * Notes-import bookkeeping (NOT a ledger): one JSON map, src-id → NotesSource + * record (contracts/graph.ts) — the file path + import time + the run's usage + * log. Read-modify-write like a chat file, not append-only like the ledgers. + */ +export const SOURCES_FILE = path.join(PATHS.graph, "sources.json"); +/** The layout ledger (read-modify-write like sources.json): nodeId → + * {x,y} resting positions from the incremental settle (core/layout.ts). + * Carries an updatedAt stamp for future live-physics conflict handling. */ +export const POSITIONS_FILE = path.join(PATHS.graph, "positions.json"); + +/** The embedding ledger: one L2-normalized vector per node id, all from + * ONE model (swapping models re-embeds everything — vectors from + * different models live in incompatible spaces). Read-modify-write + * JSON like positions/sources. */ +export const EMBEDDINGS_FILE = path.join(PATHS.graph, "embeddings.json"); + +/** + * Graph render settings (the settings-panel controls): one small JSON blob, + * read-modify-write. Shared shape/sanitizer: src/core/render-settings.ts. + */ +export const RENDER_SETTINGS_FILE = path.join(PATHS.graph, "render-settings.json"); diff --git a/docs/research/graph-rendering.md b/docs/research/graph-rendering.md new file mode 100644 index 0000000..78273e6 --- /dev/null +++ b/docs/research/graph-rendering.md @@ -0,0 +1,227 @@ +# Graph rendering & cluster-first layout — research findings + +*Compiled 2026-09-13 from four research passes (library fact-check ×2, prior-art ×2), all +claims source-verified by subagents against shipped package code (`npm pack` tarballs of +`sigma@3.0.3`, `@sigma/layer-webgl@3.0.0`), the sigma.js GitHub storybook source, and +primary docs. Speculation is marked where it appears. This document is the decision +basis for the rendering rebuild; nothing has been implemented yet.* + +--- + +## 1. Requirements (as of this research) + +Current symptoms to eliminate: +- Hull layers lag behind nodes during pan (desync). +- Settings changes shift hulls up-left until the next interaction (canvas-sizing race). +- GPU ~50–100% during pan/zoom (redundant full-canvas 2D repaints per camera event + + WebGL re-render per frame; the earlier `ctx.filter: blur` per frame caused coil whine). +- Stale/garbage boundary shapes at extreme zoom-out. + +Feature set: +- Circles + directed arrows; precomputed layout (no runtime physics). +- Cluster regions: hull boundary around each community, dashed stroke, soft translucent + fill, stable color per region, region name label. +- Inverse zoom gates: node labels when zoomed IN, region labels when zoomed OUT, + cross-fading over a shared span. +- Pan, zoom-to-cursor, hover glow, click → detail card, edge-type filtering, camera + survives UI changes. 60fps, minimal GPU, ordinary laptop. + +New direction (user, 2026-09-13): +- **Cluster-first placement**: layout depends on clustering — clusters placed as units, + nodes within a cluster close together (compact, natural hull shapes as a consequence). +- **Freeform experimentation**: a dev-only playground to try layouts/shapes/styles + against real and synthetic data; renderer must be dumb, data-driven, swappable. + +Scale: 338 nodes / 211 edges today; low thousands expected (notes imports). + +## 2. Verified facts about the current stack (sigma.js v3) + +- **v3.0.3 ships a PUBLIC custom-layer API** — `createLayer` / `createCanvas` / + `createCanvasContext` / `killLayer` (typed in shipped `.d.ts`). Our current + `firstChild` DOM injection was improvising a worse version of a supported API. +- **Sigma sizes and DPR-scales ALL registered canvases** in its own `resize()` — + using the public API eliminates the canvas-sizing race by construction. +- **`afterRender` fires once per rendered frame** (rAF-debounced `scheduleRender()`). + Camera motion → full re-render each rAF. **Hover is cheap**: it only redraws the + `hoverNodes` WebGL layer + `edgeLabels` canvas via a separate rAF; no `afterRender`, + no full re-render. Sigma is strictly event-driven: no continuous loop, no rendering + when idle (verified: zero `setInterval`, all rAF sites are schedulers). +- **`renderer.graphToViewport()` is public and typed** — the sanctioned graph→pixel + path. `renderer.matrix` is **private**: building a `ctx.setTransform()` from it would + be guessing at internals; per-vertex `graphToViewport()` is the supported route. +- **The official cluster-label storybook example is exactly our pattern**: + custom DOM layer + `afterRender` + per-cluster `graphToViewport()` re-projection, + maintained in the sigma repo. Our minimal-change fix path is a demonstrated usage. +- Real v3 layer order (verified in source, NOT in docs): + `edges` (WebGL) → `edgeLabels` → `nodes` (WebGL) → `labels` → `hovers` → + `hoverNodes` (WebGL) → `mouse`. + +### `@sigma/layer-webgl@3.0.0` (official background-layer package) +- `bindWebGLLayer` inserts a background WebGL layer (`beforeLayer: "edges"`), redrawn + on `afterRender` with the same camera matrix — fully in-loop, no desync. +- **`createContoursProgram` renders METABALL-style implicit fields, not hull polygons** + (verified from the fragment shader: fullscreen quad, summed radial falloffs, + analytic contour lines via `dFdx/dFdy`). Translucent fill: yes (arbitrary CSS color + with alpha). **Dashed stroke: NO** — solid anti-aliased line only; no dash parameter + exists. Per-community color: yes, but one program + one WebGL context per community, + and each runs a fullscreen fragment pass per frame (browsers cap ~16 WebGL contexts — + a problem for dozens of regions). +- **Decisive fact**: our exact visual (hull polygon + translucent fill + dashed + stroke) is **NOT achievable with shipped layer-webgl programs**. It would require a + custom `WebGLLayerProgram` (supported extension point, moderate effort, dashes DIY in + GLSL) — whereas Canvas 2D gives it for free with `ctx.setLineDash()`. +- v4-era programs (heatmap, density-splat, dashed/fill layer factories) are NOT in the + v3 package. + +### sigma v4 +- Real but **beta** (4.0.0-beta.5; docs site self-describes as alpha). Single `stage` + canvas + public `createLayer`, GPU picking with downsizing, SDF labels, depth + layers. Breaking migration. Not a production bet today. + +### Pan GPU cost at our scale +- Structural facts verified (event-driven idle; full re-render per camera frame; + synchronous `gl.readPixels` picking per mousemove pre-#1546, since fixed). +- **Absolute GPU cost of panning 338–3k nodes in v3: UNVERIFIED** — no public + benchmark exists; nobody has measured it. If it matters, we measure it ourselves. + +## 3. Verified facts about the candidates + +| Candidate | Renderer | True hull fill+dash | Picking | Camera | Maintenance | License | Fit | +|---|---|---|---|---|---|---|---| +| sigma v3 + public 2D canvas layer | WebGL (+our 2D) | yes (our canvas, free) | GPU color-pick | built-in | active (3.0.3, 2026-04) | MIT | 7/10 | +| sigma v3 + layer-webgl contours | WebGL | **no** (metaballs, solid border) | — | built-in | active | MIT | 6/10 | +| sigma v4 + layers | WebGL, SDF | via custom program (beta risk) | GPU picking | built-in | **beta** | MIT | 6 (risky) | +| d3-zoom + Canvas 2D (hand-rolled loop) | Canvas 2D | yes, trivially | distance check | d3-zoom (verified math) | d3 very active | ISC | **8/10** | +| PixiJS v8 + pixi-viewport | WebGL/WebGPU/Canvas | yes (Graphics; dashes via shim `pixi-dashed-line`) | built-in events | pixi-viewport 6.0.3 | very active | MIT | 7/10 | +| cytoscape.js 3.34 | Canvas 2D | via bubblesets/layers extensions | built-in | built-in | active | MIT | 5/10 | +| vis-network 10.1 | Canvas 2D | **no** — clustering = supernodes (collapse, opposite of our regions) | built-in | built-in | sporadic | MIT | 2/10 | +| @cosmos.gl/graph (Cosmograph engine) | WebGL (luma.gl) | no (cluster = layout force only) | GPU | built-in | active | MIT | 3/10 | + +Fact nuances we must not over-trust: +- "Canvas 2D is GPU-rasterized": **half-true**. Skia may raster on CPU or GPU depending + on browser/OS config. Either way the cost driver is **repaint count × pixel area + (DPR²)**, not "Skia = GPU". +- Canvas 2D at 60fps for hundreds–low-thousands of elements: **supported by practitioner + evidence** (cytoscape is fully canvas-based into the thousands; 10k animated canvas + elements ≈ 55fps in one 2025/26 benchmark). Pixel area matters more than node count + at our scale. +- Obsidian's graph view renderer: **UNVERIFIED** — no primary source. Do not cite. +- PixiJS exact bundle size: unmeasured. yFiles: commercial, unverified details. + +## 4. Prior art for our specific features + +### Region/hull visuals +- Nobody in the graph-tool world ships our region aesthetic: Gephi desktop, Gephi Lite + (the flagship sigma app — zero hull code in its tree), Obsidian — all partition by + **node coloring only**. The region feature is a differentiator, not a copied pattern. +- The aesthetic reference: ggforce `geom_mark_hull` (expand + round the hull, low-alpha + fill, integrated label placement). +- **Bostock's thick-stroke hull trick**: one path, wide stroke (e.g. 32px) + + `linejoin: round` = padded, rounded hull with zero polygon-offset math. Replaces our + offset-normal code outright. +- Named-region UX validation: Excalidraw frames / tldraw regions (dashed titled + container around content). +- Closest feature twin: **Cosmograph** — documents "cluster labels when zoomed out, + point labels when zoomed in" as a first-class use case (our exact inverse gate). + +### Zoom-gated labels +- Consensus pattern = map-style per-layer zoom ranges (Mapbox/MapLibre + `minzoom`/`maxzoom` on symbol layers). State labels at far zoom, city labels at near + zoom is the standard cartographic solution; our inverse gates are the same mechanism + with opposite ranges. Sigma's own `labelRenderedSizeThreshold` is the zoom-in gate. +- Cross-fade + hysteresis over a shared span: no single canonical writeup (medium + confidence); nearest established practice is map label collision systems. + +### The sync pattern (fixes our desync class structurally) +- Canonical and repeatedly documented: **one rAF loop with a dirty flag; input/DOM + events only mutate state.** All layers drawn in the same frame, under the same + camera transform — either same canvas (`setTransform` once, draw back-to-front) or + stacked canvases all redrawn by the same loop. + Sources: browser.engineering, MDN "Optimizing canvas", web.dev canvas performance. + +### Picking +- Canvas 2D at our scale: nearest-node distance check per pointer event is the norm, + trivially fast (µs for 3k nodes; spatial grid only past ~10k). + +## 5. Cluster-first layout (the new direction) + +- **Two-level pattern is well-trodden**: place cluster anchors first (quotient graph — + one super-node per community, Gephi's "Groups by Partition" plugin is the desktop + prior art), layout nodes within each cluster, compose by translating local layouts + to cluster anchors. Bostock's Clustered Force Layout lineage; cosmos.gl's cluster + force verified from source (per-frame center-of-mass pull, or user-pinned positions — + cluster anchors as data independent of member layout). +- **graphology FA2 has NO cluster-aware settings** (full settings list read; gravity is + global). **LinLog mode** (`linLogMode: true`) is the documented "communities separate + tightly" switch — a one-afternoon A/B candidate before any custom pipeline. +- **No turnkey JS package does the composed two-level pipeline** — assemble from + graphology primitives (~150 lines of glue: quotient build, per-cluster FA2 with + circular init, compose transform, `noverlap` cleanup). The combination is our + synthesis from verified primitives. +- Deterministic given fixed init positions (no RNG dependency in FA2). +- **Server-side at extraction time is the better fit** for our architecture: positions + become data, rendering stays dumb. Browser worker fallback for live re-layout. +- Harness precedents: yFiles layout playground (polished), Observable/bl.ocks lineage + (notebook = graph + sliders + canvas), react-sigma layouts example. No canonical OSS + playground for our stack — copy the config-driven pattern (algorithm dropdown + + parameter sliders + seed input + live canvas). + +## 6. Options (the decision) + +**Option A — Stay on sigma v3; mount the hull canvas via the public +`createCanvas` API; redraw on `afterRender` via `graphToViewport()`.** +Supported, minimal-change, demonstrated by sigma's own cluster-label storybook. Kills +the desync, the sizing race, and the multi-event redraws (once per frame, sigma-sized). +Our exact visuals stay (dashes free in 2D). Keeps sigma's camera/labels/picking +machinery. Residual: full-canvas 2D repaint per interaction frame remains (raster cost, +mitigable with move-gating like sigma's own `hideLabelsOnMove`); WebGL + 2D raster +still both run per frame. Est: ~1 day. + +**Option A′ — sigma + custom `WebGLLayerProgram` for hulls.** Supported extension +point, but dashes are DIY GLSL and we'd rebuild in shaders what Canvas 2D gives free. +No reason given A exists. Est: several days, for a worse iteration loop. + +**Option B — d3-zoom + hand-rolled single Canvas 2D renderer.** +One canvas, one rAF + dirty flag, world-space geometry + one transform, all layers +back-to-front in the same frame. Lowest GPU/CPU, total control, perfectly matches the +new requirements: renderer is a dumb draw function over data (layout experiments can't +fight it), playground is trivial, cluster-first layout slots in as pure data. Costs +~600 lines + camera feel + picking + label gating owned by us. Est: ~1 focused day. + +**Option C — PixiJS v8 + pixi-viewport.** Cleanest scene-graph answer (one ticker, +sync by construction), WebGPU headroom. But: dashes need a third-party shim, no graph +semantics, bundle, more machinery than Canvas 2D needs at our scale. Est: 1–2 days. + +**Recommendation**: **Option B as the target** — it is the only option that structurally +matches all three requirement sets (bugs, cluster-first placement, freeform +experimentation), scored highest (8/10), and removes the vendor-internals class of +problems permanently. **Option A is a legitimate ~1-day stabilization** if we want the +current bugs dead immediately while deferring the rebuild; it reuses all our existing +visual code and is the officially sanctioned pattern, but it keeps us on sigma's +opinionated machinery and the experimentation direction will keep pushing against it. + +**Layout work rides along independent of renderer choice**: first A/B +LinLog-only FA2 (one afternoon), then the two-level pipeline +(quotient → per-cluster FA2 → compose → noverlap) if needed, developed in the playground. + +## 7. Facts we should NOT rely on + +- "Obsidian's graph view uses renderer X" — unverified. +- "Sigma's documented canvas DOM order" — source-only, implementation detail, changed + in v4. +- "Canvas 2D is always GPU-rasterized" — false as stated. +- "Sigma v4 is ready" — beta. +- "Sigma pan cost at our scale is X%" — nobody has measured it publicly. +- "@cosmograph/cosmos is the maintained Cosmograph engine" — stale npm channel (Dec + 2024); current engine is `@cosmos.gl/graph` v3. +- Any PixiJS bundle-size number — unmeasured. + +## 8. Open items for the implementation phase + +- Measure pan GPU/CPU ourselves (Chrome profiler) before/after whichever path is chosen. +- Playground design: `/playground` dev-only route, config-driven (layout algorithm + dropdown, parameter sliders, seed input, synthetic + real datasets), same renderer. +- Hull styling decision inside Option B: Bostock thick-stroke trick vs offset polygons + (geometry already unit-tested in `components/cluster-geometry.ts`). +- Label-gate hysteresis: small dead-band in log-zoom to avoid threshold flicker. +- Whether layout runs server-side at extraction (persisted as data) or in-browser. diff --git a/docs/vision.md b/docs/vision.md index a220a41..965c606 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -43,18 +43,60 @@ Every node records every **(chat, message) point** where it came up; every edge Three stages, one pipeline — the sole graph writer: -- **Extractor (LLM) — session-only scope.** Sees ONLY the session transcript (tree-blind, with message ids). Reports freely, with verbatim evidence: propositions from anywhere in the session (one reasoning step each; prose counts as much as anything the teacher formalized), edges as the transcript articulates them, message-granular provenance, retroactive quiz attribution, and per proposition an **understanding level** — how strongly the learner accepted/understood it in this session, a CLOSED named-word scale (`solid` / `shaky` / `accepted` / `resisted` / `not yet engaged`; 2026-09-08: schema-enforced — exactly one word, no numbers, no justifications, which are dropped rather than stored). The levels are data for the future state model; their treatment is TBD. +- **Extractor (LLM) — single-source scope.** Sees ONLY the source text — a session transcript (tree-blind, with message ids) or raw notes (the front door's second prompt, above). Reports freely, with verbatim evidence: propositions from anywhere in the session (one reasoning step each; prose counts as much as anything the teacher formalized), edges as the transcript articulates them, message-granular provenance, retroactive quiz attribution, and per proposition an **understanding level** — how strongly the learner accepted/understood it in this session, a CLOSED named-word scale (`solid` / `shaky` / `accepted` / `resisted` / `not yet engaged`; 2026-09-08: schema-enforced — exactly one word, no numbers, no justifications, which are dropped rather than stored). The levels are data for the future state model; their treatment is TBD. - **Auditor (LLM) — the global-graph scope.** Sees the extractor's output AND the existing graph. Two jobs: the faithfulness net (every candidate verified against verbatim evidence — unchanged), and the **identity judgment**: link each proposition to an existing node (recording the session's phrasing as an alias) or create it new. Conservative: duplicates over wrong merges. - **Merger (deterministic).** Append-only ledger writes; quiz event kinds/refs rebuilt mechanically from quiz tool parts. Every judgment is an LLM's; everything mechanical stays code. **Event ledger (amended):** quiz outcomes are rebuilt deterministically from quiz tool parts (correct / miss / don't-know, plus the learner's note). **There is no `taught` event** — no marking-things-done. The extractor's per-proposition understanding levels join the ledger as session-scoped observations (one closed-scale word per node per session). Identity is no longer exact-string matching — it is the auditor's judgment, with aliases as the phrasing record. +### The format-agnostic front door — notes imports (DECIDED 2026-09-13) + +The same three-stage pipeline (extractor → auditor → merger) runs over **raw +personal notes in arbitrary formats** — an Obsidian vault, plain markdown, +anything textual. The extractor is format-blind by construction (it always +saw rendered text, never ChatRecord JSON), so the notes path needs only a +second extractor prompt (general-purpose extraction; the understanding word +judges the **author's** grasp from their own commentary) and a source +abstraction: each imported file becomes a `NotesSource` record (deterministic +id from the path hash — re-imports are idempotent), playing the `chatId` role +in mappings, checkpoints, and observation events. + +Decided edges: + +- **No session shaping.** Notes are never bent into teacher/learner + transcripts; quiz outcomes inside notes surface only via the extractor's + understanding observations (the deterministic quiz-event rebuild is a + session-pipeline feature — it needs structured quiz tool parts). +- **Static provenance.** Imported nodes show the file path in the sessions + list as a static entry — never a link. Understanding-assessment levels + apply to personal notes as to sessions. +- **Same single-flight registry and observability** as session runs: live + status, usage accounting per stage, true abort. One pipeline instance, + two front doors (quiz-triggered sessions; POST /api/graph/import). + ### Understanding is derived (amended 2026-09-02) Known vs. understood, event ledger as source of truth, derived state recomputed on read — carried over from the design doc in spirit. The state model itself is now an explicit design discussion: **node states should be session-independent and computed from the depth and strength of dependencies** (the `follows`/`supports` topology) plus quiz events and understanding notes. That redesign is pending ideation; the transitional five-class derivation (taught / known / missed / don't-know / candidate) was **removed** (2026-09-08) — the graph view renders state-neutral nodes until the state model is designed. The anchored calibration rubric stays parked until real data. +### Clusters — regions of the world model (DECIDED 2026-09-13, ideation) + +When the graph fills up, the canvas needs a readable at-a-glance structure: **clusters** — objective regions of the world model (domains like "Lacanian psychoanalysis" or "photosynthesis"), discovered by **community detection** on the `follows`/`supports` edges. A community = nodes linked to each other far more than to the rest (modularity). Everything about them is **entirely inferred from the graph data** — the extraction/audit pipeline never knows clusters exist. + +- **Algorithm: Louvain (modularity maximization), seeded for determinism.** In-stack via graphology's ecosystem, near-linear, scales far beyond our sizes. Determinism gives reproducibility (same graph → same clusters); the seed/graphology choice also keeps the upgrade path open: Louvain's dendrogram contains **hierarchy** — "math ⊃ differential calculus ⊃ chain rule" is one level down — deliberately NOT built now (multi-scale clusters stay parked). Dismissed: Leiden (better resolution guarantees, no JS port), Infomap (heuristic-strong, heavier, weaker JS support), Girvan–Newman (O(n³)), label propagation (unstable), spectral (needs k handed in). +- **Structure, not layout.** Clusters are a pure function of the *edges*, never of the visual layout. The force-directed layout already places structurally-close nodes together, so cluster hulls will look contiguous — but cluster identity must not inherit the layout's seed dependence and drift. +- **Derived, cached, auto-updated.** Clusters recompute when the ledger changes, cached against a graph version stamp — same doctrine as states and strengths: derived never stored, nothing hand-maintained. +- **Size-gated meaning.** Only communities ≥ ~3 nodes render as clusters (threshold tunable on real data) — a visible blob always means "a real domain"; lone nodes and tiny communities stay plain. +- **Naming: one cheap LLM call per new/changed cluster** — a single call (no agent loop) with the members' names + statements, producing a short domain label. Never part of extraction; a separate, cached seam. +- **Labeling persistence — the core-survival rule.** Each labeling record (append-only, like everything we persist) stores the label AND the member set it was produced with (the **core**). A recomputed community **keeps** its label while ≥ half its members are from that core; when under half are, it **relabels** with all current nodes (a new record is appended — old records stay as history: an audit trail of how the domains were named over time). Single successor: on splits/merges the strongest-overlap community inherits, the others get fresh labelings; a core with no ≥50% successor simply has no living heir. Cluster color keys to the labeling record — color survives exactly as long as the label does. +- **Rendering: glow blobs behind everything.** A dashed boundary encloses the cluster's hull; the fill is blurred, low-opacity, and sits *behind* the whole graph (edges and nodes float above it, so the blur naturally fogs the inter-cluster space). Per-cluster stable hue. **Cluster labels are inverse-gated** against the existing camera-ratio span: zoomed out = **domain view** (cluster labels on, node labels off), zoomed in = **proposition view** (node labels on, cluster labels faded) — one symmetric transition between the two views of one canvas. +- **Boundary geometry: computed at draw time, never stored** — a pure function of the member positions. This is the constraint that keeps future physics/moving nodes (likely not implemented soon) from invalidating anything: convex hull + padding + rounded corners is cheap per-frame; organic metaball hulls are the same family. Secondary until then. +- **Inter-cluster edges stay visible.** Bridges between domains are meaningful (Nietzsche's will-to-power *follows* Schopenhauer's will) — one of the most informative reads of the clustered view, not clutter. +- **Honest risk: sparsity.** Community detection wants density contrast, and the current real graph is thin — the 2k-node synthetic benchmark (open question 1) is the *primary* validation of clustering quality, not just of layout/render speed. + ### Rendering (DECIDED — research report 2026-08-22) +**Render settings (2026-09-13):** four basic controls in the graph view — node size, edge width, arrowhead length, and **the zoom-level control** (the label-visibility threshold; 1× = fit zoom) — persisted per browser. The zoom control is deliberately THE single zoom knob: cluster labels gate inversely on it over the same fade span (implemented: node labels fade out over the span as cluster labels fade in — they never coexist). Settings changes rebuild only the renderer: layout positions and the camera never move. (Edge-length / spacing controls deliberately parked until a cluster-aware spacing design exists.) + - **Library: sigma.js v3 + graphology + graphology-layout-forceatlas2.** ~40 KB gzipped all-in — half of vis-network, a third of cytoscape, far below react-force-graph (~250 KB deps) and cosmos.gl (luma.gl stack). - **Layout: synchronous run-then-freeze.** `forceAtlas2.assign(graph, { iterations: 100, settings: inferSettings(graph) })` runs once, writes x/y, and no animation loop ever starts — near-zero idle CPU (sigma only redraws on camera/graph events). <150 ms at hundreds of nodes; at ~2k enable `barnesHutOptimize`, then the bundled Web Worker, then server-cached positions — in that order, only if latency bites. - **Labels at zoom:** sigma's `labelRenderedSizeThreshold` replicates Obsidian's "text fade threshold" exactly. @@ -73,7 +115,7 @@ Every session starts with the learner's knowledge graph as base context for the ### Home page -The graph view replaces the current placeholder. v1: all nodes from all sessions, node color/shape by state class (the current five classes are transitional — see "Understanding is derived"), pan/zoom, labels at zoom. Clicking a node → the exact (session, message) points where it came up (message-granular provenance, deep-linked). Everything richer (frontier glow, rot map, blast radius) comes after the state model exists. +The graph view replaces the current placeholder. v1: all nodes from all sessions, pan/zoom, labels at zoom, state-neutral nodes (see "Understanding is derived"), and — once implemented — cluster glow-blobs with dashed hulls behind the graph, domain↔proposition as the two zoom-gated views (see "Clusters"). Clicking a node → the exact (session, message) points where it came up (message-granular provenance, deep-linked). Everything richer (frontier glow, rot map, blast radius) comes after the state model exists. --- @@ -213,6 +255,8 @@ The teacher's context window and attention are precious — it has the lesson to 4. Home-graph v1 interactions beyond pan/zoom (node click → session list? — presumed cheap yes). 5. Grain mismatch frequency (`subsumes`-style relations) — per original doc, measured by data, not pre-solved. 6. Aggregation semantics for understanding (min / mean / two-level) — fit on real data, per original doc. +7. Cluster visual details — boundary aesthetic (convex vs. organic hull), label placement/typography, palette — decide during visual testing with real blobs (§1 Clusters; theory DECIDED 2026-09-13). +8. Cluster size threshold (~3) — tune on the synthetic benchmark + real data (§1 Clusters). --- diff --git a/next.config.ts b/next.config.ts index 5d5d02e..c3236a4 100644 --- a/next.config.ts +++ b/next.config.ts @@ -4,7 +4,13 @@ const nextConfig: NextConfig = { // Framework-free core lives in src/ (plain TS, imported by routes and CLI alike). // The pi engine must stay external: its providers are runtime-loaded Node // code (undici streams, catalog data) that breaks when bundled into routes. - serverExternalPackages: ["@earendil-works/pi-ai", "@earendil-works/pi-agent-core"], + // transformers.js (embeddings) is runtime-loaded ONNX — same reason as + // the pi engine: bundling onnxruntime-node breaks route builds. + serverExternalPackages: [ + "@earendil-works/pi-ai", + "@earendil-works/pi-agent-core", + "@huggingface/transformers", + ], }; export default nextConfig; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index f9278b2..d6c41f0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "@ai-sdk/react": "^4.0.79", "@earendil-works/pi-agent-core": "^0.85.1", "@earendil-works/pi-ai": "^0.85.1", + "@huggingface/transformers": "^4.2.0", "@mozilla/readability": "^0.6.0", "@radix-ui/react-use-controllable-state": "^1.2.6", "@rive-app/react-webgl2": "^4.32.1", @@ -24,9 +25,10 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", + "d3-force": "^3.0.0", + "d3-selection": "^3.0.0", + "d3-zoom": "^3.0.0", "embla-carousel-react": "^8.6.0", - "graphology": "^0.26.0", - "graphology-layout-forceatlas2": "^0.10.1", "katex": "^0.16.47", "linkedom": "^0.18.13", "lucide-react": "^1.33.0", @@ -42,7 +44,6 @@ "react-dom": "^19.2.8", "react-jsx-parser": "^2.4.1", "remark-breaks": "^4.0.0", - "sigma": "^3.0.3", "streamdown": "^2.5.0", "tailwind-merge": "^3.6.0", "tokenlens": "^1.3.1", @@ -1910,6 +1911,582 @@ } } }, + "node_modules/@huggingface/jinja": { + "version": "0.5.10", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.10.tgz", + "integrity": "sha512-SgS1D1bglQ94ceD4ZCL6eayUDy9uV1xuyk61OjgSxU5GJh7upqZCKII0JoTYMc6wWsg3JUgaPRX0ElQzXPk3Cw==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/tokenizers": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@huggingface/tokenizers/-/tokenizers-0.1.3.tgz", + "integrity": "sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA==", + "license": "Apache-2.0" + }, + "node_modules/@huggingface/transformers": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-4.2.0.tgz", + "integrity": "sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ==", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/jinja": "^0.5.6", + "@huggingface/tokenizers": "^0.1.3", + "onnxruntime-node": "1.24.3", + "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c", + "sharp": "^0.34.5" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, "node_modules/@humanfs/core": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", @@ -1998,7 +2575,6 @@ "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", "license": "MIT", - "optional": true, "engines": { "node": ">=18" } @@ -5962,6 +6538,15 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/adm-zip": { + "version": "0.5.18", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.18.tgz", + "integrity": "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -6291,6 +6876,13 @@ "url": "https://github.com/sponsors/fb55" } }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT" + }, "node_modules/bowser": { "version": "2.14.1", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", @@ -7340,7 +7932,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "dev": true, "license": "MIT", "dependencies": { "es-define-property": "^1.0.0", @@ -7358,7 +7949,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", - "dev": true, "license": "MIT", "dependencies": { "define-data-property": "^1.0.1", @@ -7394,12 +7984,17 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=8" } }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "license": "MIT" + }, "node_modules/detect-node-es": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", @@ -7711,7 +8306,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -7721,7 +8315,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -7802,6 +8395,12 @@ "tests/browser-compat" ] }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "license": "MIT" + }, "node_modules/esbuild": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", @@ -8301,15 +8900,6 @@ "node": ">=0.10.0" } }, - "node_modules/events": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", - "license": "MIT", - "engines": { - "node": ">=0.8.x" - } - }, "node_modules/eventsource-parser": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", @@ -8481,6 +9071,12 @@ "node": ">=16" } }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, "node_modules/flatted": { "version": "3.4.4", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", @@ -8749,6 +9345,23 @@ "node": ">=10.13.0" } }, + "node_modules/global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, "node_modules/globals": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", @@ -8766,7 +9379,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", - "dev": true, "license": "MIT", "dependencies": { "define-properties": "^1.2.1", @@ -8809,7 +9421,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -8825,45 +9436,11 @@ "dev": true, "license": "ISC" }, - "node_modules/graphology": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/graphology/-/graphology-0.26.0.tgz", - "integrity": "sha512-8SSImzgUUYC89Z042s+0r/vMibY7GX/Emz4LDO5e7jYXhuoWfHISPFJYjpRLUSJGq6UQ6xlenvX1p/hJdfXuXg==", - "license": "MIT", - "dependencies": { - "events": "^3.3.0" - }, - "peerDependencies": { - "graphology-types": ">=0.24.0" - } - }, - "node_modules/graphology-layout-forceatlas2": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/graphology-layout-forceatlas2/-/graphology-layout-forceatlas2-0.10.1.tgz", - "integrity": "sha512-ogzBeF1FvWzjkikrIFwxhlZXvD2+wlY54lqhsrWprcdPjopM2J9HoMweUmIgwaTvY4bUYVimpSsOdvDv1gPRFQ==", - "license": "MIT", - "dependencies": { - "graphology-utils": "^2.1.0" - }, - "peerDependencies": { - "graphology-types": ">=0.19.0" - } - }, - "node_modules/graphology-types": { - "version": "0.24.8", - "resolved": "https://registry.npmjs.org/graphology-types/-/graphology-types-0.24.8.tgz", - "integrity": "sha512-hDRKYXa8TsoZHjgEaysSRyPdT6uB78Ci8WnjgbStlQysz7xR52PInxNsmnB7IBOM1BhikxkNyCVEFgmPKnpx3Q==", - "license": "MIT", - "peer": true - }, - "node_modules/graphology-utils": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/graphology-utils/-/graphology-utils-2.5.2.tgz", - "integrity": "sha512-ckHg8MXrXJkOARk56ZaSCM1g1Wihe2d6iTmz1enGOz4W/l831MBCKSayeFQfowgF8wd+PQ4rlch/56Vs/VZLDQ==", - "license": "MIT", - "peerDependencies": { - "graphology-types": ">=0.23.0" - } + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" }, "node_modules/hachure-fill": { "version": "0.5.2", @@ -8898,7 +9475,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "dev": true, "license": "MIT", "dependencies": { "es-define-property": "^1.0.0" @@ -10024,6 +10600,12 @@ "dev": true, "license": "MIT" }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "license": "ISC" + }, "node_modules/json5": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", @@ -10489,6 +11071,30 @@ "node": ">= 20" } }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/matcher/node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -11897,7 +12503,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -12010,6 +12615,49 @@ "regex-recursion": "^6.0.2" } }, + "node_modules/onnxruntime-common": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.3.tgz", + "integrity": "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.24.3.tgz", + "integrity": "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==", + "hasInstallScript": true, + "license": "MIT", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "adm-zip": "^0.5.16", + "global-agent": "^3.0.0", + "onnxruntime-common": "1.24.3" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.26.0-dev.20260416-b7804b056c", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.26.0-dev.20260416-b7804b056c.tgz", + "integrity": "sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.24.0-dev.20251116-b39e144322", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { + "version": "1.24.0-dev.20251116-b39e144322", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.0-dev.20251116-b39e144322.tgz", + "integrity": "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==", + "license": "MIT" + }, "node_modules/openai": { "version": "6.40.0", "resolved": "https://registry.npmjs.org/openai/-/openai-6.40.0.tgz", @@ -12233,6 +12881,12 @@ "node": ">=16.20.0" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, "node_modules/points-on-curve": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/points-on-curve/-/points-on-curve-0.2.0.tgz", @@ -12943,6 +13597,23 @@ "node": ">=0.10.0" } }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, "node_modules/robust-predicates": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.3.tgz", @@ -13082,7 +13753,6 @@ "version": "7.8.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "devOptional": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -13091,6 +13761,27 @@ "node": ">=10" } }, + "node_modules/semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", + "license": "MIT" + }, + "node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", @@ -13289,16 +13980,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/sigma": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/sigma/-/sigma-3.0.3.tgz", - "integrity": "sha512-5H0zFlx6/NTQpqBg4Rm569ZOpnBOXMaS25UQThIWMU3XyzI5AhmorK/gnl87BvJBLhQd0tW4C0LIp3enWzMoNw==", - "license": "MIT", - "dependencies": { - "events": "^3.3.0", - "graphology-utils": "^2.5.2" - } - }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", @@ -13318,6 +13999,12 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "license": "BSD-3-Clause" + }, "node_modules/stable-hash-x": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/stable-hash-x/-/stable-hash-x-0.2.0.tgz", @@ -13849,6 +14536,18 @@ "node": ">= 0.8.0" } }, + "node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/typebox": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.3.7.tgz", diff --git a/package.json b/package.json index 5eb7a45..6969412 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "@ai-sdk/react": "^4.0.79", "@earendil-works/pi-agent-core": "^0.85.1", "@earendil-works/pi-ai": "^0.85.1", + "@huggingface/transformers": "^4.2.0", "@mozilla/readability": "^0.6.0", "@radix-ui/react-use-controllable-state": "^1.2.6", "@rive-app/react-webgl2": "^4.32.1", @@ -29,9 +30,10 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", + "d3-force": "^3.0.0", + "d3-selection": "^3.0.0", + "d3-zoom": "^3.0.0", "embla-carousel-react": "^8.6.0", - "graphology": "^0.26.0", - "graphology-layout-forceatlas2": "^0.10.1", "katex": "^0.16.47", "linkedom": "^0.18.13", "lucide-react": "^1.33.0", @@ -47,7 +49,6 @@ "react-dom": "^19.2.8", "react-jsx-parser": "^2.4.1", "remark-breaks": "^4.0.0", - "sigma": "^3.0.3", "streamdown": "^2.5.0", "tailwind-merge": "^3.6.0", "tokenlens": "^1.3.1", diff --git a/scripts/backfill-graph.ts b/scripts/backfill-graph.ts index 194addc..9eefef0 100644 --- a/scripts/backfill-graph.ts +++ b/scripts/backfill-graph.ts @@ -8,13 +8,14 @@ import { chatStore } from "../src/store/chats.ts"; import { graphStore } from "../src/store/graph.ts"; import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts"; +import { embedNewNodes } from "../src/server/embeddings.ts"; import { getGraph } from "../src/core/graph.ts"; async function main() { const only = process.argv[2]; // optional: extract just one chat id const chats = only ? chatStore.listChats().filter((c) => c.id === only) : chatStore.listChats(); console.log(`[backfill] ${chats.length} chats to extract`); - const pipeline = createExtractionPipeline({ chatStore, graphStore }); + const pipeline = createExtractionPipeline({ chatStore, graphStore, embedder: embedNewNodes }); for (const chat of chats) { const t0 = Date.now(); try { diff --git a/scripts/bench-graph.ts b/scripts/bench-graph.ts deleted file mode 100644 index a3a2d12..0000000 --- a/scripts/bench-graph.ts +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Layout benchmark (W4; vision §1 "Rendering (DECIDED)" — "Pre-implementation - * validation: 2k-node synthetic benchmark to confirm layout time"). - * - * npx tsx scripts/bench-graph.ts - * - * Two scales: the vision's target regime (hundreds of nodes, <150 ms budget - * with a synchronous run-then-freeze pass) and the stress case (2,000 nodes, - * ~4,000 edges), the latter timed with and without barnesHutOptimize. - * - * VERDICT (2026-08-22, Apple silicon, Node 24): hundreds of nodes are - * comfortably inside the budget (see the measured lines — ~25 ms at 100, - * ~90 ms at 300). At 2k nodes the naive O(n²) pass takes seconds, and - * barnesHutOptimize cuts it roughly in half — still over budget, which is - * exactly the vision's escalation ladder: Barnes-Hut is already switched on - * automatically above ~500 nodes in components/graph-view.tsx, and the - * bundled Web Worker / server-cached positions stay available as later - * rungs if real graphs ever get there. v1 learner graphs are dozens of - * nodes; the stress case is preparation, not expectation. - */ - -import Graph from "graphology"; -import forceAtlas2 from "graphology-layout-forceatlas2"; - -const ITERATIONS = 100; - -/** A plausible learning graph: mostly-local chains plus long-range cross-links. */ -function synthesize(nodeCount: number, edgeCount: number): Graph { - const graph = new Graph(); - for (let i = 0; i < nodeCount; i++) { - graph.addNode(`n${i}`, { - label: `node ${i}`, - size: 6, - x: Math.random() * 100, - y: Math.random() * 100, - }); - } - let added = 0; - for (let i = 0; added < edgeCount && i < edgeCount * 3; i++) { - const from = Math.floor(Math.random() * nodeCount); - const span = Math.random() < 0.8 ? 1 + Math.floor(Math.random() * 10) : Math.floor(Math.random() * nodeCount); - const to = (from + span) % nodeCount; - if (from === to || graph.hasEdge(`n${from}`, `n${to}`)) continue; - graph.addEdge(`n${from}`, `n${to}`, { type: "arrow", size: 1 }); - added++; - } - return graph; -} - -function bench(label: string, nodeCount: number, edgeCount: number, settings: Record): void { - const graph = synthesize(nodeCount, edgeCount); - const start = performance.now(); - forceAtlas2.assign(graph, { - iterations: ITERATIONS, - settings: { ...forceAtlas2.inferSettings(graph), ...settings }, - }); - const elapsed = performance.now() - start; - console.log( - `${label}: ${elapsed.toFixed(1)} ms (${graph.order} nodes, ${graph.size} edges, ${ITERATIONS} iterations)`, - ); -} - -console.log(`Target regime — synchronous run-then-freeze (budget: <150 ms):\n`); -bench(" 100 nodes, ~200 edges", 100, 200, {}); -bench(" 300 nodes, ~600 edges", 300, 600, {}); -bench(" 500 nodes, ~1000 edges", 500, 1000, {}); -console.log(`\nStress case — 2k nodes (~4000 edges), the vision's escalation ladder:\n`); -bench(" naive repulsion (O(n²)) ", 2000, 4000, {}); -bench(" barnesHutOptimize ", 2000, 4000, { barnesHutOptimize: true }); -bench(" barnesHutOptimize (warmed JIT)", 2000, 4000, { barnesHutOptimize: true }); diff --git a/scripts/embed-graph.ts b/scripts/embed-graph.ts new file mode 100644 index 0000000..04514a0 --- /dev/null +++ b/scripts/embed-graph.ts @@ -0,0 +1,23 @@ +/** + * Embed the whole graph (backfill / cold start): one-time pass that + * fills the embedding ledger for every node — the same embedNewNodes the + * post-merge hook runs, just with an empty (or stale-model) ledger so + * every node is "new". First run downloads the model weights (~300MB) + * to the local HF cache; subsequent runs embed only what's missing. + * + * npx tsx scripts/embed-graph.ts + */ +import { graphStore } from "../src/store/graph.ts"; +import { embedNewNodes, EMBEDDING_MODEL, EMBEDDING_DIM } from "../src/server/embeddings.ts"; + +async function main() { + const t0 = Date.now(); + console.log(`[embed] model: ${EMBEDDING_MODEL} (dim ${EMBEDDING_DIM}, q8)`); + const { embedded, total } = await embedNewNodes(graphStore); + console.log(`[embed] embedded ${embedded} nodes → ledger holds ${total} vectors (${((Date.now() - t0) / 1000).toFixed(1)}s)`); +} + +main().catch((err) => { + console.error("[embed] failed:", err); + process.exit(1); +}); diff --git a/scripts/import-notes.ts b/scripts/import-notes.ts new file mode 100644 index 0000000..f1c7acf --- /dev/null +++ b/scripts/import-notes.ts @@ -0,0 +1,70 @@ +/** + * Import raw notes files through the format-agnostic front door (vision §1, + * 2026-09-13) — every *.md / *.txt file in a directory becomes one + * extraction run (extractor → auditor → merger) over its raw text. Run + * from this worktree only — data never crosses trees: + * + * npx tsx --env-file=.env.local scripts/import-notes.ts [concurrency] + * + * Default dir: real-data-extraction/learn-sessions (the prototype + * learning-session exports — deliberately NOT imported as chats: they run + * through the notes path, raw). Runs in this process: the dev server's + * status chips do NOT see them (registry is per-process) — the dev server + * sees runs triggered through /api/graph/import. + */ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts"; +import { embedNewNodes } from "../src/server/embeddings.ts"; +import { clusterNamingSettled } from "../src/agent/subagents/cluster-namer.ts"; +import { getGraph } from "../src/core/graph.ts"; +import { stores } from "../src/server/stores.ts"; + +async function main() { + const dir = process.argv[2] ?? "real-data-extraction/learn-sessions"; + const limit = Number(process.argv[3] ?? 4); + const files = fs + .readdirSync(dir) + .filter((f) => f.endsWith(".md") || f.endsWith(".markdown") || f.endsWith(".txt")) + .sort(); + console.log(`[import-notes] ${files.length} note files from ${dir} (concurrency ${limit})`); + + const pipeline = createExtractionPipeline({ ...stores, embedder: embedNewNodes }); + const queue = [...files]; + const totals = { nodes: 0, edges: 0, events: 0, failed: 0 }; + const workers = Array.from({ length: Math.min(limit, files.length) }, async () => { + for (;;) { + const file = queue.shift(); + if (!file) return; + const filePath = path.join(dir, file); + const t0 = Date.now(); + try { + const stats = await pipeline.importNotes({ filePath, text: fs.readFileSync(filePath, "utf-8") }); + totals.nodes += stats.nodes; + totals.edges += stats.edges; + totals.events += stats.events; + console.log( + `[import-notes] ${file}: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events in ${((Date.now() - t0) / 1000).toFixed(1)}s`, + ); + } catch (err) { + totals.failed++; + console.error(`[import-notes] ${file} FAILED:`, err); + } + } + }); + await Promise.all(workers); + + // The post-merge cluster naming is fire-and-forget inside the pipeline; + // settle it before the process exits so labels actually land. + await clusterNamingSettled(); + + const graph = getGraph(stores.graphStore); + const labels = stores.graphStore.readClusterLabels().length; + console.log( + `[import-notes] done: +${totals.nodes} nodes, +${totals.edges} edges, +${totals.events} events, ${totals.failed} failed — ` + + `graph now ${graph.nodes.length} nodes, ${graph.edges.length} edges, ${graph.events.length} events, ` + + `${labels} named clusters`, + ); +} + +main(); \ No newline at end of file diff --git a/scripts/replay-layout.ts b/scripts/replay-layout.ts new file mode 100644 index 0000000..ef71041 --- /dev/null +++ b/scripts/replay-layout.ts @@ -0,0 +1,125 @@ +/** + * Option A simulation: replay the CURRENT graph through the incremental + * layout as if it had arrived one imported-notes file at a time — the + * organic growth path the user described (one dependency structure / + * cluster at a time, orphans collecting on the rim). + * + * Reads ONLY the ledgers (nodes / edges / mappings / sources); writes ONLY + * data/graph/positions.json — no LLM calls, no extraction, no chat records. + * + * npx tsx scripts/replay-layout.ts [--depth=vertical|radial] + * + * --depth regenerates the ledger WITH inference-depth anchoring (the + * playground mode baked into the rest state); omit it for the free layout. + */ +import { graphStore } from "../src/store/graph.ts"; +// --depth=vertical|radial: bake the playground depth mode into the ledger. +const depthArg = process.argv.find((a) => a.startsWith("--depth=")); +const depthMode: "off" | "vertical" | "radial" = + depthArg === "--depth=vertical" ? "vertical" : depthArg === "--depth=radial" ? "radial" : "off"; +import { incrementalLayout } from "../src/core/layout.ts"; +import type { GraphNode, NodeMapping, Position } from "../src/core/contracts.ts"; + +function main() { + const t0 = Date.now(); + + // Unique nodes (lines may repeat an id — first defines). + const seen = new Set(); + const nodes: GraphNode[] = []; + for (const line of graphStore.readNodes()) { + if (seen.has(line.id)) continue; + seen.add(line.id); + nodes.push(line); + } + const edges = graphStore.readEdges(); + // The semantic-topic source: the embedding ledger (run + // scripts/embed-graph.ts first if missing/stale). + const embeddings = graphStore.readEmbeddings(); + if (Object.keys(embeddings.vectors).length === 0) { + console.warn("[replay] embedding ledger is empty — clusters will be absent (run scripts/embed-graph.ts)"); + } + + // Batches: each imported file, in real import order (sources.json + // importedAt). A node belongs to the file whose mapping first claims it + // (mapping line order = arrival); nodes no mapping covers (legacy) ride + // the final batch. + const sources = [...graphStore.readSources()].sort((a, b) => a.importedAt.localeCompare(b.importedAt)); + const mappings: NodeMapping[] = graphStore.readMappings(); + const batchOf = new Map(); // nodeId → srcId + for (const m of mappings) { + if (!batchOf.has(m.graphNodeId)) batchOf.set(m.graphNodeId, m.chatId); + } + + const delivered = new Set(); + let positions = new Map(Object.entries(graphStore.readPositions())); + if (positions.size > 0) { + console.log("[replay] discarding existing positions ledger — full replay from empty"); + positions = new Map(); + } + + const byId = new Map(nodes.map((n) => [n.id, n])); + const unmappedFallback = sources[sources.length - 1]?.id ?? ""; + + console.log(`[replay] ${nodes.length} unique nodes, ${edges.length} edge lines, ${sources.length} sources`); + let maxOldShift = 0; + let shiftReport = ""; + for (const source of sources) { + const batch = nodes.filter((n) => batchOf.get(n.id) === source.id); + const isLast = source.id === sources[sources.length - 1]!.id; + if (isLast) { + // Sweep in any never-mapped nodes with the last batch. + for (const n of nodes) { + if (!batchOf.has(n.id) && !delivered.has(n.id)) { + batch.push(n); + batchOf.set(n.id, unmappedFallback); + } + } + } + for (const n of batch) delivered.add(n.id); + + // Edges activate once BOTH endpoints exist (a later file can claim + // one endpoint — the edge lights up then). + const activeEdges = edges.filter((e) => delivered.has(e.from) && delivered.has(e.to)); + + const before = new Map(positions); + const batchNodes = nodes.filter((n) => delivered.has(n.id)); + positions = incrementalLayout(batchNodes, activeEdges, positions, { + // The same partition the real settle would see for this batch: + // the persisted semantic topics, filtered to delivered nodes + // (never recomputed — the agglomerative pass is ~1.5s at scale). + clusters: embeddings.clusters + .map((c) => ({ members: c.members.filter((id) => delivered.has(id)) })) + .filter((c) => c.members.length > 0), + ...(depthMode === "off" + ? {} + : { + depthAnchor: { + mode: depthMode, + strength: 0.8, + spacing: 60, + ...(depthMode === "radial" ? { baseRadius: 220 } : {}), + }, + }), + }); + for (const [id, prev] of before) { + const now = positions.get(id)!; + const shift = Math.hypot(prev.x - now.x, prev.y - now.y); + if (shift > maxOldShift) { + maxOldShift = shift; + shiftReport = `${id} ${prev.x.toFixed(0)},${prev.y.toFixed(0)} -> ${now.x.toFixed(0)},${now.y.toFixed(0)} (batch ${batch.length} new, ${activeEdges.length} edges)`; + } + } + const label = source.filePath.split("/").pop() ?? source.id; + console.log( + `[replay] +${String(batch.length).padStart(3)} nodes ${String(activeEdges.length).padStart(4)} edges active (${label})`, + ); + } + + graphStore.writePositions(Object.fromEntries(positions)); + console.log( + `[replay] done: ${positions.size} positions written in ${((Date.now() - t0) / 1000).toFixed(1)}s` + + ` (max old-node shift within a batch: ${maxOldShift.toFixed(1)}: ${shiftReport})`, + ); +} + +main(); \ No newline at end of file diff --git a/src/agent/lh-tool.ts b/src/agent/lh-tool.ts index 3a2a14d..24a7b45 100644 --- a/src/agent/lh-tool.ts +++ b/src/agent/lh-tool.ts @@ -66,7 +66,10 @@ export function toPiTool(def: LhTool): AgentTool { name: def.name, label: def.name, description: def.description, - parameters: z.toJSONSchema(def.schema as z.ZodType) as never, + // io: "input" — the declaration describes what the MODEL must send, + // so .default() fields are optional in the schema (omission = the + // default); required-empty arrays are a known weak-model trap. + parameters: z.toJSONSchema(def.schema as z.ZodType, { io: "input" }) as never, // Question tools opt into SEQUENTIAL execution: pi runs the whole // batch in call order when any called tool opts in, so the // tool-input-available chunks arrive in the model's call order — the @@ -110,6 +113,9 @@ export function piToolDeclarations(registry: Record>) { name, label: name, description: def.description, - parameters: z.toJSONSchema(def.schema as z.ZodType) as never, + // io: "input" — the declaration describes what the MODEL must send, + // so .default() fields are optional in the schema (omission = the + // default); required-empty arrays are a known weak-model trap. + parameters: z.toJSONSchema(def.schema as z.ZodType, { io: "input" }) as never, })); } \ No newline at end of file diff --git a/src/agent/subagents/cluster-namer.ts b/src/agent/subagents/cluster-namer.ts new file mode 100644 index 0000000..ba4ef8c --- /dev/null +++ b/src/agent/subagents/cluster-namer.ts @@ -0,0 +1,198 @@ +/** + * CLUSTER NAMING (vision §1, DECIDED 2026-09-13): the garnish layer over + * community detection. Detection is purely structural and deterministic + * (core/clusters.ts); naming is ONE singular cheap LLM call that labels + * every NEW/CHANGED community at once, given member names + statements. + * + * Core-survival: a community keeps an existing labeling while at least + * half its core survives in it (matchClusterLabelings); only communities + * with no surviving label AND at least MIN_CLUSTER_SIZE members are sent + * to the namer. Records are append-only and idempotent (content-hashed + * ids dedupe at the store seam), so a naming race or a re-run appends + * nothing new. + * + * Runs POST-EXTRACTION (user-decided 2026-09-13): both front doors call + * refreshClusterLabels after a successful merge. Failure is NON-FATAL — + * the graph and hulls render fine unlabeled; the next run retries. + */ +import { z } from "zod"; +import { lhTool, toPiTools } from "../lh-tool.ts"; +import type { PiEngine } from "../models.ts"; +import { runSubagent } from "./pi-run.ts"; +import type { GraphNode } from "../../core/contracts.ts"; +import { getGraph, clusterContentId } from "../../core/graph.ts"; +import { matchClusterLabelings, MIN_CLUSTER_SIZE, type Community } from "../../core/clusters.ts"; +import type { GraphStore } from "../../core/ports.ts"; + +/** How many member propositions the namer sees per cluster — enough to + * recognize the domain without paying for the whole region. */ +const MAX_MEMBERS_PER_CLUSTER = 12; + +const CLUSTER_NAMER_INSTRUCTIONS = `You name communities of propositions in a personal knowledge graph. + +You are given clusters. Each cluster is a set of related propositions +(theory fragments the author is learning) plus, for context, a sketch of +every OTHER cluster already named. For each UNNAMED cluster, mint ONE +short DOMAIN LABEL: 1-4 words naming the region of the world model those +propositions belong to together — like a section heading in a textbook. +Use the members' own vocabulary ("Differential forms", "Sales +psychology", "Server architecture"), not generic words ("Topics", +"Various concepts"). If the propositions are unrelated, that is fine: +name the best common denominator anyway — a reader should recognize the +region at a glance. + +Your entire deliverable is ONE submit_labels call. Include EVERY +top-level field even when empty (labels: [] when nothing to name).`; + +const labelsSchema = z.object({ + labels: z + .array( + z.object({ + /** The cluster key from the prompt ("c1", "c2", ...). */ + key: z.string(), + label: z.string(), + }), + ) + .default([]), +}); + +export interface UnnamedCluster { + key: string; + members: string[]; + nodes: Pick[]; +} + +/** One single-submit namer run (the same shape as every extraction stage): + * the whole deliverable is ONE zod-validated submit call. */ +async function nameClusters( + engine: PiEngine | undefined, + signal: AbortSignal | undefined, + unnamed: UnnamedCluster[], +): Promise<{ key: string; label: string }[]> { + let submitted: { key: string; label: string }[] | undefined; + await runSubagent( + toPiTools({ + submit_labels: lhTool({ + name: "submit_labels", + description: "Submit the cluster labels.", + schema: labelsSchema, + execute: async (labels) => { + submitted = labels.labels; + return labels; + }, + }), + }), + { + role: "labeler", + engine, + systemPrompt: CLUSTER_NAMER_INSTRUCTIONS, + prompt: JSON.stringify( + { + unnamedClusters: unnamed.map((c) => ({ + key: c.key, + members: c.nodes.map((n) => ({ name: n.name, statement: n.statement })), + })), + }, + null, + 2, + ), + maxTurns: 4, + abortSignal: signal, + shouldStop: (_message, toolResults) => + toolResults.some((r) => r.toolName === "submit_labels" && !r.isError), + }, + ); + if (!submitted) throw new Error("the cluster namer never submitted labels"); + return submitted; +} + +export interface ClusterRefreshStats { + /** Communities the current graph divides into (all sizes). */ + detected: number; + /** Communities that kept a label via core-survival. */ + claimed: number; + /** Communities sent to the namer (unnamed, size >= MIN_CLUSTER_SIZE). */ + named: number; + /** Labeling records actually appended (idempotent dedupe may shrink this). */ + appended: number; +} + +/** + * The post-extraction garnish: detect communities on the CURRENT graph, + * join persisted labelings (core-survival), and name what is new — one + * singular LLM call. Appends ClusterLabeling records; the NEXT snapshot + * read joins them (the payload is derived, never stored). + * + * Serialized: concurrent merges (parallel notes imports) chain here, so + * two runs never mint duplicate labels for the same community. + */ +async function doRefresh(deps: { + graphStore: GraphStore; + /** Test seam: a scripted engine (production uses the labeler role). */ + engine?: PiEngine; + signal?: AbortSignal; +}): Promise { + const { graphStore, engine, signal } = deps; + const { nodes } = getGraph(graphStore); + // Semantic topics: the partition the embedder persisted in the + // embedding ledger (same source the snapshot renders — never + // recomputed here; the agglomerative pass is ~1.5s at scale). + const communities = graphStore.readEmbeddings().clusters; + const labelings = graphStore.readClusterLabels(); + const claims = matchClusterLabelings(communities, labelings); + const byId = new Map(nodes.map((n) => [n.id, n])); + + const claimed = claims.filter((c) => c !== null).length; + const unnamed = communities + .filter((c, i) => !claims[i] && c.members.length >= MIN_CLUSTER_SIZE) + .map((c: Community, i) => { + const nodesFor = c.members + .map((id) => byId.get(id)) + .filter((n): n is GraphNode => n !== undefined) + .slice(0, MAX_MEMBERS_PER_CLUSTER) + .map((n) => ({ name: n.name, statement: n.statement })); + return { key: `c${i}`, members: c.members, nodes: nodesFor }; + }); + + if (unnamed.length === 0) return { detected: communities.length, claimed, named: 0, appended: 0 }; + + const labels = await nameClusters(engine, signal, unnamed); + let appended = 0; + const seen = new Set(); + for (const { key, members } of unnamed) { + if (seen.has(key)) continue; + const label = labels.find((l) => l.key === key)?.label?.trim(); + if (!label) continue; + seen.add(key); + graphStore.appendClusterLabel({ + id: clusterContentId(members), + label, + core: members, + createdAt: new Date().toISOString(), + }); + appended++; + } + return { detected: communities.length, claimed, named: unnamed.length, appended }; +} + +let namingTail: Promise = Promise.resolve(); + +/** + * Fire the refresh, serialized behind any refresh already running (a + * parallel import's merge waits its turn; each run reads fresh ledgers). + * Waivable: call sites may fire-and-forget; scripts await the return. + */ +export function refreshClusterLabels(deps: { + graphStore: GraphStore; + engine?: PiEngine; + signal?: AbortSignal; +}): Promise { + const run = namingTail.then(() => doRefresh(deps)); + namingTail = run.catch(() => {}); + return run; +} + +/** Await every refresh fired so far (scripts: don't exit mid-naming). */ +export async function clusterNamingSettled(): Promise { + await namingTail; +} diff --git a/src/agent/subagents/extraction.ts b/src/agent/subagents/extraction.ts index 42bd98e..67157f6 100644 --- a/src/agent/subagents/extraction.ts +++ b/src/agent/subagents/extraction.ts @@ -1,7 +1,9 @@ +import * as crypto from "node:crypto"; import type { UIMessage } from "ai"; import type { Usage } from "@earendil-works/pi-ai"; import { z } from "zod"; import { lhTool, toPiTools } from "../lh-tool.ts"; +import type { ModelRole } from "../../../config.ts"; import type { PiEngine } from "../models.ts"; import { runSubagent } from "./pi-run.ts"; import type { @@ -14,6 +16,9 @@ import type { SessionFindings, } from "../../core/contracts.ts"; import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts"; +import type { GraphStore } from "../../core/ports.ts"; +import { settleGraphLayout } from "../../core/layout-settle.ts"; +import { refreshClusterLabels } from "./cluster-namer.ts"; /** * The extraction pipeline (vision §1 construction, 2026-09-02 redesign) — the @@ -42,6 +47,17 @@ import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../ * whatever the in-flight run missed). On failure nothing is checkpointed — * the next trigger re-runs the whole thing. * + * The FORMAT-AGNOSTIC FRONT DOOR (2026-09-13): importNotes() runs the SAME + * three stages over raw personal notes (arbitrary format — no chat, no + * session, no message ids). A notes-mode extractor prompt replaces the + * session one; the auditor and merger are shared (identity + deterministic + * appends don't care where text came from). Provenance is the file itself: + * source ids ("src_" + path hash) play the chatId role in mappings, events + * and checkpoints, and the source record (graphStore sources seam) carries + * the file path + the run's usage. No quiz events are ever built from + * notes — quiz outcomes inside the text surface only through the + * extractor's understanding observations (user-decided 2026-09-13). + * * Observability (§5 baseline): every stage logs one line per step, its * start, and its duration; the pipeline logs a failure line before * rethrowing, so the server log reads start → stage timings → merged @@ -154,7 +170,9 @@ const findingsSchema = z.object({ .describe("Every distinct proposition the session touched — true statements, misconception-free."), edges: z .array(edgeSchema) - .describe("Relations between propositions: follows (derivable) / supports (grounds without entailing)."), + .describe("Relations between propositions: follows (derivable) / supports (grounds without entailing). Pass [] when there are none — never omit the field.") + .default([]), // weak models omit required-but-empty arrays; omission means "none" + attribution: z .object({ quiz: z @@ -163,9 +181,51 @@ const findingsSchema = z.object({ "toolCallId of EVERY quiz line → the proposition id it was really testing. Attribute RETROACTIVELY — the whole transcript is visible; map each quiz to the proposition it tests (often articulated elsewhere in the session). null when genuinely unattributable.", ), }) - .describe("Quiz attribution — your only event judgment."), + .describe("Quiz attribution — your only event judgment. Pass { quiz: {} } when the session has no quiz lines — never omit the field.") + .default({ quiz: {} }), // weak models omit empty objects too }); +const NOTES_EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE source — +the user's own NOTES, in an ARBITRARY format (personal notes, an exported conversation, a document — anything) — and +report what those notes show the author engaging with, as structured findings for their global knowledge graph. + +There is no session and no chat structure here: the source is raw text, the author's own material. Read it as such. + +Report, with verbatim evidence quotes for everything: + +NODE THEORY — the graph is a set of TRUE propositions about the world, full stop. A node is never anything about the +author or the source. The author's relationship to a proposition is expressed ENTIRELY by your understanding note; it +never leaks into the node itself. So no "the author understands X", no "mentioned in these notes": the node is just X. +A misconception is NOT a node of its own wrong claim — state the TRUE correction as the proposition. + +1. PROPOSITIONS — every distinct reasoning-step-sized proposition the notes actually engage with: + - Each has a "name" — a VERY short label, a few words, a compact handle like a glossary headword ("Will to power", + "Divine command theory"). Not a sentence; it becomes the node's label in the graph, so keep it glanceable. + - And a "statement" — the self-contained TRUE proposition at full precision. + - Notes may explain things at length or quote material the author is studying: extract the propositions themselves, + one reasoning step each — never a topic blob, never a sub-step. + - "messages": ALWAYS an empty list — provenance is the file itself. + - "aliases": verbatim phrasings the notes used for the same proposition. + - "understanding": ONE word from the fixed scale — "solid", "shaky", "accepted", "resisted", or "not yet engaged" — + judging the AUTHOR's grasp of the proposition as the notes show it: their own commentary, self-assessments, + recorded quiz outcomes, the way they write about it. Strictly one word, no numbers, no justification (a + wrong-shaped note is rejected by the schema). When the notes give no signal, omit the field. + +2. EDGES between propositions: + - "follows": to is DERIVABLE from from (understanding from makes understanding to achievable). + - "supports": from GROUNDS to without entailing it (helpful context, not a prerequisite). + - Only relations the notes actually articulate — with evidence. + +There is no quiz attribution for notes — the source has no quiz events, and the schema has no attribution field. + +Evidence discipline: every quote must appear VERBATIM in the source. The auditor rejects anything unbacked. When in +doubt, report; the auditor decides. + +Include EVERY top-level field of the call even when empty — "edges": [] when there are no relations. NEVER omit +a field because it is empty. + +Your entire deliverable is ONE submit_findings call.`; + const EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE session transcript and report what was learned — as structured findings for the learner's global knowledge graph. @@ -208,25 +268,35 @@ quiz outcomes carry that the learner believed the wrong version. Evidence discipline: every quote must appear VERBATIM in the transcript view (a QUIZ line counts as the transcript). The auditor rejects anything unbacked. When in doubt, report; the auditor decides. +Include EVERY top-level field of the call even when empty — "edges": [], "attribution": { quiz: {} } when there +are no quizzes. NEVER omit a field because it is empty. + Your entire deliverable is ONE submit_findings call.`; -/** One extractor run: the whole deliverable is ONE zod-validated submit - * call (a pi agent purely so the model can retry a schema-invalid call). - * Returns the submitted findings — captured from the execute seam, which +/** One single-submit agent run — the shared shape of EVERY LLM stage + * here (session + notes): the whole deliverable is ONE zod-validated + * submit call (a pi agent purely so the model can retry a schema-invalid + * call). The submitted value is captured from the execute seam, which * only fires on a schema-valid call. */ -async function runExtractor( - engine: PiEngine | undefined, - prompt: string, - signal: AbortSignal | undefined, - onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void, -): Promise { - let submitted: SessionFindings | undefined; +async function runSubmitAgent(opts: { + role: ModelRole; + toolName: string; + toolDescription: string; + schema: z.ZodType; + systemPrompt: string; + prompt: string; + engine: PiEngine | undefined; + signal: AbortSignal | undefined; + onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void; + neverSubmitted: string; +}): Promise { + let submitted: T | undefined; const result = await runSubagent( toPiTools({ - submit_findings: lhTool({ - name: "submit_findings", - description: "Submit the session findings.", - schema: findingsSchema, + [opts.toolName]: lhTool({ + name: opts.toolName, + description: opts.toolDescription, + schema: opts.schema, execute: async (findings) => { submitted = findings; return findings; @@ -234,28 +304,71 @@ async function runExtractor( }), }), { - role: "extractor", - engine, - systemPrompt: EXTRACTOR_INSTRUCTIONS, - prompt, + role: opts.role, + engine: opts.engine, + systemPrompt: opts.systemPrompt, + prompt: opts.prompt, maxTurns: 4, - abortSignal: signal, + abortSignal: opts.signal, onAssistantMessage: (message, step) => - onStep( + opts.onStep( message.usage, message.model, step, message.stopReason, message.content.filter((c) => c.type === "toolCall").map((c) => (c as { name: string }).name).join(", ") || "(none)", ), - shouldStop: (_message, toolResults) => toolResults.some((r) => r.toolName === "submit_findings" && !r.isError), + shouldStop: (_message, toolResults) => toolResults.some((r) => r.toolName === opts.toolName && !r.isError), }, ); void result; - if (!submitted) throw new Error("the extractor never submitted findings"); + if (!submitted) throw new Error(opts.neverSubmitted); return submitted; } +/** The extractor over ONE session transcript (the chat front door). */ +function runExtractor( + engine: PiEngine | undefined, + prompt: string, + signal: AbortSignal | undefined, + onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void, +): Promise { + return runSubmitAgent({ + role: "extractor", + toolName: "submit_findings", + toolDescription: "Submit the session findings.", + schema: findingsSchema, + systemPrompt: EXTRACTOR_INSTRUCTIONS, + prompt, + engine, + signal, + onStep, + neverSubmitted: "the extractor never submitted findings", + }); +} + +/** The extractor over ONE source of raw personal notes (the format-agnostic + * front door): same single-submit shape, notes-mode prompt + schema. */ +function runNotesExtractor( + engine: PiEngine | undefined, + prompt: string, + signal: AbortSignal | undefined, + onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void, +): Promise { + return runSubmitAgent({ + role: "extractor", + toolName: "submit_findings", + toolDescription: "Submit the notes findings.", + schema: notesFindingsSchema, + systemPrompt: NOTES_EXTRACTOR_INSTRUCTIONS, + prompt, + engine, + signal, + onStep, + neverSubmitted: "the extractor never submitted findings", + }); +} + // ─── the auditor ─────────────────────────────────────────────────────────── const auditSchema = findingsSchema.extend({ @@ -276,14 +389,25 @@ const auditSchema = findingsSchema.extend({ reason: z.string(), }), ) - .describe("Everything you dropped, with the reason — the audit trail."), + .describe("Everything you dropped, with the reason — the audit trail. Pass [] when you dropped nothing — never omit the field.") + .default([]), // same weak-model tolerance as findings.edges }); -const AUDITOR_INSTRUCTIONS = `You are the audit stage of a learning application's extraction pipeline. You receive -ONE session transcript, a set of proposed findings (propositions, edges, quiz attribution), and the CURRENT -knowledge graph. Your jobs: the faithfulness net, and identity. +// The notes front door's shapes: everything session findings have, minus +// quiz attribution — raw notes have no quiz tool parts, so there is +// nothing to attribute (and no event to build: outcomes inside the text +// surface only through understanding observations, user-decided). +const notesFindingsSchema = findingsSchema.omit({ attribution: true }); +const notesAuditSchema = auditSchema.omit({ attribution: true }); +type NotesFindings = Omit; +type NotesAuditResult = Omit; -FAITHFULNESS — verify every candidate against the transcript: +const AUDITOR_INSTRUCTIONS = `You are the audit stage of a learning application's extraction pipeline. You receive +ONE source — a session transcript, or the author's personal notes in an ARBITRARY format (the notes front door) — a +set of proposed findings (propositions, edges, and quiz attribution where the source has quiz lines), and the +CURRENT knowledge graph. Your jobs: the faithfulness net, and identity. + +FAITHFULNESS — verify every candidate against the source text: - PROPOSITIONS: every evidence quote must appear (near-)verbatim in the transcript view, and the statement must be a faithful reading of it. The node theory applies: a statement must be a TRUE, self-contained, world-facing fact — reword learner- or session-relative statements ("the learner understands X" → "X"), state @@ -293,8 +417,10 @@ FAITHFULNESS — verify every candidate against the transcript: - EDGES: the evidence must actually articulate the relation, in the direction claimed. Drop direction-swapped or speculative edges. - ATTRIBUTION: the QUIZ line's question must genuinely test the attributed proposition. Drop wrong - attributions to null, or re-point them to the proposition the question actually tests. -- UNDERSTANDING levels: verify against the quiz outcomes and the session's evidence; each note must be + attributions to null, or re-point them to the proposition the question actually tests. (Session + sources only — notes findings carry no attribution; there is nothing to verify.) +- UNDERSTANDING levels: verify against the quiz outcomes and the session's evidence — or, for notes, + the author's own words; each note must be exactly one word from the fixed scale ("solid", "shaky", "accepted", "resisted", "not yet engaged") — reword or drop anything else. No justifications. @@ -311,7 +437,10 @@ Also drop propositions not referenced by anything else ONLY if they have no othe well-evidenced proposition is still knowledge. Never invent new candidates. Your entire deliverable is ONE submit_review call containing the verified findings -(the same shapes you received, with linkTo added) plus the dropped list.`; +(the same shapes you received, with linkTo added) plus the dropped list. + +Include EVERY top-level field of the call even when empty — "edges": [], "dropped": []. NEVER omit a field +because it is empty.`; /** The current graph, compact — the auditor's identity context. */ function graphView(nodes: GraphNode[]): string { @@ -324,47 +453,47 @@ function graphView(nodes: GraphNode[]): string { return lines.join("\n"); } -/** One auditor run — same single-submit shape as the extractor. */ -async function runAuditor( +/** The auditor over session findings (the chat front door). */ +function runAuditor( engine: PiEngine | undefined, prompt: string, signal: AbortSignal | undefined, onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void, ): Promise { - let submitted: AuditResult | undefined; - const result = await runSubagent( - toPiTools({ - submit_review: lhTool({ - name: "submit_review", - description: "Submit the verified findings subset (with identity verdicts) + the dropped list.", - schema: auditSchema, - execute: async (review) => { - submitted = review; - return review; - }, - }), - }), - { - role: "auditor", - engine, - systemPrompt: AUDITOR_INSTRUCTIONS, - prompt, - maxTurns: 4, - abortSignal: signal, - onAssistantMessage: (message, step) => - onStep( - message.usage, - message.model, - step, - message.stopReason, - message.content.filter((c) => c.type === "toolCall").map((c) => (c as { name: string }).name).join(", ") || "(none)", - ), - shouldStop: (_message, toolResults) => toolResults.some((r) => r.toolName === "submit_review" && !r.isError), - }, - ); - void result; - if (!submitted) throw new Error("the auditor never submitted a review"); - return submitted; + return runSubmitAgent({ + role: "auditor", + toolName: "submit_review", + toolDescription: "Submit the verified findings subset (with identity verdicts) + the dropped list.", + schema: auditSchema, + systemPrompt: AUDITOR_INSTRUCTIONS, + prompt, + engine, + signal, + onStep, + neverSubmitted: "the auditor never submitted a review", + }); +} + +/** The auditor over notes findings (the format-agnostic front door): same + * faithfulness + identity job, notes-mode schema (no attribution). */ +function runNotesAuditor( + engine: PiEngine | undefined, + prompt: string, + signal: AbortSignal | undefined, + onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void, +): Promise { + return runSubmitAgent({ + role: "auditor", + toolName: "submit_review", + toolDescription: "Submit the verified findings subset (with identity verdicts) + the dropped list.", + schema: notesAuditSchema, + systemPrompt: AUDITOR_INSTRUCTIONS, + prompt, + engine, + signal, + onStep, + neverSubmitted: "the auditor never submitted a review", + }); } // ─── the pipeline status registry ───────────────────────────────────────────── @@ -376,6 +505,13 @@ export type ExtractionPhase = "extracting" | "auditing" | "merging"; * Exported for the status payload's consumer typing (the dock). */ export interface RunningExtraction { chatId: string; + /** Which front door the run came through — label + usage-persistence + * depend on it (chats bill their chat record; notes bill the source + * record). */ + kind: "chat" | "notes"; + /** Display label (the file path for notes imports); the status route + * falls back to the chat store for chat runs. */ + label?: string; phase: ExtractionPhase; startedAt: number; /** Live per-stage usage, accumulated as steps bill (the dock's cost / @@ -388,6 +524,10 @@ export interface RunningExtraction { const runningExtractions = new Map(); +/** Last minted startedAt — kept monotonic so two runs starting in the same + * millisecond never collide on the usage idempotency key. */ +let lastStartedAt = 0; + /** In-flight pipeline runs (oldest first) — the /api/graph/status payload. */ export function extractionStatuses(): RunningExtraction[] { return [...runningExtractions.values()].sort((a, b) => a.startedAt - b.startedAt); @@ -411,26 +551,51 @@ export interface ExtractionModels { /** Test seam: a scripted engine per stage (production uses the roles). */ extractor?: PiEngine; auditor?: PiEngine; + /** Test seam for the cluster namer (post-extraction garnish). */ + labeler?: PiEngine; } -export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {} +export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels { + /** The post-merge embedding batch (semantic clusters run on it). + * INJECTED by the composition root (the agent layer never imports + * the server layer — architecture law): production passes the real + * local-model embedder (server/embeddings.ts embedNewNodes), tests + * fake it. Failure inside it is non-fatal by contract. */ + embedder: (graphStore: GraphStore) => Promise; +} + +export interface NotesImportInput { + /** The source's file path — its identity (deterministic src id) AND its + * provenance label in the detail card. */ + filePath: string; + /** The file's raw content, VERBATIM — any format. */ + text: string; +} export interface ExtractionPipeline { /** Extract + audit + merge ONE chat. Single-flight per chat; resolves to * the merger's append stats (or throws — no checkpoint on failure). */ runExtraction(chatId: string): Promise; + /** The format-agnostic front door (2026-09-13): run the same pipeline + * over ONE raw notes file. The source id is deterministic per file + * path, so re-importing the same file re-runs extraction against the + * same source identity (mappings + events dedupe by it). */ + importNotes(input: NotesImportInput): Promise; } export function createExtractionPipeline(deps: ExtractionPipelineDeps): ExtractionPipeline { - const { chatStore, graphStore } = deps; + const { chatStore, graphStore, embedder: embed } = deps; const inFlight = new Set(); - function setPhase(chatId: string, phase: ExtractionPhase): void { + function setPhase(chatId: string, phase: ExtractionPhase, kind: "chat" | "notes", label?: string): void { const prev = runningExtractions.get(chatId); + if (!prev) lastStartedAt = Math.max(Date.now(), lastStartedAt + 1); runningExtractions.set(chatId, { chatId, + kind, + label: prev?.label ?? label, // phase changes never lose the label phase, - startedAt: prev?.startedAt ?? Date.now(), + startedAt: prev?.startedAt ?? lastStartedAt, usage: prev?.usage ?? {}, // phase changes never lose usage controller: prev?.controller ?? new AbortController(), }); @@ -525,7 +690,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti function persistRunUsage(chatId: string, durationMs: number, stats: EmissionStats | undefined): void { const run = runningExtractions.get(chatId); if (!run) return; - chatStore.appendExtractionUsage(chatId, { + const entry: ExtractionRunUsage = { startedAt: run.startedAt, at: new Date().toISOString(), durationMs, @@ -534,7 +699,9 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti extractor: run.usage.extractor ?? { model: "unknown", steps: [] }, auditor: run.usage.auditor ?? { model: "unknown", steps: [] }, }, - }); + }; + if (run.kind === "notes") graphStore.appendSourceUsage(chatId, entry); + else chatStore.appendExtractionUsage(chatId, entry); } async function runExtraction(chatId: string): Promise { @@ -543,19 +710,41 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti const t0 = performance.now(); try { console.info(`[extraction ${chatId}] pipeline start`); - setPhase(chatId, "extracting"); + setPhase(chatId, "extracting", "chat"); const findings = await extract(chatId); - setPhase(chatId, "auditing"); + setPhase(chatId, "auditing", "chat"); const auditResult = await audit(chatId, findings); - setPhase(chatId, "merging"); - // The merger re-reads the chat record itself — events are rebuilt - // from the transcript, never from stale copies. - const stats = mergeFindings(chatId, auditResult, { chatStore, graphStore }); + setPhase(chatId, "merging", "chat"); + // The merger merges from the CURRENT record — events are rebuilt + // from the live transcript, never from stale copies. + const record = chatStore.getChat(chatId); + if (!record) throw new Error(`Chat not found: ${chatId}`); + const stats = mergeFindings(chatId, auditResult, { chatStore, graphStore }, { + messages: record.messages as UIMessage[], + foldedAt: record.updatedAt, + }); + // Semantic embeddings (DECIDED 2026-09-13): topics come from + // MEANING, so embed the new propositions BEFORE the settle and + // the namer look at clusters. Failure is non-fatal — clustering + // runs without the new nodes until the next merge retries. + await embed(graphStore).catch((err) => { + console.error(`[extraction ${chatId}] embedding failed (non-fatal):`, err); + }); + // Option A: the layout ledger evolves with the merge (sync — the + // next snapshot fetch must already reflect the new nodes). + settleGraphLayout(graphStore); persistRunUsage(chatId, Math.round(performance.now() - t0), stats); console.info( `[extraction ${chatId}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` + `(${Math.round(performance.now() - t0)}ms total)`, ); + // Post-extraction garnish (DECIDED 2026-09-13): name new/changed + // clusters. Fire-and-forget — naming never fails the run. + refreshClusterLabels({ + graphStore, + engine: deps.labeler, + signal: runningExtractions.get(chatId)?.controller.signal, + }).catch(() => {}); return stats; } catch (err) { persistRunUsage(chatId, Math.round(performance.now() - t0), undefined); @@ -577,6 +766,125 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti } } - return { runExtraction }; + // ── the notes front door (format-agnostic: raw text in, same graph out) ── + + /** The raw source view the notes stages see: the file path + the text + * VERBATIM. No message ids — provenance is the file itself. */ + function notesView(input: { filePath: string; text: string }): string { + return `SOURCE (the author's personal notes): ${input.filePath}\n\n${input.text}`; + } + + async function notesExtract(input: { id: string; filePath: string; text: string }): Promise { + const signal = runningExtractions.get(input.id)?.controller.signal; + signal?.throwIfAborted(); // an abort during a sync gap ends the run here + const view = notesView(input); + console.info(`[extraction ${input.id}] notes extractor starting (${view.split("\n").length} source lines)`); + const t0 = performance.now(); + const findings = await runNotesExtractor( + deps.extractor, + view, + signal, + (usage, model, step, finishReason, tools) => { + recordStep(input.id, "extractor", callUsage(usage), model); + console.info(`[extraction] extractor step ${step}: ${finishReason}, tools: ${tools}`); + }, + ); + console.info(`[extraction ${input.id}] notes extractor done in ${Math.round(performance.now() - t0)}ms`); + return findings; + } + + async function notesAudit(input: { id: string; filePath: string; text: string }, findings: NotesFindings): Promise { + const signal = runningExtractions.get(input.id)?.controller.signal; + signal?.throwIfAborted(); // an abort during the extract→audit gap ends here + const graph = getGraph(graphStore); + console.info( + `[extraction ${input.id}] notes auditor starting (${findings.propositions.length} propositions, ` + + `${findings.edges.length} edges, ${graph.nodes.length} graph nodes)`, + ); + const t0 = performance.now(); + const auditResult = await runNotesAuditor( + deps.auditor, + `Source:\n\n${notesView(input)}\n\nProposed findings:\n\n${JSON.stringify(findings, null, 2)}\n\n` + + `Current knowledge graph (${graph.nodes.length} nodes):\n\n${graphView(graph.nodes)}`, + signal, + (usage, model, step, finishReason, tools) => { + recordStep(input.id, "auditor", callUsage(usage), model); + console.info(`[extraction] auditor step ${step}: ${finishReason}, tools: ${tools}`); + }, + ); + if (auditResult.dropped.length > 0) { + console.info( + `[extraction ${input.id}] audit dropped ${auditResult.dropped.length}: ` + + auditResult.dropped.map((d) => `${d.ref} (${d.reason})`).join("; "), + ); + } + console.info( + `[extraction ${input.id}] audit kept ${auditResult.propositions.length}/${findings.propositions.length} propositions, ` + + `${auditResult.propositions.filter((p) => p.linkTo !== undefined).length} linked to existing nodes`, + ); + console.info(`[extraction ${input.id}] notes auditor done in ${Math.round(performance.now() - t0)}ms`); + return auditResult; + } + + async function runNotesExtraction(input: { id: string; filePath: string; text: string; importedAt: string }): Promise { + const { id } = input; + if (inFlight.has(id)) return { nodes: 0, edges: 0, events: 0 }; + inFlight.add(id); + const t0 = performance.now(); + try { + console.info(`[extraction ${id}] notes pipeline start (${input.filePath})`); + setPhase(id, "extracting", "notes", input.filePath); + const findings = await notesExtract(input); + setPhase(id, "auditing", "notes", input.filePath); + const notesResult = await notesAudit(input, findings); + setPhase(id, "merging", "notes", input.filePath); + // Normalize to the merger's findings shape: notes carry no quiz + // attribution, and the merger scans no quiz parts for notes + // (messages: null) — nothing to attribute, nothing to build. + const auditResult: AuditResult = { ...notesResult, attribution: { quiz: {} } }; + const stats = mergeFindings(id, auditResult, { chatStore, graphStore }, { messages: null, foldedAt: input.importedAt }); + // Same post-merge chain as the chat path: embeddings first + // (topics from meaning), then the settle, then the namer. + await embed(graphStore).catch((err) => { + console.error(`[extraction ${id}] embedding failed (non-fatal):`, err); + }); + // Option A: same post-merge settle as the chat path. + settleGraphLayout(graphStore); + persistRunUsage(id, Math.round(performance.now() - t0), stats); + console.info( + `[extraction ${id}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` + + `(${Math.round(performance.now() - t0)}ms total)`, + ); + refreshClusterLabels({ + graphStore, + engine: deps.labeler, + signal: runningExtractions.get(id)?.controller.signal, + }).catch(() => {}); + return stats; + } catch (err) { + persistRunUsage(id, Math.round(performance.now() - t0), undefined); + if (err instanceof Error && err.name === "AbortError") { + console.info(`[extraction ${id}] aborted (${Math.round(performance.now() - t0)}ms) — nothing merged, nothing checkpointed`); + return { nodes: 0, edges: 0, events: 0 }; + } + console.error(`[extraction ${id}] notes pipeline failed (${Math.round(performance.now() - t0)}ms):`, err); + throw err; + } finally { + inFlight.delete(id); + runningExtractions.delete(id); // done or failed — no longer running + } + } + + /** Import ONE raw notes file: register the source (deterministic id per + * file path — re-import keeps the same identity and its usage history) + * and run the pipeline over it. */ + async function importNotes(input: NotesImportInput): Promise { + const id = `src_${crypto.createHash("sha256").update(input.filePath).digest("hex").slice(0, 12)}`; + const importedAt = new Date().toISOString(); + graphStore.upsertSource({ id, filePath: input.filePath, importedAt }); + return runNotesExtraction({ id, ...input, importedAt }); + } + + return { runExtraction, importNotes }; } diff --git a/src/core/clusters.ts b/src/core/clusters.ts new file mode 100644 index 0000000..7bc17b2 --- /dev/null +++ b/src/core/clusters.ts @@ -0,0 +1,72 @@ +/** + * CLUSTER LABEL MATCHING (vision §1). DETECTION moved to semantic topic + * clustering over embeddings (core/semantic-clusters.ts, DECIDED + * 2026-09-13) — Louvain over the edge structure is retired: it only ever + * found the reasoning chains (the graph is disconnected by nature), and + * edge-orphans could never join anything. What REMAINS here is the + * cluster contract + core-survival label matching, which is + * detection-agnostic: it consumes the same { members } shape semantic + * clustering emits. + * + * Labelings key to sorted MEMBER SETS, never to detection indices — + * a labeling record outlives any reshuffle of the partition. + * + * Browser-safe pure logic (no node builtins, no fs): the server runs + * matching for the snapshot payload, tests run it directly. + */ +import type { ClusterLabeling } from "./contracts.ts"; + +/** A detected community. Members are SORTED node ids — the member set IS + * the identity a labeling record can outlive a reshuffle with. */ +export interface Community { + members: string[]; +} + +/** Communities below this size are not rendered, not named — presentation + * rule (user-decided 2026-09-13): a region of the world model needs at + * least three propositions to be worth a hull and a label. */ +export const MIN_CLUSTER_SIZE = 3; + +/** + * Claim labeling records for detected communities — CORE-SURVIVAL (DECIDED + * 2026-09-13): a community keeps a record's label while AT LEAST HALF of + * the record's core survives in it (overlap * 2 >= core.length; exactly + * half survives). Both sides are single-successor: one community holds at + * most one label, one record is claimed by at most one community — on a + * split, only the branch carrying the majority of the core keeps the name; + * on a merge, the best-overlapping record wins. + * + * Greedy assignment by overlap size (largest first; ties broken by newer + * record). Returns a claim (or null) per community, in input order — + * unclaimed communities are the namer's input. Pure: same inputs, same + * claims. + */ +export function matchClusterLabelings( + communities: readonly Community[], + labelings: readonly ClusterLabeling[], +): (ClusterLabeling | null)[] { + interface Pair { + community: number; + labeling: ClusterLabeling; + overlap: number; + } + const pairs: Pair[] = []; + communities.forEach((community, ci) => { + const members = new Set(community.members); + for (const labeling of labelings) { + const overlap = labeling.core.reduce((n, id) => n + (members.has(id) ? 1 : 0), 0); + if (overlap * 2 >= labeling.core.length && overlap > 0) { + pairs.push({ community: ci, labeling, overlap }); + } + } + }); + pairs.sort((a, b) => b.overlap - a.overlap || (a.labeling.createdAt < b.labeling.createdAt ? 1 : -1)); + const claims: (ClusterLabeling | null)[] = communities.map(() => null); + const takenRecords = new Set(); + for (const pair of pairs) { + if (claims[pair.community] || takenRecords.has(pair.labeling.id)) continue; + claims[pair.community] = pair.labeling; + takenRecords.add(pair.labeling.id); + } + return claims; +} diff --git a/src/core/contracts/graph.ts b/src/core/contracts/graph.ts index 24ea899..3a69258 100644 --- a/src/core/contracts/graph.ts +++ b/src/core/contracts/graph.ts @@ -20,6 +20,8 @@ /** Two relations only (DECIDED 2026-09-02): follows = b is derivable from a; * supports = a grounds b without entailing it. No scalars, no encoded * strength — strength is derived at read time from topology. */ +import type { ExtractionRunUsage } from "./chat.ts"; + export type EdgeType = "follows" | "supports"; /** What one extraction run merged into the graph ledgers — the merger's @@ -37,10 +39,32 @@ export interface EmissionStats { * edges record every (chat, message) where they came up, deep-linkable to * the exact transcript point. */ export interface ProvenancePoint { + /** The chat this point anchors to — or a notes source id ("src_…"): + * imports carry no message anchors (provenance is the file itself, + * recorded through the mapping line + the source record). */ chatId: string; messageId: string; } +/** One imported notes file — the format-agnostic front door's bookkeeping + * (DECIDED 2026-09-13): personal notes in ARBITRARY formats run through the + * same extractor → auditor → merger pipeline, with the file itself as the + * provenance anchor. There is no chat: the detail card shows the file path + * as a STATIC entry (not a link), and understanding observations still + * apply (judged from the author's own voice in the notes). The run's usage + * log lives here, mirroring ChatRecord.extractions. Ids are "src_" + hash + * of the file path — deterministic, so re-importing the same file re-runs + * extraction against the same source id (mappings/events dedupe by it). */ +export interface NotesSource { + id: string; + filePath: string; + /** ISO — when this file was imported (the checkpoint's foldedAt). */ + importedAt: string; + /** The run's usage log — same append-only discipline as + * ChatRecord.extractions, idempotent per startedAt. */ + extractions?: ExtractionRunUsage[]; +} + export interface GraphNode { /** Opaque stable id, store-assigned: "n_" + nanoid-like. */ id: string; @@ -107,6 +131,8 @@ export interface LedgerEvent { /** null = not yet attributed. */ nodeId: string | null; kind: LedgerEventKind; + /** The chat that produced this evidence — or a notes source id + * ("src_…") for imported-notes observations. */ chatId: string; at: string; /** Idempotency key: the quiz part's toolCallId, or `obs:` for @@ -244,11 +270,66 @@ export interface AuditResult { } /** The GET /api/graph payload — everything the home view's PURE renderer needs. */ +/** cluster-labels.jsonl line — one LLM-named community (vision §1, + * DECIDED 2026-09-13). Append-only history: records are immutable, never + * edited or deleted; dead labelings remain as history. IDENTITY is the + * sorted member set ("core"), hashed into the id, so re-naming the same + * community dedupes and a labeling survives graph edits as long as at + * least half its core survives in some single community (core-survival; + * under half → the community is renamed, splits can keep only one + * successor). */ +/** The embedding ledger shape (config.EMBEDDINGS_FILE): vectors for + * EVERY embedded node, all produced by one model at one dimensionality. */ +export interface EmbeddingsLedger { + /** The embedding model id (e.g. the HF repo) — mismatch = stale + * vectors, the writer re-embeds everything. */ + model: string; + /** Vector dimensionality (MRL-truncated on write). */ + dim: number; + /** nodeId → L2-normalized vector (cosine similarity = dot product). */ + vectors: Record; + /** The topic partition over those vectors, computed by the SAME + * embedder run (the user-decided trigger: recluster at the end of + * every embedding batch, never per read — the agglomerative pass is + * ~1.5s at current scale). Consumers (snapshot, settle, namer) read + * this instead of recomputing. Empty when nothing is embedded. */ + clusters: Array<{ members: string[] }>; + /** ISO timestamp of the last write. */ + updatedAt: string; +} + +export interface ClusterLabeling { + /** "clu_" + content hash of the sorted core. */ + id: string; + /** The short domain label (1–4 words), e.g. "Differential forms". */ + label: string; + /** The member node ids the label was minted for, sorted — the survival core. */ + core: string[]; + createdAt: string; +} + +/** One rendered cluster in the snapshot payload (vision §1). Members are + * node ids; the renderer draws the hull from member positions at draw + * time and gates the label inversely on the node-label zoom control. */ +export interface Cluster { + /** The labeling record id when claimed; unlabeled clusters still + * appear (hull, no name) with a member-hash id. */ + id: string; + /** null until the naming call has minted/claimed a labeling. */ + label: string | null; + /** Node ids, sorted. Hull + centroid are renderer concerns. */ + members: string[]; +} + export interface GraphSnapshot { nodes: GraphNode[]; edges: GraphEdge[]; /** chatId → title, for every chat that touched the graph. */ sessions: Record; + /** srcId → file path, for every IMPORTED NOTES file that touched the + * graph (the format-agnostic front door): a static provenance entry — + * the detail card shows the path as plain text, not a link. */ + fileSources: Record; /** nodeId → chatIds that touched the node (click-through provenance). */ nodeSessions: Record; /** nodeId → chatId → that session's understanding observation (the @@ -256,6 +337,20 @@ export interface GraphSnapshot { * shown next to the session list — the confidence history). Multiple * notes for one (node, chat) collapse to the latest. */ nodeObservations: Record>; + /** Structural communities (Louvain over follows+supports), size ≥ 3, + * joined with persisted LLM labelings. Purely structural — never + * depends on layout. */ + clusters: Cluster[]; + /** Resting layout positions per node id, when a positions ledger + * exists (core/layout.ts settle writes it). Absent on a fresh graph — + * the view then falls back to its own one-shot forceAtlas2 pass. */ + positions?: Record; +} + +/** A node's resting position in layout space (the positions ledger). */ +export interface Position { + x: number; + y: number; } /** Derive a short chat title from the learner's first message (pure). diff --git a/src/core/graph.ts b/src/core/graph.ts index c24bd3a..1c4863c 100644 --- a/src/core/graph.ts +++ b/src/core/graph.ts @@ -38,10 +38,11 @@ * adapters in. */ -import { randomBytes } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import { isToolUIPart, type UIMessage } from "ai"; import type { AuditResult, + Cluster, EmissionStats, GraphEdge, GraphNode, @@ -50,6 +51,7 @@ import type { LedgerEvent, LedgerEventKind, } from "./contracts.ts"; +import { matchClusterLabelings, MIN_CLUSTER_SIZE } from "./clusters.ts"; import type { QuestionAnswerData } from "./question-core.ts"; import type { ChatStore, GraphStore } from "./ports.ts"; @@ -64,6 +66,17 @@ function newGraphNodeId(): string { return `n_${randomBytes(6).toString("hex")}`; } +/** What one merge run merges FROM (the caller supplies it — the merger never + * reads the chat store): a session's transcript + its `updatedAt` + * checkpoint, or — for the format-agnostic notes front door (2026-09-13) — + * `messages: null`: raw notes carry no tool parts, so no quiz events are + * rebuilt (quiz outcomes inside notes surface only through the extractor's + * understanding observations), and the checkpoint stamps the import. */ +export interface MergeSource { + messages: UIMessage[] | null; + foldedAt: string; +} + // ─── transcript scanning (shared by the merger + trigger) ───────────────────── /** A flattened, order-preserving view of every tool part in a transcript. */ @@ -126,11 +139,15 @@ const samePoint = (a: ProvenancePoint, b: ProvenancePoint): boolean => * parts, nodeId from the extractor's attribution through this run's * mapping) plus one observation event per proposition carrying an * understanding note (ref `obs:`). No taught events, ever. + * + * `chatId` is the run's source key: a real chat, or a notes source id + * ("src_…") for the format-agnostic import path — same ledgers, same + * idempotency, one graph. The `source` parameter carries the transcript + * being merged (null messages = raw notes: no quiz events) and the + * checkpoint stamp; the merger itself is chat-store-free. */ -export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDeps): EmissionStats { - const { graphStore: ledgers, chatStore } = deps; - const record = chatStore.getChat(chatId); - if (!record) return { nodes: 0, edges: 0, events: 0 }; +export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDeps, source: MergeSource): EmissionStats { + const { graphStore: ledgers } = deps; const stats: EmissionStats = { nodes: 0, edges: 0, events: 0 }; // 1. Resolution: this run's propositions → graph node ids. Existing @@ -214,7 +231,7 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep for (const event of ledgers.readEvents()) { if (event.chatId === chatId && event.ref) seenRefs.add(event.ref); } - for (const part of toolParts(record.messages)) { + for (const part of source.messages ? toolParts(source.messages) : []) { if (part.name !== "quiz" || part.state !== "output-available") continue; const event = quizEvent(part, chatId, audit, mapping, now); if (event?.ref !== undefined && !seenRefs.has(event.ref)) { @@ -241,8 +258,9 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep stats.events++; } - // 4. Checkpoint: this chat is extracted through its current updatedAt. - ledgers.appendCheckpoint({ chatId, foldedAt: record.updatedAt }); + // 4. Checkpoint: this source is extracted through its foldedAt (the + // chat's updatedAt, or the import time for notes). + ledgers.appendCheckpoint({ chatId, foldedAt: source.foldedAt }); return stats; } @@ -314,6 +332,16 @@ export function pendingQuizAnswers(chatId: string, deps: GraphDeps): number { // ─── derivation (reads only — everything below is recomputable) ───────────── +/** The content identity of a community: "clu_" + sha256 of the sorted + * member ids (12 hex). Deterministic — the same member set always mints + * the same id — so re-naming races and idempotent re-runs dedupe at the + * store seam (appendClusterLabel). Lives here (server-only core), not in + * clusters.ts, because clusters.ts is barrel-exported and must stay + * browser-safe (no node:crypto). */ +export function clusterContentId(members: readonly string[]): string { + return "clu_" + createHash("sha256").update([...members].sort().join("\n")).digest("hex").slice(0, 12); +} + export interface DerivedGraph { nodes: GraphNode[]; edges: GraphEdge[]; @@ -423,20 +451,28 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot { if (event.nodeId !== null) touch(event.nodeId, event.chatId); } - // Titles for every chat the graph can point back at. - const chatIds = new Set(); + // Labels for every source the graph can point back at: real chats + // (title, deep-linkable) and imported notes files (path, static — the + // format-agnostic front door's provenance). + const sourceIds = new Set(); for (const chats of nodeChats.values()) { - for (const id of chats) chatIds.add(id); + for (const id of chats) sourceIds.add(id); } const sessions: Record = {}; - for (const id of chatIds) { + const fileSources: Record = {}; + for (const id of sourceIds) { const chat = deps.chatStore.getChat(id); - if (chat) sessions[id] = chat.title; + if (chat) { + sessions[id] = chat.title; + continue; + } + const source = deps.graphStore.readSources().find((s) => s.id === id); + if (source) fileSources[id] = source.filePath; } const nodeSessions: Record = {}; for (const [nodeId, chats] of nodeChats) { - const ids = [...chats].filter((id) => sessions[id] !== undefined); + const ids = [...chats].filter((id) => sessions[id] !== undefined || fileSources[id] !== undefined); if (ids.length > 0) nodeSessions[nodeId] = ids; } @@ -461,7 +497,41 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot { (nodeObservations[nodeId] ??= {})[chatId] = note; } - return { nodes, edges, sessions, nodeSessions, nodeObservations }; + // Semantic topic communities (DECIDED 2026-09-13, replacing structural + // Louvain): average-linkage clustering over embeddings — related + // topics, not reasoning chains. Edge-orphans join topics by meaning; + // stragglers stay loose (filtered at the presentation threshold + // below). The partition is computed ONCE per embedding batch by the + // embedder (the agglomerative pass is ~1.5s at current scale) and + // PERSISTED in the embedding ledger — reads just load it. Nodes whose + // embedding hasn't landed yet simply sit unclustered until the next + // read after the embedder runs. + const communities = deps.graphStore.readEmbeddings().clusters; + const claims = matchClusterLabelings(communities, deps.graphStore.readClusterLabels()); + const clusters: Cluster[] = communities + .map((community, index) => ({ community, claim: claims[index] })) + .filter(({ community }) => community.members.length >= MIN_CLUSTER_SIZE) + .map(({ community, claim }) => ({ + id: claim?.id ?? clusterContentId(community.members), + label: claim?.label ?? null, + members: community.members, + })); + + // Resting layout (Option A): the positions ledger, when the settle has + // ever run. Absent on a fresh graph — the view falls back to its own + // one-shot pass until the first merge writes the ledger. + const positions = deps.graphStore.readPositions(); + + return { + nodes, + edges, + sessions, + fileSources, + nodeSessions, + nodeObservations, + clusters, + ...(Object.keys(positions).length > 0 ? { positions } : {}), + }; } // Re-export: the contract owns the shape; legacy importers still reach it. diff --git a/src/core/index.ts b/src/core/index.ts index 84d1676..5e1a4bf 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -17,8 +17,15 @@ export * from "./contracts.ts"; export * from "./ports.ts"; +export * from "./render-settings.ts"; export * from "./question-core.ts"; +// clusters.ts (Louvain) is deliberately NOT re-exported: nothing in the +// shell runs community detection — the snapshot carries the partition +// from the server. Keeping it out keeps the heavy clustering deps out of the +// client bundle (verified 2026-09-13: chunks contained it while this +// line existed). Same logic as layout-settle.ts below. +export * from "./layout.ts"; export * from "./plan-tree.ts"; export * from "./phases.ts"; export * from "./pending.ts"; diff --git a/src/core/layout-settle.ts b/src/core/layout-settle.ts new file mode 100644 index 0000000..8a405d6 --- /dev/null +++ b/src/core/layout-settle.ts @@ -0,0 +1,44 @@ +import type { EmbeddingsLedger, GraphEdge, GraphNode } from "./contracts.ts"; +import { incrementalLayout } from "./layout.ts"; +import type { Position } from "./contracts.ts"; + +/** + * The merge-time settle: recompute the resting layout from the CURRENT + * graph + the positions ledger, and replace the ledger. Called by the + * extraction pipeline after every merge (the sole writer) and by the + * positions-commit endpoint when the graph grew under a live-physics + * commit (client positions become the anchor set). Synchronous — it must + * finish before the next /api/graph fetch reflects the change. + * + * SPLIT FROM layout.ts deliberately (2026-09-13): layout.ts is in the + * browser barrel, and the settle needs the Louvain partition for the + * cluster centroid pull — the heavy deps must not ship in the client + * bundle. Server code imports this module DIRECTLY; it is never + * re-exported through src/core/index.ts. + */ +export function settleGraphLayout(graphStore: { + readNodes(): GraphNode[]; + readEdges(): GraphEdge[]; + readPositions(): Record; + writePositions(positions: Record): void; + readEmbeddings(): EmbeddingsLedger; +}): void { + const nodes = graphStore.readNodes(); + if (nodes.length === 0) return; + // Node LINES may repeat an id (link-appends) — dedupe, first defines. + const seen = new Set(); + const unique = nodes.filter((n) => { + if (seen.has(n.id)) return false; + seen.add(n.id); + return true; + }); + const edges = graphStore.readEdges(); + const previous = new Map(Object.entries(graphStore.readPositions())); + // The cluster partition for the centroid pull: the SAME persisted + // semantic topics the snapshot renders (computed by the embedder + // once per batch, stored in the embedding ledger) — one cluster + // concept everywhere. + const communities = graphStore.readEmbeddings().clusters; + const next = incrementalLayout(unique, edges, previous, { clusters: communities }); + graphStore.writePositions(Object.fromEntries(next)); +} \ No newline at end of file diff --git a/src/core/layout.ts b/src/core/layout.ts new file mode 100644 index 0000000..df388db --- /dev/null +++ b/src/core/layout.ts @@ -0,0 +1,629 @@ +/** + * The layout engine — d3-force, everywhere (DECIDED 2026-09-13). + * + * One engine owns the equilibrium: the merge-time incremental settle AND + * the live drag physics run the SAME force set (links as springs, charge + * repulsion, weak gravity), so the positions ledger and the transient + * simulation agree on what "resting" means. (FA2 was replaced after a + * research round: it has no annealing/pinning/settle model and mixing it + * with a different live engine visibly re-flows the graph on every drag.) + * + * Layout EVOLVES instead of reshuffling (Option A): each settle warm-starts + * from the positions ledger and spring-ties every previously-positioned + * node toward its anchor (a per-node anchor force). New nodes seed near + * their positioned neighbors; orphans seed on the rim OUTSIDE the core and + * are anchored there, so they read as "not yet connected knowledge". + * + * Design constraints (kept deliberately, for live physics): + * - PURE: no fs, no server types — this module runs server-side (the + * merge settle) and in the browser (the live simulation, via the core + * barrel). A pinned node is just fx/fy; an anchored node is a spring. + * - DETERMINISTIC: seeds derive from node id; d3-force ticks are + * deterministic — same inputs, same layout. + */ + +import { + forceSimulation, + forceLink, + forceManyBody, + forceX, + forceY, + type Simulation, + type SimulationLinkDatum, + type SimulationNodeDatum, +} from "d3-force"; +import type { GraphEdge, GraphNode, Position } from "./contracts.ts"; +import { seededRandom } from "./question-core.ts"; + +/** A node the d3 simulation mutates (x/y/vx/vy; fx/fy = pinned). */ +export interface LayoutNode extends SimulationNodeDatum { + id: string; +} + +/** A link as d3 sees it (id strings resolve to node refs on init). */ +type SimLink = SimulationLinkDatum; + +/** How the inference rank is expressed spatially (playground, 2026-09-13): + * the graph's `follows`/`supports` edges encode derivation (premise → + * conclusion), so depth-0 nodes (no incoming edges — the fundamentals) + * can anchor a shared baseline and every derivation level rises from it. + * `vertical`: levels are horizontal rows, arrows point up. + * `radial`: levels are concentric rings, foundations at the center, the + * outer boundary IS the edge of the user's knowledge. + * `off`: today's free layout. */ +export type DepthAnchorMode = "off" | "vertical" | "radial"; + +export interface DepthAnchor { + mode: DepthAnchorMode; + /** Anchor pull toward the level (same units as clusterPull). */ + strength: number; + /** Distance between consecutive inference levels (world units). */ + spacing: number; + /** Baseline point: vertical mode uses .y as the depth-0 line; radial + * mode uses the full point as the ring center. Default (0,0). */ + center?: { x: number; y: number }; + /** Inner ring radius (radial only; depth-0 nodes sit at this ring). */ + baseRadius?: number; +} + +/** Inference depth per node: the longest chain of directed edges from a + * base node. CYCLE-SAFE (2026-09-13, user requirement — the DAG property + * is typical, not guaranteed): Tarjan SCC condensation collapses each + * strongly connected component to one super-node, ranks the (always + * acyclic) condensation, and every cycle member shares its component's + * depth — mutual support reads as equal footing. Iterative Tarjan (no + * recursion depth limits), O(V+E), deterministic. */ +export function inferenceDepths( + nodes: ReadonlyArray>, + edges: ReadonlyArray, +): Map { + const valid = new Set(nodes.map((n) => n.id)); + const adj = new Map(); + for (const e of edges) { + if (e.from === e.to || !valid.has(e.from) || !valid.has(e.to)) continue; + let list = adj.get(e.from); + if (!list) adj.set(e.from, (list = [])); + list.push(e.to); + } + + // Iterative Tarjan SCC. + const num = new Map(); + const low = new Map(); + const onStack = new Set(); + const stack: string[] = []; + const comp = new Map(); + let counter = 0; + let compCount = 0; + for (const start of valid) { + if (num.has(start)) continue; + num.set(start, counter); + low.set(start, counter); + counter++; + stack.push(start); + onStack.add(start); + const work: [string, number][] = [[start, 0]]; + while (work.length > 0) { + const frame = work[work.length - 1]!; + const v = frame[0]; + const outs = adj.get(v) ?? []; + if (frame[1] < outs.length) { + const i = frame[1]; + frame[1] = i + 1; + const w = outs[i]!; + if (!num.has(w)) { + num.set(w, counter); + low.set(w, counter); + counter++; + stack.push(w); + onStack.add(w); + work.push([w, 0]); + } else if (onStack.has(w)) { + low.set(v, Math.min(low.get(v)!, num.get(w)!)); + } + } else { + work.pop(); + if (low.get(v) === num.get(v)) { + let w: string; + do { + w = stack.pop()!; + onStack.delete(w); + comp.set(w, compCount); + } while (w !== v); + compCount++; + } + if (work.length > 0) { + const parent = work[work.length - 1]![0]; + low.set(parent, Math.min(low.get(parent)!, low.get(v)!)); + } + } + } + } + + // Condensation predecessors, then longest path. Tarjan emits sinks + // first, so descending comp index = topological order (sources first). + const preds: Set[] = Array.from({ length: compCount }, () => new Set()); + for (const e of edges) { + const a = comp.get(e.from); + const b = comp.get(e.to); + if (a == null || b == null || a === b) continue; + preds[b]!.add(a); + } + const depth = new Array(compCount).fill(0); + for (let c = compCount - 1; c >= 0; c--) { + let d = 0; + for (const p of preds[c]!) d = Math.max(d, depth[p]! + 1); + depth[c] = d; + } + const out = new Map(); + for (const n of nodes) out.set(n.id, depth[comp.get(n.id)!] ?? 0); + return out; +} + +export interface SimOptions { + /** Link spring rest length (world units). */ + linkDistance?: number; + /** Link spring stiffness MULTIPLIER (1 = d3's default + * 1/min(degree) stiffness). */ + linkStrength?: number; + /** Many-body repulsion strength (negative). */ + chargeStrength?: number; + /** Center-of-mass gravity: pulls loose components inward. Weak — + * anchors, not gravity, hold the evolved layout's frame. */ + gravity?: { x: number; y: number; strength: number }; + /** Alpha decay per tick — higher = snappier settle. The merge settle + * uses d3's default 0.0228 (~300 ticks); the live sim passes ~0.05. */ + alphaDecay?: number; + /** Velocity damping per tick (d3 default 0.4). Higher = more friction. */ + velocityDecay?: number; + /** Inference-depth anchoring (see DepthAnchor). Off by default. */ + depthAnchor?: DepthAnchor; + /** Cluster centroid pull (the "virtual node" without the node): + * members of each cluster accelerate toward their own cluster's + * centroid. Two O(n) passes per tick — cheaper than a real virtual + * node (which would need one extra link spring per member plus a + * maintained position). Strength 0 or absent = force off. + * Callers supply the partition (Louvain output shape). */ + clusterPull?: { + clusters: ReadonlyArray<{ members: ReadonlyArray }>; + strength: number; + }; +} + +/** Link input: edges are undirected for layout purposes. */ +export interface LayoutEdge { + from: string; + to: string; +} + +export const LINK_DISTANCE = 40; +export const CHARGE_STRENGTH = -120; +/** Per-tick anchor re-blend: fraction of an anchored node's displacement + * from its previous position that SURVIVES each tick. 0.85 = gentle + * evolution where forces are strong, stillness where they are not — + * the exact FA2-era constant. Position-space (alpha-independent), so it + * cannot be overpowered by d3's long-range link springs the way a force- + * space anchor can (forceLink is a Hooke spring: strength*(d-L)*alpha). */ +const ANCHOR_RETENTION = 0.85; +/** Gravity strengths. d3 manyBody repulsion falls off as 1/d, so two + * 20-node blobs at distance d push with ~n1*n2*|charge|/d; cluster + * spacing settles where that equals g*d*n/2 — g=0.03 lands clusters + * ~400 world units apart (the FA2-era density), and the anchor re-blend + * (position-space, alpha-independent) keeps gravity's transient sag + * from ever moving an anchored node at rest. */ +const BOOTSTRAP_GRAVITY = 0.2; +/** Exported: the live sim reuses this so its rest state matches the + * server settle's equilibrium (one engine, one rest). */ +export const INCREMENTAL_GRAVITY = 0.12; +/** Exported: default cluster-centroid pull strength (tuned by replay — + * compact enough to read as blobs, weak enough that cluster shape is + * still the edge network's). */ +export const CLUSTER_PULL_STRENGTH = 0.15; + +/** + * The one force set. Returns a STOPPED simulation — callers drive it + * manually (tick-per-frame live, or a bounded tick loop when settling). + * Deterministic; mutates `nodes` in place (d3 semantics: impure by design, + * zero per-tick allocation). + */ +export function createGraphSimulation( + nodes: LayoutNode[], + edges: ReadonlyArray, + options: SimOptions = {}, +): Simulation { + // Link stiffness: d3's default is 1/min(deg) (degree-scaled springs). + // linkStrength multiplies it, so 1 = unchanged default behavior. + const degree = new Map(); + for (const e of edges) { + degree.set(e.from, (degree.get(e.from) ?? 0) + 1); + degree.set(e.to, (degree.get(e.to) ?? 0) + 1); + } + const linkStrength = options.linkStrength ?? 1; + + const sim = forceSimulation(nodes) + .force( + "link", + forceLink( + edges.map((e) => ({ source: e.from, target: e.to })), + ) + .id((d) => d.id) + .distance(options.linkDistance ?? LINK_DISTANCE) + .strength( + (l) => + (linkStrength * 1) / + Math.min(degree.get((l.source as LayoutNode).id) ?? 1, degree.get((l.target as LayoutNode).id) ?? 1), + ), + ) + .force("charge", forceManyBody().strength(options.chargeStrength ?? CHARGE_STRENGTH)) + .alphaDecay(options.alphaDecay ?? 0.0228) + .velocityDecay(options.velocityDecay ?? 0.4) + .stop(); + + // Gravity, mode-projected: in vertical mode the depth force OWNS the + // y axis, so gravity goes x-only (it would fight the level lines); in + // radial mode a center pull is purely radial and would fight the + // rings, so it is dropped entirely (rings already bound the radius). + // Depth mode projections (DECIDED 2026-09-13), with the ORPHAN RULE + // (user-decided, same day): a node with no edges at all has no + // dependency structure — depth is a property of inference chains, so + // stamping unconnected propositions onto the baseline / base ring + // assigns them meaning they don't have. Orphans are exempt from the + // depth force and keep the REGULAR view's forces instead (full center + // gravity + full cluster pull, unprojected). + const mode = options.depthAnchor?.mode ?? "off"; + let orphan: Uint8Array | null = null; + if (mode !== "off") { + const connected = new Set(); + const valid = new Set(nodes.map((n) => n.id)); + for (const e of edges) { + // Self-loops and dangling endpoints are not a structure (same + // guards as inferenceDepths). + if (e.from === e.to || !valid.has(e.from) || !valid.has(e.to)) continue; + connected.add(e.from); + connected.add(e.to); + } + orphan = new Uint8Array(nodes.length); + nodes.forEach((n, i) => { + if (!connected.has(n.id)) orphan![i] = 1; + }); + } + const g = options.gravity; + if (g && mode === "off") { + sim.force("x", forceX(g.x).strength(g.strength)); + sim.force("y", forceY(g.y).strength(g.strength)); + } else if (g) { + // Mode-projected gravity, EXACTLY d3's forceX/forceY math (restoring + // force × strength × alpha): structured nodes get x-only in vertical + // mode (the depth force owns y) and none in radial mode (the rings + // own the radius); ORPHANS get the full regular-view pull in every + // mode. + const orphanMask = orphan!; + sim.force("gravity", (alpha: number) => { + for (let i = 0; i < nodes.length; i++) { + const n = nodes[i]!; + if (orphanMask[i]) { + n.vx = (n.vx ?? 0) + (g.x - (n.x ?? 0)) * g.strength * alpha; + n.vy = (n.vy ?? 0) + (g.y - (n.y ?? 0)) * g.strength * alpha; + } else if (mode === "vertical") { + n.vx = (n.vx ?? 0) + (g.x - (n.x ?? 0)) * g.strength * alpha; + } + // radial + structured: nothing — the rings own the radius. + } + }); + } + + // Cluster centroid pull: the membership index is resolved ONCE (nodes + // don't change clusters mid-sim); each tick is two allocation-free + // O(n) passes (sum, then pull). Singletons (<2 members) skip themselves. + const cp = options.clusterPull; + if (cp && cp.strength > 0 && cp.clusters.length > 0) { + const clusterIdx = new Map(); + cp.clusters.forEach((c, i) => { + for (const id of c.members) clusterIdx.set(id, i); + }); + const nodeCluster = nodes.map((n) => clusterIdx.get(n.id) ?? -1); + const k = cp.clusters.length; + const sumX = new Float64Array(k); + const sumY = new Float64Array(k); + const cnt = new Float64Array(k); + const strength = cp.strength; + // The cluster centroid pull also respects the depth mode: vertical + // → x-only (the depth force owns y); radial → tangential only + // (the radial component would drag members off their rings; the + // tangential component still tightens each cluster's angular + // spread, which is all clustering needs to do there). + const ringCx = options.depthAnchor?.center?.x ?? 0; + const ringCy = options.depthAnchor?.center?.y ?? 0; + sim.force("clusterPull", (alpha: number) => { + sumX.fill(0); + sumY.fill(0); + cnt.fill(0); + for (let i = 0; i < nodes.length; i++) { + const ci = nodeCluster[i]!; + if (ci < 0) continue; + const n = nodes[i]!; + sumX[ci] = sumX[ci]! + (n.x ?? 0); + sumY[ci] = sumY[ci]! + (n.y ?? 0); + cnt[ci] = cnt[ci]! + 1; + } + for (let i = 0; i < nodes.length; i++) { + const ci = nodeCluster[i]!; + if (ci < 0 || (cnt[ci] ?? 0) < 2) continue; + const n = nodes[i]!; + const px = n.x ?? 0; + const py = n.y ?? 0; + const ux = sumX[ci]! / cnt[ci]! - px; + const uy = sumY[ci]! / cnt[ci]! - py; + if (orphan && orphan[i]) { + // Orphans: full regular-view pull (see the ORPHAN RULE above). + n.vx = (n.vx ?? 0) + ux * strength * alpha; + n.vy = (n.vy ?? 0) + uy * strength * alpha; + } else if (mode === "vertical") { + n.vx = (n.vx ?? 0) + ux * strength * alpha; + } else if (mode === "radial") { + const rx = px - ringCx; + const ry = py - ringCy; + const r = Math.hypot(rx, ry); + if (r > 1e-9) { + // Tangential component: u − (u·r̂)r̂ + const dot = (ux * rx + uy * ry) / (r * r); + n.vx = (n.vx ?? 0) + (ux - dot * rx) * strength * alpha; + n.vy = (n.vy ?? 0) + (uy - dot * ry) * strength * alpha; + } + } else { + n.vx = (n.vx ?? 0) + ux * strength * alpha; + n.vy = (n.vy ?? 0) + uy * strength * alpha; + } + } + }); + } + + // ── Inference-depth anchoring ──────────────────────────────────────── + // The rank is COMPUTED (not simulated): per-node target scalar derived + // from inferenceDepths, then one O(n) pull pass per tick. Soft knob — + // high strength approaches pinned rows/rings while low strength leaves + // organic sag. + const da = options.depthAnchor; + if (da && da.mode !== "off" && da.strength > 0) { + const depths = inferenceDepths(nodes, edges); + const strength = da.strength; + const spacing = da.spacing; + const c = da.center ?? { x: 0, y: 0 }; + const orphanMask = orphan; // ORPHAN RULE: depth skips unstructured nodes + if (da.mode === "vertical") { + const baseY = c.y; + sim.force("depth", (alpha: number) => { + for (let i = 0; i < nodes.length; i++) { + if (orphanMask?.[i]) continue; + const n = nodes[i]!; + const targetY = baseY - (depths.get(n.id) ?? 0) * spacing; + n.vy = (n.vy ?? 0) + (targetY - (n.y ?? 0)) * strength * alpha; + } + }); + } else { + const baseR = da.baseRadius ?? 0; + const cx0 = c.x; + const cy0 = c.y; + sim.force("depth", (alpha: number) => { + for (let i = 0; i < nodes.length; i++) { + if (orphanMask?.[i]) continue; + const n = nodes[i]!; + const dx = (n.x ?? 0) - cx0; + const dy = (n.y ?? 0) - cy0; + // Degenerate direction at the exact ring center: nudge + // outward on a deterministic axis. + const r = Math.hypot(dx, dy); + const ux = r > 1e-9 ? dx / r : 1; + const uy = r > 1e-9 ? dy / r : 0; + const target = baseR + (depths.get(n.id) ?? 0) * spacing; + const mag = (target - r) * strength * alpha; + n.vx = (n.vx ?? 0) + ux * mag; + n.vy = (n.vy ?? 0) + uy * mag; + } + }); + } + } + return sim; +} + +export interface IncrementalLayoutOptions { + /** Manual ticks for the settle (default 300 — d3's anneal to alphaMin). */ + ticks?: number; + /** Per-tick anchor re-blend (default 0.85; 1 = fully pinned). */ + anchorRetention?: number; + /** Center-of-mass gravity strength override (tuning/A-B experiments; + * defaults to the internal bootstrap/incremental constants). */ + gravityStrength?: number; + /** Cluster partition (Louvain output shape). Supplied by callers so + * this module stays louvain-free (the browser bundle must not carry + * embedding ledger; absent = no cluster pull. */ + clusters?: ReadonlyArray<{ members: ReadonlyArray }>; + /** Cluster centroid pull strength (default CLUSTER_PULL_STRENGTH). */ + clusterPullStrength?: number; + /** Inference-depth anchoring (see DepthAnchor). `center` defaults to + * the anchored core's centroid. */ + depthAnchor?: DepthAnchor; +} + +/** + * One incremental settle. `nodes`/`edges` describe the CURRENT graph; + * `previous` is the positions ledger (empty on the very first settle — + * then this degrades to a seeded-scatter bootstrap). Returns positions + * for EVERY node in `nodes`. + */ +export function incrementalLayout( + nodes: ReadonlyArray>, + edges: ReadonlyArray, + previous: ReadonlyMap, + options: IncrementalLayoutOptions = {}, +): Map { + const ticks = options.ticks ?? 300; + const retention = options.anchorRetention ?? ANCHOR_RETENTION; + + // Anchors: the warm-start set (the previous rest state). + const anchors = new Map(); + for (const node of nodes) { + const prev = previous.get(node.id); + if (prev) anchors.set(node.id, { x: prev.x, y: prev.y }); + } + + // The anchored core's bounding box + centroid, for rim seeds. + let minX = Infinity; + let maxX = -Infinity; + let minY = Infinity; + let maxY = -Infinity; + let cx = 0; + let cy = 0; + for (const a of anchors.values()) { + minX = Math.min(minX, a.x); + maxX = Math.max(maxX, a.x); + minY = Math.min(minY, a.y); + maxY = Math.max(maxY, a.y); + cx += a.x; + cy += a.y; + } + const hasCore = anchors.size > 0; + if (hasCore) { + cx /= anchors.size; + cy /= anchors.size; + } + const coreRadius = hasCore ? Math.max(maxX - minX, maxY - minY) / 2 : 1; + + // Adjacency for neighbor seeding (undirected: edges include assertions). + const adjacency = new Map(); + const linkTo = (a: string, b: string): void => { + if (a === b) return; + let list = adjacency.get(a); + if (!list) adjacency.set(a, (list = [])); + list.push(b); + }; + for (const e of edges) { + linkTo(e.from, e.to); + linkTo(e.to, e.from); + } + + // The rim radius derives from the CONNECTED core's mean radius — orphans + // never inflate it (a max-span-based radius feeds back: each batch's + // rim pushes the next batch's rim farther out, and the layout runs + // away). 1.7x matches the density the FA2-era settle produced + // (orphan mean r ~338 vs connected ~194). + let rimRadius = 0; + let connectedAnchored = 0; + for (const [id, a] of anchors) { + if ((adjacency.get(id)?.length ?? 0) === 0) continue; + connectedAnchored++; + rimRadius += Math.hypot(a.x - cx, a.y - cy); + } + if (connectedAnchored > 0) rimRadius = (rimRadius / connectedAnchored) * 1.7; + else if (hasCore) rimRadius = coreRadius * 1.15; // no connected anchors yet: bbox fallback + rimRadius += 4; + + // Seed every node. + const simNodes: LayoutNode[] = nodes.map((node) => { + const rand = seededRandom(node.id); + const anchor = anchors.get(node.id); + let x: number; + let y: number; + if (anchor) { + // Warm start: resume from the previous rest position. + x = anchor.x; + y = anchor.y; + } else { + // New node: seed near its positioned neighbors (the cluster it + // arrived with), else on the rim outside the core, else scatter. + let sx = 0; + let sy = 0; + let found = 0; + for (const nbr of adjacency.get(node.id) ?? []) { + const na = anchors.get(nbr); + if (na) { + sx += na.x; + sy += na.y; + found++; + } + } + if (found > 0) { + x = sx / found + (rand() - 0.5) * 8; + y = sy / found + (rand() - 0.5) * 8; + } else if (hasCore) { + const angle = rand() * Math.PI * 2; + const radius = rimRadius + rand() * 8; + x = cx + Math.cos(angle) * radius; + y = cy + Math.sin(angle) * radius; + // An orphan's intended rest spot IS the rim seed: anchor it + // there so gravity cannot sink it into the core. (A later + // batch linking it to the graph pulls it in naturally — + // but then it also has a previous position, and its anchor + // becomes that previous position, exactly like any node.) + anchors.set(node.id, { x, y }); + } else { + // Bootstrap: the classic seeded scatter. + x = (rand() - 0.5) * 100; + y = (rand() - 0.5) * 100; + } + } + // d3 is fine with coincident seeds (no division), but nudge exact + // overlaps of a brand-new node onto its only neighbor anyway. + if (x === 0 && y === 0) x = 0.5; + return { id: node.id, x, y }; + }); + + const sim = createGraphSimulation(simNodes, edges, { + gravity: { + x: cx, + y: cy, + strength: options.gravityStrength ?? (hasCore ? INCREMENTAL_GRAVITY : BOOTSTRAP_GRAVITY), + }, + clusterPull: options.clusters + ? { clusters: options.clusters, strength: options.clusterPullStrength ?? CLUSTER_PULL_STRENGTH } + : undefined, + depthAnchor: + options.depthAnchor && options.depthAnchor.mode !== "off" + ? { ...options.depthAnchor, center: options.depthAnchor.center ?? { x: cx, y: cy } } + : options.depthAnchor, + }); + const dMode = options.depthAnchor?.mode ?? "off"; + const dCenter = options.depthAnchor?.center ?? { x: cx, y: cy }; + const byId = new Map(simNodes.map((n) => [n.id, n])); + for (let i = 0; i < ticks; i++) { + sim.tick(); + // Anchor re-blend (position space, alpha-independent — see the + // ANCHOR_RETENTION note). Pinned nodes (fx/fy) are left to d3. + // DEPTH-MODE PROJECTION: the re-blend preserves the user's + // arrangement in the FREE dimension only — x in vertical mode, the + // angle in radial mode — because the depth force owns the + // constrained dimension (a node's level/ring is meaning, not + // memory). Off-mode: full 2D blend as before. + if (hasCore && retention < 1) { + for (const [id, a] of anchors) { + const n = byId.get(id); + if (!n || n.fx != null || n.fy != null) continue; + const x = n.x ?? 0; + const y = n.y ?? 0; + if (dMode === "vertical") { + n.x = a.x + (x - a.x) * retention; + } else if (dMode === "radial") { + const rx = x - dCenter.x; + const ry = y - dCenter.y; + const r = Math.hypot(rx, ry); + const ax = a.x - dCenter.x; + const ay = a.y - dCenter.y; + const ar = Math.hypot(ax, ay); + if (r > 1e-9 && ar > 1e-9) { + const theta = Math.atan2(ry, rx); + const thetaA = Math.atan2(ay, ax); + const blended = thetaA + Math.atan2(Math.sin(theta - thetaA), Math.cos(theta - thetaA)) * retention; + n.x = dCenter.x + r * Math.cos(blended); + n.y = dCenter.y + r * Math.sin(blended); + } + } else { + n.x = a.x + (x - a.x) * retention; + n.y = a.y + (y - a.y) * retention; + } + } + } + } + + const out = new Map(); + for (const n of simNodes) out.set(n.id, { x: n.x ?? 0, y: n.y ?? 0 }); + return out; +} diff --git a/src/core/ports.ts b/src/core/ports.ts index 6343a2e..9f9d893 100644 --- a/src/core/ports.ts +++ b/src/core/ports.ts @@ -22,8 +22,12 @@ import type { ExtractionRunUsage, NodeMapping, PlanTree, + Position, SessionCheckpoint, TurnUsage, + NotesSource, + ClusterLabeling, + EmbeddingsLedger, } from "./contracts.ts"; // ─── ChatStore — the session record's home (data/chats) ───────────────────── @@ -80,6 +84,34 @@ export interface GraphStore { /** Append one emission checkpoint. */ appendCheckpoint(checkpoint: SessionCheckpoint): void; readCheckpoints(): SessionCheckpoint[]; + + /** Append one cluster labeling — idempotent by id (the id hashes the + * sorted core, so identical member sets dedupe). */ + appendClusterLabel(labeling: ClusterLabeling): void; + readClusterLabels(): ClusterLabeling[]; + + /** Upsert one imported-notes source record (config.SOURCES_FILE — a + * JSON map, not a ledger; an upsert keeps the existing usage log). */ + upsertSource(source: NotesSource): void; + /** Every notes source record, by id order-insensitive. */ + readSources(): NotesSource[]; + /** Append one run's usage to a source record — idempotent per + * startedAt, mirroring ChatRecord.extractions. */ + appendSourceUsage(id: string, entry: ExtractionRunUsage): void; + + /** The layout ledger (config.POSITIONS_FILE): nodeId → resting {x,y}. + * WRITTEN ONLY by the incremental settle after a merge (core/layout.ts + * settleGraphLayout) — read-only for everything else. */ + readPositions(): Record; + /** Replace the whole ledger (the settle is a full-graph computation). */ + writePositions(positions: Record): void; + + /** The embedding ledger (config.EMBEDDINGS_FILE). WRITTEN ONLY by + * the embedder (server/embeddings.ts) after a merge — read-only for + * clustering consumers (snapshot, settle, namer). */ + readEmbeddings(): EmbeddingsLedger; + /** Replace the whole ledger (embedding is a full-batch computation). */ + writeEmbeddings(ledger: EmbeddingsLedger): void; } // ─── JobStore — background sub-agent jobs (data/jobs) ───────────────────────── diff --git a/src/core/render-settings.ts b/src/core/render-settings.ts new file mode 100644 index 0000000..1002985 --- /dev/null +++ b/src/core/render-settings.ts @@ -0,0 +1,280 @@ +/** + * Graph render settings: the shape, the defaults, the sanitizer, and the + * slider mapping (log-scaled, default-centered) — shared by the client (components/graph-canvas.ts, + * graph-view.tsx), the server store (src/store/render-settings.ts, which + * persists them to data/render-settings.json), and the API route. + * + * This module is pure: no DOM, no d3, no node APIs — safe everywhere. + * + * Settings are server-persisted (render-settings.json); the localStorage era + * is over. Whatever arrives from disk or the network is passed through + * sanitizeRenderSettings before use — never trusted raw. + * + * The labelZoom semantics: the sigma-compatible "camera ratio" threshold + * (1 = fit zoom, SMALLER = zoomed in). Node labels show at ratio <= labelZoom + * and fade over LABELS_FADE_SPAN; cluster region labels fade in inversely. + * Persisted values keep these units across the renderer swap. + */ + +import { + CHARGE_STRENGTH, + CLUSTER_PULL_STRENGTH, + INCREMENTAL_GRAVITY, + LINK_DISTANCE, + type DepthAnchorMode, +} from "./layout.ts"; + +export interface GraphRenderSettings { + nodeSize: number; + edgeWidth: number; + arrowLength: number; + /** Camera-ratio threshold for node labels (see module doc). */ + labelZoom: number; + /** Node + hover label font size, px at fit zoom (scales with camera). */ + labelSize: number; + /** Toggle for the cluster region labels (hulls always draw). */ + clusterLabels: boolean; + /** Region rendering (the blurred cluster tints). Server-persisted + * since 2026-09-13 (user request) — was session-only dev state. */ + regions: RegionTuning; + /** Live-sim physics + inference-depth anchoring. Server-persisted + * since 2026-09-13 (user request) — was session-only dev state. */ + physics: PhysicsTuning; +} + + +const clamp = (v: number, min: number, max: number) => Math.min(Math.max(v, min), max); + +/** + * Coerce arbitrary JSON (server file, network body, localStorage leftovers) + * into valid settings: missing keys fall back to defaults, out-of-range + * values clamp, wrong types reset. Never throws. + */ +export function sanitizeRenderSettings(raw: unknown): GraphRenderSettings { + const r = (raw ?? {}) as Partial; + const num = (v: unknown, fallback: number, min: number, max: number) => { + const n = typeof v === "number" && Number.isFinite(v) ? v : fallback; + return clamp(n, min, max); + }; + return { + nodeSize: num(r.nodeSize, DEFAULT_RENDER_SETTINGS.nodeSize, NODE_SIZE_RANGE.min, NODE_SIZE_RANGE.max), + edgeWidth: num(r.edgeWidth, DEFAULT_RENDER_SETTINGS.edgeWidth, EDGE_WIDTH_RANGE.min, EDGE_WIDTH_RANGE.max), + arrowLength: num(r.arrowLength, DEFAULT_RENDER_SETTINGS.arrowLength, ARROW_LENGTH_RANGE.min, ARROW_LENGTH_RANGE.max), + // Range matches the log slider's span [LABEL_ZOOM_MIN, LABEL_ZOOM_MAX]. + labelZoom: num(r.labelZoom, DEFAULT_RENDER_SETTINGS.labelZoom, LABEL_ZOOM_MIN, LABEL_ZOOM_MAX), + labelSize: num(r.labelSize, DEFAULT_RENDER_SETTINGS.labelSize, LABEL_SIZE_RANGE.min, LABEL_SIZE_RANGE.max), + clusterLabels: + typeof r.clusterLabels === "boolean" ? r.clusterLabels : DEFAULT_RENDER_SETTINGS.clusterLabels, + // Nested sections (2026-09-13): old files lack them → defaults. + regions: sanitizeRegionTuning(r.regions), + physics: sanitizePhysicsTuning(r.physics), + }; +} + +// ─── Region + physics tuning (moved from the renderer, 2026-09-13) ───────────── +// +// The renderer's region/physics controls became USER-FACING persisted +// settings: one shape + one sanitizer in core, consumed by the settings +// store (this file's schema), the panel, and the renderer's +// setRegionTuning/setPhysicsTuning. Every clamp below is a SWEEP-VALIDATED +// safe range (2026-09-13, real graph, renderer's exact sim): each field at +// both range ends AND all four range corners settles cleanly — no NaN, no +// explosion (linkStrength ≥ 8 explodes the layout, 5 wobbles; cap 4), +// no non-settle (velocityDecay 0 never damps; floor 0.1), no 3x sprawl +// (gravity 0 / charge -1000 excluded). + +/** Region rendering knobs (the blurred cluster tints). */ +export interface RegionTuning { + /** Dilation half-width of the round-cap edge strokes, screen px at + * fit zoom (the expansion of the cluster skeleton). */ + dilation: number; + /** Box-blur radius in bitmap px (3 passes ≈ Gaussian). Fixed set: + * see REGION_BLUR_OPTIONS (dropdown in the panel — non-integer + * radii broke the rendering, so the control is integer-only). */ + blur: number; + /** Max bitmap dimension (cell = worldSpan / this). Fixed set: + * see REGION_BITMAP_OPTIONS (dropdown in the panel). */ + bitmap: number; + /** Region alpha, applied ONCE after the blur (dilate at full + * opacity → blur → scale). Constant interior, soft edges; overlaps + * of any count never stack (DECIDED 2026-09-14 — per-stroke alpha + * accumulated through source-over and double-counted orphans). */ + edgeAlpha: number; +} + +export const REGION_BITMAP_OPTIONS = [128, 192, 256, 384, 512] as const; +export const REGION_BLUR_OPTIONS = [2, 4, 6, 8, 10] as const; + +export const DEFAULT_REGION_TUNING: RegionTuning = { + dilation: 28, + blur: 6, + bitmap: 384, + edgeAlpha: 0.35, +}; + +/** Nearest member of a fixed option set (dropdowns have no free values). */ +function snapToOption(value: number, options: readonly number[]): number { + return options.reduce((best, o) => (Math.abs(o - value) < Math.abs(best - value) ? o : best), options[0] as number); +} + +/** Clamp/validate a region tuning. `bitmap` and `blur` snap to the nearest + * allowed option (they're fixed sets — dropdowns, not free numbers). */ +export function sanitizeRegionTuning(input: Partial | null | undefined): RegionTuning { + const d = DEFAULT_REGION_TUNING; + const num = (v: unknown, fallback: number): number => + typeof v === "number" && Number.isFinite(v) ? v : fallback; + const bitmap = num(input?.bitmap, d.bitmap); + return { + dilation: Math.min(Math.max(num(input?.dilation, d.dilation), REGION_DILATION_RANGE.min), REGION_DILATION_RANGE.max), + blur: snapToOption(num(input?.blur, d.blur), REGION_BLUR_OPTIONS), + bitmap: snapToOption(bitmap, REGION_BITMAP_OPTIONS), + edgeAlpha: Math.min(Math.max(num(input?.edgeAlpha, d.edgeAlpha), REGION_EDGE_ALPHA_RANGE.min), 1), + }; +} + +/** Live-sim physics controls (user-facing since 2026-09-13): every value + * feeds createGraphSimulation — the SAME engine the server settle runs — + * so tuning the feel changes exactly what a drag relaxes toward. */ +export interface PhysicsTuning { + /** Node repulsion (d3 charge, negative = push apart). */ + chargeStrength: number; + /** Edge attraction: link-spring stiffness multiplier (1 = d3 default). */ + linkStrength: number; + /** Link spring rest length (world units). */ + linkDistance: number; + /** Central pull toward the graph centroid. */ + gravity: number; + /** Per-cluster pull toward the cluster centroid. 0 = off. */ + clusterPull: number; + /** Per-tick velocity damping (d3 default 0.4). */ + velocityDecay: number; + /** Inference-depth anchoring mode (see DepthAnchor in core). */ + depthMode: DepthAnchorMode; + /** Depth-anchor pull strength. */ + depthStrength: number; + /** Spacing between inference levels (world units). */ + depthSpacing: number; + /** Inner ring radius, radial mode (world units). */ + depthBaseRadius: number; +} + +export const DEFAULT_PHYSICS_TUNING: PhysicsTuning = { + chargeStrength: CHARGE_STRENGTH, + linkStrength: 1, + linkDistance: LINK_DISTANCE, + gravity: INCREMENTAL_GRAVITY, + clusterPull: CLUSTER_PULL_STRENGTH, + velocityDecay: 0.4, + depthMode: "off", + // 0.8 by sweep on the real graph: 215/216 arrows read upward with + // organic sag (mean |dev| from level ~34u at spacing 60); 1.5 gives + // crisp rows (dev ~18) if the promoted default wants them. + depthStrength: 0.8, + depthSpacing: 60, + depthBaseRadius: 220, +}; + +/** Clamp anything into a usable tuning (sliders/JSON can be empty/NaN). */ +export function sanitizePhysicsTuning(input: Partial | null | undefined): PhysicsTuning { + const d = DEFAULT_PHYSICS_TUNING; + const num = (v: unknown, fallback: number): number => + typeof v === "number" && Number.isFinite(v) ? v : fallback; + return { + chargeStrength: Math.min(Math.max(num(input?.chargeStrength, d.chargeStrength), CHARGE_RANGE.min), CHARGE_RANGE.max), + linkStrength: Math.min(Math.max(num(input?.linkStrength, d.linkStrength), LINK_STRENGTH_RANGE.min), LINK_STRENGTH_RANGE.max), + linkDistance: Math.min(Math.max(num(input?.linkDistance, d.linkDistance), LINK_DISTANCE_RANGE.min), LINK_DISTANCE_RANGE.max), + gravity: Math.min(Math.max(num(input?.gravity, d.gravity), GRAVITY_RANGE.min), GRAVITY_RANGE.max), + clusterPull: Math.min(Math.max(num(input?.clusterPull, d.clusterPull), 0), CLUSTER_PULL_RANGE.max), + velocityDecay: Math.min(Math.max(num(input?.velocityDecay, d.velocityDecay), VELOCITY_DECAY_RANGE.min), VELOCITY_DECAY_RANGE.max), + depthMode: input?.depthMode === "vertical" || input?.depthMode === "radial" ? input.depthMode : "off", + depthStrength: Math.min(Math.max(num(input?.depthStrength, d.depthStrength), DEPTH_STRENGTH_RANGE.min), DEPTH_STRENGTH_RANGE.max), + depthSpacing: Math.min(Math.max(num(input?.depthSpacing, d.depthSpacing), DEPTH_SPACING_RANGE.min), DEPTH_SPACING_RANGE.max), + depthBaseRadius: Math.min(Math.max(num(input?.depthBaseRadius, d.depthBaseRadius), DEPTH_BASE_RADIUS_RANGE.min), DEPTH_BASE_RADIUS_RANGE.max), + }; +} + +// Sizes are screen px AT FIT ZOOM (the world-space renderer scales them +// with the camera, so a label reads at labelSize x k/kFit on screen). +// Defaults were re-tuned for world-scaling on 2026-09-13: the old values +// were calibrated in the screen-fixed era, so everything rendered ~2x +// too big once sizes started scaling with the graph. +export const DEFAULT_RENDER_SETTINGS: GraphRenderSettings = { + nodeSize: 2, + edgeWidth: 1, + arrowLength: 2, + labelZoom: 0.5, + labelSize: 2, + clusterLabels: true, + regions: DEFAULT_REGION_TUNING, + physics: DEFAULT_PHYSICS_TUNING, +}; + +// ─── The slider mapping ──────────────────────────────────────────────────────── +// +// EVERY slider is log-scaled over its setting's range with the DEFAULT at +// position 50 (dead center), per the user's spec: centered defaults, coarse +// steps (the UI steps by 5 → 21 stops), no fine-grained linear tracks. +// Ranges are chosen so each default is the geometric mean of its ends. + +export const NODE_SIZE_RANGE = { min: 0.5, max: 8 } as const; +export const EDGE_WIDTH_RANGE = { min: 0.25, max: 4 } as const; +export const ARROW_LENGTH_RANGE = { min: 0.5, max: 8 } as const; +export const LABEL_SIZE_RANGE = { min: 0.5, max: 8 } as const; +export const LABEL_ZOOM_MIN = 0.0625; // slider far left: labels only very deep in +export const LABEL_ZOOM_MAX = 4; // slider far right: labels visible at fit zoom + +// ─── Region + physics slider ranges ────────────────────────────────────────── +// +// Same laws as the size sliders: log-scaled, DEFAULT AT 50 (each default is +// the geometric mean of its ends), coarse 5-step tracks. Signed/zeroed +// fields get special treatment: +// chargeStrength is NEGATIVE — the slider maps |charge| and displays "-x"; +// clusterPull has an OFF zone — position 0 is exactly 0 (see +// toOffSliderPosition), the log range starts one step in. Blur used to +// have one too; it's a dropdown since 2026-09-13 (non-integers broke). + +export const REGION_DILATION_RANGE = { min: 7, max: 112 } as const; // gm = 28 +export const REGION_EDGE_ALPHA_RANGE = { min: 0.12, max: 1 } as const; // gm ≈ 0.35 + +export const CHARGE_RANGE = { min: -480, max: -30 } as const; // |gm| = 120 +export const LINK_STRENGTH_RANGE = { min: 0.25, max: 4 } as const; // gm = 1 +export const LINK_DISTANCE_RANGE = { min: 10, max: 160 } as const; // gm = 40 +export const GRAVITY_RANGE = { min: 0.03, max: 0.48 } as const; // gm = 0.12 +export const CLUSTER_PULL_RANGE = { min: 0.0225, max: 1 } as const; // gm = 0.15; 0 = off +export const VELOCITY_DECAY_RANGE = { min: 0.18, max: 0.9 } as const; // gm ≈ 0.40 +export const DEPTH_STRENGTH_RANGE = { min: 0.2, max: 3.2 } as const; // gm = 0.8 +export const DEPTH_SPACING_RANGE = { min: 15, max: 240 } as const; // gm = 60 +export const DEPTH_BASE_RADIUS_RANGE = { min: 55, max: 880 } as const; // gm = 220 + +/** Off-aware slider mapping (position 0 → value 0 = OFF; the log range + * starts one step in). Inverse of fromOffSliderPosition. */ +export function toOffSliderPosition(value: number, min: number, max: number): number { + if (value <= 0) return 0; + return toSliderPosition(value, min, max); +} + +/** Off-aware slider mapping: position 0 → 0 (off), else log-scaled. */ +export function fromOffSliderPosition(position: number, min: number, max: number): number { + if (position <= 0) return 0; + return fromSliderPosition(position, min, max); +} + +/** Setting value → slider position [0, 100] (log scale, default at 50). */ +export function toSliderPosition(value: number, min: number, max: number): number { + const v = clamp(value, min, max); + const logSpan = Math.log(max / min); + return (100 * Math.log(v / min)) / logSpan; +} + +/** Slider position [0, 100] → setting value (inverse of toSliderPosition). */ +export function fromSliderPosition(position: number, min: number, max: number): number { + const p = clamp(position, 0, 100) / 100; + const logSpan = Math.log(max / min); + return min * Math.exp(p * logSpan); +} + +/** Display units for labelZoom: normalized so the default (0.5) reads as 1. */ +export function labelZoomDisplay(labelZoom: number): string { + const v = Math.round((labelZoom / DEFAULT_RENDER_SETTINGS.labelZoom) * 100) / 100; + return `${v}×`; +} diff --git a/src/core/semantic-clusters.ts b/src/core/semantic-clusters.ts new file mode 100644 index 0000000..7772c60 --- /dev/null +++ b/src/core/semantic-clusters.ts @@ -0,0 +1,141 @@ +// Semantic topic clustering — the REPLACEMENT for Louvain community +// detection (DECIDED 2026-09-13): structural clusters were just the +// reasoning chains (the graph is disconnected by nature — every connected +// component became its own "cluster"), and edge-orphans could never join +// anything. Clusters are now RELATED TOPICS: nodes whose proposition +// MEANING is close, computed over the embedding ledger +// (config.EMBEDDINGS_FILE). This pulls orphans into topics by meaning +// and groups small chains with the large chains they discuss. +// +// Algorithm: threshold AGGLOMERATIVE (average linkage over cosine +// similarity). No JS HDBSCAN is trustworthy (the npm package is an +// abandoned alpha), and at proposition scale (~100s-1000s of short texts) +// brute-force pairwise cosine is sub-second — so the honest ~60-line +// hand-roll wins. "Stragglers" (nodes that fit nothing) fall out +// naturally: they simply never merge, and stay singletons. + +/** The default similarity threshold, SWEPT ON THE REAL GRAPH + * (2026-09-13): 0.60 → 26 topics of ~20 propositions, 74 of the 90 + * edge-orphans land in real topics, stragglers 10. Higher (0.65-0.68) + * gives tighter micro-topics with far more stragglers; lower (0.55) + * collapses into ~13 mega-topics. */ +export const SEMANTIC_SIM_THRESHOLD = 0.6; + +/** Cosine similarity of two vectors (defensively normalizes inputs; + * ledger vectors are already L2-normalized, so this is a dot product). */ +export function cosineSimilarity(a: readonly number[], b: readonly number[]): number { + let dot = 0; + let ma = 0; + let mb = 0; + for (let i = 0; i < a.length; i++) { + dot += a[i]! * b[i]!; + ma += a[i]! * a[i]!; + mb += b[i]! * b[i]!; + } + if (ma === 0 || mb === 0) return 0; + return dot / (Math.sqrt(ma) * Math.sqrt(mb)); +} + +export interface SemanticClusterOptions { + /** Merge while the best cluster pair's average-linkage similarity is + * >= this (default SEMANTIC_SIM_THRESHOLD). */ + similarityThreshold?: number; + /** After the cut, force every singleton into its nearest topic + * ("all-in" mode) instead of leaving stragglers loose. Default + * false: some propositions genuinely fit nothing. */ + attachStragglers?: boolean; +} + +/** Average-linkage agglomerative clustering over embedding vectors. + * Pure and deterministic (stable Map iteration, greedy by best + * similarity, first-found wins ties). Returns clusters size-desc — the + * same { members } shape Louvain produced, so label matching and the + * renderer contract are unchanged. An empty/missing ledger returns [] + * (nothing embedded yet: no clusters, every node loose — the snapshot + * degrades gracefully until the first embed). */ +export function semanticClusters( + vectors: Readonly>, + options: SemanticClusterOptions = {}, +): Array<{ members: string[] }> { + const ids = Object.keys(vectors); + if (ids.length === 0) return []; + const simThreshold = options.similarityThreshold ?? SEMANTIC_SIM_THRESHOLD; + const attach = options.attachStragglers ?? false; + + // Sum vectors per cluster: average-linkage similarity between clusters + // A and B is dot(sumA, sumB) / (|A|·|B|) — the exact mean of pairwise + // dots — so the merge loop never revisits the raw O(n²) matrix. + const sums = new Map(); + const members = new Map(); + ids.forEach((id, i) => { + sums.set(i, [...vectors[id]!]); + members.set(i, [id]); + }); + + // Greedy merge: repeatedly join the pair with the highest + // average-linkage similarity until it drops below the threshold. + // O(k²) pair scan per merge — trivial at proposition scale. + for (;;) { + let bestA = -1; + let bestB = -1; + let bestSim = -Infinity; + for (const [a, sumA] of sums) { + for (const [b, sumB] of sums) { + if (a >= b) continue; + let dot = 0; + for (let i = 0; i < sumA.length; i++) dot += sumA[i]! * sumB[i]!; + const sim = dot / (members.get(a)!.length * members.get(b)!.length); + if (sim > bestSim) { + bestSim = sim; + bestA = a; + bestB = b; + } + } + } + if (bestA < 0 || bestSim < simThreshold) break; + // Merge B into A (deterministic direction: lower cluster index). + const sumA = sums.get(bestA)!; + const sumB = sums.get(bestB)!; + for (let i = 0; i < sumA.length; i++) sumA[i] = sumA[i]! + sumB[i]!; + members.get(bestA)!.push(...members.get(bestB)!); + sums.delete(bestB); + members.delete(bestB); + } + + // Optional all-in pass: singletons join their nearest topic regardless + // of threshold — exactly the "forced membership" mode. The difference + // from stragglers mode is this one pass. Singleton-vs-cluster + // average-linkage similarity is dot(sumS, sumK) / |K|. + if (attach) { + const singletons = [...members.entries()].filter(([, m]) => m.length === 1).map(([k]) => k); + for (const s of singletons) { + if (!members.has(s)) continue; // absorbed by an earlier attach + let bestTarget = -1; + let bestSim = -Infinity; + for (const [k, m] of members) { + if (k === s || m.length < 2) continue; + let dot = 0; + const sumS = sums.get(s)!; + const sumK = sums.get(k)!; + for (let i = 0; i < sumS.length; i++) dot += sumS[i]! * sumK[i]!; + const sim = dot / m.length; + if (sim > bestSim) { + bestSim = sim; + bestTarget = k; + } + } + if (bestTarget >= 0) { + const sumT = sums.get(bestTarget)!; + const sumS = sums.get(s)!; + for (let i = 0; i < sumT.length; i++) sumT[i] = sumT[i]! + sumS[i]!; + members.get(bestTarget)!.push(...members.get(s)!); + sums.delete(s); + members.delete(s); + } + } + } + + return [...members.values()] + .map((m) => ({ members: [...m].sort() })) + .sort((a, b) => b.members.length - a.members.length || (a.members[0] ?? "").localeCompare(b.members[0] ?? "")); +} diff --git a/src/server/chat-handler.ts b/src/server/chat-handler.ts index a4a3436..a160b9f 100644 --- a/src/server/chat-handler.ts +++ b/src/server/chat-handler.ts @@ -4,6 +4,7 @@ import { createProberAgent, createTeacherAgent, type TeachingAgent } from "../ag import { piEngineFor } from "../agent/models.ts"; import { createJobRunner } from "../agent/subagents/runner.ts"; import { createExtractionPipeline } from "../agent/subagents/extraction.ts"; +import { embedNewNodes } from "./embeddings.ts"; import type { ModelRole } from "../../config.ts"; import { claimSteers, deliverJobResult, setJobRespawn, setTurnRunner } from "./jobs.ts"; import { recordToPiMessages } from "./pi/record-adapter.ts"; @@ -37,7 +38,15 @@ const jobRunner = createJobRunner({ jobStore: stores.jobStore, deliver: deliverJobResult, }); -const extraction = createExtractionPipeline(stores); +/** The app's ONE extraction pipeline instance — both front doors share it: + * quiz-triggered session runs (turn-service) and notes imports + * (/api/graph/import) hit the same single-flight registry. */ +export const extraction = createExtractionPipeline({ + ...stores, + // The REAL embedder (local transformer model): post-merge embedding + // batch — semantic clusters are computed and persisted here. + embedder: embedNewNodes, +}); // Wake-pipeline wiring (contracts.ts "THE ASYNC PROTOCOL" §4): the pipeline // fires its server-side teacher turns through THIS composition root, and diff --git a/src/server/commit-positions.ts b/src/server/commit-positions.ts new file mode 100644 index 0000000..a31232e --- /dev/null +++ b/src/server/commit-positions.ts @@ -0,0 +1,57 @@ +/** + * Server-side commit for the live physics drag (PUT /api/graph/positions). + * The client's renderer anneals a drag to rest and PUTs the new rest + * state; the server trusts it for the nodes it knows about and, if the + * graph grew while the user was dragging (a merge landed), runs ONE + * incremental settle (settleGraphLayout) so the fresh nodes are absorbed + * into the frame the user just arranged — client positions act as + * anchors, never as a clobber. + * + * Every body is sanitized: garbage coordinates clamp or drop, unknown + * ids are discarded, nothing throws. + */ +import type { Position } from "../core/contracts.ts"; +import { getGraph } from "../core/graph.ts"; +import { settleGraphLayout } from "../core/layout-settle.ts"; +import { stores } from "./stores.ts"; + +/** Max |x| or |y| accepted — NaN/Infinity clamp here, absurd too. */ +const MAX_COORD = 1e6; + +/** + * Accepts the raw PUT body (a Record of id → {x, y}) and returns the + * sanitized positions actually persisted (empty object = nothing written, + * e.g. a body with no known ids — the caller reports but never fails). + */ +export function commitPositions(input: unknown): Record { + if (typeof input !== "object" || input === null) return {}; + const raw = input as Record; + + // Only ids the graph actually has (first line wins, like getGraph). + const known = new Map(); + for (const line of stores.graphStore.readNodes()) { + if (!known.has(line.id)) known.set(line.id, true); + } + + const positions: Record = {}; + for (const [id, value] of Object.entries(raw)) { + if (!known.has(id)) continue; + if (typeof value !== "object" || value === null) continue; + const { x, y } = value as Record; + if (typeof x !== "number" || typeof y !== "number" || !Number.isFinite(x) || !Number.isFinite(y)) + continue; + positions[id] = { + x: Math.min(Math.max(x, -MAX_COORD), MAX_COORD), + y: Math.min(Math.max(y, -MAX_COORD), MAX_COORD), + }; + } + if (Object.keys(positions).length === 0) return {}; + + stores.graphStore.writePositions(positions); + + // The graph grew past what the client knew (a merge landed mid-drag): + // absorb the new nodes with the client's rest state as anchors. + const unique = getGraph(stores.graphStore).nodes.length; + if (unique > Object.keys(positions).length) settleGraphLayout(stores.graphStore); + return positions; +} \ No newline at end of file diff --git a/src/server/embeddings.ts b/src/server/embeddings.ts new file mode 100644 index 0000000..0d9194f --- /dev/null +++ b/src/server/embeddings.ts @@ -0,0 +1,125 @@ +/** + * THE EMBEDDER (DECIDED 2026-09-13): semantic topic clustering needs + * vectors, and vectors need a model running ON the user's machine — no + * external services, no API bills, no network dependency for re-embedding. + * + * Runtime: in-process transformers.js (@huggingface/transformers v4 over + * onnxruntime-node — native Apple Silicon acceleration). The model + * weights download ONCE to the local HF cache on first run (~300MB). + * + * Model: EmbeddingGemma-300M (q8) — best quality-per-megabyte small + * embedding model as of late 2025 (MTEB-en 68.36), built for on-device + * use, with MRL truncation so we store 512 of its 768 dims. Every input + * uses the model card's document format ("title: none | text: …"), + * consistently — mixing prefixed and unprefixed inputs degrades quality. + * + * Model swaps re-embed EVERYTHING: vectors from different models live in + * incompatible spaces. The ledger records model + dim; a mismatch marks + * the whole ledger stale and this module re-embeds from scratch (minutes + * at current scale). + */ +import { pipeline, type FeatureExtractionPipeline } from "@huggingface/transformers"; +import type { EmbeddingsLedger, GraphNode } from "../core/contracts.ts"; +import { semanticClusters, SEMANTIC_SIM_THRESHOLD } from "../core/semantic-clusters.ts"; + +export const EMBEDDING_MODEL = "onnx-community/embeddinggemma-300m-ONNX"; +/** MRL truncation of the native 768 — negligible quality loss, half the + * ledger size (2026-09-13 researcher sweep). */ +export const EMBEDDING_DIM = 512; +/** Batch size bounds peak memory; propositions are tiny so 32 per call + * keeps the pipeline warm without thrashing. */ +const EMBED_BATCH = 32; + +let pipelinePromise: Promise | null = null; + +function getPipeline(): Promise { + pipelinePromise ??= pipeline("feature-extraction", EMBEDDING_MODEL, { dtype: "q8" }); + return pipelinePromise; +} + +/** EmbeddingGemma's required document format (model card). */ +function embedInput(statement: string): string { + return `title: none | text: ${statement}`; +} + +/** Truncate an L2-normalized vector to EMBEDDING_DIM (MRL: leading dims + * carry most of the signal) and re-normalize so cosine = dot product. */ +function truncateNormalize(vec: readonly number[]): number[] { + const out = vec.slice(0, EMBEDDING_DIM); + let mag = 0; + for (const x of out) mag += x * x; + mag = Math.sqrt(mag); + return mag > 0 ? out.map((x) => x / mag) : out; +} + +/** + * Embed every node the ledger is missing — the post-merge hook (new + * propositions only, typically a handful) AND the cold-start backfill + * (empty ledger = every node) in one function. Model/dim mismatch voids + * the ledger first (full re-embed). Also prunes vectors for nodes that + * no longer exist. Writes the whole ledger and returns counts for logs. + * Non-fatal by contract: the caller treats failure as "clustering runs + * without the new nodes until the next merge retries". + */ +export async function embedNewNodes( + graphStore: { + readNodes(): GraphNode[]; + readEmbeddings(): EmbeddingsLedger; + writeEmbeddings(ledger: EmbeddingsLedger): void; + }, +): Promise<{ embedded: number; total: number }> { + const ledger = graphStore.readEmbeddings(); + const stale = ledger.model !== EMBEDDING_MODEL || ledger.dim !== EMBEDDING_DIM; + const vectors: Record = stale ? {} : { ...ledger.vectors }; + + // Node LINES may repeat an id — dedupe, first defines (getGraph rule). + const seen = new Set(); + const todo: GraphNode[] = []; + for (const node of graphStore.readNodes()) { + if (seen.has(node.id)) continue; + seen.add(node.id); + if (!vectors[node.id]) todo.push(node); + } + + if (todo.length > 0) { + const extractor = await getPipeline(); + for (let i = 0; i < todo.length; i += EMBED_BATCH) { + const batch = todo.slice(i, i + EMBED_BATCH); + const output = await extractor(batch.map((n) => embedInput(n.statement)), { + pooling: "mean", + normalize: true, + }); + const rows = output.tolist() as number[][]; + for (let j = 0; j < batch.length; j++) { + vectors[batch[j]!.id] = truncateNormalize(rows[j]!); + } + } + } + + // Prune ghosts: vectors for node ids that no longer exist. + let pruned = 0; + for (const id of Object.keys(vectors)) { + if (!seen.has(id)) { + delete vectors[id]; + pruned++; + } + } + + // RECLUSTER AT THE END OF THE BATCH (user-decided 2026-09-13): the + // agglomerative pass is ~1.5s at current scale, so it runs HERE — + // once per embedding batch — and consumers (snapshot, settle, namer) + // read the persisted partition instead of recomputing per read. + const clusters = semanticClusters(vectors, { similarityThreshold: SEMANTIC_SIM_THRESHOLD }); + + const wrote = todo.length > 0 || pruned > 0 || stale || ledger.clusters.length !== clusters.length; + if (wrote) { + graphStore.writeEmbeddings({ + model: EMBEDDING_MODEL, + dim: EMBEDDING_DIM, + vectors, + clusters, + updatedAt: new Date().toISOString(), + }); + } + return { embedded: todo.length, total: Object.keys(vectors).length }; +} diff --git a/src/server/index.ts b/src/server/index.ts index bf5b3b6..b482018 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -7,8 +7,10 @@ export * from "./chat-handler.ts"; export * from "./chats.ts"; export * from "./graph.ts"; +export * from "./commit-positions.ts"; export * from "./jobs.ts"; export * from "./stores.ts"; +export { readRenderSettings, writeRenderSettings } from "../store/render-settings.ts"; export * from "./turn-liveness.ts"; export * from "./observability.ts"; export * from "./model-catalog.ts"; diff --git a/src/store/graph.ts b/src/store/graph.ts index ac86c45..8c72007 100644 --- a/src/store/graph.ts +++ b/src/store/graph.ts @@ -15,6 +15,10 @@ * (chatId, sourceId) → graphNodeId * sessions.jsonl SessionCheckpoint— emission checkpoints (foldedAt) * + * Plus config.SOURCES_FILE (sources.json): the imported-notes bookkeeping + * (NOT a ledger) — one JSON map, src-id → NotesSource, read-modify-write + * like a chat file. The format-agnostic front door (2026-09-13). + * * Implements core/ports.ts GraphStore. data/graph/* is exclusively this * adapter's. Reads are strict against the contracts.ts shapes (the 2026-09-02 * graph redesign): corrupt lines are skipped, never guessed at — the ledgers @@ -25,13 +29,18 @@ import * as fs from "node:fs"; import * as path from "node:path"; -import { LEDGER_FILES } from "../../config.ts"; +import { EMBEDDINGS_FILE, LEDGER_FILES, POSITIONS_FILE, SOURCES_FILE } from "../../config.ts"; import type { + ClusterLabeling, + EmbeddingsLedger, GraphEdge, GraphNode, + ExtractionRunUsage, LedgerEvent, LedgerEventKind, + NotesSource, NodeMapping, + Position, ProvenancePoint, SessionCheckpoint, } from "../core/contracts.ts"; @@ -80,6 +89,14 @@ function readValid(file: string, isValid: (v: unknown) => v is T): T[] { const isProvenancePoint = (v: unknown): v is ProvenancePoint => isRecord(v) && isString(v.chatId) && isString(v.messageId); +const isClusterLabeling = (v: unknown): v is ClusterLabeling => + isRecord(v) && + isString(v.id) && + isString(v.label) && + Array.isArray(v.core) && + v.core.every(isString) && + isString(v.createdAt); + const isGraphNode = (v: unknown): v is GraphNode => isRecord(v) && isString(v.id) && @@ -157,4 +174,171 @@ export const graphStore: GraphStore = { readCheckpoints(): SessionCheckpoint[] { return readValid(LEDGER_FILES.sessions, isCheckpoint); }, + + // ── cluster labelings (config.LEDGER_FILES.clusterLabels) ── + + /** Idempotent by id: the id is a hash of the sorted core, so re-appending + * the same member set (re-naming races, idempotent re-runs) is a no-op. */ + appendClusterLabel(labeling: ClusterLabeling): void { + if (this.readClusterLabels().some((l) => l.id === labeling.id)) return; + appendLine(LEDGER_FILES.clusterLabels, labeling); + }, + readClusterLabels(): ClusterLabeling[] { + return readValid(LEDGER_FILES.clusterLabels, isClusterLabeling); + }, + + // ── imported-notes bookkeeping (config.SOURCES_FILE — a JSON map) ── + + /** Upsert: keeps the existing record's usage log when re-importing. */ + upsertSource(source: NotesSource): void { + const all = readSourceMap(); + const prev = all[source.id]; + all[source.id] = prev ? { ...source, extractions: prev.extractions } : source; + writeSourceMap(all); + }, + readSources(): NotesSource[] { + return Object.values(readSourceMap()); + }, + /** Idempotent per startedAt — same discipline as chats.appendExtractionUsage. */ + appendSourceUsage(id: string, entry: ExtractionRunUsage): void { + const all = readSourceMap(); + const source = all[id]; + if (!source) return; // no such source — nothing to bill against + source.extractions = source.extractions?.some((r) => r.startedAt === entry.startedAt) + ? source.extractions + : [...(source.extractions ?? []), entry]; + writeSourceMap(all); + }, + + // ── the layout ledger (config.POSITIONS_FILE — a JSON map) ── + readPositions(): Record { + return readPositionMap(); + }, + writePositions(positions: Record): void { + writePositionMap(positions); + }, + readEmbeddings(): EmbeddingsLedger { + return readEmbeddingsLedger(); + }, + writeEmbeddings(ledger: EmbeddingsLedger): void { + writeEmbeddingsLedger(ledger); + }, }; + +/** positions.json — { positions, updatedAt }. Read-modify-write, NOT + * append-only: the settle is a full-graph computation, so it replaces the + * whole map. The updatedAt stamp exists for future live-physics conflict + * handling (a browser commit must be able to detect "the graph changed + * underneath me"); reads just return the map. */ +function readPositionMap(): Record { + let raw: string; + try { + raw = fs.readFileSync(POSITIONS_FILE, "utf-8"); + } catch { + return {}; // no ledger yet — nothing has ever settled + } + try { + const parsed: unknown = JSON.parse(raw); + if (!isRecord(parsed) || !isRecord(parsed.positions)) return {}; + const out: Record = {}; + for (const [id, value] of Object.entries(parsed.positions)) { + if (isPosition(value)) out[id] = { x: value.x, y: value.y }; + } + return out; + } catch { + return {}; + } +} + +function writePositionMap(positions: Record): void { + fs.mkdirSync(path.dirname(POSITIONS_FILE), { recursive: true }); + fs.writeFileSync( + POSITIONS_FILE, + JSON.stringify({ positions, updatedAt: new Date().toISOString() }, null, "\t"), + "utf-8", + ); +} + +/** embeddings.json — the whole EmbeddingsLedger at once. Missing/corrupt + * file reads as an EMPTY ledger (model "") so consumers see "nothing + * embedded yet" rather than an error; the embedder backfills. */ +function readEmbeddingsLedger(): EmbeddingsLedger { + let raw: string; + try { + raw = fs.readFileSync(EMBEDDINGS_FILE, "utf-8"); + } catch { + return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }; + } + try { + const parsed: unknown = JSON.parse(raw); + if (!isRecord(parsed)) return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }; + const model = typeof parsed.model === "string" ? parsed.model : ""; + const dim = typeof parsed.dim === "number" && Number.isFinite(parsed.dim) ? parsed.dim : 0; + const vectors: Record = {}; + if (isRecord(parsed.vectors)) { + for (const [id, value] of Object.entries(parsed.vectors)) { + if ( + Array.isArray(value) && + value.length > 0 && + value.every((x) => typeof x === "number" && Number.isFinite(x)) + ) { + vectors[id] = value as number[]; + } + } + } + const clusters: Array<{ members: string[] }> = []; + if (Array.isArray(parsed.clusters)) { + for (const c of parsed.clusters) { + if ( + isRecord(c) && + Array.isArray(c.members) && + c.members.length > 0 && + c.members.every((m) => typeof m === "string") + ) { + clusters.push({ members: [...(c.members as string[])] }); + } + } + } + return { model, dim, vectors, clusters, updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : "" }; + } catch { + return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }; + } +} + +function writeEmbeddingsLedger(ledger: EmbeddingsLedger): void { + fs.mkdirSync(path.dirname(EMBEDDINGS_FILE), { recursive: true }); + fs.writeFileSync(EMBEDDINGS_FILE, JSON.stringify(ledger, null, "\t"), "utf-8"); +} + +const isPosition = (v: unknown): v is Position => + isRecord(v) && typeof v.x === "number" && Number.isFinite(v.x) && typeof v.y === "number" && Number.isFinite(v.y); + +/** sources.json — { id: NotesSource }. Read-modify-write, NOT append-only: + * the usage log lives inside the record (like a chat's JSON file). */ +function readSourceMap(): Record { + let raw: string; + try { + raw = fs.readFileSync(SOURCES_FILE, "utf-8"); + } catch { + return {}; + } + try { + const parsed: unknown = JSON.parse(raw); + if (!isRecord(parsed)) return {}; + const out: Record = {}; + for (const [id, value] of Object.entries(parsed)) { + if (isNotesSource(value)) out[id] = value; + } + return out; + } catch { + return {}; + } +} + +function writeSourceMap(map: Record): void { + fs.mkdirSync(path.dirname(SOURCES_FILE), { recursive: true }); + fs.writeFileSync(SOURCES_FILE, JSON.stringify(map, null, "\t"), "utf-8"); +} + +const isNotesSource = (v: unknown): v is NotesSource => + isRecord(v) && isString(v.id) && isString(v.filePath) && isString(v.importedAt); diff --git a/src/store/render-settings.ts b/src/store/render-settings.ts new file mode 100644 index 0000000..754135c --- /dev/null +++ b/src/store/render-settings.ts @@ -0,0 +1,40 @@ +/** + * Server-side persistence for the graph render settings (the settings-panel + * controls): data/render-settings.json — read-modify-write, NOT a ledger. + * + * The shape, defaults, and sanitizer live in src/core/render-settings.ts + * (pure, shared with the client); this module only moves bytes. Whatever + * comes off disk is sanitized by the caller on load — and we sanitize on + * write too, so the file can never hold an out-of-range value that would + * silently shift a slider. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { RENDER_SETTINGS_FILE } from "../../config.ts"; +import { DEFAULT_RENDER_SETTINGS, sanitizeRenderSettings, type GraphRenderSettings } from "../core/render-settings.ts"; + +/** The persisted settings, or null when no file exists yet (first boot). */ +export function readRenderSettings(): GraphRenderSettings | null { + let raw: string; + try { + raw = fs.readFileSync(RENDER_SETTINGS_FILE, "utf-8"); + } catch { + return null; // never booted / unreadable → caller uses defaults + } + try { + return sanitizeRenderSettings(JSON.parse(raw)); + } catch { + return null; // corrupt → defaults, never crash the view + } +} + +/** Persist settings (sanitized). Returns what was written. */ +export function writeRenderSettings(raw: unknown): GraphRenderSettings { + const settings = sanitizeRenderSettings(raw); + fs.mkdirSync(path.dirname(RENDER_SETTINGS_FILE), { recursive: true }); + fs.writeFileSync(RENDER_SETTINGS_FILE, JSON.stringify(settings, null, "\t"), "utf-8"); + return settings; +} + +export { DEFAULT_RENDER_SETTINGS }; diff --git a/tests/clusters.test.ts b/tests/clusters.test.ts new file mode 100644 index 0000000..b7a4ffc --- /dev/null +++ b/tests/clusters.test.ts @@ -0,0 +1,85 @@ +/** + * Community detection tests (vision §1, DECIDED 2026-09-13) — pure + * structural Louvain over follows+supports, deterministic via seed. + * Assertions are on SHAPE and MEMBER SETS, never community indices + * (indices reshuffle; member sets are what labelings key to). + */ +import { strict as assert } from "node:assert"; +import { describe, test } from "node:test"; +import { matchClusterLabelings, MIN_CLUSTER_SIZE, type Community } from "../src/core/clusters.ts"; +import { clusterHue } from "../components/cluster-geometry.ts"; +import type { ClusterLabeling } from "../src/core/contracts.ts"; + +const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${prefix}${i}`); + +/** A clique on the given ids. */ +const clique = (nodeIds: string[]) => + nodeIds.flatMap((a, i) => nodeIds.slice(i + 1).map((b) => ({ from: a, to: b }))); + +const labeling = (id: string, core: string[], createdAt = "2026-09-01T00:00:00.000Z"): ClusterLabeling => ({ + id, + label: `label-${id}`, + core, + createdAt, +}); + +describe("matchClusterLabelings", () => { + test("claims a labeling while at least half its core survives in the community", () => { + const community: Community = { members: ["a", "b", "c", "d", "e"] }; + const records = [labeling("clu_1", ["a", "b", "x", "y"])]; // 2 of 4 survive — exactly half + const [claim] = matchClusterLabelings([community], records); + assert.equal(claim?.id, "clu_1"); + // One fewer survivor: under half → relabel. + const [dead] = matchClusterLabelings([{ members: ["a", "c", "d", "e", "f"] }], records); + assert.equal(dead, null); + }); + + test("single successor: a split core keeps its label in only one community", () => { + const records = [labeling("clu_1", ["a", "b", "c", "d"])]; + const left: Community = { members: ["a", "b", "z"] }; // 2 of 4 — half + const right: Community = { members: ["c", "d", "w"] }; // 2 of 4 — half + // Tie: larger overlap wins; equal → both eligible, greedy takes first come. + const claims = matchClusterLabelings([left, right], records); + const claimed = claims.filter((c) => c !== null); + assert.equal(claimed.length, 1); + assert.equal(claimed[0]?.id, "clu_1"); + }); + + test("greedy: the best-overlapping community wins the record", () => { + const records = [labeling("clu_1", ["a", "b", "c", "d"])]; + const weak: Community = { members: ["a", "b", "q", "r", "s"] }; // 2/4 + const strong: Community = { members: ["a", "b", "c", "x", "y"] }; // 3/4 + const claims = matchClusterLabelings([weak, strong], records); + assert.equal(claims[0], null); + assert.equal(claims[1]?.id, "clu_1"); + }); + + test("one record per community; newest record breaks ties", () => { + const oldR = labeling("clu_old", ["a", "b"], "2026-09-01T00:00:00.000Z"); + const newR = labeling("clu_new", ["a", "b"], "2026-09-02T00:00:00.000Z"); + const community: Community = { members: ["a", "b", "c"] }; + const [claim] = matchClusterLabelings([community], [oldR, newR]); + assert.equal(claim?.id, "clu_new"); + }); + + test("no labelings: every community unclaimed; empty communities array ok", () => { + assert.deepEqual(matchClusterLabelings([{ members: ["a", "b"] }], []), [null]); + assert.deepEqual(matchClusterLabelings([], [labeling("clu_1", ["a"])]), []); + }); + + test("MIN_CLUSTER_SIZE is 3 (the render/naming threshold)", () => { + assert.equal(MIN_CLUSTER_SIZE, 3); + }); +}); + +describe("clusterHue", () => { + test("is stable per id and spreads", () => { + // Same id → same hue, always; different ids → usually different hues. + assert.equal(clusterHue("cluster-a"), clusterHue("cluster-a")); + const hues = new Set(); + for (let i = 0; i < 40; i++) hues.add(clusterHue(`cluster-${i}`)); + assert.ok(hues.size > 20, `expected spread, got ${hues.size} distinct hues for 40 ids`); + for (const h of hues) assert.ok(h >= 0 && h < 360); + }); +}); + diff --git a/tests/extraction.test.ts b/tests/extraction.test.ts index 3b58964..24e90ca 100644 --- a/tests/extraction.test.ts +++ b/tests/extraction.test.ts @@ -27,15 +27,20 @@ import type { UIMessage } from "ai"; import type { PiEngine } from "../src/agent/models.ts"; import type { AuditResult, + ClusterLabeling, + ExtractionRunUsage, GraphEdge, GraphNode, LedgerEvent, NodeMapping, + NotesSource, SessionCheckpoint, SessionFindings, } from "../src/core/contracts.ts"; import type { GraphStore } from "../src/core/ports.ts"; import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts"; +import { clusterNamingSettled, refreshClusterLabels } from "../src/agent/subagents/cluster-namer.ts"; +import { semanticClusters } from "../src/core/semantic-clusters.ts"; import { chatStore } from "../src/store/chats.ts"; import { PATHS } from "../config.ts"; @@ -60,6 +65,16 @@ function memoryLedgers() { const events: LedgerEvent[] = []; const mappings: NodeMapping[] = []; const checkpoints: SessionCheckpoint[] = []; + const sources = new Map(); + const clusterLabels: ClusterLabeling[] = []; + const positions = new Map(); + const embeddings = { + model: "", + dim: 0, + updatedAt: "", + vectors: new Map(), + clusters: new Array>(), + }; return { appendNode: (n: GraphNode) => void nodes.push(n), readNodes: () => nodes.map((n) => ({ ...n })), @@ -71,6 +86,40 @@ function memoryLedgers() { readMappings: () => mappings.map((m) => ({ ...m })), appendCheckpoint: (c: SessionCheckpoint) => void checkpoints.push(c), readCheckpoints: () => checkpoints.map((c) => ({ ...c })), + upsertSource: (s: NotesSource) => { + // Mirrors the real store: an upsert KEEPS the existing usage log. + const prev = sources.get(s.id); + sources.set(s.id, { ...s, extractions: [...(prev?.extractions ?? [])] }); + }, + readSources: () => [...sources.values()].map((s) => ({ ...s, extractions: [...(s.extractions ?? [])] })), + appendSourceUsage: (id: string, entry: ExtractionRunUsage) => { + const source = sources.get(id); + if (!source) return; + if (!source.extractions?.some((r) => r.startedAt === entry.startedAt)) { + source.extractions = [...(source.extractions ?? []), entry]; + } + }, + // Cluster labelings — mirrors the real store: append dedupes by id. + appendClusterLabel: (l: ClusterLabeling) => { + if (!clusterLabels.some((x) => x.id === l.id)) clusterLabels.push(l); + }, + readClusterLabels: () => clusterLabels.map((l) => ({ ...l, core: [...l.core] })), + // The layout ledger — settleGraphLayout writes it after each merge. + readPositions: () => Object.fromEntries([...positions].map(([id, p]) => [id, { ...p }])), + writePositions: (next) => { + positions.clear(); + for (const [id, p] of Object.entries(next)) positions.set(id, { x: p.x, y: p.y }); + }, + // The embedding ledger — the embedder writes it after a merge. + readEmbeddings: () => ({ model: embeddings.model, dim: embeddings.dim, vectors: Object.fromEntries([...embeddings.vectors].map(([id, v]) => [id, [...v]])), clusters: [...embeddings.clusters.map((c) => ({ members: [...c] }))], updatedAt: embeddings.updatedAt }), + writeEmbeddings: (next) => { + embeddings.model = next.model; + embeddings.dim = next.dim; + embeddings.updatedAt = next.updatedAt; + embeddings.vectors.clear(); + for (const [id, v] of Object.entries(next.vectors)) embeddings.vectors.set(id, [...v]); + embeddings.clusters = next.clusters.map((c) => [...c.members]); + }, } satisfies GraphStore; } @@ -288,8 +337,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: submitModel("submit_findings", FINDINGS), auditor: submitModel("submit_review", verdict), + labeler: submitModel("submit_labels", { labels: [] }), }); const stats = await pipeline.runExtraction(chatId); @@ -328,8 +379,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: submitModel("submit_findings", FINDINGS), auditor: silentModel(), // never submits the review + labeler: submitModel("submit_labels", { labels: [] }), }); await assert.rejects(pipeline.runExtraction(chatId), /the auditor never submitted a review/); assert.equal(ledgers.readNodes().length, 0); @@ -338,8 +391,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const failingExtractor = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: silentModel(), auditor: submitModel("submit_review", { ...FINDINGS, dropped: [] }), + labeler: submitModel("submit_labels", { labels: [] }), }); await assert.rejects(failingExtractor.runExtraction(chatId), /the extractor never submitted findings/); assert.equal(ledgers.readCheckpoints().length, 0); @@ -368,10 +423,12 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, // Each stage's first generate is the observation point: the // registry must already carry this chat at the right phase. extractor: submitModel("submit_findings", FINDINGS, () => seen.push(phaseOf())), auditor: submitModel("submit_review", verdict, () => seen.push(phaseOf())), + labeler: submitModel("submit_labels", { labels: [] }), }); await pipeline.runExtraction(chatId); assert.deepEqual(seen, ["extracting", "auditing"]); @@ -400,12 +457,14 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, // During the auditor's generate, the registry must already carry // the extractor's fully accumulated live usage (2 steps × 10/5). extractor: submitModel("submit_findings", FINDINGS), auditor: submitModel("submit_review", verdict, () => usageSeen.push(extractionStatuses().find((r) => r.chatId === chatId)?.usage.extractor?.steps.at(-1)?.inputTokens), ), + labeler: submitModel("submit_labels", { labels: [] }), }); await pipeline.runExtraction(chatId); @@ -440,8 +499,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: hangingModel(), // hangs until aborted — no step finishes auditor: submitModel("submit_review", { propositions: [], edges: [], attribution: { quiz: {} }, dropped: [] }), + labeler: submitModel("submit_labels", { labels: [] }), }); const runPromise = pipeline.runExtraction(chatId); @@ -490,8 +551,10 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: gatedExtractor, auditor: silentModel(), + labeler: submitModel("submit_labels", { labels: [] }), }); const first = pipeline.runExtraction(chatId); @@ -507,9 +570,224 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: silentModel(), auditor: silentModel(), + labeler: submitModel("submit_labels", { labels: [] }), }); await assert.rejects(pipeline.runExtraction("chat_does_not_exist"), /Chat not found/); }); }); + +// ── the notes front door (format-agnostic import) ──────────────────────────── + +/** Raw personal notes — arbitrary format, no session, no message ids. */ +const NOTES_TEXT = `> [!quote] YOU +> teach me vim + +> [!abstract] PI +> Vim always starts in Normal mode — keys are commands, not text.`; + +/** What the notes-mode extractor submits: same shapes minus attribution, + * and messages always empty (the file is the provenance). */ +const NOTES_FINDINGS = { + propositions: [ + { + id: "p1", + name: "Vim normal mode", + statement: "Vim starts in Normal mode, where keys are commands.", + aliases: [], + messages: [], + evidence: ["Vim always starts in Normal mode — keys are commands, not text."], + understanding: "solid", + }, + ], + edges: [], +} as const; + +const NOTES_VERDICT = { + propositions: [NOTES_FINDINGS.propositions[0]], + edges: [], + dropped: [], +} as const; + +const CLUSTER_FINDINGS = { + propositions: [ + { id: "p1", name: "Modal editing", statement: "Vim edits text through modal command modes.", aliases: [], messages: [], evidence: ["Normal mode keys are commands."], understanding: "solid" }, + { id: "p2", name: "Verb-object grammar", statement: "Vim commands compose as verb and object.", aliases: [], messages: [], evidence: ["d then motion deletes."], understanding: "solid" }, + { id: "p3", name: "Operator-pending", statement: "Vim enters operator-pending mode between verb and object.", aliases: [], messages: [], evidence: ["After d, a motion is expected."], understanding: "solid" }, + ], + edges: [ + { from: "p1", to: "p2", type: "follows", messages: [], evidence: ["e"] }, + { from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] }, + ], +} as const; + +const CLUSTER_VERDICT = { + propositions: CLUSTER_FINDINGS.propositions, + edges: CLUSTER_FINDINGS.edges, + dropped: [], +} as const; + +/** Test seam for the embedder: fakes the post-merge embedding batch — + * one identical unit vector per node → semanticClusters merges them all + * into ONE topic — then persists the partition like the real embedder. */ +const fakeEmbedder = async (store: GraphStore) => { + const seen = new Set(); + const ids: string[] = []; + for (const n of store.readNodes()) { + if (seen.has(n.id)) continue; + seen.add(n.id); + ids.push(n.id); + } + const vectors = Object.fromEntries(ids.map((id) => [id, [1, 0, 0]])); + store.writeEmbeddings({ + model: "test/embedder", + dim: 3, + vectors, + clusters: semanticClusters(vectors), + updatedAt: "2026-09-13T00:00:00.000Z", + }); +}; + +describe("cluster naming (the post-extraction garnish)", () => { + it("names a new community after a notes merge — one record, claimed on the next refresh", async () => { + const ledgers = memoryLedgers(); + const namerCalls: number[] = []; + const labeler = submitModel("submit_labels", { labels: [{ key: "c0", label: "Vim modal grammar" }] }, () => namerCalls.push(1)); + const pipeline = createExtractionPipeline({ + chatStore, + graphStore: ledgers, + embedder: fakeEmbedder, + extractor: submitModel("submit_findings", CLUSTER_FINDINGS), + auditor: submitModel("submit_review", CLUSTER_VERDICT), + labeler, + }); + const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes, any format" }); + assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 }); + // The refresh is fire-and-forget inside the pipeline — settle it. + await clusterNamingSettled(); + assert.equal(namerCalls.length, 1, "ONE singular naming call"); + const records = ledgers.readClusterLabels(); + assert.equal(records.length, 1); + assert.equal(records[0]!.label, "Vim modal grammar"); + assert.equal(records[0]!.core.length, 3); + assert.match(records[0]!.id, /^clu_/); + // Second refresh: core-survival claims it — nothing new to name. + const again = await refreshClusterLabels({ graphStore: ledgers, engine: labeler }); + assert.equal(again.named, 0); + assert.equal(again.appended, 0); + assert.ok(again.claimed >= 1); + assert.equal(ledgers.readClusterLabels().length, 1, "no duplicate records"); + }); + + it("naming failure never fails the run", async () => { + const ledgers = memoryLedgers(); + const pipeline = createExtractionPipeline({ + chatStore, + graphStore: ledgers, + embedder: fakeEmbedder, + extractor: submitModel("submit_findings", CLUSTER_FINDINGS), + auditor: submitModel("submit_review", CLUSTER_VERDICT), + labeler: silentModel(), // never submits — the namer throws + }); + const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes" }); + assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 }, "the merge itself succeeded"); + await clusterNamingSettled(); + assert.equal(ledgers.readClusterLabels().length, 0, "nothing named, nothing appended"); + }); +}); + +describe("the notes front door (importNotes — format-agnostic)", () => { + it("imports raw notes end to end: source record, file-scoped provenance, no quiz events", async () => { + const ledgers = memoryLedgers(); + const seen: { kind?: string; label?: string }[] = []; + const pipeline = createExtractionPipeline({ + chatStore, + graphStore: ledgers, + embedder: fakeEmbedder, + extractor: submitModel("submit_findings", NOTES_FINDINGS), + // During the auditor's call the registry must show the notes run. + auditor: submitModel("submit_review", NOTES_VERDICT, () => { + const run = extractionStatuses().find((r) => r.chatId.startsWith("src_")); + seen.push({ kind: run?.kind, label: run?.label }); + }), + labeler: submitModel("submit_labels", { labels: [] }), + }); + const stats = await pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT }); + + assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 }); + // The run surfaced as a NOTES run labeled with the file path. + assert.deepEqual(seen, [{ kind: "notes", label: "/vault/learn vim.md" }]); + assert.equal(extractionStatuses().length, 0, "the registry clears when the run ends"); + + // The source record: deterministic src id, the file path, the usage log. + const source = ledgers.readSources()[0]!; + assert.match(source.id, /^src_/); + assert.equal(source.filePath, "/vault/learn vim.md"); + assert.ok(Date.parse(source.importedAt)); + assert.equal(source.extractions?.length, 1, "the run's usage persisted into the source record"); + assert.equal(source.extractions![0]!.stats?.nodes, 1); + assert.ok(source.extractions![0]!.stages.extractor.steps.length > 0, "faux usage captured"); + + // File-scoped provenance: the mapping line carries the source id; + // the node carries NO message anchors. + const node = ledgers.readNodes()[0]!; + assert.equal(node.statement, "Vim starts in Normal mode, where keys are commands."); + assert.deepEqual(node.provenance, []); + assert.deepEqual(ledgers.readMappings(), [{ chatId: source.id, sourceId: "p1", graphNodeId: node.id }]); + + // One observation event, zero quiz events — outcomes inside notes + // surface only through understanding observations (user-decided). + const events = ledgers.readEvents(); + assert.deepEqual( + events.map((e) => [e.kind, e.chatId, e.ref, e.note]), + [["observation", source.id, "obs:p1", "solid"]], + ); + // The checkpoint stamps the import. + assert.equal(ledgers.readCheckpoints()[0]!.chatId, source.id); + }); + + it("re-importing the same file is idempotent: same source identity, refs dedupe, usage history grows", async () => { + const ledgers = memoryLedgers(); + const mk = () => + createExtractionPipeline({ + chatStore, + graphStore: ledgers, + embedder: fakeEmbedder, + extractor: submitModel("submit_findings", NOTES_FINDINGS), + auditor: submitModel("submit_review", NOTES_VERDICT), + labeler: submitModel("submit_labels", { labels: [] }), + }); + const first = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT }); + assert.deepEqual(first, { nodes: 1, edges: 0, events: 1 }); + + const second = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT }); + assert.deepEqual(second, { nodes: 0, edges: 0, events: 0 }, "the same source id dedupes everything"); + + assert.equal(ledgers.readSources().length, 1, "one source identity — the path hashes to the same id"); + assert.equal(ledgers.readSources()[0]!.extractions?.length, 2, "both runs billed their usage"); + assert.equal(ledgers.readNodes().length, 1, "no duplicate node"); + }); + + it("aborts like a chat run: true cancellation, nothing merged, partial usage in the source record", async () => { + const ledgers = memoryLedgers(); + const pipeline = createExtractionPipeline({ + chatStore, + graphStore: ledgers, + embedder: fakeEmbedder, + extractor: hangingModel(), + auditor: submitModel("submit_review", NOTES_VERDICT), + labeler: submitModel("submit_labels", { labels: [] }), + }); + const runPromise = pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT }); + + const srcId = extractionStatuses().find((r) => r.chatId.startsWith("src_"))!.chatId; + assert.equal(abortExtraction(srcId), true); + assert.deepEqual(await runPromise, { nodes: 0, edges: 0, events: 0 }); + assert.equal(ledgers.readNodes().length, 0, "nothing merged"); + assert.equal(ledgers.readCheckpoints().length, 0, "nothing checkpointed"); + assert.equal(ledgers.readSources()[0]!.extractions?.length, 1, "partial usage still persisted"); + assert.equal(ledgers.readSources()[0]!.extractions![0]!.stats, undefined, "never merged → no stats"); + }); +}); diff --git a/tests/graph-canvas.test.ts b/tests/graph-canvas.test.ts new file mode 100644 index 0000000..b51ea74 --- /dev/null +++ b/tests/graph-canvas.test.ts @@ -0,0 +1,227 @@ +/** + * Pure-helper tests for the hand-rolled canvas renderer + * (components/graph-canvas.ts). The renderer class itself is DOM-bound + * (canvas, rAF, ResizeObserver) and is verified live in the browser; the + * math it runs on every frame must be exact, so it is pinned here. + * + * The camera-ratio convention is the load-bearing contract with the + * persisted settings: sigma's ratio (1 = fit, SMALLER = zoomed in) is kept + * verbatim so stored labelZoom values survive the renderer swap. + */ + +import assert from "node:assert/strict"; +import { describe, it as test } from "node:test"; +import { + arrowGeometry, + boxBlurRound, + fitTransform, + labelAlpha, + pickNearest, + zoomRatioOf, + LABELS_FADE_SPAN, +} from "../components/graph-canvas.ts"; +import { + sanitizePhysicsTuning, + sanitizeRegionTuning, + DEFAULT_PHYSICS_TUNING, + DEFAULT_REGION_TUNING, + REGION_BITMAP_OPTIONS, +} from "../src/core/render-settings.ts"; + +describe("zoomRatioOf (the sigma-compatible camera ratio)", () => { + test("1 at fit zoom, smaller when zoomed in, larger when zoomed out", () => { + assert.equal(zoomRatioOf(2, 2), 1); // at fit + assert.equal(zoomRatioOf(2, 4), 0.5); // zoomed in 2x -> ratio halves + assert.equal(zoomRatioOf(2, 1), 2); // zoomed out 2x -> ratio doubles + }); +}); + +describe("labelAlpha (the single zoom gate)", () => { + test("fully visible at/below the threshold, faded out after the span", () => { + assert.equal(labelAlpha(0.5, 0.5), 1); // at the threshold + assert.equal(labelAlpha(0.25, 0.5), 1); // zoomed further in: stays 1 + assert.equal(labelAlpha(0.75, 0.5), 0.5); // mid-fade + assert.equal(labelAlpha(1.0, 0.5), 0); // one span out: gone + assert.equal(labelAlpha(2.0, 0.5), 0); // far out: clamped + }); + + test("node labels and cluster labels cross-fade over the exact same span", () => { + // The renderer draws cluster labels at 1 - labelAlpha; the two must + // always sum to 1 within the fade and never coexist at full opacity. + for (const ratio of [0, 0.25, 0.5, 0.6, 0.75, 0.9, 1, 2, 4]) { + const node = labelAlpha(ratio, 0.5); + const cluster = 1 - node; + assert.ok(node + cluster > 0.999 && node + cluster < 1.001, `sum at ratio ${ratio}`); + assert.ok(node <= 1 && cluster <= 1, `clamped at ratio ${ratio}`); + } + }); + + test("the span constant is unchanged from the sigma era (stored settings depend on it)", () => { + assert.equal(LABELS_FADE_SPAN, 0.5); + }); +}); + +describe("fitTransform", () => { + test("centers the bbox and respects padding", () => { + // A 100x50 graph in a 500x300 viewport, 40px padding: + // available 420x220 -> k = min(4.2, 4.4) = 4.2 + const t = fitTransform({ minX: -50, maxX: 50, minY: -25, maxY: 25 }, 500, 300, 40); + assert.ok(Math.abs(t.k - 4.2) < 1e-9, `k=${t.k}`); + assert.ok(Math.abs(t.x - 250) < 1e-9, `x=${t.x}`); // centered + assert.ok(Math.abs(t.y - 150) < 1e-9, `y=${t.y}`); + // World center maps to viewport center: + assert.ok(Math.abs(0 * t.k + t.x - 250) < 1e-9); + }); + + test("never returns a non-positive scale for degenerate bboxes", () => { + const t = fitTransform({ minX: 5, maxX: 5, minY: 5, maxY: 5 }, 100, 100, 10); + assert.ok(t.k > 0 && Number.isFinite(t.k)); + }); +}); + +describe("arrowGeometry", () => { + test("the tip lands exactly on the target disc edge", () => { + const g = arrowGeometry({ x: 0, y: 0 }, { x: 100, y: 0 }, 6, 6, 5); + assert.ok(g, "non-degenerate edge"); + assert.ok(Math.abs(g!.tip.x - 94) < 1e-9 && Math.abs(g!.tip.y) < 1e-9); + }); + + test("the shaft starts outside the source disc", () => { + const g = arrowGeometry({ x: 0, y: 0 }, { x: 100, y: 0 }, 6, 6, 5)!; + assert.ok(Math.abs(g.shaftFrom.x - 6) < 1e-9 && Math.abs(g.shaftFrom.y) < 1e-9); + // Shaft ends at the head base; head is drawn from base to tip. + assert.ok(Math.abs(g.shaftTo.x - 89) < 1e-9); + }); + + test("the head triangle is symmetric around the shaft axis (0.8x wideness)", () => { + const g = arrowGeometry({ x: 0, y: 0 }, { x: 100, y: 0 }, 6, 6, 5)!; + // halfW = 0.4 x headLength = 2; both base corners share x, mirrored y. + assert.ok(Math.abs(Math.abs(g.left.y) - 2) < 1e-9 && Math.abs(Math.abs(g.right.y) - 2) < 1e-9); + assert.ok(Math.abs(g.left.y + g.right.y) < 1e-9); // mirror + assert.ok(Math.abs(g.left.x - g.right.x) < 1e-9); + }); + + test("works at arbitrary angles (diagonal stays normalized)", () => { + const a = { x: 0, y: 0 }; + const b = { x: 60, y: 80 }; // len 100, dir (0.6, 0.8) + const g = arrowGeometry(a, b, 10, 10, 8)!; + assert.ok(Math.abs(g.tip.x - (60 - 0.6 * 10)) < 1e-9 && Math.abs(g.tip.y - (80 - 0.8 * 10)) < 1e-9); + }); + + test("null for coincident endpoints (never a NaN arrow)", () => { + assert.equal(arrowGeometry({ x: 5, y: 5 }, { x: 5, y: 5 }, 6, 6, 5), null); + }); +}); + +describe("pickNearest (distance-check picking)", () => { + const nodes = [ + { id: "a", x: 10, y: 10 }, + { id: "b", x: 50, y: 50 }, + { id: "c", x: 90, y: 10 }, + ]; + + test("picks the nearest node within the pad", () => { + assert.equal(pickNearest(nodes, 12, 12, 16)?.id, "a"); + assert.equal(pickNearest(nodes, 48, 52, 16)?.id, "b"); + assert.equal(pickNearest(nodes, 88, 8, 16)?.id, "c"); + }); + + test("the picking pad extends the hitbox beyond the disc", () => { + // 20px away, only reachable because the pad covers it. + assert.equal(pickNearest(nodes, 28, 18, 25)?.id, "a"); + // Beyond the pad: nothing. + assert.equal(pickNearest(nodes, 30, 40, 3), null); + }); + + test("ties resolve deterministically (first scanned wins)", () => { + const pair = [ + { id: "first", x: 0, y: 0 }, + { id: "second", x: 10, y: 0 }, + ]; + assert.equal(pickNearest(pair, 5, 0, 10)?.id, "first"); + }); + + test("empty node list picks nothing", () => { + assert.equal(pickNearest([], 0, 0, 100), null); + }); +}); + +describe("sanitizeRegionTuning (dev number inputs)", () => { + test("defaults survive untouched; partial input keeps the rest", () => { + const t = sanitizeRegionTuning(undefined); + assert.deepEqual(t, DEFAULT_REGION_TUNING); + const half = sanitizeRegionTuning({ blur: 7.2 }); + assert.equal(half.blur, 8); // snaps into the fixed set + assert.equal(half.dilation, DEFAULT_REGION_TUNING.dilation); + }); + + test("garbage clamps: NaN falls back, out-of-range clamps", () => { + const t = sanitizeRegionTuning({ dilation: Number.NaN, blur: -5, bitmap: 99999, edgeAlpha: 42 }); + assert.equal(t.dilation, DEFAULT_REGION_TUNING.dilation); // NaN -> default + assert.equal(t.blur, 2); // snaps into the fixed set + assert.equal(t.bitmap, REGION_BITMAP_OPTIONS[REGION_BITMAP_OPTIONS.length - 1]); // 99999 snaps to the largest option + assert.equal(t.edgeAlpha, 1); + }); + + test("empty string / null / undefined values fall back per-field", () => { + const t = sanitizeRegionTuning({ edgeAlpha: Number.NaN }); + assert.equal(t.edgeAlpha, DEFAULT_REGION_TUNING.edgeAlpha); + }); +}); + +describe("boxBlurRound (premultiplied region blur)", () => { + test("spreads a single opaque px into its neighborhood and conserves sum", () => { + // 5x5 RGBA, one white opaque pixel at (2,2) + const w = 5, h = 5; + const a = new Float32Array(w * h * 4); + const center = (2 * w + 2) * 4; + a[center] = 255; a[center + 1] = 255; a[center + 2] = 255; a[center + 3] = 255; + const tmp = new Float32Array(a.length); + const b = new Float32Array(a.length); + boxBlurRound(a, tmp, b, w, h, 1); + // Alpha conservation: total alpha unchanged. + let sumA = 0; + for (let i = 3; i < b.length; i += 4) sumA += b[i]!; + assert.ok(Math.abs(sumA - 255) < 1e-3, `alpha sum ${sumA}`); + // The center dimmed, the 8 neighbors picked some up. + assert.ok(b[center + 3]! < 255 && b[center + 3]! > 0); + const up = ((1 * w) + 2) * 4 + 3; + assert.ok(b[up]! > 0, "neighbor got alpha"); + // Far corner (4,4) untouched. + assert.equal(b[(4 * 5 + 4) * 4 + 3], 0); + }); + + test("radius 0 is a plain copy", () => { + const a = new Float32Array([1, 2, 3, 4, 5, 6, 7, 8]); + const tmp = new Float32Array(8); + const b = new Float32Array(8); + boxBlurRound(a, tmp, b, 1, 2, 0); + assert.deepEqual([...b], [1, 2, 3, 4, 5, 6, 7, 8]); + }); +}); + +describe("sanitizePhysicsTuning (physics controls)", () => { + test("defaults survive untouched; partial input keeps the rest", () => { + assert.deepEqual(sanitizePhysicsTuning(undefined), DEFAULT_PHYSICS_TUNING); + const half = sanitizePhysicsTuning({ clusterPull: 0.6 }); + assert.equal(half.clusterPull, 0.6); + assert.equal(half.chargeStrength, DEFAULT_PHYSICS_TUNING.chargeStrength); + }); + + test("garbage clamps: NaN falls back, out-of-range clamps", () => { + const t = sanitizePhysicsTuning({ + chargeStrength: Number.NaN, + linkStrength: 99, + linkDistance: -5, + gravity: 7, + clusterPull: -1, + velocityDecay: 3, + }); + assert.equal(t.chargeStrength, DEFAULT_PHYSICS_TUNING.chargeStrength); + assert.equal(t.linkStrength, 4); // sweep-validated cap (8 explodes, 5 wobbles) + assert.equal(t.linkDistance, 10); + assert.equal(t.gravity, 0.48); + assert.equal(t.clusterPull, 0); + assert.equal(t.velocityDecay, 0.9); + }); +}); diff --git a/tests/graph.test.ts b/tests/graph.test.ts index 2a104d1..2bfc79c 100644 --- a/tests/graph.test.ts +++ b/tests/graph.test.ts @@ -20,17 +20,27 @@ import { after, describe, it } from "node:test"; import type { UIMessage } from "ai"; import type { AuditResult } from "../src/core/contracts.ts"; import type { QuestionAnswerData } from "../src/core/question-core.ts"; -import { getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts"; +import { clusterContentId, getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts"; import { graphStore } from "../src/store/graph.ts"; import { chatStore } from "../src/store/chats.ts"; -import { LEDGER_FILES, PATHS } from "../config.ts"; +import { LEDGER_FILES, PATHS, SOURCES_FILE, EMBEDDINGS_FILE } from "../config.ts"; /** The wired fs adapters, exactly what src/server/stores.ts passes in production. */ const graphDeps = { chatStore, graphStore } as const satisfies GraphDeps; +/** The session merge source for a chat — what the pipeline passes: the live + * transcript + its updatedAt checkpoint stamp. (Notes imports pass + * messages: null — see the notes-merger test below.) */ +function sessionSource(chatId: string): { messages: UIMessage[]; foldedAt: string } { + const record = chatStore.getChat(chatId)!; + return { messages: record.messages as UIMessage[], foldedAt: record.updatedAt }; +} + // ─── ledger isolation (backup-and-clear of data/graph/*.jsonl) ────────────── -const LEDGER_PATHS = Object.values(LEDGER_FILES); +// EMBEDDINGS_FILE too: the semantic-cluster source (read by graphSnapshot) +// — isolated like the rest so this suite never touches the real vectors. +const LEDGER_PATHS = [...Object.values(LEDGER_FILES), SOURCES_FILE, EMBEDDINGS_FILE]; const backups: Array<{ file: string; bak: string }> = []; const createdChats: string[] = []; @@ -233,7 +243,7 @@ function scopedGraph(chatIds: string[]) { describe("the merger (mergeFindings)", () => { it("creates nodes from the verdicts, rebuilds quiz events deterministically, records observations", () => { - const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps); + const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id)); assert.deepEqual(stats, { nodes: 3, edges: 1, events: 6 }); const graph = scopedGraph([chat1.id]); @@ -302,7 +312,7 @@ describe("the merger (mergeFindings)", () => { }); it("is idempotent — re-merging the same verdicts appends nothing", () => { - const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps); + const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id)); assert.deepEqual(stats, { nodes: 0, edges: 0, events: 0 }); const graph = scopedGraph([chat1.id]); assert.equal(graph.nodes.length, 3); @@ -346,6 +356,7 @@ describe("the merger (mergeFindings)", () => { dropped: [], }, graphDeps, + sessionSource(chat2.id), ); assert.deepEqual(stats, { nodes: 0, edges: 0, events: 1 }); @@ -363,6 +374,100 @@ describe("the merger (mergeFindings)", () => { assert.deepEqual(m, { chatId: chat2.id, sourceId: "q1", graphNodeId: alphaId }); }); + it("resolves imported-notes ids to their file paths in the snapshot (static provenance)", () => { + // The notes front door in miniature: a source record + a merge under + // its src id — the snapshot must label it with the file path (a + // static entry), never as a chat session. + graphStore.upsertSource({ id: "src_snapfixture", filePath: "/vault/learn vim.md", importedAt: "2026-09-13T00:00:00.000Z" }); + mergeFindings( + "src_snapfixture", + { + propositions: [ + { + id: "p1", + name: "Vim registers", + statement: "Vim registers map single keys to editing commands.", + aliases: [], + messages: [], + evidence: ["fixture evidence"], + understanding: "solid", + }, + ], + edges: [], + attribution: { quiz: {} }, + dropped: [], + }, + graphDeps, + { messages: null, foldedAt: "2026-09-13T00:00:00.000Z" }, + ); + const snap = graphSnapshot(graphDeps); + const node = snap.nodes.find((n) => n.statement === "Vim registers map single keys to editing commands.")!; + assert.equal(snap.fileSources["src_snapfixture"], "/vault/learn vim.md"); + assert.deepEqual(snap.nodeSessions[node.id], ["src_snapfixture"], "the file entry joins the node's sessions list"); + assert.equal(snap.sessions["src_snapfixture"], undefined, "a notes source is never a chat session"); + // The understanding observation joins under the source id like any session's. + assert.equal(snap.nodeObservations[node.id]!["src_snapfixture"], "solid"); + }); + + it("derives semantic clusters in graphSnapshot, joined with persisted labelings", () => { + // Three propositions; the cluster comes from the EMBEDDING ledger + // (semantic topics, DECIDED 2026-09-13) — the embedder persists the + // partition, the snapshot just reads it. Here we play the embedder. + const tri = ["Alpha.", "Beta.", "Gamma cluster."] as const; + const merge = mergeFindings( + "src_clusterfixture", + { + propositions: tri.map((statement, i) => ({ + id: `p${i + 1}`, + name: `Cluster node ${i + 1}`, + statement, + aliases: [], + messages: [], + evidence: ["fixture evidence"], + })), + edges: [ + { from: "p1", to: "p2", type: "follows", messages: [], evidence: ["e"] }, + { from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] }, + { from: "p1", to: "p3", type: "follows", messages: [], evidence: ["e"] }, + ], + attribution: { quiz: {} }, + dropped: [], + }, + graphDeps, + { messages: null, foldedAt: "2026-09-13T00:00:00.000Z" }, + ); + assert.equal(merge.nodes, 3); + // The embedder's contract: recluster at the end of the batch and + // PERSIST the partition in the embeddings ledger. + const ids = tri.map((s) => getGraph(graphStore).nodes.find((n) => n.statement === s)!.id); + graphStore.writeEmbeddings({ + model: "test/embedder", + dim: 2, + vectors: {}, + clusters: [{ members: ids }], + updatedAt: "2026-09-13T00:00:00.000Z", + }); + const snap = graphSnapshot(graphDeps); + // Presentation threshold: every cluster is >= 3 members. + assert.ok(snap.clusters.every((c) => c.members.length >= 3)); + // Our trio is exactly one cluster, unlabeled until the namer runs. + const mine = snap.clusters.find((c) => c.members.length === 3 && tri.every((s) => c.members.includes(snap.nodes.find((n) => n.statement === s)!.id))); + assert.ok(mine, "the 3-clique is a cluster"); + assert.equal(mine!.label, null); + assert.match(mine!.id, /^clu_/); + // Mint the labeling the namer would append — the next read joins it. + graphStore.appendClusterLabel({ + id: clusterContentId(mine!.members), + label: "Test region", + core: mine!.members, + createdAt: "2026-09-13T00:00:00.000Z", + }); + const snap2 = graphSnapshot(graphDeps); + const claimed = snap2.clusters.find((c) => c.members.length === 3 && tri.every((s) => c.members.includes(snap2.nodes.find((n) => n.statement === s)!.id)))!; + assert.equal(claimed.label, "Test region"); + assert.equal(claimed.id, clusterContentId(mine!.members), "the claimed cluster carries the record's id"); + }); + it("derives sessions and per-session understanding observations in graphSnapshot", () => { const snap = graphSnapshot(graphDeps); const idByStatement = new Map(snap.nodes.map((n) => [n.statement, n.id])); @@ -408,17 +513,54 @@ describe("the merger (mergeFindings)", () => { dropped: [], }, graphDeps, + sessionSource(chat3.id), ); assert.equal(stats.nodes, 1, "created — the merger does not match strings"); assert.equal(getGraph(graphStore).nodes.filter((n) => n.statement === "Alpha intuition").length, 2); }); - it("returns zeros for unknown chats", () => { - assert.deepEqual(mergeFindings("chat_does_not_exist", fixtureAudit(), graphDeps), { - nodes: 0, - edges: 0, - events: 0, + it("merges the notes front door: source id as chatId, no quiz events, checkpoint at import time", () => { + // Raw personal notes carry no transcript — the merger gets no + // message parts, so NO quiz events are rebuilt; the understanding + // observation still lands, keyed to the source id. + const stats = mergeFindings( + "src_notesfixture", + { + propositions: [ + { + id: "p1", + name: "Vim normal mode", + statement: "Vim starts in Normal mode, where keys are commands.", + aliases: [], + messages: [], // notes carry no message anchors — the file is the provenance + evidence: ["fixture evidence"], + understanding: "solid", + }, + ], + edges: [], + attribution: { quiz: {} }, // normalized by the notes pipeline — nothing to attribute + dropped: [], + }, + graphDeps, + { messages: null, foldedAt: "2026-09-13T00:00:00.000Z" }, + ); + assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 }); + + const node = getGraph(graphStore).nodes.find((n) => n.statement.startsWith("Vim starts"))!; + assert.deepEqual(node.provenance, [], "no message anchors — the mapping line carries the source"); + assert.deepEqual(graphStore.readMappings().find((m) => m.chatId === "src_notesfixture"), { + chatId: "src_notesfixture", + sourceId: "p1", + graphNodeId: node.id, }); + // The observation event is file-scoped; no quiz events exist. + const events = graphStore.readEvents().filter((e) => e.chatId === "src_notesfixture"); + assert.deepEqual( + events.map((e) => [e.kind, e.ref, e.note]), + [["observation", "obs:p1", "solid"]], + ); + // The checkpoint stamps the import time. + assert.equal(graphStore.readCheckpoints().find((c) => c.chatId === "src_notesfixture")?.foldedAt, "2026-09-13T00:00:00.000Z"); }); }); @@ -452,7 +594,7 @@ describe("the trigger helper (pendingQuizAnswers)", () => { assert.equal(pendingQuizAnswers(chat.id, graphDeps), 1, "one answered quiz pending"); assert.equal(pendingQuizAnswers("chat_does_not_exist", graphDeps), 0); - mergeFindings(chat.id, fixtureAudit(), graphDeps); + mergeFindings(chat.id, fixtureAudit(), graphDeps, sessionSource(chat.id)); // call-s is unattributed here (fixtureAudit has no call-s attribution) // — but it still becomes an event, so nothing stays pending. assert.equal(pendingQuizAnswers(chat.id, graphDeps), 0, "extracted → nothing pending"); diff --git a/tests/layout.test.ts b/tests/layout.test.ts new file mode 100644 index 0000000..f9e1a11 --- /dev/null +++ b/tests/layout.test.ts @@ -0,0 +1,315 @@ +/** + * Option A — incremental anchored force layout (core/layout.ts). The + * properties the user asked for: the layout EVOLVES (old placements stay + * recognizable), new nodes land near their cluster, orphans collect on + * the rim, and everything is deterministic. + */ +import assert from "node:assert/strict"; +import { describe, it } from "node:test"; +import { createGraphSimulation, incrementalLayout, inferenceDepths } from "../src/core/layout.ts"; +import { settleGraphLayout } from "../src/core/layout-settle.ts"; +import type { GraphNode, GraphEdge, Position } from "../src/core/contracts.ts"; + +const node = (id: string): GraphNode => ({ + id, + name: id, + statement: id, + aliases: [], + createdAt: "2026-09-13T00:00:00.000Z", + provenance: [], +}); +const edge = (from: string, to: string): GraphEdge => ({ from, to, type: "supports", provenance: [] }); + +const mean = (positions: Map, ids: string[]): { x: number; y: number } => ({ + x: ids.reduce((s, id) => s + positions.get(id)!.x, 0) / ids.length, + y: ids.reduce((s, id) => s + positions.get(id)!.y, 0) / ids.length, +}); + +describe("incrementalLayout", () => { + it("bootstraps deterministically from an empty ledger", () => { + const nodes = ["a", "b", "c", "d"].map(node); + const edges = [edge("a", "b"), edge("b", "c"), edge("c", "d")]; + const first = incrementalLayout(nodes, edges, new Map()); + const second = incrementalLayout(nodes, edges, new Map()); + assert.equal(first.size, 4); + for (const [id, p] of first) { + assert.ok(Number.isFinite(p.x) && Number.isFinite(p.y), `finite coords for ${id}`); + assert.deepEqual(p, second.get(id), `deterministic position for ${id}`); + } + }); + + it("rest state is stable: a re-settle with nothing new barely moves nodes", () => { + const nodes = ["a", "b", "c", "d", "e"].map(node); + const edges = [edge("a", "b"), edge("b", "c"), edge("c", "d"), edge("d", "e"), edge("a", "e")]; + const rest = incrementalLayout(nodes, edges, new Map()); + const span = + Math.max(...[...rest.values()].map((p) => p.x)) - Math.min(...[...rest.values()].map((p) => p.x)); + const again = incrementalLayout(nodes, edges, rest); + let maxShift = 0; + for (const [id, p] of rest) { + const q = again.get(id)!; + maxShift = Math.max(maxShift, Math.hypot(p.x - q.x, p.y - q.y)); + } + assert.ok(maxShift < span * 0.1, `re-settle drift ${maxShift.toFixed(2)} should be < 10% of span ${span.toFixed(2)}`); + }); + + it("evolves: an anchored node stays near its anchor while a new neighbor joins it", () => { + const core = ["a", "b", "c"].map(node); + const coreEdges = [edge("a", "b"), edge("b", "c")]; + const previous = incrementalLayout(core, coreEdges, new Map()); + const anchor = previous.get("b")!; + const dist = Math.hypot; + const nearby = ["d", "e"].map(node); + const next = incrementalLayout( + [...core, ...nearby], + [...coreEdges, edge("b", "d"), edge("d", "e")], + previous, + ); + // The anchor keeps its neighborhood (bounded drift), and the new + // nodes landed near it rather than being flung elsewhere. + const coreSpanBefore = + Math.max(...[...previous.values()].map((p) => p.x)) - Math.min(...[...previous.values()].map((p) => p.x)); + assert.ok( + dist(next.get("b")!.x - anchor.x, next.get("b")!.y - anchor.y) < coreSpanBefore, + "anchored node b stays near its previous position", + ); + const centroid = mean(next, ["b", "d", "e"]); + assert.ok( + dist(next.get("d")!.x - centroid.x, next.get("d")!.y - centroid.y) < coreSpanBefore * 2, + "new neighbor d landed near the cluster it arrived with", + ); + }); + + it("rim: an orphan ends up outside the anchored core", () => { + const core = ["a", "b", "c", "d", "e"].map(node); + const coreEdges = [ + edge("a", "b"), + edge("b", "c"), + edge("c", "d"), + edge("d", "e"), + edge("a", "e"), + edge("b", "d"), + ]; + const previous = incrementalLayout(core, coreEdges, new Map()); + const next = incrementalLayout([...core, node("orphan")], coreEdges, previous); + const c = mean(previous, ["a", "b", "c", "d", "e"]); + const coreMax = Math.max( + ...[...next.entries()].filter(([id]) => id !== "orphan").map(([, p]) => Math.hypot(p.x - c.x, p.y - c.y)), + ); + const orphanR = Math.hypot(next.get("orphan")!.x - c.x, next.get("orphan")!.y - c.y); + assert.ok(orphanR > coreMax, `orphan at r=${orphanR.toFixed(1)} should sit outside core max r=${coreMax.toFixed(1)}`); + }); +}); + +describe("settleGraphLayout", () => { + it("dedupes repeated node lines, runs, and writes the whole ledger", () => { + const written: Record[] = []; + const lines = [node("a"), node("b"), node("a")]; // "a" appended twice + const store = { + readNodes: () => lines.map((l) => ({ ...l })), + readEdges: () => [edge("a", "b")], + readPositions: () => ({} as Record), + writePositions: (p: Record) => void written.push(p), + readEmbeddings: () => ({ model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }), + }; + settleGraphLayout(store); + assert.equal(written.length, 1); + assert.deepEqual(Object.keys(written[0]!).sort(), ["a", "b"], "one position per UNIQUE node"); + }); + + it("does nothing on an empty graph", () => { + const written: Record[] = []; + settleGraphLayout({ + readNodes: () => [], + readEdges: () => [], + readPositions: () => ({} as Record), + writePositions: (p) => void written.push(p), + readEmbeddings: () => ({ model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }), + }); + assert.equal(written.length, 0); + }); +}); +describe("clusterPull (the centroid force)", () => { + it("compacts each cluster toward its OWN centroid; strength 0 = off", () => { + // Two loose chains, far apart, edges only WITHIN each chain. + const mk = (p: string, baseX: number) => + Array.from({ length: 8 }, (_, i) => ({ id: `${p}${i}`, x: baseX + i * 40, y: i * 15 })); + const a = mk("a", -400); + const b = mk("b", 400); + const nodes = [...a, ...b]; + const edges = [ + ...a.slice(0, -1).map((n, i) => ({ from: n.id, to: a[i + 1]!.id })), + ...b.slice(0, -1).map((n, i) => ({ from: n.id, to: b[i + 1]!.id })), + ]; + const clusters = [{ members: a.map((n) => n.id) }, { members: b.map((n) => n.id) }]; + + const spreadOf = (ns: { id: string; x: number; y: number }[]): number => { + let sum = 0; + for (const group of [ns.filter((n) => n.id[0] === "a"), ns.filter((n) => n.id[0] === "b")]) { + const cx = group.reduce((s, n) => s + n.x, 0) / group.length; + const cy = group.reduce((s, n) => s + n.y, 0) / group.length; + sum += group.reduce((s, n) => s + Math.hypot(n.x - cx, n.y - cy), 0) / group.length; + } + return sum; + }; + + const run = (strength: number) => { + const sim = createGraphSimulation( + nodes.map((n) => ({ ...n })), + edges, + { clusterPull: { clusters, strength } }, + ); + for (let i = 0; i < 150; i++) sim.tick(); + return spreadOf(sim.nodes() as unknown as { id: string; x: number; y: number }[]); + }; + const pulled = run(0.15); + const unpulled = run(0); + // The force compacts (well under the unpulled baseline)... + assert.ok(pulled < unpulled * 0.8, `pulled spread ${pulled.toFixed(1)} should be well under ${unpulled.toFixed(1)}`); + // ...and both groups survive as separate blobs (no merge). + assert.ok(pulled > 0, "spread is finite/positive"); + }); +}); + +describe("inferenceDepths (SCC-safe ranking)", () => { + const nodes = (ids: string[]) => ids.map((id) => ({ id })); + + it("ranks a chain, a diamond takes the longest path, orphans are depth 0", () => { + // chain a->b->c; diamond a->d, b->d, c->d (d reachable at 1 and 2 and 3) + const ns = nodes(["a", "b", "c", "d", "loner"]); + const edges = [ + { from: "a", to: "b" }, + { from: "b", to: "c" }, + { from: "a", to: "d" }, + { from: "b", to: "d" }, + { from: "c", to: "d" }, + ]; + const d = inferenceDepths(ns, edges); + assert.equal(d.get("a"), 0); + assert.equal(d.get("b"), 1); + assert.equal(d.get("c"), 2); + assert.equal(d.get("d"), 3); // longest path, not shortest + assert.equal(d.get("loner"), 0); + }); + + it("cycles share a depth (mutual support = equal footing)", () => { + // x <-> y (a cycle), fed by s; z follows x. + const ns = nodes(["s", "x", "y", "z"]); + const edges = [ + { from: "s", to: "x" }, + { from: "x", to: "y" }, + { from: "y", to: "x" }, + { from: "x", to: "z" }, + ]; + const d = inferenceDepths(ns, edges); + assert.equal(d.get("s"), 0); + assert.equal(d.get("x"), d.get("y"), "cycle members share a depth"); + assert.equal(d.get("x"), 1); + assert.equal(d.get("z"), 2); + // Deterministic across calls. + assert.deepEqual([...inferenceDepths(ns, edges).entries()], [...d.entries()]); + }); +}); + +describe("depthAnchor force", () => { + const nodes = (ids: string[]) => ids.map((id) => ({ id, x: 0, y: 0 })); + + it("vertical: the premise lands below its conclusion", () => { + const ns = nodes(["base", "derived"]); + const edges = [{ from: "base", to: "derived" }]; + const sim = createGraphSimulation(ns, edges, { + depthAnchor: { mode: "vertical", strength: 1.2, spacing: 100, center: { x: 0, y: 0 } }, + }); + for (let i = 0; i < 300; i++) sim.tick(); + const byId = new Map(sim.nodes().map((n) => [n.id, n])); + const baseY = byId.get("base")!.y!; + const derivedY = byId.get("derived")!.y!; + assert.ok(baseY > derivedY, `base (${baseY.toFixed(0)}) should sit BELOW derived (${derivedY.toFixed(0)})`); + assert.ok( + baseY - derivedY > 60, + `level separation should be substantial (got ${(baseY - derivedY).toFixed(0)})`, + ); + }); + + it("radial: inference depth becomes radius", () => { + const ns = nodes(["base", "mid", "top"]); + const edges = [ + { from: "base", to: "mid" }, + { from: "mid", to: "top" }, + ]; + const sim = createGraphSimulation(ns, edges, { + depthAnchor: { mode: "radial", strength: 1.2, spacing: 80, baseRadius: 100, center: { x: 0, y: 0 } }, + }); + for (let i = 0; i < 300; i++) sim.tick(); + const r = (id: string) => { + const n = sim.nodes().find((x) => x.id === id)!; + return Math.hypot(n.x!, n.y!); + }; + assert.ok(r("base") < r("mid"), `base ring (${r("base").toFixed(0)}) inside mid (${r("mid").toFixed(0)})`); + assert.ok(r("mid") < r("top"), `mid ring (${r("mid").toFixed(0)}) inside top (${r("top").toFixed(0)})`); + }); + + it("settle with depthAnchor: old nodes keep x memory but snap to their level", () => { + // Node "old" was previously placed at y=300 (off-level). With + // vertical anchoring a re-settle must move it to the baseline + // while preserving (roughly) its x arrangement. + const previous = new Map([["old", { x: 250, y: 300 }], ["peer", { x: 260, y: 290 }]]); + const next = incrementalLayout( + [{ id: "old" }, { id: "peer" }, { id: "child" }], + [ + { from: "old", to: "child" }, + { from: "peer", to: "child" }, + ], + previous, + { depthAnchor: { mode: "vertical", strength: 0.6, spacing: 80 } }, + ); + const old = next.get("old")!; + // The baseline defaults to the anchor centroid: (300+290)/2 = 295. + assert.ok(Math.abs(old.y - 295) < 20, `old node snaps to the baseline (y=${old.y.toFixed(1)})`); + assert.ok(old.x > 150, `x memory preserved (x=${old.x.toFixed(1)})`); + const child = next.get("child")!; + assert.ok(child.y < old.y - 40, `child rises above its premises (child=${child.y.toFixed(1)}, old=${old.y.toFixed(1)})`); + }); + + it("orphans are exempt from depth anchoring — regular forces govern them (ORPHAN RULE)", () => { + // A premise→conclusion chain plus an unconnected loner. Gravity's + // center is FAR from the baseline, so the two are distinguishable: + // a depth-stamped orphan would sit on the baseline; an exempt one + // follows gravity instead. + const ns = [ + { id: "base", x: 0, y: 0 }, + { id: "derived", x: 10, y: 0 }, + { id: "loner", x: 0, y: 0 }, + ]; + const sim = createGraphSimulation(ns, [{ from: "base", to: "derived" }], { + gravity: { x: 0, y: 500, strength: 0.1 }, + depthAnchor: { mode: "vertical", strength: 1.5, spacing: 120, center: { x: 0, y: 0 } }, + }); + for (let i = 0; i < 400; i++) sim.tick(); + const byId = new Map(sim.nodes().map((n) => [n.id, n])); + assert.ok(Math.abs(byId.get("base")!.y!) < 40, "structured base sits on the baseline"); + assert.ok(byId.get("derived")!.y! < -60, "derived rises a level"); + const lonerY = byId.get("loner")!.y!; + assert.ok(lonerY > 250, `the orphan follows gravity, not the baseline (y=${lonerY.toFixed(0)})`); + assert.ok(Math.abs(lonerY - 500) < 120, `the orphan converges on gravity's center (y=${lonerY.toFixed(0)})`); + }); + + it("radial: orphans are exempt — they gather at the center, not on the base ring", () => { + const ns = [ + { id: "base", x: 0, y: 0 }, + { id: "mid", x: 10, y: 0 }, + { id: "loner", x: 300, y: 300 }, + ]; + const sim = createGraphSimulation(ns, [{ from: "base", to: "mid" }], { + gravity: { x: 0, y: 0, strength: 0.1 }, + depthAnchor: { mode: "radial", strength: 1.5, spacing: 80, baseRadius: 150, center: { x: 0, y: 0 } }, + }); + for (let i = 0; i < 400; i++) sim.tick(); + const r = (id: string) => { + const n = sim.nodes().find((x) => x.id === id)!; + return Math.hypot(n.x!, n.y!); + }; + assert.ok(Math.abs(r("base") - 150) < 40, `base sits on the base ring (r=${r("base").toFixed(0)})`); + assert.ok(r("loner") < 60, `the orphan follows gravity to the center, not the ring (r=${r("loner").toFixed(0)})`); + }); +}); diff --git a/tests/render-settings.test.ts b/tests/render-settings.test.ts new file mode 100644 index 0000000..4d613bc --- /dev/null +++ b/tests/render-settings.test.ts @@ -0,0 +1,217 @@ +/** + * Tests for src/core/render-settings.ts — the shared settings contract + * (shape, defaults, sanitizer, label-zoom slider mapping). Pure module, + * pure tests: whatever JSON arrives from disk or the network goes through + * sanitizeRenderSettings and must come out valid, never throw. + */ + +import assert from "node:assert/strict"; +import { describe, it as test } from "node:test"; +import { + ARROW_LENGTH_RANGE, + CHARGE_RANGE, + CLUSTER_PULL_RANGE, + DEFAULT_RENDER_SETTINGS, + DEPTH_BASE_RADIUS_RANGE, + DEPTH_SPACING_RANGE, + DEPTH_STRENGTH_RANGE, + EDGE_WIDTH_RANGE, + GRAVITY_RANGE, + LABEL_SIZE_RANGE, + LABEL_ZOOM_MAX, + LABEL_ZOOM_MIN, + LINK_DISTANCE_RANGE, + LINK_STRENGTH_RANGE, + NODE_SIZE_RANGE, + REGION_BLUR_OPTIONS, + sanitizeRegionTuning, + REGION_DILATION_RANGE, + REGION_EDGE_ALPHA_RANGE, + VELOCITY_DECAY_RANGE, + DEFAULT_PHYSICS_TUNING, + DEFAULT_REGION_TUNING, + fromOffSliderPosition, + fromSliderPosition, + labelZoomDisplay, + sanitizeRenderSettings, + toOffSliderPosition, + toSliderPosition, +} from "../src/core/render-settings.ts"; + +describe("sanitizeRenderSettings", () => { + test("garbage in → full defaults out", () => { + assert.deepEqual(sanitizeRenderSettings(null), DEFAULT_RENDER_SETTINGS); + assert.deepEqual(sanitizeRenderSettings(undefined), DEFAULT_RENDER_SETTINGS); + assert.deepEqual(sanitizeRenderSettings("junk"), DEFAULT_RENDER_SETTINGS); + assert.deepEqual(sanitizeRenderSettings(42), DEFAULT_RENDER_SETTINGS); + }); + + test("partial input → defaults fill the gaps", () => { + const s = sanitizeRenderSettings({ nodeSize: 6 }); + assert.equal(s.nodeSize, 6); + assert.equal(s.edgeWidth, DEFAULT_RENDER_SETTINGS.edgeWidth); + assert.equal(s.labelSize, DEFAULT_RENDER_SETTINGS.labelSize); + assert.equal(s.labelZoom, DEFAULT_RENDER_SETTINGS.labelZoom); + assert.equal(s.clusterLabels, DEFAULT_RENDER_SETTINGS.clusterLabels); + }); + + test("out-of-range values clamp; wrong types reset", () => { + const s = sanitizeRenderSettings({ + nodeSize: 999, + edgeWidth: 0, + arrowLength: -5, + labelZoom: 100, + labelSize: "big" as unknown as number, + clusterLabels: "yes" as unknown as boolean, + }); + assert.equal(s.nodeSize, NODE_SIZE_RANGE.max); // clamped to max + assert.equal(s.edgeWidth, EDGE_WIDTH_RANGE.min); // clamped to min + assert.equal(s.arrowLength, ARROW_LENGTH_RANGE.min); + assert.equal(s.labelZoom, LABEL_ZOOM_MAX); + assert.equal(s.labelSize, DEFAULT_RENDER_SETTINGS.labelSize); // NaN-safe reset + assert.equal(s.clusterLabels, DEFAULT_RENDER_SETTINGS.clusterLabels); + }); + + test("the low ends accept small sizes (log-slider floors)", () => { + const s = sanitizeRenderSettings({ nodeSize: 0.5, edgeWidth: 0.25, arrowLength: 0.5, labelSize: 0.5 }); + assert.deepEqual( + { nodeSize: s.nodeSize, edgeWidth: s.edgeWidth, arrowLength: s.arrowLength, labelSize: s.labelSize }, + { nodeSize: 0.5, edgeWidth: 0.25, arrowLength: 0.5, labelSize: 0.5 }, + ); + }); + + test("a fully valid object passes through unchanged", () => { + const valid = { + nodeSize: 4, + edgeWidth: 2, + arrowLength: 3, + labelZoom: 0.25, + labelSize: 4, + clusterLabels: false, + regions: DEFAULT_REGION_TUNING, + physics: DEFAULT_PHYSICS_TUNING, + }; + assert.deepEqual(sanitizeRenderSettings(valid), valid); + }); + + test("the pre-2026-09-13 stored shape (flat, no regions/physics) still loads", () => { + // data/render-settings.json on disk predates the nested sections — + // those loads must get defaults, not crash, not shift anything. + const s = sanitizeRenderSettings({ nodeSize: 4, edgeWidth: 1, arrowLength: 5, labelZoom: 0.5, clusterLabels: true }); + assert.deepEqual(s.regions, DEFAULT_REGION_TUNING); + assert.deepEqual(s.physics, DEFAULT_PHYSICS_TUNING); + assert.equal(s.nodeSize, 4); + }); + + test("nested sections sanitize: garbage clamps inside them without touching the rest", () => { + const s = sanitizeRenderSettings({ + nodeSize: 3, + regions: { blur: -5, bitmap: 999, edgeAlpha: 9 }, + physics: { linkStrength: 99, gravity: 7, depthMode: "sideways" }, + }); + assert.equal(s.nodeSize, 3); + assert.equal(s.regions.blur, 2); // snaps into the fixed set {2,4,6,8,10} + assert.equal(s.regions.bitmap, 512); // snaps to the nearest fixed option + assert.equal(s.regions.edgeAlpha, 1); + assert.equal(s.regions.dilation, DEFAULT_REGION_TUNING.dilation); + assert.equal(s.physics.linkStrength, 4); // sweep-validated cap + assert.equal(s.physics.gravity, 0.48); + assert.equal(s.physics.depthMode, "off"); // unknown mode → off + assert.equal(s.physics.chargeStrength, DEFAULT_PHYSICS_TUNING.chargeStrength); + }); + + test("the pre-wave stored shape (no labelSize) still loads", () => { + // localStorage-era records lacked labelSize — must not shift anything. + const s = sanitizeRenderSettings({ nodeSize: 4, edgeWidth: 1, arrowLength: 5, labelZoom: 0.5, clusterLabels: true }); + assert.equal(s.labelSize, DEFAULT_RENDER_SETTINGS.labelSize); + assert.equal(s.nodeSize, 4); + }); +}); + +describe("slider mapping (log scale, default-centered)", () => { + const RANGES: [string, number, number, number][] = [ + ["nodeSize", NODE_SIZE_RANGE.min, NODE_SIZE_RANGE.max, DEFAULT_RENDER_SETTINGS.nodeSize], + ["edgeWidth", EDGE_WIDTH_RANGE.min, EDGE_WIDTH_RANGE.max, DEFAULT_RENDER_SETTINGS.edgeWidth], + ["arrowLength", ARROW_LENGTH_RANGE.min, ARROW_LENGTH_RANGE.max, DEFAULT_RENDER_SETTINGS.arrowLength], + ["labelSize", LABEL_SIZE_RANGE.min, LABEL_SIZE_RANGE.max, DEFAULT_RENDER_SETTINGS.labelSize], + ["labelZoom", LABEL_ZOOM_MIN, LABEL_ZOOM_MAX, DEFAULT_RENDER_SETTINGS.labelZoom], + ]; + + test("every default sits dead-center at position 50", () => { + for (const [name, min, max, dflt] of RANGES) + assert.equal(Math.round(toSliderPosition(dflt, min, max)), 50, name); + }); + + test("round-trips both directions across the whole span", () => { + for (const [name, min, max] of RANGES) { + for (let p = 0; p <= 100; p += 5) { + const v = fromSliderPosition(p, min, max); + assert.ok(Math.abs(toSliderPosition(v, min, max) - p) < 1e-9, `${name} @ ${p}`); + } + } + }); + + test("slider extremes map to the range ends and clamp beyond", () => { + for (const [name, min, max] of RANGES) { + assert.equal(fromSliderPosition(0, min, max), min, name); + assert.ok(Math.abs(fromSliderPosition(100, min, max) - max) < 1e-9, name); // exp/log float noise + assert.equal(toSliderPosition(min * 0.1, min, max), 0, name); // clamps + assert.equal(toSliderPosition(max * 10, min, max), 100, name); + } + }); + + test("display units normalize the default to 1", () => { + assert.equal(labelZoomDisplay(0.5), "1×"); + assert.equal(labelZoomDisplay(0.25), "0.5×"); + assert.equal(labelZoomDisplay(4), "8×"); + assert.equal(labelZoomDisplay(0.0625), "0.13×"); // rounded to 2 decimals + }); + + test("every region/physics default sits dead-center at position 50", () => { + // The same law for the 2026-09-13 sliders: default = geometric mean + // of the range ends (repulsion maps |charge|; friction tolerates + // the sub-0.5° drift). + const pairs: [number, number, number][] = [ + [REGION_DILATION_RANGE.min, REGION_DILATION_RANGE.max, DEFAULT_REGION_TUNING.dilation], + [REGION_EDGE_ALPHA_RANGE.min, REGION_EDGE_ALPHA_RANGE.max, DEFAULT_REGION_TUNING.edgeAlpha], + // |charge| in [30, 480] (the range stores negative ends). + [-CHARGE_RANGE.max, -CHARGE_RANGE.min, -DEFAULT_PHYSICS_TUNING.chargeStrength], + [LINK_STRENGTH_RANGE.min, LINK_STRENGTH_RANGE.max, DEFAULT_PHYSICS_TUNING.linkStrength], + [LINK_DISTANCE_RANGE.min, LINK_DISTANCE_RANGE.max, DEFAULT_PHYSICS_TUNING.linkDistance], + [GRAVITY_RANGE.min, GRAVITY_RANGE.max, DEFAULT_PHYSICS_TUNING.gravity], + [VELOCITY_DECAY_RANGE.min, VELOCITY_DECAY_RANGE.max, DEFAULT_PHYSICS_TUNING.velocityDecay], + [DEPTH_STRENGTH_RANGE.min, DEPTH_STRENGTH_RANGE.max, DEFAULT_PHYSICS_TUNING.depthStrength], + [DEPTH_SPACING_RANGE.min, DEPTH_SPACING_RANGE.max, DEFAULT_PHYSICS_TUNING.depthSpacing], + [DEPTH_BASE_RADIUS_RANGE.min, DEPTH_BASE_RADIUS_RANGE.max, DEFAULT_PHYSICS_TUNING.depthBaseRadius], + ]; + for (const [min, max, d] of pairs) { + assert.ok(Math.abs(toSliderPosition(d, min, max) - 50) < 0.6, `default ${d} in [${min}, ${max}]`); + } + // Off-zone fields: the default sits at 50 INSIDE the on-part. + assert.ok(Math.abs(toOffSliderPosition(DEFAULT_PHYSICS_TUNING.clusterPull, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max) - 50) < 0.6); + }); + + test("off-slider mapping: position 0 ↔ value 0; the log part round-trips", () => { + assert.equal(toOffSliderPosition(0, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max), 0); + assert.equal(fromOffSliderPosition(0, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max), 0); + const v = fromOffSliderPosition(toOffSliderPosition(0.15, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max), CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max); + assert.ok(Math.abs(v - 0.15) < 1e-9); + // One step in (position 5) sits just above the log floor: + // min * (max/min)^(5/100), NOT the floor itself (that's position 0, + // the off zone). + const stepIn = CLUSTER_PULL_RANGE.min * (CLUSTER_PULL_RANGE.max / CLUSTER_PULL_RANGE.min) ** (5 / 100); + assert.ok(Math.abs(fromOffSliderPosition(5, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max) - stepIn) < 1e-9); + }); + + test("blur is a fixed set: the default is 6, non-members snap to the nearest", () => { + // Non-integer radii broke the rendering, so the control became a + // dropdown — the sanitizer enforces the set for ANY input. + assert.equal(DEFAULT_REGION_TUNING.blur, 6); + assert.equal(sanitizeRegionTuning({ blur: 3.7 }).blur, 4); + assert.equal(sanitizeRegionTuning({ blur: 7.2 }).blur, 8); + assert.equal(sanitizeRegionTuning({ blur: -5 }).blur, 2); + assert.equal(sanitizeRegionTuning({ blur: 99 }).blur, 10); + // Stored blur 0 (the old off zone) snaps back into the set. + assert.equal(sanitizeRegionTuning({ blur: 0 }).blur, 2); + }); +}); diff --git a/tests/semantic-clusters.test.ts b/tests/semantic-clusters.test.ts new file mode 100644 index 0000000..54f7fb2 --- /dev/null +++ b/tests/semantic-clusters.test.ts @@ -0,0 +1,118 @@ +/** + * Semantic topic clustering (the Louvain REPLACEMENT, DECIDED + * 2026-09-13): average-linkage agglomerative clustering over the + * embedding ledger — related TOPICS, not reasoning chains. Pure and + * deterministic, so these tests are exact. + */ +import { strict as assert } from "node:assert"; +import { describe, test } from "node:test"; +import { semanticClusters, SEMANTIC_SIM_THRESHOLD, cosineSimilarity } from "../src/core/semantic-clusters.ts"; + +/** Two clearly-related one-hot-ish vectors (high cosine similarity). */ +function near(vec: number[], target: number[], amount = 0.2): number[] { + let dot = 0; + for (let i = 0; i < vec.length; i++) dot += vec[i]! * target[i]!; + const mixed = vec.map((x, i) => x + amount * target[i]!); + let mag = 0; + for (const x of mixed) mag += x * x; + mag = Math.sqrt(mag); + return mixed.map((x) => x / mag); +} + +function axis(dim: number, k: number): number[] { + return Array.from({ length: dim }, (_, i) => (i === k ? 1 : 0)); +} + +describe("cosineSimilarity", () => { + test("identical unit vectors → 1; orthogonal → 0; defensive normalization", () => { + const a = axis(4, 0); + const b = axis(4, 1); + assert.ok(cosineSimilarity(a, a) > 0.999); + assert.ok(Math.abs(cosineSimilarity(a, b)) < 1e-9); + // Unnormalized twins still match. + assert.ok(cosineSimilarity([3, 0], [5, 0]) > 0.999); + }); +}); + +describe("semanticClusters", () => { + test("empty ledger: zero clusters (nothing embedded yet)", () => { + assert.deepEqual(semanticClusters({}), []); + }); + + test("related vectors merge; distant ones stay separate (threshold)", () => { + const dim = 8; + const topicA = axis(dim, 0); + const topicB = axis(dim, 1); + const vectors = { + a1: near(topicA, topicB, 0.1), + a2: near(topicA, topicB, 0.15), + a3: near(topicA, topicB, 0.05), + b1: near(topicB, topicA, 0.1), + b2: near(topicB, topicA, 0.12), + }; + const clusters = semanticClusters(vectors); + // Both topics form; average-linkage similarity within each is well + // above the default threshold, across is well below. + const big = clusters.filter((c) => c.members.length >= 2); + assert.equal(big.length, 2); + assert.deepEqual( + big.map((c) => c.members.sort()).sort((x, y) => x[0]!.localeCompare(y[0]!)), + [["a1", "a2", "a3"], ["b1", "b2"]], + ); + }); + + test("threshold is the granularity knob: higher splits, lower merges", () => { + const dim = 6; + const vectors = { + x: near(axis(dim, 0), axis(dim, 1), 0.3), + y: near(axis(dim, 1), axis(dim, 0), 0.3), + }; + // Similarity ~0.72: merges below that, splits above. + const sim = cosineSimilarity(vectors.x!, vectors.y!); + assert.equal(semanticClusters(vectors, { similarityThreshold: sim - 0.01 }).length, 1); + assert.equal(semanticClusters(vectors, { similarityThreshold: sim + 0.01 }).length, 2); + }); + + test("stragglers stay singletons by default; attachStragglers forces membership", () => { + const dim = 6; + const topic = axis(dim, 0); + const vectors = { + t1: near(topic, axis(dim, 1), 0.1), + t2: near(topic, axis(dim, 2), 0.1), + t3: near(topic, axis(dim, 3), 0.1), + loner: axis(dim, 5), // orthogonal: fits nothing + }; + const loose = semanticClusters(vectors); + const looseLoner = loose.find((c) => c.members.includes("loner")); + assert.equal(looseLoner?.members.length, 1, "default: loner is a straggler"); + assert.equal(loose.find((c) => c.members.length >= 3)?.members.length, 3); + + const allIn = semanticClusters(vectors, { attachStragglers: true }); + assert.equal(allIn.length, 1, "attach: the loner joins the nearest topic"); + assert.ok(allIn[0]!.members.includes("loner")); + }); + + test("members sorted; clusters ordered by size then id (deterministic)", () => { + const dim = 4; + const vectors = { + solo: axis(dim, 3), + p1: near(axis(dim, 0), axis(dim, 1), 0.05), + p2: near(axis(dim, 0), axis(dim, 2), 0.05), + q1: near(axis(dim, 1), axis(dim, 0), 0.05), + q2: near(axis(dim, 1), axis(dim, 2), 0.05), + }; + const a = semanticClusters(vectors); + const b = semanticClusters(vectors); + assert.deepEqual(a, b, "same input → identical partition"); + for (let i = 1; i < a.length; i++) { + assert.ok(a[i - 1]!.members.length >= a[i]!.members.length, "size-descending"); + } + for (const c of a) { + assert.deepEqual(c.members, [...c.members].sort(), "members sorted"); + } + }); + + test("default threshold is the swept real-graph value", () => { + assert.equal(SEMANTIC_SIM_THRESHOLD, 0.6); + }); +}); diff --git a/tests/usage.test.ts b/tests/usage.test.ts index 1a2651d..11a5391 100644 --- a/tests/usage.test.ts +++ b/tests/usage.test.ts @@ -10,7 +10,8 @@ import test from "node:test"; import * as fs from "node:fs"; import { chatStore } from "../src/store/chats.ts"; -import { PATHS } from "../config.ts"; +import { graphStore } from "../src/store/graph.ts"; +import { PATHS, SOURCES_FILE } from "../config.ts"; import { callCost, contextWindow, jobCost, sessionCost, sumSteps, turnCost } from "../src/core/prices.ts"; import type { CallUsage, ExtractionRunUsage, JobRecord, TurnUsage } from "../src/core/contracts.ts"; @@ -170,3 +171,40 @@ test("appendExtractionUsage: appends, idempotent per startedAt, not in list payl fs.unlinkSync(`${PATHS.chats}/${record.id}.json`); }); + +test("appendSourceUsage: appends to the source record, idempotent per startedAt, upsert keeps history", () => { + // The real fs graphStore — sources.json backed up and restored around + // this test (same discipline as the chat fixture files). + fs.mkdirSync(PATHS.graph, { recursive: true }); + let bak: string | undefined; + if (fs.existsSync(SOURCES_FILE)) bak = `${SOURCES_FILE}.w5test.bak`; + if (bak) fs.renameSync(SOURCES_FILE, bak); + try { + const runEntry = (startedAt: number): ExtractionRunUsage => ({ + startedAt, + at: "2026-09-13T00:00:00Z", + durationMs: 42_000, + stats: { nodes: 1, edges: 2, events: 3 }, + stages: { + extractor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] }, + auditor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] }, + }, + }); + const id = "src_usagetest01"; + graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-13T00:00:00Z" }); + graphStore.appendSourceUsage(id, runEntry(1)); + graphStore.appendSourceUsage(id, runEntry(1)); // same run appended twice → no-op + graphStore.appendSourceUsage(id, runEntry(2)); // a retried run is a NEW spend + assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2); + + // An upsert (re-import) KEEPS the usage history. + graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-14T00:00:00Z" }); + assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2); + + // Unknown ids are a silent no-op (nothing to bill against). + graphStore.appendSourceUsage("src_nope", runEntry(1)); + } finally { + fs.rmSync(SOURCES_FILE, { force: true }); + if (bak) fs.renameSync(bak, SOURCES_FILE); + } +});