From b769d32bc60dd3406de60ede50389f45a929cac5 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 01:25:31 +0300 Subject: [PATCH 01/34] =?UTF-8?q?Vision=20=C2=A71:=20clusters=20=E2=80=94?= =?UTF-8?q?=20regions=20of=20the=20world=20model=20(ideation=20concluded?= =?UTF-8?q?=202026-09-13)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Community detection as the graph's at-a-glance structure: Louvain on graphology, seeded; structure-based (never layout-based); entirely inferred from the graph, entirely separate from extraction; size- gated (>= ~3 nodes); named by one cheap cached LLM call per new/changed cluster. Labeling persistence — the core-survival rule (user-decided): each append-only labeling record stores the label + the member set it was produced with (the core); a recomputed community keeps its label while >= half its members are from that core, relabels under half (new record appended, old records stay as history); single successor on splits/merges; color keys to the labeling record. Rendering: dashed hull + blurred low-opacity fill behind everything, per-cluster stable hue, cluster labels inverse-gated against the node label span (domain view zoomed out / proposition view zoomed in); boundary geometry computed at draw time (pure function of member positions) so future physics never invalidates it; inter-cluster edges stay visible (meaningful bridges). Multi-scale clusters (hierarchical Louvain dendrogram) deliberately parked. Open questions: boundary aesthetic/label placement (visual testing), threshold tuning (synthetic benchmark — now the primary clustering-quality validation, sparsity being the honest risk). --- docs/vision.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/docs/vision.md b/docs/vision.md index a220a41..ac83348 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -53,6 +53,21 @@ Three stages, one pipeline — the sole graph writer: Known vs. understood, event ledger as source of truth, derived state recomputed on read — carried over from the design doc in spirit. The state model itself is now an explicit design discussion: **node states should be session-independent and computed from the depth and strength of dependencies** (the `follows`/`supports` topology) plus quiz events and understanding notes. That redesign is pending ideation; the transitional five-class derivation (taught / known / missed / don't-know / candidate) was **removed** (2026-09-08) — the graph view renders state-neutral nodes until the state model is designed. The anchored calibration rubric stays parked until real data. +### Clusters — regions of the world model (DECIDED 2026-09-13, ideation) + +When the graph fills up, the canvas needs a readable at-a-glance structure: **clusters** — objective regions of the world model (domains like "Lacanian psychoanalysis" or "photosynthesis"), discovered by **community detection** on the `follows`/`supports` edges. A community = nodes linked to each other far more than to the rest (modularity). Everything about them is **entirely inferred from the graph data** — the extraction/audit pipeline never knows clusters exist. + +- **Algorithm: Louvain (modularity maximization), seeded for determinism.** In-stack via graphology's ecosystem, near-linear, scales far beyond our sizes. Determinism gives reproducibility (same graph → same clusters); the seed/graphology choice also keeps the upgrade path open: Louvain's dendrogram contains **hierarchy** — "math ⊃ differential calculus ⊃ chain rule" is one level down — deliberately NOT built now (multi-scale clusters stay parked). Dismissed: Leiden (better resolution guarantees, no JS port), Infomap (heuristic-strong, heavier, weaker JS support), Girvan–Newman (O(n³)), label propagation (unstable), spectral (needs k handed in). +- **Structure, not layout.** Clusters are a pure function of the *edges*, never of the visual layout. The force-directed layout already places structurally-close nodes together, so cluster hulls will look contiguous — but cluster identity must not inherit the layout's seed dependence and drift. +- **Derived, cached, auto-updated.** Clusters recompute when the ledger changes, cached against a graph version stamp — same doctrine as states and strengths: derived never stored, nothing hand-maintained. +- **Size-gated meaning.** Only communities ≥ ~3 nodes render as clusters (threshold tunable on real data) — a visible blob always means "a real domain"; lone nodes and tiny communities stay plain. +- **Naming: one cheap LLM call per new/changed cluster** — a single call (no agent loop) with the members' names + statements, producing a short domain label. Never part of extraction; a separate, cached seam. +- **Labeling persistence — the core-survival rule.** Each labeling record (append-only, like everything we persist) stores the label AND the member set it was produced with (the **core**). A recomputed community **keeps** its label while ≥ half its members are from that core; when under half are, it **relabels** with all current nodes (a new record is appended — old records stay as history: an audit trail of how the domains were named over time). Single successor: on splits/merges the strongest-overlap community inherits, the others get fresh labelings; a core with no ≥50% successor simply has no living heir. Cluster color keys to the labeling record — color survives exactly as long as the label does. +- **Rendering: glow blobs behind everything.** A dashed boundary encloses the cluster's hull; the fill is blurred, low-opacity, and sits *behind* the whole graph (edges and nodes float above it, so the blur naturally fogs the inter-cluster space). Per-cluster stable hue. **Cluster labels are inverse-gated** against the existing camera-ratio span: zoomed out = **domain view** (cluster labels on, node labels off), zoomed in = **proposition view** (node labels on, cluster labels faded) — one symmetric transition between the two views of one canvas. +- **Boundary geometry: computed at draw time, never stored** — a pure function of the member positions. This is the constraint that keeps future physics/moving nodes (likely not implemented soon) from invalidating anything: convex hull + padding + rounded corners is cheap per-frame; organic metaball hulls are the same family. Secondary until then. +- **Inter-cluster edges stay visible.** Bridges between domains are meaningful (Nietzsche's will-to-power *follows* Schopenhauer's will) — one of the most informative reads of the clustered view, not clutter. +- **Honest risk: sparsity.** Community detection wants density contrast, and the current real graph is thin — the 2k-node synthetic benchmark (open question 1) is the *primary* validation of clustering quality, not just of layout/render speed. + ### Rendering (DECIDED — research report 2026-08-22) - **Library: sigma.js v3 + graphology + graphology-layout-forceatlas2.** ~40 KB gzipped all-in — half of vis-network, a third of cytoscape, far below react-force-graph (~250 KB deps) and cosmos.gl (luma.gl stack). @@ -73,7 +88,7 @@ Every session starts with the learner's knowledge graph as base context for the ### Home page -The graph view replaces the current placeholder. v1: all nodes from all sessions, node color/shape by state class (the current five classes are transitional — see "Understanding is derived"), pan/zoom, labels at zoom. Clicking a node → the exact (session, message) points where it came up (message-granular provenance, deep-linked). Everything richer (frontier glow, rot map, blast radius) comes after the state model exists. +The graph view replaces the current placeholder. v1: all nodes from all sessions, pan/zoom, labels at zoom, state-neutral nodes (see "Understanding is derived"), and — once implemented — cluster glow-blobs with dashed hulls behind the graph, domain↔proposition as the two zoom-gated views (see "Clusters"). Clicking a node → the exact (session, message) points where it came up (message-granular provenance, deep-linked). Everything richer (frontier glow, rot map, blast radius) comes after the state model exists. --- @@ -213,6 +228,8 @@ The teacher's context window and attention are precious — it has the lesson to 4. Home-graph v1 interactions beyond pan/zoom (node click → session list? — presumed cheap yes). 5. Grain mismatch frequency (`subsumes`-style relations) — per original doc, measured by data, not pre-solved. 6. Aggregation semantics for understanding (min / mean / two-level) — fit on real data, per original doc. +7. Cluster visual details — boundary aesthetic (convex vs. organic hull), label placement/typography, palette — decide during visual testing with real blobs (§1 Clusters; theory DECIDED 2026-09-13). +8. Cluster size threshold (~3) — tune on the synthetic benchmark + real data (§1 Clusters). --- From 543cf337891e2fd9e2baeb1e9bd90a65380975b6 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 02:08:55 +0300 Subject: [PATCH 02/34] Notes front door: format-agnostic import into the graph pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The same extractor→auditor→merger pipeline now runs over raw personal notes in arbitrary formats (vision §1, DECIDED 2026-09-13): - NotesSource records (data/graph/sources.json, deterministic id from path hash → idempotent re-imports) play the chatId role in mappings, checkpoints, and observation events; usage lands on the source record (appendSourceUsage, idempotent per startedAt, kept across upserts). - mergeFindings is chat-store-free: callers pass MergeSource {messages, foldedAt} — sessions pass the chat record, notes pass messages:null (no quiz-event rebuild; a deterministic rebuild needs structured quiz tool parts). - Second extractor prompt (NOTES_EXTRACTOR_INSTRUCTIONS) for general- purpose extraction; understanding judges the AUTHOR's grasp. Auditor wording input-agnostic. extract/audit share one generic runSubmitAgent. - graphSnapshot.fileSources (srcId→filePath); notes sources render as static path entries in the detail card (FileTextIcon, never a link). - One shared pipeline instance (exported from chat-handler): same single-flight registry, live status chips (label = file path), usage accounting, and true abort for both front doors. - Transport: POST /api/graph/import (multipart, 3-wide worker pool) + Import notes… button in the graph view; scripts/import-notes.ts for batch runs (concurrency 4). - real-data-extraction/ gitignored (raw import sources never committed). 355/355 tests; typecheck, lint, build green. --- .gitignore | 2 + app/api/graph/import/route.ts | 52 ++++ app/api/graph/status/route.ts | 4 +- components/graph-view.tsx | 82 +++++- config.ts | 7 + docs/vision.md | 27 +- scripts/import-notes.ts | 62 +++++ src/agent/subagents/extraction.ts | 404 ++++++++++++++++++++++++------ src/core/contracts/graph.ts | 30 +++ src/core/graph.ts | 52 ++-- src/core/ports.ts | 10 + src/server/chat-handler.ts | 5 +- src/store/graph.ts | 61 ++++- tests/extraction.test.ts | 136 ++++++++++ tests/graph.test.ts | 101 +++++++- tests/usage.test.ts | 40 ++- 16 files changed, 961 insertions(+), 114 deletions(-) create mode 100644 app/api/graph/import/route.ts create mode 100644 scripts/import-notes.ts diff --git a/.gitignore b/.gitignore index 1143e7e..d1030a5 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ data/ .env.local .memory/ SCOPE.md +# raw real-data import sources (user directive 2026-09-13: never commit) +real-data-extraction/ diff --git a/app/api/graph/import/route.ts b/app/api/graph/import/route.ts new file mode 100644 index 0000000..7c44afd --- /dev/null +++ b/app/api/graph/import/route.ts @@ -0,0 +1,52 @@ +/** + * POST /api/graph/import — the format-agnostic front door's transport + * (vision §1, 2026-09-13): raw personal-notes files in, one extraction run + * per file through the SAME pipeline the sessions use. Multipart form data + * (field "files", any text format — the extractor is format-agnostic); the + * pipeline mints a deterministic source id per file path, persists the + * source record (the file's provenance + the run's usage), and runs + * extractor → auditor → merger over the raw text. + * + * Small concurrency cap: the model calls are the slow part, and a burst of + * 25 simultaneous runs would trip provider rate limits — a 3-wide worker + * pool keeps the run registry (the dock's status chips) and the ledgers + * healthy. The dock's chips + graph refetch track progress live through + * /api/graph/status, as with session runs. + */ +import { extraction } from "@/src/server"; +import type { EmissionStats } from "@/src/core"; + +export const dynamic = "force-dynamic"; + +interface ImportResult { + filePath: string; + ok: boolean; + stats?: EmissionStats; + error?: string; +} + +export async function POST(request: Request) { + const form = await request.formData(); + const files = form.getAll("files").filter((f): f is File => f instanceof File); + if (files.length === 0) { + return Response.json({ results: [] as ImportResult[], error: "no files" }, { status: 400 }); + } + + const results: ImportResult[] = []; + const queue = [...files]; + const workers = Array.from({ length: Math.min(3, files.length) }, async () => { + for (;;) { + const file = queue.shift(); + if (!file) return; + try { + const text = await file.text(); + const stats = await extraction.importNotes({ filePath: file.name, text }); + results.push({ filePath: file.name, ok: true, stats }); + } catch (err) { + results.push({ filePath: file.name, ok: false, error: err instanceof Error ? err.message : String(err) }); + } + } + }); + await Promise.all(workers); + return Response.json({ results }); +} \ No newline at end of file diff --git a/app/api/graph/status/route.ts b/app/api/graph/status/route.ts index 0ba43a6..e4a3866 100644 --- a/app/api/graph/status/route.ts +++ b/app/api/graph/status/route.ts @@ -12,7 +12,9 @@ export const dynamic = "force-dynamic"; export async function GET() { const runs = extractionStatuses().map((run) => ({ chatId: run.chatId, - title: chatStore.getChat(run.chatId)?.title ?? run.chatId, + // The run's own label (the file path for notes imports) wins; chat + // runs fall back to the chat store's title. + title: run.label ?? chatStore.getChat(run.chatId)?.title ?? run.chatId, phase: run.phase, startedAt: run.startedAt, // Live per-stage usage (the money channel): model + per-step CallUsage, diff --git a/components/graph-view.tsx b/components/graph-view.tsx index 017664e..cf8fbb9 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -56,7 +56,7 @@ import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import Link from "next/link"; -import { Loader2Icon, SearchIcon, WaypointsIcon } from "lucide-react"; +import { FileTextIcon, Loader2Icon, SearchIcon, UploadIcon, WaypointsIcon } from "lucide-react"; import Graph from "graphology"; import forceAtlas2 from "graphology-layout-forceatlas2"; import type { NodeHoverDrawingFunction, NodeLabelDrawingFunction } from "sigma/rendering"; @@ -322,7 +322,9 @@ function tokenHex(cssVar: string): string { interface Selection { node: GraphNode; - chatIds: string[]; + /** The sources that touched this node: chat ids (deep-linkable sessions) + * and notes-source ids ("src_…" — imported files, static entries). */ + sourceIds: string[]; /** Node position in container pixels (from graphToViewport at click time). */ anchor: { x: number; y: number }; } @@ -368,6 +370,26 @@ export function GraphView() { }; }, [refreshTick]); + // Notes import (the format-agnostic front door): file picker → POST + // /api/graph/import; the runs surface in the status chips below, and the + // graph refetches when they finish (the runs-empty transition). + const fileInputRef = useRef(null); + const [importing, setImporting] = useState(false); + const onImportFiles = (files: FileList | null) => { + if (!files || files.length === 0) return; + setImporting(true); + const form = new FormData(); + for (const file of Array.from(files)) form.append("files", file); + fetch("/api/graph/import", { method: "POST", body: form }) + .then(async (res) => { + const body = (await res.json()) as { results: { filePath: string; ok: boolean }[] }; + const failed = body.results.filter((r) => !r.ok); + if (failed.length > 0) console.error("notes import failed for:", failed); + }) + .catch((err) => console.error("import failed:", err)) + .finally(() => setImporting(false)); + }; + // Background pipeline status: poll /api/graph/status; when a run // finishes (some -> none), refetch the graph. const [pipelineRuns, setPipelineRuns] = useState<{ chatId: string; title: string; phase: string; startedAt: number }[]>([]); @@ -490,7 +512,7 @@ export function GraphView() { const anchor = renderer?.graphToViewport({ x: pos.x ?? 0, y: pos.y ?? 0 }); setSelected({ node: graphNode, - chatIds: snapshot.nodeSessions[node] ?? [], + sourceIds: snapshot.nodeSessions[node] ?? [], anchor: anchor ?? { x: 0, y: 0 }, }); }); @@ -640,6 +662,28 @@ export function GraphView() { No matching nodes. )} +
+ { + onImportFiles(e.target.files); + e.target.value = ""; // allow re-picking the same file + }} + /> + +
)} @@ -670,20 +714,38 @@ export function GraphView() { >
{selected.node.name}
{selected.node.statement}
- {selected.chatIds.length > 0 ? ( + {selected.sourceIds.length > 0 ? (
Sessions
- {selected.chatIds.map((chatId) => { - // That session's understanding observation — the + {selected.sourceIds.map((sourceId) => { + // That source's understanding observation — the // extractor's named-word note (confidence history). - const note = snapshot?.nodeObservations?.[selected.node.id]?.[chatId]; + const note = snapshot?.nodeObservations?.[selected.node.id]?.[sourceId]; + // An imported-notes file: a STATIC entry — the path, + // plain text, never a link (no session to open). + if (snapshot?.fileSources?.[sourceId] !== undefined) { + return ( +
+ + + {snapshot.fileSources[sourceId]} + + {note && ( + + {note} + + )} +
+ ); + } + // A real session: deep-linkable provenance. return ( -
+
- {snapshot?.sessions[chatId] ?? chatId} + {snapshot?.sessions[sourceId] ?? sourceId} {/* The note is a fixed-scale word (solid/shaky/...) — the session name is the main thing; the chip is bounded so diff --git a/config.ts b/config.ts index 3dbfba7..b385125 100644 --- a/config.ts +++ b/config.ts @@ -72,3 +72,10 @@ export const LEDGER_FILES = { mappings: path.join(PATHS.graph, "mappings.jsonl"), sessions: path.join(PATHS.graph, "sessions.jsonl"), } as const; + +/** + * Notes-import bookkeeping (NOT a ledger): one JSON map, src-id → NotesSource + * record (contracts/graph.ts) — the file path + import time + the run's usage + * log. Read-modify-write like a chat file, not append-only like the ledgers. + */ +export const SOURCES_FILE = path.join(PATHS.graph, "sources.json"); diff --git a/docs/vision.md b/docs/vision.md index ac83348..206d5d5 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -43,12 +43,37 @@ Every node records every **(chat, message) point** where it came up; every edge Three stages, one pipeline — the sole graph writer: -- **Extractor (LLM) — session-only scope.** Sees ONLY the session transcript (tree-blind, with message ids). Reports freely, with verbatim evidence: propositions from anywhere in the session (one reasoning step each; prose counts as much as anything the teacher formalized), edges as the transcript articulates them, message-granular provenance, retroactive quiz attribution, and per proposition an **understanding level** — how strongly the learner accepted/understood it in this session, a CLOSED named-word scale (`solid` / `shaky` / `accepted` / `resisted` / `not yet engaged`; 2026-09-08: schema-enforced — exactly one word, no numbers, no justifications, which are dropped rather than stored). The levels are data for the future state model; their treatment is TBD. +- **Extractor (LLM) — single-source scope.** Sees ONLY the source text — a session transcript (tree-blind, with message ids) or raw notes (the front door's second prompt, above). Reports freely, with verbatim evidence: propositions from anywhere in the session (one reasoning step each; prose counts as much as anything the teacher formalized), edges as the transcript articulates them, message-granular provenance, retroactive quiz attribution, and per proposition an **understanding level** — how strongly the learner accepted/understood it in this session, a CLOSED named-word scale (`solid` / `shaky` / `accepted` / `resisted` / `not yet engaged`; 2026-09-08: schema-enforced — exactly one word, no numbers, no justifications, which are dropped rather than stored). The levels are data for the future state model; their treatment is TBD. - **Auditor (LLM) — the global-graph scope.** Sees the extractor's output AND the existing graph. Two jobs: the faithfulness net (every candidate verified against verbatim evidence — unchanged), and the **identity judgment**: link each proposition to an existing node (recording the session's phrasing as an alias) or create it new. Conservative: duplicates over wrong merges. - **Merger (deterministic).** Append-only ledger writes; quiz event kinds/refs rebuilt mechanically from quiz tool parts. Every judgment is an LLM's; everything mechanical stays code. **Event ledger (amended):** quiz outcomes are rebuilt deterministically from quiz tool parts (correct / miss / don't-know, plus the learner's note). **There is no `taught` event** — no marking-things-done. The extractor's per-proposition understanding levels join the ledger as session-scoped observations (one closed-scale word per node per session). Identity is no longer exact-string matching — it is the auditor's judgment, with aliases as the phrasing record. +### The format-agnostic front door — notes imports (DECIDED 2026-09-13) + +The same three-stage pipeline (extractor → auditor → merger) runs over **raw +personal notes in arbitrary formats** — an Obsidian vault, plain markdown, +anything textual. The extractor is format-blind by construction (it always +saw rendered text, never ChatRecord JSON), so the notes path needs only a +second extractor prompt (general-purpose extraction; the understanding word +judges the **author's** grasp from their own commentary) and a source +abstraction: each imported file becomes a `NotesSource` record (deterministic +id from the path hash — re-imports are idempotent), playing the `chatId` role +in mappings, checkpoints, and observation events. + +Decided edges: + +- **No session shaping.** Notes are never bent into teacher/learner + transcripts; quiz outcomes inside notes surface only via the extractor's + understanding observations (the deterministic quiz-event rebuild is a + session-pipeline feature — it needs structured quiz tool parts). +- **Static provenance.** Imported nodes show the file path in the sessions + list as a static entry — never a link. Understanding-assessment levels + apply to personal notes as to sessions. +- **Same single-flight registry and observability** as session runs: live + status, usage accounting per stage, true abort. One pipeline instance, + two front doors (quiz-triggered sessions; POST /api/graph/import). + ### Understanding is derived (amended 2026-09-02) Known vs. understood, event ledger as source of truth, derived state recomputed on read — carried over from the design doc in spirit. The state model itself is now an explicit design discussion: **node states should be session-independent and computed from the depth and strength of dependencies** (the `follows`/`supports` topology) plus quiz events and understanding notes. That redesign is pending ideation; the transitional five-class derivation (taught / known / missed / don't-know / candidate) was **removed** (2026-09-08) — the graph view renders state-neutral nodes until the state model is designed. The anchored calibration rubric stays parked until real data. diff --git a/scripts/import-notes.ts b/scripts/import-notes.ts new file mode 100644 index 0000000..c2a9735 --- /dev/null +++ b/scripts/import-notes.ts @@ -0,0 +1,62 @@ +/** + * Import raw notes files through the format-agnostic front door (vision §1, + * 2026-09-13) — every *.md / *.txt file in a directory becomes one + * extraction run (extractor → auditor → merger) over its raw text. Run + * from this worktree only — data never crosses trees: + * + * npx tsx --env-file=.env.local scripts/import-notes.ts [concurrency] + * + * Default dir: real-data-extraction/learn-sessions (the prototype + * learning-session exports — deliberately NOT imported as chats: they run + * through the notes path, raw). Runs in this process: the dev server's + * status chips do NOT see them (registry is per-process) — the dev server + * sees runs triggered through /api/graph/import. + */ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts"; +import { getGraph } from "../src/core/graph.ts"; +import { stores } from "../src/server/stores.ts"; + +async function main() { + const dir = process.argv[2] ?? "real-data-extraction/learn-sessions"; + const limit = Number(process.argv[3] ?? 4); + const files = fs + .readdirSync(dir) + .filter((f) => f.endsWith(".md") || f.endsWith(".markdown") || f.endsWith(".txt")) + .sort(); + console.log(`[import-notes] ${files.length} note files from ${dir} (concurrency ${limit})`); + + const pipeline = createExtractionPipeline(stores); + const queue = [...files]; + const totals = { nodes: 0, edges: 0, events: 0, failed: 0 }; + const workers = Array.from({ length: Math.min(limit, files.length) }, async () => { + for (;;) { + const file = queue.shift(); + if (!file) return; + const filePath = path.join(dir, file); + const t0 = Date.now(); + try { + const stats = await pipeline.importNotes({ filePath, text: fs.readFileSync(filePath, "utf-8") }); + totals.nodes += stats.nodes; + totals.edges += stats.edges; + totals.events += stats.events; + console.log( + `[import-notes] ${file}: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events in ${((Date.now() - t0) / 1000).toFixed(1)}s`, + ); + } catch (err) { + totals.failed++; + console.error(`[import-notes] ${file} FAILED:`, err); + } + } + }); + await Promise.all(workers); + + const graph = getGraph(stores.graphStore); + console.log( + `[import-notes] done: +${totals.nodes} nodes, +${totals.edges} edges, +${totals.events} events, ${totals.failed} failed — ` + + `graph now ${graph.nodes.length} nodes, ${graph.edges.length} edges, ${graph.events.length} events`, + ); +} + +main(); \ No newline at end of file diff --git a/src/agent/subagents/extraction.ts b/src/agent/subagents/extraction.ts index 42bd98e..2f2b608 100644 --- a/src/agent/subagents/extraction.ts +++ b/src/agent/subagents/extraction.ts @@ -1,7 +1,9 @@ +import * as crypto from "node:crypto"; import type { UIMessage } from "ai"; import type { Usage } from "@earendil-works/pi-ai"; import { z } from "zod"; import { lhTool, toPiTools } from "../lh-tool.ts"; +import type { ModelRole } from "../../../config.ts"; import type { PiEngine } from "../models.ts"; import { runSubagent } from "./pi-run.ts"; import type { @@ -42,6 +44,17 @@ import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../ * whatever the in-flight run missed). On failure nothing is checkpointed — * the next trigger re-runs the whole thing. * + * The FORMAT-AGNOSTIC FRONT DOOR (2026-09-13): importNotes() runs the SAME + * three stages over raw personal notes (arbitrary format — no chat, no + * session, no message ids). A notes-mode extractor prompt replaces the + * session one; the auditor and merger are shared (identity + deterministic + * appends don't care where text came from). Provenance is the file itself: + * source ids ("src_" + path hash) play the chatId role in mappings, events + * and checkpoints, and the source record (graphStore sources seam) carries + * the file path + the run's usage. No quiz events are ever built from + * notes — quiz outcomes inside the text surface only through the + * extractor's understanding observations (user-decided 2026-09-13). + * * Observability (§5 baseline): every stage logs one line per step, its * start, and its duration; the pipeline logs a failure line before * rethrowing, so the server log reads start → stage timings → merged @@ -166,6 +179,44 @@ const findingsSchema = z.object({ .describe("Quiz attribution — your only event judgment."), }); +const NOTES_EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE source — +the user's own NOTES, in an ARBITRARY format (personal notes, an exported conversation, a document — anything) — and +report what those notes show the author engaging with, as structured findings for their global knowledge graph. + +There is no session and no chat structure here: the source is raw text, the author's own material. Read it as such. + +Report, with verbatim evidence quotes for everything: + +NODE THEORY — the graph is a set of TRUE propositions about the world, full stop. A node is never anything about the +author or the source. The author's relationship to a proposition is expressed ENTIRELY by your understanding note; it +never leaks into the node itself. So no "the author understands X", no "mentioned in these notes": the node is just X. +A misconception is NOT a node of its own wrong claim — state the TRUE correction as the proposition. + +1. PROPOSITIONS — every distinct reasoning-step-sized proposition the notes actually engage with: + - Each has a "name" — a VERY short label, a few words, a compact handle like a glossary headword ("Will to power", + "Divine command theory"). Not a sentence; it becomes the node's label in the graph, so keep it glanceable. + - And a "statement" — the self-contained TRUE proposition at full precision. + - Notes may explain things at length or quote material the author is studying: extract the propositions themselves, + one reasoning step each — never a topic blob, never a sub-step. + - "messages": ALWAYS an empty list — provenance is the file itself. + - "aliases": verbatim phrasings the notes used for the same proposition. + - "understanding": ONE word from the fixed scale — "solid", "shaky", "accepted", "resisted", or "not yet engaged" — + judging the AUTHOR's grasp of the proposition as the notes show it: their own commentary, self-assessments, + recorded quiz outcomes, the way they write about it. Strictly one word, no numbers, no justification (a + wrong-shaped note is rejected by the schema). When the notes give no signal, omit the field. + +2. EDGES between propositions: + - "follows": to is DERIVABLE from from (understanding from makes understanding to achievable). + - "supports": from GROUNDS to without entailing it (helpful context, not a prerequisite). + - Only relations the notes actually articulate — with evidence. + +There is no quiz attribution for notes — the source has no quiz events, and the schema has no attribution field. + +Evidence discipline: every quote must appear VERBATIM in the source. The auditor rejects anything unbacked. When in +doubt, report; the auditor decides. + +Your entire deliverable is ONE submit_findings call.`; + const EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE session transcript and report what was learned — as structured findings for the learner's global knowledge graph. @@ -210,23 +261,30 @@ transcript). The auditor rejects anything unbacked. When in doubt, report; the a Your entire deliverable is ONE submit_findings call.`; -/** One extractor run: the whole deliverable is ONE zod-validated submit - * call (a pi agent purely so the model can retry a schema-invalid call). - * Returns the submitted findings — captured from the execute seam, which +/** One single-submit agent run — the shared shape of EVERY LLM stage + * here (session + notes): the whole deliverable is ONE zod-validated + * submit call (a pi agent purely so the model can retry a schema-invalid + * call). The submitted value is captured from the execute seam, which * only fires on a schema-valid call. */ -async function runExtractor( - engine: PiEngine | undefined, - prompt: string, - signal: AbortSignal | undefined, - onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void, -): Promise { - let submitted: SessionFindings | undefined; +async function runSubmitAgent(opts: { + role: ModelRole; + toolName: string; + toolDescription: string; + schema: z.ZodType; + systemPrompt: string; + prompt: string; + engine: PiEngine | undefined; + signal: AbortSignal | undefined; + onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void; + neverSubmitted: string; +}): Promise { + let submitted: T | undefined; const result = await runSubagent( toPiTools({ - submit_findings: lhTool({ - name: "submit_findings", - description: "Submit the session findings.", - schema: findingsSchema, + [opts.toolName]: lhTool({ + name: opts.toolName, + description: opts.toolDescription, + schema: opts.schema, execute: async (findings) => { submitted = findings; return findings; @@ -234,28 +292,71 @@ async function runExtractor( }), }), { - role: "extractor", - engine, - systemPrompt: EXTRACTOR_INSTRUCTIONS, - prompt, + role: opts.role, + engine: opts.engine, + systemPrompt: opts.systemPrompt, + prompt: opts.prompt, maxTurns: 4, - abortSignal: signal, + abortSignal: opts.signal, onAssistantMessage: (message, step) => - onStep( + opts.onStep( message.usage, message.model, step, message.stopReason, message.content.filter((c) => c.type === "toolCall").map((c) => (c as { name: string }).name).join(", ") || "(none)", ), - shouldStop: (_message, toolResults) => toolResults.some((r) => r.toolName === "submit_findings" && !r.isError), + shouldStop: (_message, toolResults) => toolResults.some((r) => r.toolName === opts.toolName && !r.isError), }, ); void result; - if (!submitted) throw new Error("the extractor never submitted findings"); + if (!submitted) throw new Error(opts.neverSubmitted); return submitted; } +/** The extractor over ONE session transcript (the chat front door). */ +function runExtractor( + engine: PiEngine | undefined, + prompt: string, + signal: AbortSignal | undefined, + onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void, +): Promise { + return runSubmitAgent({ + role: "extractor", + toolName: "submit_findings", + toolDescription: "Submit the session findings.", + schema: findingsSchema, + systemPrompt: EXTRACTOR_INSTRUCTIONS, + prompt, + engine, + signal, + onStep, + neverSubmitted: "the extractor never submitted findings", + }); +} + +/** The extractor over ONE source of raw personal notes (the format-agnostic + * front door): same single-submit shape, notes-mode prompt + schema. */ +function runNotesExtractor( + engine: PiEngine | undefined, + prompt: string, + signal: AbortSignal | undefined, + onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void, +): Promise { + return runSubmitAgent({ + role: "extractor", + toolName: "submit_findings", + toolDescription: "Submit the notes findings.", + schema: notesFindingsSchema, + systemPrompt: NOTES_EXTRACTOR_INSTRUCTIONS, + prompt, + engine, + signal, + onStep, + neverSubmitted: "the extractor never submitted findings", + }); +} + // ─── the auditor ─────────────────────────────────────────────────────────── const auditSchema = findingsSchema.extend({ @@ -279,11 +380,21 @@ const auditSchema = findingsSchema.extend({ .describe("Everything you dropped, with the reason — the audit trail."), }); -const AUDITOR_INSTRUCTIONS = `You are the audit stage of a learning application's extraction pipeline. You receive -ONE session transcript, a set of proposed findings (propositions, edges, quiz attribution), and the CURRENT -knowledge graph. Your jobs: the faithfulness net, and identity. +// The notes front door's shapes: everything session findings have, minus +// quiz attribution — raw notes have no quiz tool parts, so there is +// nothing to attribute (and no event to build: outcomes inside the text +// surface only through understanding observations, user-decided). +const notesFindingsSchema = findingsSchema.omit({ attribution: true }); +const notesAuditSchema = auditSchema.omit({ attribution: true }); +type NotesFindings = Omit; +type NotesAuditResult = Omit; -FAITHFULNESS — verify every candidate against the transcript: +const AUDITOR_INSTRUCTIONS = `You are the audit stage of a learning application's extraction pipeline. You receive +ONE source — a session transcript, or the author's personal notes in an ARBITRARY format (the notes front door) — a +set of proposed findings (propositions, edges, and quiz attribution where the source has quiz lines), and the +CURRENT knowledge graph. Your jobs: the faithfulness net, and identity. + +FAITHFULNESS — verify every candidate against the source text: - PROPOSITIONS: every evidence quote must appear (near-)verbatim in the transcript view, and the statement must be a faithful reading of it. The node theory applies: a statement must be a TRUE, self-contained, world-facing fact — reword learner- or session-relative statements ("the learner understands X" → "X"), state @@ -293,8 +404,10 @@ FAITHFULNESS — verify every candidate against the transcript: - EDGES: the evidence must actually articulate the relation, in the direction claimed. Drop direction-swapped or speculative edges. - ATTRIBUTION: the QUIZ line's question must genuinely test the attributed proposition. Drop wrong - attributions to null, or re-point them to the proposition the question actually tests. -- UNDERSTANDING levels: verify against the quiz outcomes and the session's evidence; each note must be + attributions to null, or re-point them to the proposition the question actually tests. (Session + sources only — notes findings carry no attribution; there is nothing to verify.) +- UNDERSTANDING levels: verify against the quiz outcomes and the session's evidence — or, for notes, + the author's own words; each note must be exactly one word from the fixed scale ("solid", "shaky", "accepted", "resisted", "not yet engaged") — reword or drop anything else. No justifications. @@ -324,47 +437,47 @@ function graphView(nodes: GraphNode[]): string { return lines.join("\n"); } -/** One auditor run — same single-submit shape as the extractor. */ -async function runAuditor( +/** The auditor over session findings (the chat front door). */ +function runAuditor( engine: PiEngine | undefined, prompt: string, signal: AbortSignal | undefined, onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void, ): Promise { - let submitted: AuditResult | undefined; - const result = await runSubagent( - toPiTools({ - submit_review: lhTool({ - name: "submit_review", - description: "Submit the verified findings subset (with identity verdicts) + the dropped list.", - schema: auditSchema, - execute: async (review) => { - submitted = review; - return review; - }, - }), - }), - { - role: "auditor", - engine, - systemPrompt: AUDITOR_INSTRUCTIONS, - prompt, - maxTurns: 4, - abortSignal: signal, - onAssistantMessage: (message, step) => - onStep( - message.usage, - message.model, - step, - message.stopReason, - message.content.filter((c) => c.type === "toolCall").map((c) => (c as { name: string }).name).join(", ") || "(none)", - ), - shouldStop: (_message, toolResults) => toolResults.some((r) => r.toolName === "submit_review" && !r.isError), - }, - ); - void result; - if (!submitted) throw new Error("the auditor never submitted a review"); - return submitted; + return runSubmitAgent({ + role: "auditor", + toolName: "submit_review", + toolDescription: "Submit the verified findings subset (with identity verdicts) + the dropped list.", + schema: auditSchema, + systemPrompt: AUDITOR_INSTRUCTIONS, + prompt, + engine, + signal, + onStep, + neverSubmitted: "the auditor never submitted a review", + }); +} + +/** The auditor over notes findings (the format-agnostic front door): same + * faithfulness + identity job, notes-mode schema (no attribution). */ +function runNotesAuditor( + engine: PiEngine | undefined, + prompt: string, + signal: AbortSignal | undefined, + onStep: (usage: Usage, model: string, step: number, finishReason: string, tools: string) => void, +): Promise { + return runSubmitAgent({ + role: "auditor", + toolName: "submit_review", + toolDescription: "Submit the verified findings subset (with identity verdicts) + the dropped list.", + schema: notesAuditSchema, + systemPrompt: AUDITOR_INSTRUCTIONS, + prompt, + engine, + signal, + onStep, + neverSubmitted: "the auditor never submitted a review", + }); } // ─── the pipeline status registry ───────────────────────────────────────────── @@ -376,6 +489,13 @@ export type ExtractionPhase = "extracting" | "auditing" | "merging"; * Exported for the status payload's consumer typing (the dock). */ export interface RunningExtraction { chatId: string; + /** Which front door the run came through — label + usage-persistence + * depend on it (chats bill their chat record; notes bill the source + * record). */ + kind: "chat" | "notes"; + /** Display label (the file path for notes imports); the status route + * falls back to the chat store for chat runs. */ + label?: string; phase: ExtractionPhase; startedAt: number; /** Live per-stage usage, accumulated as steps bill (the dock's cost / @@ -388,6 +508,10 @@ export interface RunningExtraction { const runningExtractions = new Map(); +/** Last minted startedAt — kept monotonic so two runs starting in the same + * millisecond never collide on the usage idempotency key. */ +let lastStartedAt = 0; + /** In-flight pipeline runs (oldest first) — the /api/graph/status payload. */ export function extractionStatuses(): RunningExtraction[] { return [...runningExtractions.values()].sort((a, b) => a.startedAt - b.startedAt); @@ -415,22 +539,38 @@ export interface ExtractionModels { export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {} +export interface NotesImportInput { + /** The source's file path — its identity (deterministic src id) AND its + * provenance label in the detail card. */ + filePath: string; + /** The file's raw content, VERBATIM — any format. */ + text: string; +} + export interface ExtractionPipeline { /** Extract + audit + merge ONE chat. Single-flight per chat; resolves to * the merger's append stats (or throws — no checkpoint on failure). */ runExtraction(chatId: string): Promise; + /** The format-agnostic front door (2026-09-13): run the same pipeline + * over ONE raw notes file. The source id is deterministic per file + * path, so re-importing the same file re-runs extraction against the + * same source identity (mappings + events dedupe by it). */ + importNotes(input: NotesImportInput): Promise; } export function createExtractionPipeline(deps: ExtractionPipelineDeps): ExtractionPipeline { const { chatStore, graphStore } = deps; const inFlight = new Set(); - function setPhase(chatId: string, phase: ExtractionPhase): void { + function setPhase(chatId: string, phase: ExtractionPhase, kind: "chat" | "notes", label?: string): void { const prev = runningExtractions.get(chatId); + if (!prev) lastStartedAt = Math.max(Date.now(), lastStartedAt + 1); runningExtractions.set(chatId, { chatId, + kind, + label: prev?.label ?? label, // phase changes never lose the label phase, - startedAt: prev?.startedAt ?? Date.now(), + startedAt: prev?.startedAt ?? lastStartedAt, usage: prev?.usage ?? {}, // phase changes never lose usage controller: prev?.controller ?? new AbortController(), }); @@ -525,7 +665,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti function persistRunUsage(chatId: string, durationMs: number, stats: EmissionStats | undefined): void { const run = runningExtractions.get(chatId); if (!run) return; - chatStore.appendExtractionUsage(chatId, { + const entry: ExtractionRunUsage = { startedAt: run.startedAt, at: new Date().toISOString(), durationMs, @@ -534,7 +674,9 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti extractor: run.usage.extractor ?? { model: "unknown", steps: [] }, auditor: run.usage.auditor ?? { model: "unknown", steps: [] }, }, - }); + }; + if (run.kind === "notes") graphStore.appendSourceUsage(chatId, entry); + else chatStore.appendExtractionUsage(chatId, entry); } async function runExtraction(chatId: string): Promise { @@ -543,14 +685,19 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti const t0 = performance.now(); try { console.info(`[extraction ${chatId}] pipeline start`); - setPhase(chatId, "extracting"); + setPhase(chatId, "extracting", "chat"); const findings = await extract(chatId); - setPhase(chatId, "auditing"); + setPhase(chatId, "auditing", "chat"); const auditResult = await audit(chatId, findings); - setPhase(chatId, "merging"); - // The merger re-reads the chat record itself — events are rebuilt - // from the transcript, never from stale copies. - const stats = mergeFindings(chatId, auditResult, { chatStore, graphStore }); + setPhase(chatId, "merging", "chat"); + // The merger merges from the CURRENT record — events are rebuilt + // from the live transcript, never from stale copies. + const record = chatStore.getChat(chatId); + if (!record) throw new Error(`Chat not found: ${chatId}`); + const stats = mergeFindings(chatId, auditResult, { chatStore, graphStore }, { + messages: record.messages as UIMessage[], + foldedAt: record.updatedAt, + }); persistRunUsage(chatId, Math.round(performance.now() - t0), stats); console.info( `[extraction ${chatId}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` + @@ -577,6 +724,113 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti } } - return { runExtraction }; + // ── the notes front door (format-agnostic: raw text in, same graph out) ── + + /** The raw source view the notes stages see: the file path + the text + * VERBATIM. No message ids — provenance is the file itself. */ + function notesView(input: { filePath: string; text: string }): string { + return `SOURCE (the author's personal notes): ${input.filePath}\n\n${input.text}`; + } + + async function notesExtract(input: { id: string; filePath: string; text: string }): Promise { + const signal = runningExtractions.get(input.id)?.controller.signal; + signal?.throwIfAborted(); // an abort during a sync gap ends the run here + const view = notesView(input); + console.info(`[extraction ${input.id}] notes extractor starting (${view.split("\n").length} source lines)`); + const t0 = performance.now(); + const findings = await runNotesExtractor( + deps.extractor, + view, + signal, + (usage, model, step, finishReason, tools) => { + recordStep(input.id, "extractor", callUsage(usage), model); + console.info(`[extraction] extractor step ${step}: ${finishReason}, tools: ${tools}`); + }, + ); + console.info(`[extraction ${input.id}] notes extractor done in ${Math.round(performance.now() - t0)}ms`); + return findings; + } + + async function notesAudit(input: { id: string; filePath: string; text: string }, findings: NotesFindings): Promise { + const signal = runningExtractions.get(input.id)?.controller.signal; + signal?.throwIfAborted(); // an abort during the extract→audit gap ends here + const graph = getGraph(graphStore); + console.info( + `[extraction ${input.id}] notes auditor starting (${findings.propositions.length} propositions, ` + + `${findings.edges.length} edges, ${graph.nodes.length} graph nodes)`, + ); + const t0 = performance.now(); + const auditResult = await runNotesAuditor( + deps.auditor, + `Source:\n\n${notesView(input)}\n\nProposed findings:\n\n${JSON.stringify(findings, null, 2)}\n\n` + + `Current knowledge graph (${graph.nodes.length} nodes):\n\n${graphView(graph.nodes)}`, + signal, + (usage, model, step, finishReason, tools) => { + recordStep(input.id, "auditor", callUsage(usage), model); + console.info(`[extraction] auditor step ${step}: ${finishReason}, tools: ${tools}`); + }, + ); + if (auditResult.dropped.length > 0) { + console.info( + `[extraction ${input.id}] audit dropped ${auditResult.dropped.length}: ` + + auditResult.dropped.map((d) => `${d.ref} (${d.reason})`).join("; "), + ); + } + console.info( + `[extraction ${input.id}] audit kept ${auditResult.propositions.length}/${findings.propositions.length} propositions, ` + + `${auditResult.propositions.filter((p) => p.linkTo !== undefined).length} linked to existing nodes`, + ); + console.info(`[extraction ${input.id}] notes auditor done in ${Math.round(performance.now() - t0)}ms`); + return auditResult; + } + + async function runNotesExtraction(input: { id: string; filePath: string; text: string; importedAt: string }): Promise { + const { id } = input; + if (inFlight.has(id)) return { nodes: 0, edges: 0, events: 0 }; + inFlight.add(id); + const t0 = performance.now(); + try { + console.info(`[extraction ${id}] notes pipeline start (${input.filePath})`); + setPhase(id, "extracting", "notes", input.filePath); + const findings = await notesExtract(input); + setPhase(id, "auditing", "notes", input.filePath); + const notesResult = await notesAudit(input, findings); + setPhase(id, "merging", "notes", input.filePath); + // Normalize to the merger's findings shape: notes carry no quiz + // attribution, and the merger scans no quiz parts for notes + // (messages: null) — nothing to attribute, nothing to build. + const auditResult: AuditResult = { ...notesResult, attribution: { quiz: {} } }; + const stats = mergeFindings(id, auditResult, { chatStore, graphStore }, { messages: null, foldedAt: input.importedAt }); + persistRunUsage(id, Math.round(performance.now() - t0), stats); + console.info( + `[extraction ${id}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` + + `(${Math.round(performance.now() - t0)}ms total)`, + ); + return stats; + } catch (err) { + persistRunUsage(id, Math.round(performance.now() - t0), undefined); + if (err instanceof Error && err.name === "AbortError") { + console.info(`[extraction ${id}] aborted (${Math.round(performance.now() - t0)}ms) — nothing merged, nothing checkpointed`); + return { nodes: 0, edges: 0, events: 0 }; + } + console.error(`[extraction ${id}] notes pipeline failed (${Math.round(performance.now() - t0)}ms):`, err); + throw err; + } finally { + inFlight.delete(id); + runningExtractions.delete(id); // done or failed — no longer running + } + } + + /** Import ONE raw notes file: register the source (deterministic id per + * file path — re-import keeps the same identity and its usage history) + * and run the pipeline over it. */ + async function importNotes(input: NotesImportInput): Promise { + const id = `src_${crypto.createHash("sha256").update(input.filePath).digest("hex").slice(0, 12)}`; + const importedAt = new Date().toISOString(); + graphStore.upsertSource({ id, filePath: input.filePath, importedAt }); + return runNotesExtraction({ id, ...input, importedAt }); + } + + return { runExtraction, importNotes }; } diff --git a/src/core/contracts/graph.ts b/src/core/contracts/graph.ts index 24ea899..2941541 100644 --- a/src/core/contracts/graph.ts +++ b/src/core/contracts/graph.ts @@ -20,6 +20,8 @@ /** Two relations only (DECIDED 2026-09-02): follows = b is derivable from a; * supports = a grounds b without entailing it. No scalars, no encoded * strength — strength is derived at read time from topology. */ +import type { ExtractionRunUsage } from "./chat.ts"; + export type EdgeType = "follows" | "supports"; /** What one extraction run merged into the graph ledgers — the merger's @@ -37,10 +39,32 @@ export interface EmissionStats { * edges record every (chat, message) where they came up, deep-linkable to * the exact transcript point. */ export interface ProvenancePoint { + /** The chat this point anchors to — or a notes source id ("src_…"): + * imports carry no message anchors (provenance is the file itself, + * recorded through the mapping line + the source record). */ chatId: string; messageId: string; } +/** One imported notes file — the format-agnostic front door's bookkeeping + * (DECIDED 2026-09-13): personal notes in ARBITRARY formats run through the + * same extractor → auditor → merger pipeline, with the file itself as the + * provenance anchor. There is no chat: the detail card shows the file path + * as a STATIC entry (not a link), and understanding observations still + * apply (judged from the author's own voice in the notes). The run's usage + * log lives here, mirroring ChatRecord.extractions. Ids are "src_" + hash + * of the file path — deterministic, so re-importing the same file re-runs + * extraction against the same source id (mappings/events dedupe by it). */ +export interface NotesSource { + id: string; + filePath: string; + /** ISO — when this file was imported (the checkpoint's foldedAt). */ + importedAt: string; + /** The run's usage log — same append-only discipline as + * ChatRecord.extractions, idempotent per startedAt. */ + extractions?: ExtractionRunUsage[]; +} + export interface GraphNode { /** Opaque stable id, store-assigned: "n_" + nanoid-like. */ id: string; @@ -107,6 +131,8 @@ export interface LedgerEvent { /** null = not yet attributed. */ nodeId: string | null; kind: LedgerEventKind; + /** The chat that produced this evidence — or a notes source id + * ("src_…") for imported-notes observations. */ chatId: string; at: string; /** Idempotency key: the quiz part's toolCallId, or `obs:` for @@ -249,6 +275,10 @@ export interface GraphSnapshot { edges: GraphEdge[]; /** chatId → title, for every chat that touched the graph. */ sessions: Record; + /** srcId → file path, for every IMPORTED NOTES file that touched the + * graph (the format-agnostic front door): a static provenance entry — + * the detail card shows the path as plain text, not a link. */ + fileSources: Record; /** nodeId → chatIds that touched the node (click-through provenance). */ nodeSessions: Record; /** nodeId → chatId → that session's understanding observation (the diff --git a/src/core/graph.ts b/src/core/graph.ts index c24bd3a..dee329b 100644 --- a/src/core/graph.ts +++ b/src/core/graph.ts @@ -64,6 +64,17 @@ function newGraphNodeId(): string { return `n_${randomBytes(6).toString("hex")}`; } +/** What one merge run merges FROM (the caller supplies it — the merger never + * reads the chat store): a session's transcript + its `updatedAt` + * checkpoint, or — for the format-agnostic notes front door (2026-09-13) — + * `messages: null`: raw notes carry no tool parts, so no quiz events are + * rebuilt (quiz outcomes inside notes surface only through the extractor's + * understanding observations), and the checkpoint stamps the import. */ +export interface MergeSource { + messages: UIMessage[] | null; + foldedAt: string; +} + // ─── transcript scanning (shared by the merger + trigger) ───────────────────── /** A flattened, order-preserving view of every tool part in a transcript. */ @@ -126,11 +137,15 @@ const samePoint = (a: ProvenancePoint, b: ProvenancePoint): boolean => * parts, nodeId from the extractor's attribution through this run's * mapping) plus one observation event per proposition carrying an * understanding note (ref `obs:`). No taught events, ever. + * + * `chatId` is the run's source key: a real chat, or a notes source id + * ("src_…") for the format-agnostic import path — same ledgers, same + * idempotency, one graph. The `source` parameter carries the transcript + * being merged (null messages = raw notes: no quiz events) and the + * checkpoint stamp; the merger itself is chat-store-free. */ -export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDeps): EmissionStats { - const { graphStore: ledgers, chatStore } = deps; - const record = chatStore.getChat(chatId); - if (!record) return { nodes: 0, edges: 0, events: 0 }; +export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDeps, source: MergeSource): EmissionStats { + const { graphStore: ledgers } = deps; const stats: EmissionStats = { nodes: 0, edges: 0, events: 0 }; // 1. Resolution: this run's propositions → graph node ids. Existing @@ -214,7 +229,7 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep for (const event of ledgers.readEvents()) { if (event.chatId === chatId && event.ref) seenRefs.add(event.ref); } - for (const part of toolParts(record.messages)) { + for (const part of source.messages ? toolParts(source.messages) : []) { if (part.name !== "quiz" || part.state !== "output-available") continue; const event = quizEvent(part, chatId, audit, mapping, now); if (event?.ref !== undefined && !seenRefs.has(event.ref)) { @@ -241,8 +256,9 @@ export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDep stats.events++; } - // 4. Checkpoint: this chat is extracted through its current updatedAt. - ledgers.appendCheckpoint({ chatId, foldedAt: record.updatedAt }); + // 4. Checkpoint: this source is extracted through its foldedAt (the + // chat's updatedAt, or the import time for notes). + ledgers.appendCheckpoint({ chatId, foldedAt: source.foldedAt }); return stats; } @@ -423,20 +439,28 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot { if (event.nodeId !== null) touch(event.nodeId, event.chatId); } - // Titles for every chat the graph can point back at. - const chatIds = new Set(); + // Labels for every source the graph can point back at: real chats + // (title, deep-linkable) and imported notes files (path, static — the + // format-agnostic front door's provenance). + const sourceIds = new Set(); for (const chats of nodeChats.values()) { - for (const id of chats) chatIds.add(id); + for (const id of chats) sourceIds.add(id); } const sessions: Record = {}; - for (const id of chatIds) { + const fileSources: Record = {}; + for (const id of sourceIds) { const chat = deps.chatStore.getChat(id); - if (chat) sessions[id] = chat.title; + if (chat) { + sessions[id] = chat.title; + continue; + } + const source = deps.graphStore.readSources().find((s) => s.id === id); + if (source) fileSources[id] = source.filePath; } const nodeSessions: Record = {}; for (const [nodeId, chats] of nodeChats) { - const ids = [...chats].filter((id) => sessions[id] !== undefined); + const ids = [...chats].filter((id) => sessions[id] !== undefined || fileSources[id] !== undefined); if (ids.length > 0) nodeSessions[nodeId] = ids; } @@ -461,7 +485,7 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot { (nodeObservations[nodeId] ??= {})[chatId] = note; } - return { nodes, edges, sessions, nodeSessions, nodeObservations }; + return { nodes, edges, sessions, fileSources, nodeSessions, nodeObservations }; } // Re-export: the contract owns the shape; legacy importers still reach it. diff --git a/src/core/ports.ts b/src/core/ports.ts index 6343a2e..e1e6219 100644 --- a/src/core/ports.ts +++ b/src/core/ports.ts @@ -24,6 +24,7 @@ import type { PlanTree, SessionCheckpoint, TurnUsage, + NotesSource, } from "./contracts.ts"; // ─── ChatStore — the session record's home (data/chats) ───────────────────── @@ -80,6 +81,15 @@ export interface GraphStore { /** Append one emission checkpoint. */ appendCheckpoint(checkpoint: SessionCheckpoint): void; readCheckpoints(): SessionCheckpoint[]; + + /** Upsert one imported-notes source record (config.SOURCES_FILE — a + * JSON map, not a ledger; an upsert keeps the existing usage log). */ + upsertSource(source: NotesSource): void; + /** Every notes source record, by id order-insensitive. */ + readSources(): NotesSource[]; + /** Append one run's usage to a source record — idempotent per + * startedAt, mirroring ChatRecord.extractions. */ + appendSourceUsage(id: string, entry: ExtractionRunUsage): void; } // ─── JobStore — background sub-agent jobs (data/jobs) ───────────────────────── diff --git a/src/server/chat-handler.ts b/src/server/chat-handler.ts index a4a3436..589516f 100644 --- a/src/server/chat-handler.ts +++ b/src/server/chat-handler.ts @@ -37,7 +37,10 @@ const jobRunner = createJobRunner({ jobStore: stores.jobStore, deliver: deliverJobResult, }); -const extraction = createExtractionPipeline(stores); +/** The app's ONE extraction pipeline instance — both front doors share it: + * quiz-triggered session runs (turn-service) and notes imports + * (/api/graph/import) hit the same single-flight registry. */ +export const extraction = createExtractionPipeline(stores); // Wake-pipeline wiring (contracts.ts "THE ASYNC PROTOCOL" §4): the pipeline // fires its server-side teacher turns through THIS composition root, and diff --git a/src/store/graph.ts b/src/store/graph.ts index ac86c45..b67c884 100644 --- a/src/store/graph.ts +++ b/src/store/graph.ts @@ -15,6 +15,10 @@ * (chatId, sourceId) → graphNodeId * sessions.jsonl SessionCheckpoint— emission checkpoints (foldedAt) * + * Plus config.SOURCES_FILE (sources.json): the imported-notes bookkeeping + * (NOT a ledger) — one JSON map, src-id → NotesSource, read-modify-write + * like a chat file. The format-agnostic front door (2026-09-13). + * * Implements core/ports.ts GraphStore. data/graph/* is exclusively this * adapter's. Reads are strict against the contracts.ts shapes (the 2026-09-02 * graph redesign): corrupt lines are skipped, never guessed at — the ledgers @@ -25,12 +29,14 @@ import * as fs from "node:fs"; import * as path from "node:path"; -import { LEDGER_FILES } from "../../config.ts"; +import { LEDGER_FILES, SOURCES_FILE } from "../../config.ts"; import type { GraphEdge, GraphNode, + ExtractionRunUsage, LedgerEvent, LedgerEventKind, + NotesSource, NodeMapping, ProvenancePoint, SessionCheckpoint, @@ -157,4 +163,57 @@ export const graphStore: GraphStore = { readCheckpoints(): SessionCheckpoint[] { return readValid(LEDGER_FILES.sessions, isCheckpoint); }, + + // ── imported-notes bookkeeping (config.SOURCES_FILE — a JSON map) ── + + /** Upsert: keeps the existing record's usage log when re-importing. */ + upsertSource(source: NotesSource): void { + const all = readSourceMap(); + const prev = all[source.id]; + all[source.id] = prev ? { ...source, extractions: prev.extractions } : source; + writeSourceMap(all); + }, + readSources(): NotesSource[] { + return Object.values(readSourceMap()); + }, + /** Idempotent per startedAt — same discipline as chats.appendExtractionUsage. */ + appendSourceUsage(id: string, entry: ExtractionRunUsage): void { + const all = readSourceMap(); + const source = all[id]; + if (!source) return; // no such source — nothing to bill against + source.extractions = source.extractions?.some((r) => r.startedAt === entry.startedAt) + ? source.extractions + : [...(source.extractions ?? []), entry]; + writeSourceMap(all); + }, }; + +/** sources.json — { id: NotesSource }. Read-modify-write, NOT append-only: + * the usage log lives inside the record (like a chat's JSON file). */ +function readSourceMap(): Record { + let raw: string; + try { + raw = fs.readFileSync(SOURCES_FILE, "utf-8"); + } catch { + return {}; + } + try { + const parsed: unknown = JSON.parse(raw); + if (!isRecord(parsed)) return {}; + const out: Record = {}; + for (const [id, value] of Object.entries(parsed)) { + if (isNotesSource(value)) out[id] = value; + } + return out; + } catch { + return {}; + } +} + +function writeSourceMap(map: Record): void { + fs.mkdirSync(path.dirname(SOURCES_FILE), { recursive: true }); + fs.writeFileSync(SOURCES_FILE, JSON.stringify(map, null, "\t"), "utf-8"); +} + +const isNotesSource = (v: unknown): v is NotesSource => + isRecord(v) && isString(v.id) && isString(v.filePath) && isString(v.importedAt); diff --git a/tests/extraction.test.ts b/tests/extraction.test.ts index 3b58964..18a0488 100644 --- a/tests/extraction.test.ts +++ b/tests/extraction.test.ts @@ -27,10 +27,12 @@ import type { UIMessage } from "ai"; import type { PiEngine } from "../src/agent/models.ts"; import type { AuditResult, + ExtractionRunUsage, GraphEdge, GraphNode, LedgerEvent, NodeMapping, + NotesSource, SessionCheckpoint, SessionFindings, } from "../src/core/contracts.ts"; @@ -60,6 +62,7 @@ function memoryLedgers() { const events: LedgerEvent[] = []; const mappings: NodeMapping[] = []; const checkpoints: SessionCheckpoint[] = []; + const sources = new Map(); return { appendNode: (n: GraphNode) => void nodes.push(n), readNodes: () => nodes.map((n) => ({ ...n })), @@ -71,6 +74,19 @@ function memoryLedgers() { readMappings: () => mappings.map((m) => ({ ...m })), appendCheckpoint: (c: SessionCheckpoint) => void checkpoints.push(c), readCheckpoints: () => checkpoints.map((c) => ({ ...c })), + upsertSource: (s: NotesSource) => { + // Mirrors the real store: an upsert KEEPS the existing usage log. + const prev = sources.get(s.id); + sources.set(s.id, { ...s, extractions: [...(prev?.extractions ?? [])] }); + }, + readSources: () => [...sources.values()].map((s) => ({ ...s, extractions: [...(s.extractions ?? [])] })), + appendSourceUsage: (id: string, entry: ExtractionRunUsage) => { + const source = sources.get(id); + if (!source) return; + if (!source.extractions?.some((r) => r.startedAt === entry.startedAt)) { + source.extractions = [...(source.extractions ?? []), entry]; + } + }, } satisfies GraphStore; } @@ -513,3 +529,123 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { await assert.rejects(pipeline.runExtraction("chat_does_not_exist"), /Chat not found/); }); }); + +// ── the notes front door (format-agnostic import) ──────────────────────────── + +/** Raw personal notes — arbitrary format, no session, no message ids. */ +const NOTES_TEXT = `> [!quote] YOU +> teach me vim + +> [!abstract] PI +> Vim always starts in Normal mode — keys are commands, not text.`; + +/** What the notes-mode extractor submits: same shapes minus attribution, + * and messages always empty (the file is the provenance). */ +const NOTES_FINDINGS = { + propositions: [ + { + id: "p1", + name: "Vim normal mode", + statement: "Vim starts in Normal mode, where keys are commands.", + aliases: [], + messages: [], + evidence: ["Vim always starts in Normal mode — keys are commands, not text."], + understanding: "solid", + }, + ], + edges: [], +} as const; + +const NOTES_VERDICT = { + propositions: [NOTES_FINDINGS.propositions[0]], + edges: [], + dropped: [], +} as const; + +describe("the notes front door (importNotes — format-agnostic)", () => { + it("imports raw notes end to end: source record, file-scoped provenance, no quiz events", async () => { + const ledgers = memoryLedgers(); + const seen: { kind?: string; label?: string }[] = []; + const pipeline = createExtractionPipeline({ + chatStore, + graphStore: ledgers, + extractor: submitModel("submit_findings", NOTES_FINDINGS), + // During the auditor's call the registry must show the notes run. + auditor: submitModel("submit_review", NOTES_VERDICT, () => { + const run = extractionStatuses().find((r) => r.chatId.startsWith("src_")); + seen.push({ kind: run?.kind, label: run?.label }); + }), + }); + const stats = await pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT }); + + assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 }); + // The run surfaced as a NOTES run labeled with the file path. + assert.deepEqual(seen, [{ kind: "notes", label: "/vault/learn vim.md" }]); + assert.equal(extractionStatuses().length, 0, "the registry clears when the run ends"); + + // The source record: deterministic src id, the file path, the usage log. + const source = ledgers.readSources()[0]!; + assert.match(source.id, /^src_/); + assert.equal(source.filePath, "/vault/learn vim.md"); + assert.ok(Date.parse(source.importedAt)); + assert.equal(source.extractions?.length, 1, "the run's usage persisted into the source record"); + assert.equal(source.extractions![0]!.stats?.nodes, 1); + assert.ok(source.extractions![0]!.stages.extractor.steps.length > 0, "faux usage captured"); + + // File-scoped provenance: the mapping line carries the source id; + // the node carries NO message anchors. + const node = ledgers.readNodes()[0]!; + assert.equal(node.statement, "Vim starts in Normal mode, where keys are commands."); + assert.deepEqual(node.provenance, []); + assert.deepEqual(ledgers.readMappings(), [{ chatId: source.id, sourceId: "p1", graphNodeId: node.id }]); + + // One observation event, zero quiz events — outcomes inside notes + // surface only through understanding observations (user-decided). + const events = ledgers.readEvents(); + assert.deepEqual( + events.map((e) => [e.kind, e.chatId, e.ref, e.note]), + [["observation", source.id, "obs:p1", "solid"]], + ); + // The checkpoint stamps the import. + assert.equal(ledgers.readCheckpoints()[0]!.chatId, source.id); + }); + + it("re-importing the same file is idempotent: same source identity, refs dedupe, usage history grows", async () => { + const ledgers = memoryLedgers(); + const mk = () => + createExtractionPipeline({ + chatStore, + graphStore: ledgers, + extractor: submitModel("submit_findings", NOTES_FINDINGS), + auditor: submitModel("submit_review", NOTES_VERDICT), + }); + const first = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT }); + assert.deepEqual(first, { nodes: 1, edges: 0, events: 1 }); + + const second = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT }); + assert.deepEqual(second, { nodes: 0, edges: 0, events: 0 }, "the same source id dedupes everything"); + + assert.equal(ledgers.readSources().length, 1, "one source identity — the path hashes to the same id"); + assert.equal(ledgers.readSources()[0]!.extractions?.length, 2, "both runs billed their usage"); + assert.equal(ledgers.readNodes().length, 1, "no duplicate node"); + }); + + it("aborts like a chat run: true cancellation, nothing merged, partial usage in the source record", async () => { + const ledgers = memoryLedgers(); + const pipeline = createExtractionPipeline({ + chatStore, + graphStore: ledgers, + extractor: hangingModel(), + auditor: submitModel("submit_review", NOTES_VERDICT), + }); + const runPromise = pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT }); + + const srcId = extractionStatuses().find((r) => r.chatId.startsWith("src_"))!.chatId; + assert.equal(abortExtraction(srcId), true); + assert.deepEqual(await runPromise, { nodes: 0, edges: 0, events: 0 }); + assert.equal(ledgers.readNodes().length, 0, "nothing merged"); + assert.equal(ledgers.readCheckpoints().length, 0, "nothing checkpointed"); + assert.equal(ledgers.readSources()[0]!.extractions?.length, 1, "partial usage still persisted"); + assert.equal(ledgers.readSources()[0]!.extractions![0]!.stats, undefined, "never merged → no stats"); + }); +}); diff --git a/tests/graph.test.ts b/tests/graph.test.ts index 2a104d1..9d27d11 100644 --- a/tests/graph.test.ts +++ b/tests/graph.test.ts @@ -23,14 +23,22 @@ import type { QuestionAnswerData } from "../src/core/question-core.ts"; import { getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts"; import { graphStore } from "../src/store/graph.ts"; import { chatStore } from "../src/store/chats.ts"; -import { LEDGER_FILES, PATHS } from "../config.ts"; +import { LEDGER_FILES, PATHS, SOURCES_FILE } from "../config.ts"; /** The wired fs adapters, exactly what src/server/stores.ts passes in production. */ const graphDeps = { chatStore, graphStore } as const satisfies GraphDeps; +/** The session merge source for a chat — what the pipeline passes: the live + * transcript + its updatedAt checkpoint stamp. (Notes imports pass + * messages: null — see the notes-merger test below.) */ +function sessionSource(chatId: string): { messages: UIMessage[]; foldedAt: string } { + const record = chatStore.getChat(chatId)!; + return { messages: record.messages as UIMessage[], foldedAt: record.updatedAt }; +} + // ─── ledger isolation (backup-and-clear of data/graph/*.jsonl) ────────────── -const LEDGER_PATHS = Object.values(LEDGER_FILES); +const LEDGER_PATHS = [...Object.values(LEDGER_FILES), SOURCES_FILE]; const backups: Array<{ file: string; bak: string }> = []; const createdChats: string[] = []; @@ -233,7 +241,7 @@ function scopedGraph(chatIds: string[]) { describe("the merger (mergeFindings)", () => { it("creates nodes from the verdicts, rebuilds quiz events deterministically, records observations", () => { - const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps); + const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id)); assert.deepEqual(stats, { nodes: 3, edges: 1, events: 6 }); const graph = scopedGraph([chat1.id]); @@ -302,7 +310,7 @@ describe("the merger (mergeFindings)", () => { }); it("is idempotent — re-merging the same verdicts appends nothing", () => { - const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps); + const stats = mergeFindings(chat1.id, fixtureAudit(), graphDeps, sessionSource(chat1.id)); assert.deepEqual(stats, { nodes: 0, edges: 0, events: 0 }); const graph = scopedGraph([chat1.id]); assert.equal(graph.nodes.length, 3); @@ -346,6 +354,7 @@ describe("the merger (mergeFindings)", () => { dropped: [], }, graphDeps, + sessionSource(chat2.id), ); assert.deepEqual(stats, { nodes: 0, edges: 0, events: 1 }); @@ -363,6 +372,41 @@ describe("the merger (mergeFindings)", () => { assert.deepEqual(m, { chatId: chat2.id, sourceId: "q1", graphNodeId: alphaId }); }); + it("resolves imported-notes ids to their file paths in the snapshot (static provenance)", () => { + // The notes front door in miniature: a source record + a merge under + // its src id — the snapshot must label it with the file path (a + // static entry), never as a chat session. + graphStore.upsertSource({ id: "src_snapfixture", filePath: "/vault/learn vim.md", importedAt: "2026-09-13T00:00:00.000Z" }); + mergeFindings( + "src_snapfixture", + { + propositions: [ + { + id: "p1", + name: "Vim registers", + statement: "Vim registers map single keys to editing commands.", + aliases: [], + messages: [], + evidence: ["fixture evidence"], + understanding: "solid", + }, + ], + edges: [], + attribution: { quiz: {} }, + dropped: [], + }, + graphDeps, + { messages: null, foldedAt: "2026-09-13T00:00:00.000Z" }, + ); + const snap = graphSnapshot(graphDeps); + const node = snap.nodes.find((n) => n.statement === "Vim registers map single keys to editing commands.")!; + assert.equal(snap.fileSources["src_snapfixture"], "/vault/learn vim.md"); + assert.deepEqual(snap.nodeSessions[node.id], ["src_snapfixture"], "the file entry joins the node's sessions list"); + assert.equal(snap.sessions["src_snapfixture"], undefined, "a notes source is never a chat session"); + // The understanding observation joins under the source id like any session's. + assert.equal(snap.nodeObservations[node.id]!["src_snapfixture"], "solid"); + }); + it("derives sessions and per-session understanding observations in graphSnapshot", () => { const snap = graphSnapshot(graphDeps); const idByStatement = new Map(snap.nodes.map((n) => [n.statement, n.id])); @@ -408,17 +452,54 @@ describe("the merger (mergeFindings)", () => { dropped: [], }, graphDeps, + sessionSource(chat3.id), ); assert.equal(stats.nodes, 1, "created — the merger does not match strings"); assert.equal(getGraph(graphStore).nodes.filter((n) => n.statement === "Alpha intuition").length, 2); }); - it("returns zeros for unknown chats", () => { - assert.deepEqual(mergeFindings("chat_does_not_exist", fixtureAudit(), graphDeps), { - nodes: 0, - edges: 0, - events: 0, + it("merges the notes front door: source id as chatId, no quiz events, checkpoint at import time", () => { + // Raw personal notes carry no transcript — the merger gets no + // message parts, so NO quiz events are rebuilt; the understanding + // observation still lands, keyed to the source id. + const stats = mergeFindings( + "src_notesfixture", + { + propositions: [ + { + id: "p1", + name: "Vim normal mode", + statement: "Vim starts in Normal mode, where keys are commands.", + aliases: [], + messages: [], // notes carry no message anchors — the file is the provenance + evidence: ["fixture evidence"], + understanding: "solid", + }, + ], + edges: [], + attribution: { quiz: {} }, // normalized by the notes pipeline — nothing to attribute + dropped: [], + }, + graphDeps, + { messages: null, foldedAt: "2026-09-13T00:00:00.000Z" }, + ); + assert.deepEqual(stats, { nodes: 1, edges: 0, events: 1 }); + + const node = getGraph(graphStore).nodes.find((n) => n.statement.startsWith("Vim starts"))!; + assert.deepEqual(node.provenance, [], "no message anchors — the mapping line carries the source"); + assert.deepEqual(graphStore.readMappings().find((m) => m.chatId === "src_notesfixture"), { + chatId: "src_notesfixture", + sourceId: "p1", + graphNodeId: node.id, }); + // The observation event is file-scoped; no quiz events exist. + const events = graphStore.readEvents().filter((e) => e.chatId === "src_notesfixture"); + assert.deepEqual( + events.map((e) => [e.kind, e.ref, e.note]), + [["observation", "obs:p1", "solid"]], + ); + // The checkpoint stamps the import time. + assert.equal(graphStore.readCheckpoints().find((c) => c.chatId === "src_notesfixture")?.foldedAt, "2026-09-13T00:00:00.000Z"); }); }); @@ -452,7 +533,7 @@ describe("the trigger helper (pendingQuizAnswers)", () => { assert.equal(pendingQuizAnswers(chat.id, graphDeps), 1, "one answered quiz pending"); assert.equal(pendingQuizAnswers("chat_does_not_exist", graphDeps), 0); - mergeFindings(chat.id, fixtureAudit(), graphDeps); + mergeFindings(chat.id, fixtureAudit(), graphDeps, sessionSource(chat.id)); // call-s is unattributed here (fixtureAudit has no call-s attribution) // — but it still becomes an event, so nothing stays pending. assert.equal(pendingQuizAnswers(chat.id, graphDeps), 0, "extracted → nothing pending"); diff --git a/tests/usage.test.ts b/tests/usage.test.ts index 1a2651d..11a5391 100644 --- a/tests/usage.test.ts +++ b/tests/usage.test.ts @@ -10,7 +10,8 @@ import test from "node:test"; import * as fs from "node:fs"; import { chatStore } from "../src/store/chats.ts"; -import { PATHS } from "../config.ts"; +import { graphStore } from "../src/store/graph.ts"; +import { PATHS, SOURCES_FILE } from "../config.ts"; import { callCost, contextWindow, jobCost, sessionCost, sumSteps, turnCost } from "../src/core/prices.ts"; import type { CallUsage, ExtractionRunUsage, JobRecord, TurnUsage } from "../src/core/contracts.ts"; @@ -170,3 +171,40 @@ test("appendExtractionUsage: appends, idempotent per startedAt, not in list payl fs.unlinkSync(`${PATHS.chats}/${record.id}.json`); }); + +test("appendSourceUsage: appends to the source record, idempotent per startedAt, upsert keeps history", () => { + // The real fs graphStore — sources.json backed up and restored around + // this test (same discipline as the chat fixture files). + fs.mkdirSync(PATHS.graph, { recursive: true }); + let bak: string | undefined; + if (fs.existsSync(SOURCES_FILE)) bak = `${SOURCES_FILE}.w5test.bak`; + if (bak) fs.renameSync(SOURCES_FILE, bak); + try { + const runEntry = (startedAt: number): ExtractionRunUsage => ({ + startedAt, + at: "2026-09-13T00:00:00Z", + durationMs: 42_000, + stats: { nodes: 1, edges: 2, events: 3 }, + stages: { + extractor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] }, + auditor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] }, + }, + }); + const id = "src_usagetest01"; + graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-13T00:00:00Z" }); + graphStore.appendSourceUsage(id, runEntry(1)); + graphStore.appendSourceUsage(id, runEntry(1)); // same run appended twice → no-op + graphStore.appendSourceUsage(id, runEntry(2)); // a retried run is a NEW spend + assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2); + + // An upsert (re-import) KEEPS the usage history. + graphStore.upsertSource({ id, filePath: "/vault/usagetest.md", importedAt: "2026-09-14T00:00:00Z" }); + assert.equal(graphStore.readSources().find((s) => s.id === id)?.extractions?.length, 2); + + // Unknown ids are a silent no-op (nothing to bill against). + graphStore.appendSourceUsage("src_nope", runEntry(1)); + } finally { + fs.rmSync(SOURCES_FILE, { force: true }); + if (bak) fs.renameSync(bak, SOURCES_FILE); + } +}); From c3cc55c5858e6fcca30fc5dd9693ce81712d9491 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 02:32:20 +0300 Subject: [PATCH 03/34] Extraction tool calls: tolerate omitted-empty fields (weak-model trap) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diagnosis from the real 25-file notes import (user-pasted session logs): almost every run hit 'Validation failed for tool submit_findings/review: edges must have required properties edges' — the model (glm flash, low thinking) omits top-level fields when they're empty, and pi-ai's structural JSON-Schema validation runs BEFORE the zod layer, so each omission cost one of the extractor's maxTurns 4 (full payload regeneration) and occasionally burned out the run entirely (the two FAILED imports). The model is technically wrong, but the schema provokes it: required-but-usually-empty arrays are a known weak-model anti-pattern. - lhTool's model-facing declaration now converts with io: "input" (the declaration describes what the MODEL sends): .default() fields drop out of required. No-op for teacher tools (none use defaults). - findingsSchema.edges / auditSchema.dropped: .default([]) — omission means "none"; attribution: .default({quiz:{}}) for quiz-free sessions. - Prompts hardened in all three agents: include every top-level field, never omit because empty. Structural probe: validateToolArguments now passes the exact pasted failure shape; zod re-parse in execute applies the defaults. 355/355 tests, typecheck, lint, build green. (lh-tool.ts is the shared engine seam — flagged to master: io:"input" is the correct mode for every model-facing declaration, teacher path verified unaffected.) --- src/agent/lh-tool.ts | 10 ++++++++-- src/agent/subagents/extraction.ts | 21 +++++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/src/agent/lh-tool.ts b/src/agent/lh-tool.ts index 3a2a14d..24a7b45 100644 --- a/src/agent/lh-tool.ts +++ b/src/agent/lh-tool.ts @@ -66,7 +66,10 @@ export function toPiTool(def: LhTool): AgentTool { name: def.name, label: def.name, description: def.description, - parameters: z.toJSONSchema(def.schema as z.ZodType) as never, + // io: "input" — the declaration describes what the MODEL must send, + // so .default() fields are optional in the schema (omission = the + // default); required-empty arrays are a known weak-model trap. + parameters: z.toJSONSchema(def.schema as z.ZodType, { io: "input" }) as never, // Question tools opt into SEQUENTIAL execution: pi runs the whole // batch in call order when any called tool opts in, so the // tool-input-available chunks arrive in the model's call order — the @@ -110,6 +113,9 @@ export function piToolDeclarations(registry: Record>) { name, label: name, description: def.description, - parameters: z.toJSONSchema(def.schema as z.ZodType) as never, + // io: "input" — the declaration describes what the MODEL must send, + // so .default() fields are optional in the schema (omission = the + // default); required-empty arrays are a known weak-model trap. + parameters: z.toJSONSchema(def.schema as z.ZodType, { io: "input" }) as never, })); } \ No newline at end of file diff --git a/src/agent/subagents/extraction.ts b/src/agent/subagents/extraction.ts index 2f2b608..14f0da3 100644 --- a/src/agent/subagents/extraction.ts +++ b/src/agent/subagents/extraction.ts @@ -167,7 +167,9 @@ const findingsSchema = z.object({ .describe("Every distinct proposition the session touched — true statements, misconception-free."), edges: z .array(edgeSchema) - .describe("Relations between propositions: follows (derivable) / supports (grounds without entailing)."), + .describe("Relations between propositions: follows (derivable) / supports (grounds without entailing). Pass [] when there are none — never omit the field.") + .default([]), // weak models omit required-but-empty arrays; omission means "none" + attribution: z .object({ quiz: z @@ -176,7 +178,8 @@ const findingsSchema = z.object({ "toolCallId of EVERY quiz line → the proposition id it was really testing. Attribute RETROACTIVELY — the whole transcript is visible; map each quiz to the proposition it tests (often articulated elsewhere in the session). null when genuinely unattributable.", ), }) - .describe("Quiz attribution — your only event judgment."), + .describe("Quiz attribution — your only event judgment. Pass { quiz: {} } when the session has no quiz lines — never omit the field.") + .default({ quiz: {} }), // weak models omit empty objects too }); const NOTES_EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE source — @@ -215,6 +218,9 @@ There is no quiz attribution for notes — the source has no quiz events, and th Evidence discipline: every quote must appear VERBATIM in the source. The auditor rejects anything unbacked. When in doubt, report; the auditor decides. +Include EVERY top-level field of the call even when empty — "edges": [] when there are no relations. NEVER omit +a field because it is empty. + Your entire deliverable is ONE submit_findings call.`; const EXTRACTOR_INSTRUCTIONS = `You are the extraction specialist in a learning application. You read ONE session @@ -259,6 +265,9 @@ quiz outcomes carry that the learner believed the wrong version. Evidence discipline: every quote must appear VERBATIM in the transcript view (a QUIZ line counts as the transcript). The auditor rejects anything unbacked. When in doubt, report; the auditor decides. +Include EVERY top-level field of the call even when empty — "edges": [], "attribution": { quiz: {} } when there +are no quizzes. NEVER omit a field because it is empty. + Your entire deliverable is ONE submit_findings call.`; /** One single-submit agent run — the shared shape of EVERY LLM stage @@ -377,7 +386,8 @@ const auditSchema = findingsSchema.extend({ reason: z.string(), }), ) - .describe("Everything you dropped, with the reason — the audit trail."), + .describe("Everything you dropped, with the reason — the audit trail. Pass [] when you dropped nothing — never omit the field.") + .default([]), // same weak-model tolerance as findings.edges }); // The notes front door's shapes: everything session findings have, minus @@ -424,7 +434,10 @@ Also drop propositions not referenced by anything else ONLY if they have no othe well-evidenced proposition is still knowledge. Never invent new candidates. Your entire deliverable is ONE submit_review call containing the verified findings -(the same shapes you received, with linkTo added) plus the dropped list.`; +(the same shapes you received, with linkTo added) plus the dropped list. + +Include EVERY top-level field of the call even when empty — "edges": [], "dropped": []. NEVER omit a field +because it is empty.`; /** The current graph, compact — the auditor's identity context. */ function graphView(nodes: GraphNode[]): string { From d6b229865a38faf2a2bc0e0eee3157b02f1c06c7 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 02:54:36 +0300 Subject: [PATCH 04/34] Graph view: rendering settings panel (node size, edge width, arrowhead) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three sliders behind a gear in the search panel (user-decided 2026-09-13), persisted per browser in localStorage with clamped sanitizing on load. Default node size dropped 8 → 6 for dense graphs. Mechanics: the mount effect is split — the data effect builds the graphology graph + one frozen forceAtlas2 pass per snapshot; the render effect mounts sigma and re-runs on settings change only. Sizes are stamped onto the same graph instance (layout never recomputed), and the arrowhead program is constructed per rebuild with ratios solved from the settings (head px = thickness × lengthToThicknessRatio × 2; wideness keeps the stock 0.8 shape). The camera position carries across rebuilds via a ref, so a slider tweak never re-fits or re-lays-out the view; useDeferredValue smooths slider drags. Live-verified in the browser: sliders persist to localStorage, the renderer rebuilds with zero console errors, hover/picking survives the rebuild, and a zoomed-out camera stays zoomed out across a settings change (labels-gate opacity as the observable). 355/355 tests, typecheck, lint, build green. --- components/graph-view.tsx | 234 +++++++++++++++++++++++++++++++++----- docs/vision.md | 2 + 2 files changed, 208 insertions(+), 28 deletions(-) diff --git a/components/graph-view.tsx b/components/graph-view.tsx index cf8fbb9..aed3fe7 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -52,20 +52,55 @@ * viewport position (flipping above near the bottom edge); any camera move * (pan/zoom) dismisses it. The `requires` edges render as arrows pointing * prerequisite → dependent. + * + * Rendering settings (2026-09-13): a gear in the search panel reveals three + * sliders — node size, edge width, arrowhead length — persisted per browser + * (localStorage) and applied by REBUILDING ONLY THE RENDERER: the layout + * effect (graph + forceAtlas2) is untouched by settings, and the camera + * position carries across rebuilds, so tweaking a slider never re-fits or + * re-lays-out the view. The arrowhead is a program option (head px = + * thickness × lengthToThicknessRatio × 2), solved from the two settings. */ -import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; +import { useDeferredValue, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import Link from "next/link"; -import { FileTextIcon, Loader2Icon, SearchIcon, UploadIcon, WaypointsIcon } from "lucide-react"; +import { FileTextIcon, Loader2Icon, SearchIcon, SettingsIcon, UploadIcon, WaypointsIcon } from "lucide-react"; import Graph from "graphology"; import forceAtlas2 from "graphology-layout-forceatlas2"; import type { NodeHoverDrawingFunction, NodeLabelDrawingFunction } from "sigma/rendering"; import { seededRandom, type GraphNode, type GraphSnapshot } from "@/src/core"; /** The uniform node look (no state classes — WebGL wants hex, not oklch): - * the accent color + the old "known" size, until the state model lands. */ + * the accent color, until the state model lands. */ const NODE_COLOR_TOKEN = "--primary"; -const NODE_SIZE = 8; + +// ---- Render settings (the settings-panel controls) ------------------------ +// +// Three basic controls (user-decided 2026-09-13): node disc size, edge line +// width, and arrowhead length (px). The arrowhead program couples head size +// to thickness via lengthToThicknessRatio — the renderer derives the ratio +// from the two settings, so the controls stay independent. Defaults reflect +// the pre-panel look (edge 1px + its stock 5px head) except nodeSize, which +// dropped 8 → 6 for dense graphs (user: "nodes are quite large"). Persisted +// per browser in localStorage. + +interface RenderSettings { + nodeSize: number; + edgeWidth: number; + arrowLength: number; +} +const DEFAULT_RENDER_SETTINGS: RenderSettings = { nodeSize: 6, edgeWidth: 1, arrowLength: 5 }; +const RENDER_SETTINGS_KEY = "graph-render-settings"; +const clamp = (v: number, min: number, max: number) => Math.min(Math.max(v, min), max); +/** Defensively merge stored settings (stale/partial/corrupt → defaults). */ +function sanitizeRenderSettings(raw: unknown): RenderSettings { + const r = raw as Partial | null; + return { + nodeSize: clamp(Number(r?.nodeSize ?? DEFAULT_RENDER_SETTINGS.nodeSize), 2, 14), + edgeWidth: clamp(Number(r?.edgeWidth ?? DEFAULT_RENDER_SETTINGS.edgeWidth), 0.5, 4), + arrowLength: clamp(Number(r?.arrowLength ?? DEFAULT_RENDER_SETTINGS.arrowLength), 2, 15), + }; +} /** * Pure filter predicate for the search panel: a node is visible when the @@ -329,19 +364,80 @@ interface Selection { anchor: { x: number; y: number }; } +/** One labeled range row for the rendering-settings panel. */ +function SettingSlider({ + label, + min, + max, + step, + value, + onChange, +}: { + label: string; + min: number; + max: number; + step: number; + value: number; + onChange: (v: number) => void; +}) { + return ( + + ); +} + export function GraphView() { const containerRef = useRef(null); const cardRef = useRef(null); const [snapshot, setSnapshot] = useState(null); const [selected, setSelected] = useState(null); - // Search panel state: the query text. + // Search panel state: the query text + the settings section toggle. const [query, setQuery] = useState(""); + const [settingsOpen, setSettingsOpen] = useState(false); + // Render settings (node size / edge width / arrowhead): loaded from + // localStorage AFTER mount (an initial-state read would mismatch the + // server-rendered HTML), saved on every change. + const [settings, setSettings] = useState(DEFAULT_RENDER_SETTINGS); + useEffect(() => { + try { + const stored = localStorage.getItem(RENDER_SETTINGS_KEY); + if (stored) setSettings(sanitizeRenderSettings(JSON.parse(stored))); + } catch { + // corrupt storage → defaults, never crash the view + } + }, []); + useEffect(() => { + try { + localStorage.setItem(RENDER_SETTINGS_KEY, JSON.stringify(settings)); + } catch { + // storage full/blocked → in-memory settings still work + } + }, [settings]); + // The sigma instance and its graphology graph live inside the mount - // effect; refs expose them to the filters effect, and `rendererEpoch` - // re-triggers it when a new renderer is constructed. + // effects (data: graph+layout; render: renderer); refs expose them to + // the filters effect, and `rendererEpoch` re-triggers it when a new + // renderer is constructed. `cameraRef` carries the camera position + // across renderer rebuilds (settings changes must never re-fit the + // view), and `hoverStateRef` lets the layout effect's hover repainter + // reach the CURRENT renderer's drawers. const rendererRef = useRef(null); const graphRef = useRef(null); const [rendererEpoch, setRendererEpoch] = useState(0); + const cameraRef = useRef<{ x: number; y: number; ratio: number; angle: number } | null>(null); // The id set that survives the current search — consumed by the sigma // reducers and the "no matches" hint. @@ -414,20 +510,13 @@ export function GraphView() { }; }, []); - // Build the graphology graph, lay it out once, mount sigma. + // Build the graphology graph and lay it out ONCE per snapshot: seeded + // start positions, one synchronous forceAtlas2 pass, then freeze. The + // renderer is a SEPARATE effect — render settings change only the + // renderer; the layout (positions live in this graph instance) is + // never recomputed for a settings tweak. useEffect(() => { if (!snapshot || snapshot.nodes.length === 0) return; - const container = containerRef.current; - if (!container) return; - - const edgeColor = tokenHex("--ring"); - const labelColor = tokenHex("--muted-foreground"); - const hoverTextColor = tokenHex("--foreground"); - const nodeColor = tokenHex(NODE_COLOR_TOKEN); - // Mutable hover-transition state: 0 = resting, 1 = fully hovered. - const hover: HoverState = { fade: 0, repaint: null }; - const { drawLabel, drawHover } = makeLabelDrawers(labelColor, hoverTextColor, hover); - const graph = new Graph(); for (const node of snapshot.nodes) { // Seeded start positions (FA2 requires non-zero x/y) keep the @@ -435,8 +524,8 @@ export function GraphView() { const rand = seededRandom(node.id); graph.addNode(node.id, { label: node.name, - size: NODE_SIZE, - color: nodeColor, + size: DEFAULT_RENDER_SETTINGS.nodeSize, + color: tokenHex(NODE_COLOR_TOKEN), x: (rand() - 0.5) * 100, y: (rand() - 0.5) * 100, }); @@ -445,18 +534,56 @@ export function GraphView() { // Snapshot edges are already merged by (from,to,type); the guard // is just belt-and-braces for the simple graphology graph. if (!graph.hasEdge(edge.from, edge.to)) { - graph.addEdge(edge.from, edge.to, { type: "arrow", color: edgeColor, size: 1 }); + graph.addEdge(edge.from, edge.to, { + type: "arrow", + color: tokenHex("--ring"), + size: DEFAULT_RENDER_SETTINGS.edgeWidth, + }); } } // Run-then-freeze: one synchronous pass, writes x/y, then done. // Barnes-Hut comes on automatically above ~500 nodes (the vision's // first escalation rung — see scripts/bench-graph.ts for the numbers). - const settings = { + const layoutSettings = { ...forceAtlas2.inferSettings(graph), ...(graph.order > 500 ? { barnesHutOptimize: true } : {}), }; - forceAtlas2.assign(graph, { iterations: 100, settings }); + forceAtlas2.assign(graph, { iterations: 100, settings: layoutSettings }); + + // The render effect (declared next) builds its Sigma over THIS + // instance in the same commit; on snapshot change it re-runs too + // (snapshot is in its deps) and picks the new graph up. + graphRef.current = graph; + return () => { + graphRef.current = null; + }; + }, [snapshot]); + + // Mount sigma over the laid-out graph. Re-runs when the snapshot changes + // (new graph) or the (deferred) render settings change — nothing else. + // Sizes are stamped onto the graph here, and the arrowhead geometry is + // a PROGRAM option (head px = thickness × lengthToThicknessRatio × 2), + // so both flow through a renderer rebuild while the layout and the + // camera position (carried via cameraRef) stay put. + const deferredSettings = useDeferredValue(settings); + useEffect(() => { + if (!snapshot || snapshot.nodes.length === 0) return; + const container = containerRef.current; + const graph = graphRef.current; + if (!container || !graph) return; + + const s = deferredSettings; + // Stamp the size-controlled attributes (idempotent; the old renderer + // died in cleanup — this graph is renderer-less right now). + graph.updateEachNodeAttributes((_node, attrs) => ({ ...attrs, size: s.nodeSize })); + graph.updateEachEdgeAttributes((_edge, attrs) => ({ ...attrs, size: s.edgeWidth })); + + const labelColor = tokenHex("--muted-foreground"); + const hoverTextColor = tokenHex("--foreground"); + // Mutable hover-transition state: 0 = resting, 1 = fully hovered. + const hover: HoverState = { fade: 0, repaint: null }; + const { drawLabel, drawHover } = makeLabelDrawers(labelColor, hoverTextColor, hover); let killed = false; let renderer: import("sigma").default | null = null; @@ -466,8 +593,17 @@ export function GraphView() { // Both sigma and its rendering package touch WebGL globals at module // scope, so both load dynamically (SSR-safe; the effect is browser-only). void Promise.all([import("sigma"), import("sigma/rendering")]).then( - ([{ default: Sigma }, { NodeCircleProgram }]) => { + ([{ default: Sigma }, { NodeCircleProgram, createEdgeArrowProgram }]) => { if (killed) return; + // The arrowhead program with our ratios: solve + // lengthToThicknessRatio for the desired head length + // (head px = thickness × ratio × 2), keeping the stock head + // SHAPE (wideness = 0.8 × length, as the 2.5/2 defaults). + const thicknessRatio = s.arrowLength / (2 * s.edgeWidth); + const arrowProgram = createEdgeArrowProgram({ + lengthToThicknessRatio: thicknessRatio, + widenessToThicknessRatio: thicknessRatio * 0.8, + }); // Padded picking (see PICKING_VERTEX_SHADER): the visible disc is // pixel-identical to the stock circle program; the PICKABLE disc // gains PICKING_PAD_PX of radius, so the whole hover halo is @@ -484,6 +620,7 @@ export function GraphView() { }; renderer = new Sigma(graph, container, { nodeProgramClasses: { circle: PaddedNodeCircleProgram }, + edgeProgramClasses: { arrow: arrowProgram }, defaultEdgeType: "arrow", // All labels appear/disappear together: no per-node size gating // (threshold 0) and NO sigma label-grid declutter (density → @@ -505,6 +642,9 @@ export function GraphView() { graphRef.current = graph; rendererRef.current = renderer; setRendererEpoch((e) => e + 1); + // A renderer rebuild (settings tweak) must never re-fit the + // view: restore the carried camera position, if any. + if (cameraRef.current) renderer.getCamera().setState(cameraRef.current); renderer.on("clickNode", ({ node }) => { const graphNode = snapshot.nodes.find((n) => n.id === node); if (!graphNode) return; @@ -574,11 +714,13 @@ export function GraphView() { return () => { killed = true; if (hoverAnimId !== null) cancelAnimationFrame(hoverAnimId); + // Carry the camera across rebuilds — a settings tweak keeps the + // user's pan/zoom (and the detail-card anchors stay valid). + if (renderer) cameraRef.current = renderer.getCamera().getState(); rendererRef.current = null; - graphRef.current = null; renderer?.kill(); }; - }, [snapshot]); + }, [snapshot, deferredSettings]); // Filtering through sigma's official reducers: hidden nodes drop out of // rendering, picking, hover, AND labels; edges drop when either endpoint @@ -654,9 +796,45 @@ export function GraphView() { onChange={(e) => setQuery(e.target.value)} placeholder="Search nodes…" spellCheck={false} - className="h-8 w-full rounded-lg border bg-background pl-8 pr-2 text-xs outline-none placeholder:text-muted-foreground focus:border-ring" + className="h-8 w-full rounded-lg border bg-background pl-8 pr-8 text-xs outline-none placeholder:text-muted-foreground focus:border-ring" /> +
+ {settingsOpen && ( +
+ setSettings((cur) => ({ ...cur, nodeSize: v }))} + /> + setSettings((cur) => ({ ...cur, edgeWidth: v }))} + /> + setSettings((cur) => ({ ...cur, arrowLength: v }))} + /> +
+ )} {snapshot !== null && visibleIds.size === 0 && query.trim() !== "" && (
No matching nodes. diff --git a/docs/vision.md b/docs/vision.md index 206d5d5..d486d95 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -95,6 +95,8 @@ When the graph fills up, the canvas needs a readable at-a-glance structure: **cl ### Rendering (DECIDED — research report 2026-08-22) +**Render settings (2026-09-13):** three basic controls in the graph view — node size, edge width, arrowhead length — persisted per browser. Settings changes rebuild only the renderer: layout positions and the camera never move. (Edge-length / spacing controls deliberately parked until a cluster-aware spacing design exists.) + - **Library: sigma.js v3 + graphology + graphology-layout-forceatlas2.** ~40 KB gzipped all-in — half of vis-network, a third of cytoscape, far below react-force-graph (~250 KB deps) and cosmos.gl (luma.gl stack). - **Layout: synchronous run-then-freeze.** `forceAtlas2.assign(graph, { iterations: 100, settings: inferSettings(graph) })` runs once, writes x/y, and no animation loop ever starts — near-zero idle CPU (sigma only redraws on camera/graph events). <150 ms at hundreds of nodes; at ~2k enable `barnesHutOptimize`, then the bundled Web Worker, then server-cached positions — in that order, only if latency bites. - **Labels at zoom:** sigma's `labelRenderedSizeThreshold` replicates Obsidian's "text fade threshold" exactly. From 5144706fb95eb3f16ec03e1d147792cdb1dfd5aa Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 03:15:09 +0300 Subject: [PATCH 05/34] Graph view settings: THE zoom-level control (label threshold) The label-visibility zoom threshold (was the LABELS_CAMERA_RATIO=1 constant) becomes the fourth persisted setting: labels appear when the camera ratio is at/below it (1x = fit zoom; slider 0.5x-4x, shown with an x suffix). Deliberately THE single zoom knob per the cluster design (vision section 1): cluster labels will fade in over the same span as node labels fade out, gated inversely on this same setting. Live-verified: at a zoomed-out camera (labels previously hidden), raising the threshold to 4x rebuilds and re-shows labels without any camera movement; resetting restores the fade. 355/355 tests, typecheck, lint clean. --- components/graph-view.tsx | 37 +++++++++++++++++++++++++++++-------- docs/vision.md | 2 +- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/components/graph-view.tsx b/components/graph-view.tsx index aed3fe7..a584456 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -53,8 +53,9 @@ * (pan/zoom) dismisses it. The `requires` edges render as arrows pointing * prerequisite → dependent. * - * Rendering settings (2026-09-13): a gear in the search panel reveals three - * sliders — node size, edge width, arrowhead length — persisted per browser + * Rendering settings (2026-09-13): a gear in the search panel reveals four + * sliders — node size, edge width, arrowhead length, and THE zoom-level + * control (label visibility threshold; 1× = fit zoom) — persisted per browser * (localStorage) and applied by REBUILDING ONLY THE RENDERER: the layout * effect (graph + forceAtlas2) is untouched by settings, and the camera * position carries across rebuilds, so tweaking a slider never re-fits or @@ -88,8 +89,17 @@ interface RenderSettings { nodeSize: number; edgeWidth: number; arrowLength: number; + /** + * The single zoom-level control (camera ratio at/below which node labels + * appear; 1 = fit zoom, higher = labels visible further zoomed out). It + * will ALSO drive cluster labels (vision §1, DECIDED 2026-09-13) — + * inversely: cluster/domain labels fade in over the same span as node + * labels fade out — so the two never coexist. Kept in the persisted + * settings for exactly that reuse. + */ + labelZoom: number; } -const DEFAULT_RENDER_SETTINGS: RenderSettings = { nodeSize: 6, edgeWidth: 1, arrowLength: 5 }; +const DEFAULT_RENDER_SETTINGS: RenderSettings = { nodeSize: 6, edgeWidth: 1, arrowLength: 5, labelZoom: 1 }; const RENDER_SETTINGS_KEY = "graph-render-settings"; const clamp = (v: number, min: number, max: number) => Math.min(Math.max(v, min), max); /** Defensively merge stored settings (stale/partial/corrupt → defaults). */ @@ -99,6 +109,7 @@ function sanitizeRenderSettings(raw: unknown): RenderSettings { nodeSize: clamp(Number(r?.nodeSize ?? DEFAULT_RENDER_SETTINGS.nodeSize), 2, 14), edgeWidth: clamp(Number(r?.edgeWidth ?? DEFAULT_RENDER_SETTINGS.edgeWidth), 0.5, 4), arrowLength: clamp(Number(r?.arrowLength ?? DEFAULT_RENDER_SETTINGS.arrowLength), 2, 15), + labelZoom: clamp(Number(r?.labelZoom ?? DEFAULT_RENDER_SETTINGS.labelZoom), 0.5, 4), }; } @@ -123,8 +134,6 @@ export function isNodeVisible(node: Pick, query const LABEL_MAX_WIDTH = 170; // px — wraps long names instead of a wide strip const LABEL_GAP = 6; // px between node edge and first label line const LABEL_LINE_HEIGHT = 1.3; -/** Camera ratio at/below which ALL labels appear at once (1 = fit zoom). */ -const LABELS_CAMERA_RATIO = 1; /** Zoom span (camera-ratio units) over which labels fade out together. */ const LABELS_FADE_SPAN = 0.5; @@ -371,6 +380,7 @@ function SettingSlider({ max, step, value, + display, onChange, }: { label: string; @@ -378,13 +388,15 @@ function SettingSlider({ max: number; step: number; value: number; + /** Value display override (e.g. "1.5×" for zoom) — defaults to the raw number. */ + display?: string; onChange: (v: number) => void; }) { return (
)} {snapshot !== null && visibleIds.size === 0 && query.trim() !== "" && ( diff --git a/docs/vision.md b/docs/vision.md index d486d95..da4cbd1 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -95,7 +95,7 @@ When the graph fills up, the canvas needs a readable at-a-glance structure: **cl ### Rendering (DECIDED — research report 2026-08-22) -**Render settings (2026-09-13):** three basic controls in the graph view — node size, edge width, arrowhead length — persisted per browser. Settings changes rebuild only the renderer: layout positions and the camera never move. (Edge-length / spacing controls deliberately parked until a cluster-aware spacing design exists.) +**Render settings (2026-09-13):** four basic controls in the graph view — node size, edge width, arrowhead length, and **the zoom-level control** (the label-visibility threshold; 1× = fit zoom) — persisted per browser. The zoom control is deliberately THE single zoom knob: cluster labels (below) will gate inversely on it over the same fade span. Settings changes rebuild only the renderer: layout positions and the camera never move. (Edge-length / spacing controls deliberately parked until a cluster-aware spacing design exists.) - **Library: sigma.js v3 + graphology + graphology-layout-forceatlas2.** ~40 KB gzipped all-in — half of vis-network, a third of cytoscape, far below react-force-graph (~250 KB deps) and cosmos.gl (luma.gl stack). - **Layout: synchronous run-then-freeze.** `forceAtlas2.assign(graph, { iterations: 100, settings: inferSettings(graph) })` runs once, writes x/y, and no animation loop ever starts — near-zero idle CPU (sigma only redraws on camera/graph events). <150 ms at hundreds of nodes; at ~2k enable `barnesHutOptimize`, then the bundled Web Worker, then server-cached positions — in that order, only if latency bites. From b4245a209dd0d9a9bfadd8c86d5cc0af76720a6d Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 03:42:15 +0300 Subject: [PATCH 06/34] Graph view settings: default label threshold 0.5x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The most suitable default is the stricter end — labels only once zoomed in past 2x fit (user-decided 2026-09-13). The slider window extends below it (0.25x) so the default is no longer the floor. --- components/graph-view.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/graph-view.tsx b/components/graph-view.tsx index a584456..8bee7ed 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -99,7 +99,7 @@ interface RenderSettings { */ labelZoom: number; } -const DEFAULT_RENDER_SETTINGS: RenderSettings = { nodeSize: 6, edgeWidth: 1, arrowLength: 5, labelZoom: 1 }; +const DEFAULT_RENDER_SETTINGS: RenderSettings = { nodeSize: 6, edgeWidth: 1, arrowLength: 5, labelZoom: 0.5 }; const RENDER_SETTINGS_KEY = "graph-render-settings"; const clamp = (v: number, min: number, max: number) => Math.min(Math.max(v, min), max); /** Defensively merge stored settings (stale/partial/corrupt → defaults). */ @@ -109,7 +109,7 @@ function sanitizeRenderSettings(raw: unknown): RenderSettings { nodeSize: clamp(Number(r?.nodeSize ?? DEFAULT_RENDER_SETTINGS.nodeSize), 2, 14), edgeWidth: clamp(Number(r?.edgeWidth ?? DEFAULT_RENDER_SETTINGS.edgeWidth), 0.5, 4), arrowLength: clamp(Number(r?.arrowLength ?? DEFAULT_RENDER_SETTINGS.arrowLength), 2, 15), - labelZoom: clamp(Number(r?.labelZoom ?? DEFAULT_RENDER_SETTINGS.labelZoom), 0.5, 4), + labelZoom: clamp(Number(r?.labelZoom ?? DEFAULT_RENDER_SETTINGS.labelZoom), 0.25, 4), }; } @@ -847,7 +847,7 @@ export function GraphView() { /> Date: Sun, 13 Sep 2026 03:46:24 +0300 Subject: [PATCH 07/34] Clusters phase A: community detection core (Louvain, seeded, structural) detectCommunities(nodes, edges) in src/core/clusters.ts (new dep graphology-communities-louvain): Louvain modularity over the undirected, unweighted follows+supports structure (user-decided 2026-09-13). The random-walk traversal runs on an inlined seeded mulberry32 so the same graph always yields the same partition; community indices are still unstable across edits, which is exactly why labelings (phase B) key to sorted MEMBER SETS. follows+supports between the same pair collapse to one edge; self-loops and dangling edges are skipped defensively; no min-size filter here (the >=3 render threshold is a presentation rule). Tests: two-cliques-plus-bridge split, determinism, singleton/empty graphs, pair collapse, defensive skips, member sorting. 363/363. Real-data smoke over the current notes graph: 338 nodes / 211 edges -> 138 communities, modularity 0.95, and the big ones are exactly the learn-session topics (databases, differential forms, sales, servers, vim, copulas...). 32 communities clear the >=3 render threshold. --- package-lock.json | 53 ++++++++++++++++++ package.json | 1 + src/core/clusters.ts | 118 +++++++++++++++++++++++++++++++++++++++++ src/core/index.ts | 1 + tests/clusters.test.ts | 103 +++++++++++++++++++++++++++++++++++ 5 files changed, 276 insertions(+) create mode 100644 src/core/clusters.ts create mode 100644 tests/clusters.test.ts diff --git a/package-lock.json b/package-lock.json index f9278b2..e095625 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "cmdk": "^1.1.1", "embla-carousel-react": "^8.6.0", "graphology": "^0.26.0", + "graphology-communities-louvain": "^2.0.2", "graphology-layout-forceatlas2": "^0.10.1", "katex": "^0.16.47", "linkedom": "^0.18.13", @@ -8837,6 +8838,34 @@ "graphology-types": ">=0.24.0" } }, + "node_modules/graphology-communities-louvain": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/graphology-communities-louvain/-/graphology-communities-louvain-2.0.2.tgz", + "integrity": "sha512-zt+2hHVPYxjEquyecxWXoUoIuN/UvYzsvI7boDdMNz0rRvpESQ7+e+Ejv6wK7AThycbZXuQ6DkG8NPMCq6XwoA==", + "license": "MIT", + "dependencies": { + "graphology-indices": "^0.17.0", + "graphology-utils": "^2.4.4", + "mnemonist": "^0.39.0", + "pandemonium": "^2.4.1" + }, + "peerDependencies": { + "graphology-types": ">=0.19.0" + } + }, + "node_modules/graphology-indices": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/graphology-indices/-/graphology-indices-0.17.0.tgz", + "integrity": "sha512-A7RXuKQvdqSWOpn7ZVQo4S33O0vCfPBnUSf7FwE0zNCasqwZVUaCXePuWo5HBpWw68KJcwObZDHpFk6HKH6MYQ==", + "license": "MIT", + "dependencies": { + "graphology-utils": "^2.4.2", + "mnemonist": "^0.39.0" + }, + "peerDependencies": { + "graphology-types": ">=0.20.0" + } + }, "node_modules/graphology-layout-forceatlas2": { "version": "0.10.1", "resolved": "https://registry.npmjs.org/graphology-layout-forceatlas2/-/graphology-layout-forceatlas2-0.10.1.tgz", @@ -11595,6 +11624,15 @@ "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", "license": "MIT" }, + "node_modules/mnemonist": { + "version": "0.39.8", + "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.8.tgz", + "integrity": "sha512-vyWo2K3fjrUw8YeeZ1zF0fy6Mu59RHokURlld8ymdUPjMlD9EC9ov1/YPqTgqRvUN9nTr3Gqfz29LYAmu0PHPQ==", + "license": "MIT", + "dependencies": { + "obliterator": "^2.0.1" + } + }, "node_modules/modern-tar": { "version": "0.8.4", "resolved": "https://registry.npmjs.org/modern-tar/-/modern-tar-0.8.4.tgz", @@ -11993,6 +12031,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/obliterator": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.5.tgz", + "integrity": "sha512-42CPE9AhahZRsMNslczq0ctAEtqk8Eka26QofnqC346BZdHDySk3LWka23LI7ULIw11NmltpiLagIq8gBozxTw==", + "license": "MIT" + }, "node_modules/oniguruma-parser": { "version": "0.12.2", "resolved": "https://registry.npmjs.org/oniguruma-parser/-/oniguruma-parser-0.12.2.tgz", @@ -12116,6 +12160,15 @@ "integrity": "sha512-yQA4H19AmPEoMUeavPMDIe1higySl/gH/yaQrkT/s07Qp+7pp2hYz30N3z2l5BkjVkF9Ow6o0wjJamm2y7Sn0A==", "license": "MIT" }, + "node_modules/pandemonium": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/pandemonium/-/pandemonium-2.4.1.tgz", + "integrity": "sha512-wRqjisUyiUfXowgm7MFH2rwJzKIr20rca5FsHXCMNm1W5YPP1hCtrZfgmQ62kP7OZ7Xt+cR858aB28lu5NX55g==", + "license": "MIT", + "dependencies": { + "mnemonist": "^0.39.2" + } + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", diff --git a/package.json b/package.json index 5eb7a45..32a7514 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "cmdk": "^1.1.1", "embla-carousel-react": "^8.6.0", "graphology": "^0.26.0", + "graphology-communities-louvain": "^2.0.2", "graphology-layout-forceatlas2": "^0.10.1", "katex": "^0.16.47", "linkedom": "^0.18.13", diff --git a/src/core/clusters.ts b/src/core/clusters.ts new file mode 100644 index 0000000..edeb6b8 --- /dev/null +++ b/src/core/clusters.ts @@ -0,0 +1,118 @@ +/** + * COMMUNITY DETECTION for the knowledge graph (vision §1, DECIDED + * 2026-09-13): Louvain modularity over the undirected, unweighted + * follows+supports structure — purely structural, so it never depends on + * layout, and the extraction pipeline never knows clusters exist. + * + * Determinism: Louvain's random-walk traversal is driven by a SEEDED + * mulberry32 RNG (default seed constant below), so the same graph always + * yields the same partition. Community INDICES are still not stable + * across graph edits — which is why labelings (Phase B) key to sorted + * MEMBER SETS, never to indices. + * + * Undirected + unweighted (user-decided 2026-09-13): both edge types are + * equally real relations between propositions — direction is + * derivational-vs-grounding, not stronger-vs-weaker — and a follows AND + * a supports between the same pair collapse to one edge. + * + * This module is browser-safe pure logic (no node builtins, no fs): the + * server runs it for the snapshot payload, the renderer for nothing + * (clusters arrive precomputed), and tests run it directly. + * + * NOTE: no minimum-size filter here — detection returns every community, + * singletons included. The ≥3-node render threshold (decided) is a + * PRESENTATION rule; it lives in the payload derivation and the + * renderer, not in the algorithm. + */ +import Graph from "graphology"; +import louvain from "graphology-communities-louvain"; + +/** A detected community. Members are SORTED node ids — the member set IS + * the identity a labeling record can outlive a reshuffle with. */ +export interface Community { + members: string[]; +} + +export interface CommunityDetection { + communities: Community[]; + /** Modularity of the found partition — the one quality number. */ + modularity: number; +} + +export interface DetectOptions { + /** Louvain resolution (higher → more, smaller communities). Default 1. + * Parked for the multi-scale work; exposed now so that dial has a home. */ + resolution?: number; + /** RNG seed. Default "lh-clusters". Same seed + same graph → same + * partition, always. */ + seed?: string; +} + +/** mulberry32 seeded by a string (FNV-1a) — the same recipe as + * question-core's seededRandom, inlined so this module has no + * cross-domain dependency. */ +function seededRng(seed: string): () => number { + let h = 2166136261; + for (let i = 0; i < seed.length; i++) { + h ^= seed.charCodeAt(i); + h = Math.imul(h, 16777619); + } + let a = h >>> 0; + return () => { + a |= 0; + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** + * Detect communities over a graph's structure. Inputs are the minimal + * structural shapes of the snapshot contracts (GraphNode.id, + * GraphEdge.from/to), so both the real snapshot and test fixtures feed it. + * + * Edges referencing unknown node ids are skipped defensively (the ledgers + * are append-only; a mid-merge read should never crash detection), and + * duplicate pairs (follows+supports, or reverse direction) collapse — + * mergeEdge keeps the first. + */ +export function detectCommunities( + nodes: readonly { id: string }[], + edges: readonly { from: string; to: string }[], + opts?: DetectOptions, +): CommunityDetection { + const graph = new Graph({ type: "undirected" }); + for (const node of nodes) graph.addNode(node.id); + for (const edge of edges) { + if (!graph.hasNode(edge.from) || !graph.hasNode(edge.to)) continue; + if (edge.from === edge.to) continue; // self-loops: no community signal + graph.mergeEdge(edge.from, edge.to); + } + // Louvain on a 0-edge graph is degenerate; every node is trivially its + // own community (and louvain on an empty graph returns nothing useful). + if (graph.size === 0) { + return { + communities: nodes.map((n) => ({ members: [n.id] })).sort((a, b) => (a.members[0] ?? "").localeCompare(b.members[0] ?? "")), + modularity: 0, + }; + } + const details = louvain.detailed(graph, { + rng: seededRng(opts?.seed ?? "lh-clusters"), + resolution: opts?.resolution ?? 1, + }); + const byIndex = new Map(); + for (const [id, index] of Object.entries(details.communities)) { + const bucket = byIndex.get(index) ?? []; + bucket.push(id); + byIndex.set(index, bucket); + } + const communities = [...byIndex.values()] + .map((ids) => ({ members: ids.sort() })) + .sort( + (a, b) => + b.members.length - a.members.length || + (a.members[0] ?? "").localeCompare(b.members[0] ?? ""), + ); + return { communities, modularity: details.modularity }; +} diff --git a/src/core/index.ts b/src/core/index.ts index 84d1676..a611727 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -19,6 +19,7 @@ export * from "./contracts.ts"; export * from "./ports.ts"; export * from "./question-core.ts"; +export * from "./clusters.ts"; export * from "./plan-tree.ts"; export * from "./phases.ts"; export * from "./pending.ts"; diff --git a/tests/clusters.test.ts b/tests/clusters.test.ts new file mode 100644 index 0000000..5ff3f50 --- /dev/null +++ b/tests/clusters.test.ts @@ -0,0 +1,103 @@ +/** + * Community detection tests (vision §1, DECIDED 2026-09-13) — pure + * structural Louvain over follows+supports, deterministic via seed. + * Assertions are on SHAPE and MEMBER SETS, never community indices + * (indices reshuffle; member sets are what labelings key to). + */ +import { strict as assert } from "node:assert"; +import { describe, test } from "node:test"; +import { detectCommunities } from "../src/core/clusters.ts"; +import type { Community } from "../src/core/clusters.ts"; + +const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${prefix}${i}`); + +/** A clique on the given ids. */ +const clique = (nodeIds: string[]) => + nodeIds.flatMap((a, i) => nodeIds.slice(i + 1).map((b) => ({ from: a, to: b }))); + +describe("detectCommunities", () => { + test("two dense cliques joined by a single bridge split into two communities", () => { + const left = ids(4, "L"); + const right = ids(4, "R"); + const bridge = { from: "L0", to: "R0" }; + const result = detectCommunities([...left, ...right].map((id) => ({ id })), [...clique(left), ...clique(right), bridge]); + assert.equal(result.communities.length, 2); + assert.equal(result.modularity > 0.3, true); + // Member sets, not order: the two big communities are exactly the cliques. + const big = result.communities.filter((c) => c.members.length === 4); + assert.deepEqual(big.map((c) => c.members.sort()).sort(), [left.slice().sort(), right.slice().sort()]); + }); + + test("deterministic: same graph, same seed → identical partition", () => { + const nodes = ids(10).map((id) => ({ id })); + const edges = [...clique(ids(5)), ...clique(ids(5, "n").slice().reverse().slice(0, 5)), { from: "n0", to: "n9" }]; + const a = detectCommunities(nodes, edges); + const b = detectCommunities(nodes, edges); + assert.deepEqual(a.communities, b.communities); + assert.equal(a.modularity, b.modularity); + }); + + test("no edges: every node is its own singleton community", () => { + const result = detectCommunities(ids(3).map((id) => ({ id })), []); + assert.equal(result.communities.length, 3); + for (const c of result.communities) assert.equal(c.members.length, 1); + assert.equal(result.modularity, 0); + }); + + test("empty graph: zero communities", () => { + assert.deepEqual(detectCommunities([], []), { communities: [], modularity: 0 }); + }); + + test("duplicate and reversed pairs collapse to one edge (undirected, unweighted)", () => { + const nodes = ["a", "b", "c"].map((id) => ({ id })); + // follows a→b AND supports b→a — same pair twice, both directions. + const result = detectCommunities(nodes, [ + { from: "a", to: "b" }, + { from: "b", to: "a" }, + { from: "b", to: "c" }, + ]); + assert.equal(result.communities.length, 1); + assert.deepEqual(result.communities[0].members.sort(), ["a", "b", "c"]); + }); + + test("self-loops and unknown-endpoint edges are skipped defensively", () => { + const nodes = ["a", "b"].map((id) => ({ id })); + const result = detectCommunities(nodes, [ + { from: "a", to: "a" }, // self-loop + { from: "a", to: "ghost" }, // dangling + { from: "ghost1", to: "ghost2" }, + { from: "a", to: "b" }, // the one real edge + ]); + assert.equal(result.communities.length, 1); + assert.deepEqual(result.communities[0].members.sort(), ["a", "b"]); + }); + + test("members are sorted; communities ordered by size then id", () => { + const left = ids(4, "L"); + const right = ids(4, "R"); + const result = detectCommunities( + [...left, ...right, "iso"].map((id) => ({ id })), + [...clique(left), ...clique(right), { from: "L0", to: "R0" }], + ); + for (const c of result.communities as Community[]) { + const sorted = c.members.slice().sort(); + assert.deepEqual(c.members, sorted); + } + // iso (isolated) is last: size 1 < 4. + const last = result.communities[result.communities.length - 1]; + assert.deepEqual(last.members, ["iso"]); + }); + + test("seed override changes nothing structurally but stays deterministic", () => { + const nodes = ids(8).map((id) => ({ id })); + const edges = [...clique(["n0", "n1", "n2", "n3"]), ...clique(["n4", "n5", "n6", "n7"]), { from: "n0", to: "n4" }]; + const a = detectCommunities(nodes, edges, { seed: "one" }); + const b = detectCommunities(nodes, edges, { seed: "one" }); + const c = detectCommunities(nodes, edges, { seed: "two" }); + assert.deepEqual(a.communities, b.communities); + // A different seed partitions the same graph into member sets that + // may or may not coincide — but both runs are 2 communities here, + // because the structure dominates. + assert.equal(c.communities.length, 2); + }); +}); From f0267a5fd97c26e4f1e295658b7f639cad9e0839 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 03:59:48 +0300 Subject: [PATCH 08/34] Clusters phases B+C: labeling records, the naming call, snapshot payload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Core-survival labelings (vision section 1, DECIDED 2026-09-13): ClusterLabeling records in a new append-only ledger (data/graph/cluster-labels.jsonl), immutable, id = clu_ + sha256 of the sorted core so re-naming races and re-runs dedupe at the store seam. matchClusterLabelings (core/clusters.ts, pure): a community keeps a record's label while at least half its core survives in it; both sides single-successor (a split keeps one name; greedy by overlap, ties to the newer record). The namer (src/agent/subagents/cluster-namer.ts): ONE singular cheap LLM call (new 'labeler' role, glm-flash/low) names every new/changed community at once, given member names + statements (capped at 12 members per cluster). refreshClusterLabels runs POST-EXTRACTION (user-decided) from both front doors, fire-and-forget — naming failure never fails the run; refreshes serialize behind a chain so parallel imports never mint duplicate labels. import-notes.ts awaits clusterNamingSettled before exit. Payload: GraphSnapshot.clusters (>= MIN_CLUSTER_SIZE 3) joins detection with labelings at read time — claimed clusters carry the record's label and id; unlabeled ones carry a member-hash id and null label. Detection is deterministic so no memo in v1 (ms at current scale; revisit at vault scale). Tests (372/372): matcher half-rule/split/merge/greedy/tie cases; naming end-to-end through importNotes (one call, one record, claim on re-refresh), naming failure non-fatal; snapshot derivation with labeling join; 11 test pipelines now pin an explicit labeler seam. --- config.ts | 5 + scripts/import-notes.ts | 9 +- src/agent/subagents/cluster-namer.ts | 200 +++++++++++++++++++++++++++ src/agent/subagents/extraction.ts | 15 ++ src/core/clusters.ts | 50 +++++++ src/core/contracts/graph.ts | 35 +++++ src/core/graph.ts | 32 ++++- src/core/ports.ts | 6 + src/store/graph.ts | 21 +++ tests/clusters.test.ts | 65 ++++++++- tests/extraction.test.ts | 83 +++++++++++ tests/graph.test.ts | 49 ++++++- 12 files changed, 562 insertions(+), 8 deletions(-) create mode 100644 src/agent/subagents/cluster-namer.ts diff --git a/config.ts b/config.ts index b385125..3472ad8 100644 --- a/config.ts +++ b/config.ts @@ -47,6 +47,9 @@ export const MODELS = { extractor: { model: "z-ai/glm-5.3-flash", effort: "low" }, /** The auditor verifies evidence exists; keep it cheap. */ auditor: { model: "z-ai/glm-5.3-flash", effort: "low" }, + /** Names detected communities — one singular call, given member names + * + statements. Purely a labeling flourish; keep it cheap. */ + labeler: { model: "z-ai/glm-5.3-flash", effort: "low" }, } satisfies Record; export type ModelRole = keyof typeof MODELS; @@ -71,6 +74,8 @@ export const LEDGER_FILES = { events: path.join(PATHS.graph, "events.jsonl"), mappings: path.join(PATHS.graph, "mappings.jsonl"), sessions: path.join(PATHS.graph, "sessions.jsonl"), + /** LLM-named community records (contracts/graph.ts ClusterLabeling). */ + clusterLabels: path.join(PATHS.graph, "cluster-labels.jsonl"), } as const; /** diff --git a/scripts/import-notes.ts b/scripts/import-notes.ts index c2a9735..11cf144 100644 --- a/scripts/import-notes.ts +++ b/scripts/import-notes.ts @@ -15,6 +15,7 @@ import * as fs from "node:fs"; import * as path from "node:path"; import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts"; +import { clusterNamingSettled } from "../src/agent/subagents/cluster-namer.ts"; import { getGraph } from "../src/core/graph.ts"; import { stores } from "../src/server/stores.ts"; @@ -52,10 +53,16 @@ async function main() { }); await Promise.all(workers); + // The post-merge cluster naming is fire-and-forget inside the pipeline; + // settle it before the process exits so labels actually land. + await clusterNamingSettled(); + const graph = getGraph(stores.graphStore); + const labels = stores.graphStore.readClusterLabels().length; console.log( `[import-notes] done: +${totals.nodes} nodes, +${totals.edges} edges, +${totals.events} events, ${totals.failed} failed — ` + - `graph now ${graph.nodes.length} nodes, ${graph.edges.length} edges, ${graph.events.length} events`, + `graph now ${graph.nodes.length} nodes, ${graph.edges.length} edges, ${graph.events.length} events, ` + + `${labels} named clusters`, ); } diff --git a/src/agent/subagents/cluster-namer.ts b/src/agent/subagents/cluster-namer.ts new file mode 100644 index 0000000..4c21f5c --- /dev/null +++ b/src/agent/subagents/cluster-namer.ts @@ -0,0 +1,200 @@ +/** + * CLUSTER NAMING (vision §1, DECIDED 2026-09-13): the garnish layer over + * community detection. Detection is purely structural and deterministic + * (core/clusters.ts); naming is ONE singular cheap LLM call that labels + * every NEW/CHANGED community at once, given member names + statements. + * + * Core-survival: a community keeps an existing labeling while at least + * half its core survives in it (matchClusterLabelings); only communities + * with no surviving label AND at least MIN_CLUSTER_SIZE members are sent + * to the namer. Records are append-only and idempotent (content-hashed + * ids dedupe at the store seam), so a naming race or a re-run appends + * nothing new. + * + * Runs POST-EXTRACTION (user-decided 2026-09-13): both front doors call + * refreshClusterLabels after a successful merge. Failure is NON-FATAL — + * the graph and hulls render fine unlabeled; the next run retries. + */ +import { z } from "zod"; +import { lhTool, toPiTools } from "../lh-tool.ts"; +import type { PiEngine } from "../models.ts"; +import { runSubagent } from "./pi-run.ts"; +import type { GraphNode } from "../../core/contracts.ts"; +import { getGraph, clusterContentId } from "../../core/graph.ts"; +import { + detectCommunities, + matchClusterLabelings, + MIN_CLUSTER_SIZE, + type Community, +} from "../../core/clusters.ts"; +import type { GraphStore } from "../../core/ports.ts"; + +/** How many member propositions the namer sees per cluster — enough to + * recognize the domain without paying for the whole region. */ +const MAX_MEMBERS_PER_CLUSTER = 12; + +const CLUSTER_NAMER_INSTRUCTIONS = `You name communities of propositions in a personal knowledge graph. + +You are given clusters. Each cluster is a set of related propositions +(theory fragments the author is learning) plus, for context, a sketch of +every OTHER cluster already named. For each UNNAMED cluster, mint ONE +short DOMAIN LABEL: 1-4 words naming the region of the world model those +propositions belong to together — like a section heading in a textbook. +Use the members' own vocabulary ("Differential forms", "Sales +psychology", "Server architecture"), not generic words ("Topics", +"Various concepts"). If the propositions are unrelated, that is fine: +name the best common denominator anyway — a reader should recognize the +region at a glance. + +Your entire deliverable is ONE submit_labels call. Include EVERY +top-level field even when empty (labels: [] when nothing to name).`; + +const labelsSchema = z.object({ + labels: z + .array( + z.object({ + /** The cluster key from the prompt ("c1", "c2", ...). */ + key: z.string(), + label: z.string(), + }), + ) + .default([]), +}); + +export interface UnnamedCluster { + key: string; + members: string[]; + nodes: Pick[]; +} + +/** One single-submit namer run (the same shape as every extraction stage): + * the whole deliverable is ONE zod-validated submit call. */ +async function nameClusters( + engine: PiEngine | undefined, + signal: AbortSignal | undefined, + unnamed: UnnamedCluster[], +): Promise<{ key: string; label: string }[]> { + let submitted: { key: string; label: string }[] | undefined; + await runSubagent( + toPiTools({ + submit_labels: lhTool({ + name: "submit_labels", + description: "Submit the cluster labels.", + schema: labelsSchema, + execute: async (labels) => { + submitted = labels.labels; + return labels; + }, + }), + }), + { + role: "labeler", + engine, + systemPrompt: CLUSTER_NAMER_INSTRUCTIONS, + prompt: JSON.stringify( + { + unnamedClusters: unnamed.map((c) => ({ + key: c.key, + members: c.nodes.map((n) => ({ name: n.name, statement: n.statement })), + })), + }, + null, + 2, + ), + maxTurns: 4, + abortSignal: signal, + shouldStop: (_message, toolResults) => + toolResults.some((r) => r.toolName === "submit_labels" && !r.isError), + }, + ); + if (!submitted) throw new Error("the cluster namer never submitted labels"); + return submitted; +} + +export interface ClusterRefreshStats { + /** Communities the current graph divides into (all sizes). */ + detected: number; + /** Communities that kept a label via core-survival. */ + claimed: number; + /** Communities sent to the namer (unnamed, size >= MIN_CLUSTER_SIZE). */ + named: number; + /** Labeling records actually appended (idempotent dedupe may shrink this). */ + appended: number; +} + +/** + * The post-extraction garnish: detect communities on the CURRENT graph, + * join persisted labelings (core-survival), and name what is new — one + * singular LLM call. Appends ClusterLabeling records; the NEXT snapshot + * read joins them (the payload is derived, never stored). + * + * Serialized: concurrent merges (parallel notes imports) chain here, so + * two runs never mint duplicate labels for the same community. + */ +async function doRefresh(deps: { + graphStore: GraphStore; + /** Test seam: a scripted engine (production uses the labeler role). */ + engine?: PiEngine; + signal?: AbortSignal; +}): Promise { + const { graphStore, engine, signal } = deps; + const { nodes, edges } = getGraph(graphStore); + const detection = detectCommunities(nodes, edges); + const labelings = graphStore.readClusterLabels(); + const claims = matchClusterLabelings(detection.communities, labelings); + const byId = new Map(nodes.map((n) => [n.id, n])); + + const claimed = claims.filter((c) => c !== null).length; + const unnamed = detection.communities + .filter((c, i) => !claims[i] && c.members.length >= MIN_CLUSTER_SIZE) + .map((c: Community, i) => { + const nodesFor = c.members + .map((id) => byId.get(id)) + .filter((n): n is GraphNode => n !== undefined) + .slice(0, MAX_MEMBERS_PER_CLUSTER) + .map((n) => ({ name: n.name, statement: n.statement })); + return { key: `c${i}`, members: c.members, nodes: nodesFor }; + }); + + if (unnamed.length === 0) return { detected: detection.communities.length, claimed, named: 0, appended: 0 }; + + const labels = await nameClusters(engine, signal, unnamed); + let appended = 0; + const seen = new Set(); + for (const { key, members } of unnamed) { + if (seen.has(key)) continue; + const label = labels.find((l) => l.key === key)?.label?.trim(); + if (!label) continue; + seen.add(key); + graphStore.appendClusterLabel({ + id: clusterContentId(members), + label, + core: members, + createdAt: new Date().toISOString(), + }); + appended++; + } + return { detected: detection.communities.length, claimed, named: unnamed.length, appended }; +} + +let namingTail: Promise = Promise.resolve(); + +/** + * Fire the refresh, serialized behind any refresh already running (a + * parallel import's merge waits its turn; each run reads fresh ledgers). + * Waivable: call sites may fire-and-forget; scripts await the return. + */ +export function refreshClusterLabels(deps: { + graphStore: GraphStore; + engine?: PiEngine; + signal?: AbortSignal; +}): Promise { + const run = namingTail.then(() => doRefresh(deps)); + namingTail = run.catch(() => {}); + return run; +} + +/** Await every refresh fired so far (scripts: don't exit mid-naming). */ +export async function clusterNamingSettled(): Promise { + await namingTail; +} diff --git a/src/agent/subagents/extraction.ts b/src/agent/subagents/extraction.ts index 14f0da3..e06bb97 100644 --- a/src/agent/subagents/extraction.ts +++ b/src/agent/subagents/extraction.ts @@ -16,6 +16,7 @@ import type { SessionFindings, } from "../../core/contracts.ts"; import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts"; +import { refreshClusterLabels } from "./cluster-namer.ts"; /** * The extraction pipeline (vision §1 construction, 2026-09-02 redesign) — the @@ -548,6 +549,8 @@ export interface ExtractionModels { /** Test seam: a scripted engine per stage (production uses the roles). */ extractor?: PiEngine; auditor?: PiEngine; + /** Test seam for the cluster namer (post-extraction garnish). */ + labeler?: PiEngine; } export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {} @@ -716,6 +719,13 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti `[extraction ${chatId}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` + `(${Math.round(performance.now() - t0)}ms total)`, ); + // Post-extraction garnish (DECIDED 2026-09-13): name new/changed + // clusters. Fire-and-forget — naming never fails the run. + refreshClusterLabels({ + graphStore, + engine: deps.labeler, + signal: runningExtractions.get(chatId)?.controller.signal, + }).catch(() => {}); return stats; } catch (err) { persistRunUsage(chatId, Math.round(performance.now() - t0), undefined); @@ -819,6 +829,11 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti `[extraction ${id}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` + `(${Math.round(performance.now() - t0)}ms total)`, ); + refreshClusterLabels({ + graphStore, + engine: deps.labeler, + signal: runningExtractions.get(id)?.controller.signal, + }).catch(() => {}); return stats; } catch (err) { persistRunUsage(id, Math.round(performance.now() - t0), undefined); diff --git a/src/core/clusters.ts b/src/core/clusters.ts index edeb6b8..a762a73 100644 --- a/src/core/clusters.ts +++ b/src/core/clusters.ts @@ -26,6 +26,7 @@ */ import Graph from "graphology"; import louvain from "graphology-communities-louvain"; +import type { ClusterLabeling } from "./contracts.ts"; /** A detected community. Members are SORTED node ids — the member set IS * the identity a labeling record can outlive a reshuffle with. */ @@ -33,6 +34,11 @@ export interface Community { members: string[]; } +/** Communities below this size are not rendered, not named — presentation + * rule (user-decided 2026-09-13): a region of the world model needs at + * least three propositions to be worth a hull and a label. */ +export const MIN_CLUSTER_SIZE = 3; + export interface CommunityDetection { communities: Community[]; /** Modularity of the found partition — the one quality number. */ @@ -116,3 +122,47 @@ export function detectCommunities( ); return { communities, modularity: details.modularity }; } + +/** + * Claim labeling records for detected communities — CORE-SURVIVAL (DECIDED + * 2026-09-13): a community keeps a record's label while AT LEAST HALF of + * the record's core survives in it (overlap * 2 >= core.length; exactly + * half survives). Both sides are single-successor: one community holds at + * most one label, one record is claimed by at most one community — on a + * split, only the branch carrying the majority of the core keeps the name; + * on a merge, the best-overlapping record wins. + * + * Greedy assignment by overlap size (largest first; ties broken by newer + * record). Returns a claim (or null) per community, in input order — + * unclaimed communities are the namer's input. Pure: same inputs, same + * claims. + */ +export function matchClusterLabelings( + communities: readonly Community[], + labelings: readonly ClusterLabeling[], +): (ClusterLabeling | null)[] { + interface Pair { + community: number; + labeling: ClusterLabeling; + overlap: number; + } + const pairs: Pair[] = []; + communities.forEach((community, ci) => { + const members = new Set(community.members); + for (const labeling of labelings) { + const overlap = labeling.core.reduce((n, id) => n + (members.has(id) ? 1 : 0), 0); + if (overlap * 2 >= labeling.core.length && overlap > 0) { + pairs.push({ community: ci, labeling, overlap }); + } + } + }); + pairs.sort((a, b) => b.overlap - a.overlap || (a.labeling.createdAt < b.labeling.createdAt ? 1 : -1)); + const claims: (ClusterLabeling | null)[] = communities.map(() => null); + const takenRecords = new Set(); + for (const pair of pairs) { + if (claims[pair.community] || takenRecords.has(pair.labeling.id)) continue; + claims[pair.community] = pair.labeling; + takenRecords.add(pair.labeling.id); + } + return claims; +} diff --git a/src/core/contracts/graph.ts b/src/core/contracts/graph.ts index 2941541..77aac84 100644 --- a/src/core/contracts/graph.ts +++ b/src/core/contracts/graph.ts @@ -270,6 +270,37 @@ export interface AuditResult { } /** The GET /api/graph payload — everything the home view's PURE renderer needs. */ +/** cluster-labels.jsonl line — one LLM-named community (vision §1, + * DECIDED 2026-09-13). Append-only history: records are immutable, never + * edited or deleted; dead labelings remain as history. IDENTITY is the + * sorted member set ("core"), hashed into the id, so re-naming the same + * community dedupes and a labeling survives graph edits as long as at + * least half its core survives in some single community (core-survival; + * under half → the community is renamed, splits can keep only one + * successor). */ +export interface ClusterLabeling { + /** "clu_" + content hash of the sorted core. */ + id: string; + /** The short domain label (1–4 words), e.g. "Differential forms". */ + label: string; + /** The member node ids the label was minted for, sorted — the survival core. */ + core: string[]; + createdAt: string; +} + +/** One rendered cluster in the snapshot payload (vision §1). Members are + * node ids; the renderer draws the hull from member positions at draw + * time and gates the label inversely on the node-label zoom control. */ +export interface Cluster { + /** The labeling record id when claimed; unlabeled clusters still + * appear (hull, no name) with a member-hash id. */ + id: string; + /** null until the naming call has minted/claimed a labeling. */ + label: string | null; + /** Node ids, sorted. Hull + centroid are renderer concerns. */ + members: string[]; +} + export interface GraphSnapshot { nodes: GraphNode[]; edges: GraphEdge[]; @@ -286,6 +317,10 @@ export interface GraphSnapshot { * shown next to the session list — the confidence history). Multiple * notes for one (node, chat) collapse to the latest. */ nodeObservations: Record>; + /** Structural communities (Louvain over follows+supports), size ≥ 3, + * joined with persisted LLM labelings. Purely structural — never + * depends on layout. */ + clusters: Cluster[]; } /** Derive a short chat title from the learner's first message (pure). diff --git a/src/core/graph.ts b/src/core/graph.ts index dee329b..bebde35 100644 --- a/src/core/graph.ts +++ b/src/core/graph.ts @@ -38,10 +38,11 @@ * adapters in. */ -import { randomBytes } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import { isToolUIPart, type UIMessage } from "ai"; import type { AuditResult, + Cluster, EmissionStats, GraphEdge, GraphNode, @@ -50,6 +51,7 @@ import type { LedgerEvent, LedgerEventKind, } from "./contracts.ts"; +import { detectCommunities, matchClusterLabelings, MIN_CLUSTER_SIZE } from "./clusters.ts"; import type { QuestionAnswerData } from "./question-core.ts"; import type { ChatStore, GraphStore } from "./ports.ts"; @@ -330,6 +332,16 @@ export function pendingQuizAnswers(chatId: string, deps: GraphDeps): number { // ─── derivation (reads only — everything below is recomputable) ───────────── +/** The content identity of a community: "clu_" + sha256 of the sorted + * member ids (12 hex). Deterministic — the same member set always mints + * the same id — so re-naming races and idempotent re-runs dedupe at the + * store seam (appendClusterLabel). Lives here (server-only core), not in + * clusters.ts, because clusters.ts is barrel-exported and must stay + * browser-safe (no node:crypto). */ +export function clusterContentId(members: readonly string[]): string { + return "clu_" + createHash("sha256").update([...members].sort().join("\n")).digest("hex").slice(0, 12); +} + export interface DerivedGraph { nodes: GraphNode[]; edges: GraphEdge[]; @@ -485,7 +497,23 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot { (nodeObservations[nodeId] ??= {})[chatId] = note; } - return { nodes, edges, sessions, fileSources, nodeSessions, nodeObservations }; + // Structural communities (vision §1): Louvain over the undirected + // follows+supports structure, joined with persisted labelings — + // core-survival matching, size-filtered at the presentation threshold. + // Detection is DETERMINISTIC (seeded), so no memo in v1: a snapshot + // read recomputes it (ms at current scale; revisit at vault scale). + const detection = detectCommunities(nodes, edges); + const claims = matchClusterLabelings(detection.communities, deps.graphStore.readClusterLabels()); + const clusters: Cluster[] = detection.communities + .map((community, index) => ({ community, claim: claims[index] })) + .filter(({ community }) => community.members.length >= MIN_CLUSTER_SIZE) + .map(({ community, claim }) => ({ + id: claim?.id ?? clusterContentId(community.members), + label: claim?.label ?? null, + members: community.members, + })); + + return { nodes, edges, sessions, fileSources, nodeSessions, nodeObservations, clusters }; } // Re-export: the contract owns the shape; legacy importers still reach it. diff --git a/src/core/ports.ts b/src/core/ports.ts index e1e6219..3f96871 100644 --- a/src/core/ports.ts +++ b/src/core/ports.ts @@ -25,6 +25,7 @@ import type { SessionCheckpoint, TurnUsage, NotesSource, + ClusterLabeling, } from "./contracts.ts"; // ─── ChatStore — the session record's home (data/chats) ───────────────────── @@ -82,6 +83,11 @@ export interface GraphStore { appendCheckpoint(checkpoint: SessionCheckpoint): void; readCheckpoints(): SessionCheckpoint[]; + /** Append one cluster labeling — idempotent by id (the id hashes the + * sorted core, so identical member sets dedupe). */ + appendClusterLabel(labeling: ClusterLabeling): void; + readClusterLabels(): ClusterLabeling[]; + /** Upsert one imported-notes source record (config.SOURCES_FILE — a * JSON map, not a ledger; an upsert keeps the existing usage log). */ upsertSource(source: NotesSource): void; diff --git a/src/store/graph.ts b/src/store/graph.ts index b67c884..ea050bd 100644 --- a/src/store/graph.ts +++ b/src/store/graph.ts @@ -31,6 +31,7 @@ import * as fs from "node:fs"; import * as path from "node:path"; import { LEDGER_FILES, SOURCES_FILE } from "../../config.ts"; import type { + ClusterLabeling, GraphEdge, GraphNode, ExtractionRunUsage, @@ -86,6 +87,14 @@ function readValid(file: string, isValid: (v: unknown) => v is T): T[] { const isProvenancePoint = (v: unknown): v is ProvenancePoint => isRecord(v) && isString(v.chatId) && isString(v.messageId); +const isClusterLabeling = (v: unknown): v is ClusterLabeling => + isRecord(v) && + isString(v.id) && + isString(v.label) && + Array.isArray(v.core) && + v.core.every(isString) && + isString(v.createdAt); + const isGraphNode = (v: unknown): v is GraphNode => isRecord(v) && isString(v.id) && @@ -164,6 +173,18 @@ export const graphStore: GraphStore = { return readValid(LEDGER_FILES.sessions, isCheckpoint); }, + // ── cluster labelings (config.LEDGER_FILES.clusterLabels) ── + + /** Idempotent by id: the id is a hash of the sorted core, so re-appending + * the same member set (re-naming races, idempotent re-runs) is a no-op. */ + appendClusterLabel(labeling: ClusterLabeling): void { + if (this.readClusterLabels().some((l) => l.id === labeling.id)) return; + appendLine(LEDGER_FILES.clusterLabels, labeling); + }, + readClusterLabels(): ClusterLabeling[] { + return readValid(LEDGER_FILES.clusterLabels, isClusterLabeling); + }, + // ── imported-notes bookkeeping (config.SOURCES_FILE — a JSON map) ── /** Upsert: keeps the existing record's usage log when re-importing. */ diff --git a/tests/clusters.test.ts b/tests/clusters.test.ts index 5ff3f50..8050101 100644 --- a/tests/clusters.test.ts +++ b/tests/clusters.test.ts @@ -6,8 +6,9 @@ */ import { strict as assert } from "node:assert"; import { describe, test } from "node:test"; -import { detectCommunities } from "../src/core/clusters.ts"; +import { detectCommunities, matchClusterLabelings, MIN_CLUSTER_SIZE } from "../src/core/clusters.ts"; import type { Community } from "../src/core/clusters.ts"; +import type { ClusterLabeling } from "../src/core/contracts.ts"; const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${prefix}${i}`); @@ -15,6 +16,62 @@ const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${pref const clique = (nodeIds: string[]) => nodeIds.flatMap((a, i) => nodeIds.slice(i + 1).map((b) => ({ from: a, to: b }))); +const labeling = (id: string, core: string[], createdAt = "2026-09-01T00:00:00.000Z"): ClusterLabeling => ({ + id, + label: `label-${id}`, + core, + createdAt, +}); + +describe("matchClusterLabelings", () => { + test("claims a labeling while at least half its core survives in the community", () => { + const community: Community = { members: ["a", "b", "c", "d", "e"] }; + const records = [labeling("clu_1", ["a", "b", "x", "y"])]; // 2 of 4 survive — exactly half + const [claim] = matchClusterLabelings([community], records); + assert.equal(claim?.id, "clu_1"); + // One fewer survivor: under half → relabel. + const [dead] = matchClusterLabelings([{ members: ["a", "c", "d", "e", "f"] }], records); + assert.equal(dead, null); + }); + + test("single successor: a split core keeps its label in only one community", () => { + const records = [labeling("clu_1", ["a", "b", "c", "d"])]; + const left: Community = { members: ["a", "b", "z"] }; // 2 of 4 — half + const right: Community = { members: ["c", "d", "w"] }; // 2 of 4 — half + // Tie: larger overlap wins; equal → both eligible, greedy takes first come. + const claims = matchClusterLabelings([left, right], records); + const claimed = claims.filter((c) => c !== null); + assert.equal(claimed.length, 1); + assert.equal(claimed[0]?.id, "clu_1"); + }); + + test("greedy: the best-overlapping community wins the record", () => { + const records = [labeling("clu_1", ["a", "b", "c", "d"])]; + const weak: Community = { members: ["a", "b", "q", "r", "s"] }; // 2/4 + const strong: Community = { members: ["a", "b", "c", "x", "y"] }; // 3/4 + const claims = matchClusterLabelings([weak, strong], records); + assert.equal(claims[0], null); + assert.equal(claims[1]?.id, "clu_1"); + }); + + test("one record per community; newest record breaks ties", () => { + const oldR = labeling("clu_old", ["a", "b"], "2026-09-01T00:00:00.000Z"); + const newR = labeling("clu_new", ["a", "b"], "2026-09-02T00:00:00.000Z"); + const community: Community = { members: ["a", "b", "c"] }; + const [claim] = matchClusterLabelings([community], [oldR, newR]); + assert.equal(claim?.id, "clu_new"); + }); + + test("no labelings: every community unclaimed; empty communities array ok", () => { + assert.deepEqual(matchClusterLabelings([{ members: ["a", "b"] }], []), [null]); + assert.deepEqual(matchClusterLabelings([], [labeling("clu_1", ["a"])]), []); + }); + + test("MIN_CLUSTER_SIZE is 3 (the render/naming threshold)", () => { + assert.equal(MIN_CLUSTER_SIZE, 3); + }); +}); + describe("detectCommunities", () => { test("two dense cliques joined by a single bridge split into two communities", () => { const left = ids(4, "L"); @@ -57,7 +114,7 @@ describe("detectCommunities", () => { { from: "b", to: "c" }, ]); assert.equal(result.communities.length, 1); - assert.deepEqual(result.communities[0].members.sort(), ["a", "b", "c"]); + assert.deepEqual(result.communities[0]!.members.sort(), ["a", "b", "c"]); }); test("self-loops and unknown-endpoint edges are skipped defensively", () => { @@ -69,7 +126,7 @@ describe("detectCommunities", () => { { from: "a", to: "b" }, // the one real edge ]); assert.equal(result.communities.length, 1); - assert.deepEqual(result.communities[0].members.sort(), ["a", "b"]); + assert.deepEqual(result.communities[0]!.members.sort(), ["a", "b"]); }); test("members are sorted; communities ordered by size then id", () => { @@ -84,7 +141,7 @@ describe("detectCommunities", () => { assert.deepEqual(c.members, sorted); } // iso (isolated) is last: size 1 < 4. - const last = result.communities[result.communities.length - 1]; + const last = result.communities[result.communities.length - 1]!; assert.deepEqual(last.members, ["iso"]); }); diff --git a/tests/extraction.test.ts b/tests/extraction.test.ts index 18a0488..4ccc072 100644 --- a/tests/extraction.test.ts +++ b/tests/extraction.test.ts @@ -27,6 +27,7 @@ import type { UIMessage } from "ai"; import type { PiEngine } from "../src/agent/models.ts"; import type { AuditResult, + ClusterLabeling, ExtractionRunUsage, GraphEdge, GraphNode, @@ -38,6 +39,7 @@ import type { } from "../src/core/contracts.ts"; import type { GraphStore } from "../src/core/ports.ts"; import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts"; +import { clusterNamingSettled, refreshClusterLabels } from "../src/agent/subagents/cluster-namer.ts"; import { chatStore } from "../src/store/chats.ts"; import { PATHS } from "../config.ts"; @@ -63,6 +65,7 @@ function memoryLedgers() { const mappings: NodeMapping[] = []; const checkpoints: SessionCheckpoint[] = []; const sources = new Map(); + const clusterLabels: ClusterLabeling[] = []; return { appendNode: (n: GraphNode) => void nodes.push(n), readNodes: () => nodes.map((n) => ({ ...n })), @@ -87,6 +90,11 @@ function memoryLedgers() { source.extractions = [...(source.extractions ?? []), entry]; } }, + // Cluster labelings — mirrors the real store: append dedupes by id. + appendClusterLabel: (l: ClusterLabeling) => { + if (!clusterLabels.some((x) => x.id === l.id)) clusterLabels.push(l); + }, + readClusterLabels: () => clusterLabels.map((l) => ({ ...l, core: [...l.core] })), } satisfies GraphStore; } @@ -306,6 +314,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { graphStore: ledgers, extractor: submitModel("submit_findings", FINDINGS), auditor: submitModel("submit_review", verdict), + labeler: submitModel("submit_labels", { labels: [] }), }); const stats = await pipeline.runExtraction(chatId); @@ -346,6 +355,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { graphStore: ledgers, extractor: submitModel("submit_findings", FINDINGS), auditor: silentModel(), // never submits the review + labeler: submitModel("submit_labels", { labels: [] }), }); await assert.rejects(pipeline.runExtraction(chatId), /the auditor never submitted a review/); assert.equal(ledgers.readNodes().length, 0); @@ -356,6 +366,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { graphStore: ledgers, extractor: silentModel(), auditor: submitModel("submit_review", { ...FINDINGS, dropped: [] }), + labeler: submitModel("submit_labels", { labels: [] }), }); await assert.rejects(failingExtractor.runExtraction(chatId), /the extractor never submitted findings/); assert.equal(ledgers.readCheckpoints().length, 0); @@ -388,6 +399,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { // registry must already carry this chat at the right phase. extractor: submitModel("submit_findings", FINDINGS, () => seen.push(phaseOf())), auditor: submitModel("submit_review", verdict, () => seen.push(phaseOf())), + labeler: submitModel("submit_labels", { labels: [] }), }); await pipeline.runExtraction(chatId); assert.deepEqual(seen, ["extracting", "auditing"]); @@ -422,6 +434,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { auditor: submitModel("submit_review", verdict, () => usageSeen.push(extractionStatuses().find((r) => r.chatId === chatId)?.usage.extractor?.steps.at(-1)?.inputTokens), ), + labeler: submitModel("submit_labels", { labels: [] }), }); await pipeline.runExtraction(chatId); @@ -458,6 +471,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { graphStore: ledgers, extractor: hangingModel(), // hangs until aborted — no step finishes auditor: submitModel("submit_review", { propositions: [], edges: [], attribution: { quiz: {} }, dropped: [] }), + labeler: submitModel("submit_labels", { labels: [] }), }); const runPromise = pipeline.runExtraction(chatId); @@ -508,6 +522,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { graphStore: ledgers, extractor: gatedExtractor, auditor: silentModel(), + labeler: submitModel("submit_labels", { labels: [] }), }); const first = pipeline.runExtraction(chatId); @@ -525,6 +540,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { graphStore: ledgers, extractor: silentModel(), auditor: silentModel(), + labeler: submitModel("submit_labels", { labels: [] }), }); await assert.rejects(pipeline.runExtraction("chat_does_not_exist"), /Chat not found/); }); @@ -562,6 +578,70 @@ const NOTES_VERDICT = { dropped: [], } as const; +const CLUSTER_FINDINGS = { + propositions: [ + { id: "p1", name: "Modal editing", statement: "Vim edits text through modal command modes.", aliases: [], messages: [], evidence: ["Normal mode keys are commands."], understanding: "solid" }, + { id: "p2", name: "Verb-object grammar", statement: "Vim commands compose as verb and object.", aliases: [], messages: [], evidence: ["d then motion deletes."], understanding: "solid" }, + { id: "p3", name: "Operator-pending", statement: "Vim enters operator-pending mode between verb and object.", aliases: [], messages: [], evidence: ["After d, a motion is expected."], understanding: "solid" }, + ], + edges: [ + { from: "p1", to: "p2", type: "follows", messages: [], evidence: ["e"] }, + { from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] }, + ], +} as const; + +const CLUSTER_VERDICT = { + propositions: CLUSTER_FINDINGS.propositions, + edges: CLUSTER_FINDINGS.edges, + dropped: [], +} as const; + +describe("cluster naming (the post-extraction garnish)", () => { + it("names a new community after a notes merge — one record, claimed on the next refresh", async () => { + const ledgers = memoryLedgers(); + const namerCalls: number[] = []; + const labeler = submitModel("submit_labels", { labels: [{ key: "c0", label: "Vim modal grammar" }] }, () => namerCalls.push(1)); + const pipeline = createExtractionPipeline({ + chatStore, + graphStore: ledgers, + extractor: submitModel("submit_findings", CLUSTER_FINDINGS), + auditor: submitModel("submit_review", CLUSTER_VERDICT), + labeler, + }); + const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes, any format" }); + assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 }); + // The refresh is fire-and-forget inside the pipeline — settle it. + await clusterNamingSettled(); + assert.equal(namerCalls.length, 1, "ONE singular naming call"); + const records = ledgers.readClusterLabels(); + assert.equal(records.length, 1); + assert.equal(records[0]!.label, "Vim modal grammar"); + assert.equal(records[0]!.core.length, 3); + assert.match(records[0]!.id, /^clu_/); + // Second refresh: core-survival claims it — nothing new to name. + const again = await refreshClusterLabels({ graphStore: ledgers, engine: labeler }); + assert.equal(again.named, 0); + assert.equal(again.appended, 0); + assert.ok(again.claimed >= 1); + assert.equal(ledgers.readClusterLabels().length, 1, "no duplicate records"); + }); + + it("naming failure never fails the run", async () => { + const ledgers = memoryLedgers(); + const pipeline = createExtractionPipeline({ + chatStore, + graphStore: ledgers, + extractor: submitModel("submit_findings", CLUSTER_FINDINGS), + auditor: submitModel("submit_review", CLUSTER_VERDICT), + labeler: silentModel(), // never submits — the namer throws + }); + const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes" }); + assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 }, "the merge itself succeeded"); + await clusterNamingSettled(); + assert.equal(ledgers.readClusterLabels().length, 0, "nothing named, nothing appended"); + }); +}); + describe("the notes front door (importNotes — format-agnostic)", () => { it("imports raw notes end to end: source record, file-scoped provenance, no quiz events", async () => { const ledgers = memoryLedgers(); @@ -575,6 +655,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => { const run = extractionStatuses().find((r) => r.chatId.startsWith("src_")); seen.push({ kind: run?.kind, label: run?.label }); }), + labeler: submitModel("submit_labels", { labels: [] }), }); const stats = await pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT }); @@ -618,6 +699,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => { graphStore: ledgers, extractor: submitModel("submit_findings", NOTES_FINDINGS), auditor: submitModel("submit_review", NOTES_VERDICT), + labeler: submitModel("submit_labels", { labels: [] }), }); const first = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT }); assert.deepEqual(first, { nodes: 1, edges: 0, events: 1 }); @@ -637,6 +719,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => { graphStore: ledgers, extractor: hangingModel(), auditor: submitModel("submit_review", NOTES_VERDICT), + labeler: submitModel("submit_labels", { labels: [] }), }); const runPromise = pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT }); diff --git a/tests/graph.test.ts b/tests/graph.test.ts index 9d27d11..c695394 100644 --- a/tests/graph.test.ts +++ b/tests/graph.test.ts @@ -20,7 +20,7 @@ import { after, describe, it } from "node:test"; import type { UIMessage } from "ai"; import type { AuditResult } from "../src/core/contracts.ts"; import type { QuestionAnswerData } from "../src/core/question-core.ts"; -import { getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts"; +import { clusterContentId, getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts"; import { graphStore } from "../src/store/graph.ts"; import { chatStore } from "../src/store/chats.ts"; import { LEDGER_FILES, PATHS, SOURCES_FILE } from "../config.ts"; @@ -407,6 +407,53 @@ describe("the merger (mergeFindings)", () => { assert.equal(snap.nodeObservations[node.id]!["src_snapfixture"], "solid"); }); + it("derives structural clusters in graphSnapshot, joined with persisted labelings", () => { + // A self-contained 3-clique: three propositions, all mutually linked. + const tri = ["Alpha.", "Beta.", "Gamma cluster."] as const; + const merge = mergeFindings( + "src_clusterfixture", + { + propositions: tri.map((statement, i) => ({ + id: `p${i + 1}`, + name: `Cluster node ${i + 1}`, + statement, + aliases: [], + messages: [], + evidence: ["fixture evidence"], + })), + edges: [ + { from: "p1", to: "p2", type: "follows", messages: [], evidence: ["e"] }, + { from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] }, + { from: "p1", to: "p3", type: "follows", messages: [], evidence: ["e"] }, + ], + attribution: { quiz: {} }, + dropped: [], + }, + graphDeps, + { messages: null, foldedAt: "2026-09-13T00:00:00.000Z" }, + ); + assert.equal(merge.nodes, 3); + const snap = graphSnapshot(graphDeps); + // Presentation threshold: every cluster is >= 3 members. + assert.ok(snap.clusters.every((c) => c.members.length >= 3)); + // Our trio is exactly one cluster, unlabeled until the namer runs. + const mine = snap.clusters.find((c) => c.members.length === 3 && tri.every((s) => c.members.includes(snap.nodes.find((n) => n.statement === s)!.id))); + assert.ok(mine, "the 3-clique is a cluster"); + assert.equal(mine!.label, null); + assert.match(mine!.id, /^clu_/); + // Mint the labeling the namer would append — the next read joins it. + graphStore.appendClusterLabel({ + id: clusterContentId(mine!.members), + label: "Test region", + core: mine!.members, + createdAt: "2026-09-13T00:00:00.000Z", + }); + const snap2 = graphSnapshot(graphDeps); + const claimed = snap2.clusters.find((c) => c.members.length === 3 && tri.every((s) => c.members.includes(snap2.nodes.find((n) => n.statement === s)!.id)))!; + assert.equal(claimed.label, "Test region"); + assert.equal(claimed.id, clusterContentId(mine!.members), "the claimed cluster carries the record's id"); + }); + it("derives sessions and per-session understanding observations in graphSnapshot", () => { const snap = graphSnapshot(graphDeps); const idByStatement = new Map(snap.nodes.map((n) => [n.statement, n.id])); From 71a17c5e0d99534f8a282ba14579f858d3b55622 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 04:07:01 +0300 Subject: [PATCH 09/34] Clusters phase D: hulls, blurred fills, and inverse-gated region labels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two custom canvas layers around sigma's own stack (DOM order: hulls behind everything; region labels above nodes, below node labels). Per cluster (vision section 1): convex hull (monotone chain) of member viewport positions, offset outward 14px, rounded corners, a blurred low-opacity fill (ctx.filter, graceful degrade to flat fill) plus a dashed boundary stroke, and the LLM label at the centroid. Stable hue per cluster id (content hash) — a region keeps its color across reshuffles because the id only changes when the members do. Geometry is a pure function of positions at draw time: every afterRender re-derives hulls through the current camera, so pan/zoom/ relayout can never invalidate it (the boundary decision pays off: no stored geometry, nothing to recompute after a future physics change). The inverse gate: the cluster label layer's opacity is the exact complement of the node-label layer's over the fade span of THE zoom control (labelZoom, default 0.5x) — the two label systems never coexist. Live-verified in the browser: hulls painting behind edges; region labels above nodes; zoom-in crossfades cluster 1 -> 0 as node labels 0 -> 1 (0.176 + 0.824 mid-fade); zoom-out restores; hull pixel count changes with zoom (geometry follows the camera); zero console errors; layers removed on renderer teardown. 372/372 tests, typecheck, lint, build green. Also named the real graph: one 11.5s labeler call minted all 32 region labels for the 338-node notes graph (Relational Databases & SQL, Differential Forms, Sales as Uncertainty Reduction, ...). --- components/graph-view.tsx | 174 ++++++++++++++++++++++++++++++++++++++ docs/vision.md | 2 +- 2 files changed, 175 insertions(+), 1 deletion(-) diff --git a/components/graph-view.tsx b/components/graph-view.tsx index 8bee7ed..0797f44 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -16,6 +16,15 @@ * sigma's defaults: * - defaultDrawNodeLabel: plain wrapped text, centered below the node, * word-wrapped at a max line width — no truncation, no one-line sprawl. + * Cluster hulls (vision §1) live on TWO custom canvas layers around + * sigma's own stack: a hull layer behind everything (blurred low-opacity + * fill + dashed boundary per cluster, stable hue from the cluster id) + * and a region-label layer above nodes but below node labels (the label + * at the centroid, inverse-gated on THE zoom control — node labels fade + * out over the span as cluster labels fade in, so they never coexist). + * Geometry is a pure function of member positions at draw time: every + * afterRender re-derives hulls through the current camera, so pan/zoom + * can never invalidate it. * - defaultDrawNodeHover: NO background box (sigma's default is a * hardcoded white one). Instead the hovered node gets a soft glow * halo in the node's color at a constant radius, flush with the @@ -411,6 +420,75 @@ function SettingSlider({ ); } +// ─── cluster hulls (vision §1, DECIDED 2026-09-13) ──────────────────────────── +// Boundary geometry is a PURE FUNCTION OF POSITIONS AT DRAW TIME: every +// redraw re-derives the hull from current member positions mapped through +// the camera, so pan/zoom/relayout can never invalidate it. No metaballs +// in v1 — convex hull + outward offset + rounded corners; revisit when +// regions want concave fidelity. + +/** Stable hue per cluster id (content-hash): the region keeps its color + * across reshuffles because the id only changes when the members do. */ +function clusterHue(id: string): number { + let h = 0; + for (const ch of id) h = (h * 31 + ch.charCodeAt(0)) % 360; + return h; +} + +type Pt = { x: number; y: number }; + +/** Convex hull (Andrew monotone chain), viewport pixels. */ +function convexHull(points: Pt[]): Pt[] { + if (points.length < 3) return points; + const pts = [...points].sort((a, b) => a.x - b.x || a.y - b.y); + const cross = (o: Pt, a: Pt, b: Pt) => (a.x - o.x) * (b.y - o.y) - (a.y - o.y) * (b.x - o.x); + const lower: Pt[] = []; + for (const p of pts) { + while (lower.length >= 2 && cross(lower[lower.length - 2]!, lower[lower.length - 1]!, p) <= 0) lower.pop(); + lower.push(p); + } + const upper: Pt[] = []; + for (let i = pts.length - 1; i >= 0; i--) { + const p = pts[i]!; + while (upper.length >= 2 && cross(upper[upper.length - 2]!, upper[upper.length - 1]!, p) <= 0) upper.pop(); + upper.push(p); + } + upper.pop(); + lower.pop(); + return [...lower, ...upper]; +} + +/** Outward edge normal of segment a→b (screen coords: y grows down). */ +function outwardNormal(a: Pt, b: Pt): Pt { + const dx = b.x - a.x; + const dy = b.y - a.y; + const len = Math.hypot(dx, dy) || 1; + return { x: -dy / len, y: dx / len }; +} + +/** Offset a convex polygon outward by `pad` px (per-vertex normal sum). */ +function offsetPolygon(points: Pt[], pad: number): Pt[] { + return points.map((p, i) => { + const prev = points[(i - 1 + points.length) % points.length]!; + const next = points[(i + 1) % points.length]!; + const n1 = outwardNormal(prev, p); + const n2 = outwardNormal(p, next); + return { x: p.x + (n1.x + n2.x) * pad, y: p.y + (n1.y + n2.y) * pad }; + }); +} + +/** Path through the polygon with rounded corners (arcTo). */ +function traceRoundedPath(ctx: CanvasRenderingContext2D, points: Pt[], radius: number): void { + const first = points[0]!; + ctx.moveTo(first.x, first.y); + for (let i = 1; i <= points.length; i++) { + const corner = points[i % points.length]!; + const next = points[(i + 1) % points.length]!; + ctx.arcTo(corner.x, corner.y, next.x, next.y, radius); + } + ctx.closePath(); +} + export function GraphView() { const containerRef = useRef(null); const cardRef = useRef(null); @@ -599,6 +677,10 @@ export function GraphView() { let killed = false; let renderer: import("sigma").default | null = null; + // The custom cluster layers (hull + region label canvases) — created + // inside the dynamic-import callback, removed in this effect's cleanup. + let clusterCanvas: HTMLCanvasElement | null = null; + let clusterLabelCanvas: HTMLCanvasElement | null = null; // rAF id of the hover fade transition — the ONLY animation (see the // file header). Lives out here so the effect cleanup can cancel it. let hoverAnimId: number | null = null; @@ -657,6 +739,91 @@ export function GraphView() { // A renderer rebuild (settings tweak) must never re-fit the // view: restore the carried camera position, if any. if (cameraRef.current) renderer.getCamera().setState(cameraRef.current); + + // ── Cluster hulls (vision §1): two custom canvas layers around + // sigma's own stack (DOM order: edges, nodes, [labels], edgeLabels, + // labels, ...). The HULL layer sits behind everything; the region + // LABEL layer sits above nodes but below node labels — the two + // label systems are inverse-gated on the same zoom control, so + // they never coexist (see syncRenderLabels below). + const sigmaCanvases = renderer.getCanvases(); + // Local consts so the draw closure below stays narrowed. + const hullCanvas = (clusterCanvas = document.createElement("canvas")); + hullCanvas.style.position = "absolute"; + container.insertBefore(hullCanvas, container.firstChild ?? null); + const regionCanvas = (clusterLabelCanvas = document.createElement("canvas")); + regionCanvas.style.position = "absolute"; + // Actual DOM order (sigma v3): edges, edgeLabels, nodes, labels, … + // so "before labels" = above nodes, below node labels — the two + // label systems are inverse-gated and never coexist anyway. + sigmaCanvases.labels?.parentNode?.insertBefore(regionCanvas, sigmaCanvases.labels); + // All hull geometry is derived per draw — positions through the + // CURRENT camera — so any pan/zoom/render stays correct. + const drawClusters = () => { + const r = renderer; + if (!r || snapshot.clusters.length === 0) return; + const ref = sigmaCanvases.nodes; + if (!ref) return; + const cssW = ref.clientWidth; + const cssH = ref.clientHeight; + if (cssW === 0 || cssH === 0) return; + const dpr = ref.width / cssW; + const hullCtx = hullCanvas.getContext("2d"); + const labelCtx = regionCanvas.getContext("2d"); + if (!hullCtx || !labelCtx) return; + for (const canvas of [hullCanvas, regionCanvas]) { + if (canvas.width !== ref.width || canvas.height !== ref.height) { + canvas.width = ref.width; + canvas.height = ref.height; + } + canvas.style.width = ref.style.width || `${cssW}px`; + canvas.style.height = ref.style.height || `${cssH}px`; + } + hullCtx.setTransform(dpr, 0, 0, dpr, 0, 0); + labelCtx.setTransform(dpr, 0, 0, dpr, 0, 0); + hullCtx.clearRect(0, 0, cssW, cssH); + labelCtx.clearRect(0, 0, cssW, cssH); + labelCtx.textAlign = "center"; + labelCtx.textBaseline = "middle"; + labelCtx.font = `500 18px ${getComputedStyle(document.body).fontFamily}`; + for (const cluster of snapshot.clusters) { + const pts: Pt[] = []; + for (const id of cluster.members) { + if (!graph.hasNode(id)) continue; + const a = graph.getNodeAttributes(id); + const vp = r.graphToViewport({ x: a.x ?? 0, y: a.y ?? 0 }); + if (vp) pts.push(vp); + } + const hull = convexHull(pts); + if (hull.length < 3) continue; + const hue = clusterHue(cluster.id); + // Blurred low-opacity fill + dashed boundary stroke. + const outer = offsetPolygon(hull, 14); + hullCtx.beginPath(); + traceRoundedPath(hullCtx, outer, 9); + // ctx.filter degrades to flat fill where unsupported. + try { + hullCtx.filter = "blur(12px)"; + } catch { + /* older browsers: skip the blur */ + } + hullCtx.fillStyle = `hsla(${hue}, 65%, 60%, 0.08)`; + hullCtx.fill(); + hullCtx.filter = "none"; + hullCtx.setLineDash([7, 7]); + hullCtx.strokeStyle = `hsla(${hue}, 65%, 62%, 0.5)`; + hullCtx.lineWidth = 1.5; + hullCtx.stroke(); + hullCtx.setLineDash([]); + // The region label at the centroid (inverse-gated layer). + if (!cluster.label) continue; + const centroid = hull.reduce((acc, p) => ({ x: acc.x + p.x / hull.length, y: acc.y + p.y / hull.length }), { x: 0, y: 0 }); + labelCtx.fillStyle = `hsla(${hue}, 55%, 78%, 0.85)`; + labelCtx.fillText(cluster.label, centroid.x, centroid.y); + } + }; + renderer.on("afterRender", drawClusters); + renderer.on("clickNode", ({ node }) => { const graphNode = snapshot.nodes.find((n) => n.id === node); if (!graphNode) return; @@ -687,6 +854,9 @@ export function GraphView() { renderer?.setSetting("renderLabels", shouldRender); } if (labelsCanvas) labelsCanvas.style.opacity = alpha.toFixed(3); + // The inverse gate (vision §1): cluster labels fade in over + // the exact span node labels fade out — THE one zoom control. + regionCanvas.style.opacity = (1 - alpha).toFixed(3); }; syncRenderLabels(renderer.getCamera().getState().ratio); renderer.getCamera().on("updated", (state) => { @@ -731,6 +901,10 @@ export function GraphView() { if (renderer) cameraRef.current = renderer.getCamera().getState(); rendererRef.current = null; renderer?.kill(); + // The custom cluster layers are ours — sigma's kill() removes only + // its own canvases. + clusterCanvas?.remove(); + clusterLabelCanvas?.remove(); }; }, [snapshot, deferredSettings]); diff --git a/docs/vision.md b/docs/vision.md index da4cbd1..965c606 100644 --- a/docs/vision.md +++ b/docs/vision.md @@ -95,7 +95,7 @@ When the graph fills up, the canvas needs a readable at-a-glance structure: **cl ### Rendering (DECIDED — research report 2026-08-22) -**Render settings (2026-09-13):** four basic controls in the graph view — node size, edge width, arrowhead length, and **the zoom-level control** (the label-visibility threshold; 1× = fit zoom) — persisted per browser. The zoom control is deliberately THE single zoom knob: cluster labels (below) will gate inversely on it over the same fade span. Settings changes rebuild only the renderer: layout positions and the camera never move. (Edge-length / spacing controls deliberately parked until a cluster-aware spacing design exists.) +**Render settings (2026-09-13):** four basic controls in the graph view — node size, edge width, arrowhead length, and **the zoom-level control** (the label-visibility threshold; 1× = fit zoom) — persisted per browser. The zoom control is deliberately THE single zoom knob: cluster labels gate inversely on it over the same fade span (implemented: node labels fade out over the span as cluster labels fade in — they never coexist). Settings changes rebuild only the renderer: layout positions and the camera never move. (Edge-length / spacing controls deliberately parked until a cluster-aware spacing design exists.) - **Library: sigma.js v3 + graphology + graphology-layout-forceatlas2.** ~40 KB gzipped all-in — half of vis-network, a third of cytoscape, far below react-force-graph (~250 KB deps) and cosmos.gl (luma.gl stack). - **Layout: synchronous run-then-freeze.** `forceAtlas2.assign(graph, { iterations: 100, settings: inferSettings(graph) })` runs once, writes x/y, and no animation loop ever starts — near-zero idle CPU (sigma only redraws on camera/graph events). <150 ms at hundreds of nodes; at ~2k enable `barnesHutOptimize`, then the bundled Web Worker, then server-cached positions — in that order, only if latency bites. From 9df8044d614cf829761a1729cf5a946866f126a5 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 04:23:32 +0300 Subject: [PATCH 10/34] Cluster hulls: fix inverted offset normal, kill per-frame blur, labels toggle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three problems from the first cut (user report 2026-09-13), all rooted and fixed: 1. Boundaries reshaping on zoom — an inward offset normal. In y-down screen coordinates, monotone chain yields clockwise-on-screen hulls, whose OUTWARD normal is (dy, -dx); the code used (-dy, dx), so the 14px padding pulled every hull vertex 14-20px INWARD. Zoomed in this mostly passed; zoomed out, once a cluster projected smaller than ~2x the pad, vertices crossed the centroid and the polygon self- intersected — spiky garbage re-forming every frame. Geometry moved to components/cluster-geometry.ts (exported, unit-tested) with the corrected normal; tests pin the direction (offset moves every vertex AWAY from the centroid; stays sane when the hull is smaller than the pad; hull orientation asserted clockwise-on-screen). 2. GPU ~100% + audible coil whine on any interaction — the hull redraw was hooked to sigma's afterRender, which fires for EVERY render including ~a dozen hover-animation frames per hover, and each redraw rasterized ~32 fills through ctx.filter blur(12px), a GPU-heavy full-canvas filter. Now: redraws happen ONLY on camera updates and resizes (hover leaves the hull canvas untouched — verified live), and the blur is replaced with layered low-alpha fills (soft edge at a fraction of the cost, no canvas filter at all). 3. Cluster labels cluttering the view — a persisted 'Cluster labels' checkbox in the rendering settings (hulls stay, text toggles). 377/377 tests, typecheck, lint, build green. Live-verified: hover does not repaint hulls, zoom does; far zoom-out yields 3 contiguous tinted regions (no fragmentation); toggle clears/returns labels and persists. --- components/cluster-geometry.ts | 80 +++++++++++++++++++++ components/graph-view.tsx | 122 +++++++++++---------------------- tests/clusters.test.ts | 93 +++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 81 deletions(-) create mode 100644 components/cluster-geometry.ts diff --git a/components/cluster-geometry.ts b/components/cluster-geometry.ts new file mode 100644 index 0000000..15d9a80 --- /dev/null +++ b/components/cluster-geometry.ts @@ -0,0 +1,80 @@ +/** + * CLUSTER HULL GEOMETRY (vision §1) — pure presentation math for the + * knowledge-graph regions. Extracted from graph-view.tsx so the normal + * direction and hull correctness are UNIT-TESTED (the first version's + * offset normal pointed INWARD in screen coordinates — in y-down screen + * space, monotone chain yields clockwise-on-screen hulls, whose outward + * normal is (dy, -dx), not (-dy, dx) — which made every boundary SHRINK + * 14px per vertex and self-intersect into spiky garbage as soon as a + * cluster projected smaller than the padding; shapes churned on every + * zoom frame. See tests/clusters.test.ts: "offsetPolygon moves every + * vertex AWAY from the centroid".) + */ + +export type Pt = { x: number; y: number }; + +/** Stable hue per cluster id (content-hash): the region keeps its color + * across reshuffles because the id only changes when the members do. */ +export function clusterHue(id: string): number { + let h = 0; + for (const ch of id) h = (h * 31 + ch.charCodeAt(0)) % 360; + return h; +} + +/** Convex hull (Andrew monotone chain). In y-DOWN screen coordinates this + * yields a CLOCKWISE-on-screen ring — the orientation + * outwardNormal/offsetPolygon below are written for. */ +export function convexHull(points: Pt[]): Pt[] { + if (points.length < 3) return points; + const pts = [...points].sort((a, b) => a.x - b.x || a.y - b.y); + const cross = (o: Pt, a: Pt, b: Pt) => (a.x - o.x) * (b.y - o.y) - (a.y - o.y) * (b.x - o.x); + const lower: Pt[] = []; + for (const p of pts) { + while (lower.length >= 2 && cross(lower[lower.length - 2]!, lower[lower.length - 1]!, p) <= 0) lower.pop(); + lower.push(p); + } + const upper: Pt[] = []; + for (let i = pts.length - 1; i >= 0; i--) { + const p = pts[i]!; + while (upper.length >= 2 && cross(upper[upper.length - 2]!, upper[upper.length - 1]!, p) <= 0) upper.pop(); + upper.push(p); + } + upper.pop(); + lower.pop(); + return [...lower, ...upper]; +} + +/** Outward edge normal of segment a→b for CLOCKWISE-on-screen rings + * (y-down coords). (dy, -dx): for the top edge of a square traversed + * left→right, this points up-screen — away from the interior. */ +export function outwardNormal(a: Pt, b: Pt): Pt { + const dx = b.x - a.x; + const dy = b.y - a.y; + const len = Math.hypot(dx, dy) || 1; + return { x: dy / len, y: -dx / len }; +} + +/** Offset a convex polygon outward by `pad` px (per-vertex normal sum — + * each corner moves by up to ~1.41×pad, giving rounded-square-ish + * corners before the arc rounding is even applied). */ +export function offsetPolygon(points: Pt[], pad: number): Pt[] { + return points.map((p, i) => { + const prev = points[(i - 1 + points.length) % points.length]!; + const next = points[(i + 1) % points.length]!; + const n1 = outwardNormal(prev, p); + const n2 = outwardNormal(p, next); + return { x: p.x + (n1.x + n2.x) * pad, y: p.y + (n1.y + n2.y) * pad }; + }); +} + +/** Path through the polygon with rounded corners (arcTo). */ +export function traceRoundedPath(ctx: CanvasRenderingContext2D, points: Pt[], radius: number): void { + const first = points[0]!; + ctx.moveTo(first.x, first.y); + for (let i = 1; i <= points.length; i++) { + const corner = points[i % points.length]!; + const next = points[(i + 1) % points.length]!; + ctx.arcTo(corner.x, corner.y, next.x, next.y, radius); + } + ctx.closePath(); +} diff --git a/components/graph-view.tsx b/components/graph-view.tsx index 0797f44..ba185cd 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -79,6 +79,7 @@ import Graph from "graphology"; import forceAtlas2 from "graphology-layout-forceatlas2"; import type { NodeHoverDrawingFunction, NodeLabelDrawingFunction } from "sigma/rendering"; import { seededRandom, type GraphNode, type GraphSnapshot } from "@/src/core"; +import { clusterHue, convexHull, offsetPolygon, traceRoundedPath, type Pt } from "./cluster-geometry.ts"; /** The uniform node look (no state classes — WebGL wants hex, not oklch): * the accent color, until the state model lands. */ @@ -107,8 +108,11 @@ interface RenderSettings { * settings for exactly that reuse. */ labelZoom: number; + /** Toggle for the cluster region labels (the hulls stay) — they can + * crowd the screen at fit zoom with many regions (user, 2026-09-13). */ + clusterLabels: boolean; } -const DEFAULT_RENDER_SETTINGS: RenderSettings = { nodeSize: 6, edgeWidth: 1, arrowLength: 5, labelZoom: 0.5 }; +const DEFAULT_RENDER_SETTINGS: RenderSettings = { nodeSize: 6, edgeWidth: 1, arrowLength: 5, labelZoom: 0.5, clusterLabels: true }; const RENDER_SETTINGS_KEY = "graph-render-settings"; const clamp = (v: number, min: number, max: number) => Math.min(Math.max(v, min), max); /** Defensively merge stored settings (stale/partial/corrupt → defaults). */ @@ -119,6 +123,7 @@ function sanitizeRenderSettings(raw: unknown): RenderSettings { edgeWidth: clamp(Number(r?.edgeWidth ?? DEFAULT_RENDER_SETTINGS.edgeWidth), 0.5, 4), arrowLength: clamp(Number(r?.arrowLength ?? DEFAULT_RENDER_SETTINGS.arrowLength), 2, 15), labelZoom: clamp(Number(r?.labelZoom ?? DEFAULT_RENDER_SETTINGS.labelZoom), 0.25, 4), + clusterLabels: typeof r?.clusterLabels === "boolean" ? r.clusterLabels : DEFAULT_RENDER_SETTINGS.clusterLabels, }; } @@ -423,71 +428,9 @@ function SettingSlider({ // ─── cluster hulls (vision §1, DECIDED 2026-09-13) ──────────────────────────── // Boundary geometry is a PURE FUNCTION OF POSITIONS AT DRAW TIME: every // redraw re-derives the hull from current member positions mapped through -// the camera, so pan/zoom/relayout can never invalidate it. No metaballs -// in v1 — convex hull + outward offset + rounded corners; revisit when -// regions want concave fidelity. - -/** Stable hue per cluster id (content-hash): the region keeps its color - * across reshuffles because the id only changes when the members do. */ -function clusterHue(id: string): number { - let h = 0; - for (const ch of id) h = (h * 31 + ch.charCodeAt(0)) % 360; - return h; -} - -type Pt = { x: number; y: number }; - -/** Convex hull (Andrew monotone chain), viewport pixels. */ -function convexHull(points: Pt[]): Pt[] { - if (points.length < 3) return points; - const pts = [...points].sort((a, b) => a.x - b.x || a.y - b.y); - const cross = (o: Pt, a: Pt, b: Pt) => (a.x - o.x) * (b.y - o.y) - (a.y - o.y) * (b.x - o.x); - const lower: Pt[] = []; - for (const p of pts) { - while (lower.length >= 2 && cross(lower[lower.length - 2]!, lower[lower.length - 1]!, p) <= 0) lower.pop(); - lower.push(p); - } - const upper: Pt[] = []; - for (let i = pts.length - 1; i >= 0; i--) { - const p = pts[i]!; - while (upper.length >= 2 && cross(upper[upper.length - 2]!, upper[upper.length - 1]!, p) <= 0) upper.pop(); - upper.push(p); - } - upper.pop(); - lower.pop(); - return [...lower, ...upper]; -} - -/** Outward edge normal of segment a→b (screen coords: y grows down). */ -function outwardNormal(a: Pt, b: Pt): Pt { - const dx = b.x - a.x; - const dy = b.y - a.y; - const len = Math.hypot(dx, dy) || 1; - return { x: -dy / len, y: dx / len }; -} - -/** Offset a convex polygon outward by `pad` px (per-vertex normal sum). */ -function offsetPolygon(points: Pt[], pad: number): Pt[] { - return points.map((p, i) => { - const prev = points[(i - 1 + points.length) % points.length]!; - const next = points[(i + 1) % points.length]!; - const n1 = outwardNormal(prev, p); - const n2 = outwardNormal(p, next); - return { x: p.x + (n1.x + n2.x) * pad, y: p.y + (n1.y + n2.y) * pad }; - }); -} - -/** Path through the polygon with rounded corners (arcTo). */ -function traceRoundedPath(ctx: CanvasRenderingContext2D, points: Pt[], radius: number): void { - const first = points[0]!; - ctx.moveTo(first.x, first.y); - for (let i = 1; i <= points.length; i++) { - const corner = points[i % points.length]!; - const next = points[(i + 1) % points.length]!; - ctx.arcTo(corner.x, corner.y, next.x, next.y, radius); - } - ctx.closePath(); -} +// the camera, so pan/zoom/relayout can never invalidate it. The math +// lives in ./cluster-geometry.ts (unit-tested — see that file's header +// for the inward-normal bug this extraction is named after). export function GraphView() { const containerRef = useRef(null); @@ -797,32 +740,38 @@ export function GraphView() { const hull = convexHull(pts); if (hull.length < 3) continue; const hue = clusterHue(cluster.id); - // Blurred low-opacity fill + dashed boundary stroke. - const outer = offsetPolygon(hull, 14); - hullCtx.beginPath(); - traceRoundedPath(hullCtx, outer, 9); - // ctx.filter degrades to flat fill where unsupported. - try { - hullCtx.filter = "blur(12px)"; - } catch { - /* older browsers: skip the blur */ + // Soft translucent tint WITHOUT ctx.filter: a canvas blur + // filter rasterizes on the GPU per call — hooked to every + // render that was ~32 blurred fills per frame (sustained + // ~100% GPU rasterization + audible coil whine; reported + // 2026-09-13). Layered low-alpha fills approximate the + // same soft edge for a fraction of the cost. + for (const [pad, alpha] of [[24, 0.025], [19, 0.04], [14, 0.06]] as const) { + hullCtx.beginPath(); + traceRoundedPath(hullCtx, offsetPolygon(hull, pad), 9); + hullCtx.fillStyle = `hsla(${hue}, 65%, 60%, ${alpha})`; + hullCtx.fill(); } - hullCtx.fillStyle = `hsla(${hue}, 65%, 60%, 0.08)`; - hullCtx.fill(); - hullCtx.filter = "none"; + // Dashed boundary on the innermost offset. + hullCtx.beginPath(); + traceRoundedPath(hullCtx, offsetPolygon(hull, 14), 9); hullCtx.setLineDash([7, 7]); hullCtx.strokeStyle = `hsla(${hue}, 65%, 62%, 0.5)`; hullCtx.lineWidth = 1.5; hullCtx.stroke(); hullCtx.setLineDash([]); - // The region label at the centroid (inverse-gated layer). - if (!cluster.label) continue; + // The region label at the centroid (inverse-gated layer) — + // skippable entirely via the settings toggle. + if (!settings.clusterLabels || !cluster.label) continue; const centroid = hull.reduce((acc, p) => ({ x: acc.x + p.x / hull.length, y: acc.y + p.y / hull.length }), { x: 0, y: 0 }); labelCtx.fillStyle = `hsla(${hue}, 55%, 78%, 0.85)`; labelCtx.fillText(cluster.label, centroid.x, centroid.y); } }; - renderer.on("afterRender", drawClusters); + // Redraw ONLY when the view actually moves: camera updates and + // resizes. NOT on sigma's afterRender — hover animations re-render + // sigma many frames per interaction, and hulls don't care. + renderer.on("resize", drawClusters); renderer.on("clickNode", ({ node }) => { const graphNode = snapshot.nodes.find((n) => n.id === node); @@ -858,9 +807,11 @@ export function GraphView() { // the exact span node labels fade out — THE one zoom control. regionCanvas.style.opacity = (1 - alpha).toFixed(3); }; + drawClusters(); syncRenderLabels(renderer.getCamera().getState().ratio); renderer.getCamera().on("updated", (state) => { syncRenderLabels(state.ratio); + drawClusters(); setSelected(null); }); // The hover fade: entering schedules renders (sigma re-invokes @@ -1028,6 +979,15 @@ export function GraphView() { display={`${settings.labelZoom}×`} onChange={(v) => setSettings((cur) => ({ ...cur, labelZoom: v }))} /> +
)} {snapshot !== null && visibleIds.size === 0 && query.trim() !== "" && ( diff --git a/tests/clusters.test.ts b/tests/clusters.test.ts index 8050101..84cd467 100644 --- a/tests/clusters.test.ts +++ b/tests/clusters.test.ts @@ -7,6 +7,7 @@ import { strict as assert } from "node:assert"; import { describe, test } from "node:test"; import { detectCommunities, matchClusterLabelings, MIN_CLUSTER_SIZE } from "../src/core/clusters.ts"; +import { clusterHue, convexHull, offsetPolygon, outwardNormal } from "../components/cluster-geometry.ts"; import type { Community } from "../src/core/clusters.ts"; import type { ClusterLabeling } from "../src/core/contracts.ts"; @@ -72,6 +73,98 @@ describe("matchClusterLabelings", () => { }); }); +describe("hull geometry (the inward-normal regression)", () => { + // Screen coordinates: y grows DOWN. The first implementation used + // (-dy, dx) as the "outward" normal — which points INWARD for the + // clockwise-on-screen rings monotone chain produces here, shrinking + // every hull by the pad and inverting it entirely once a cluster + // projected smaller than ~2x the pad (spiky shape churn on zoom). + + const dist = (a: { x: number; y: number }, b: { x: number; y: number }) => Math.hypot(a.x - b.x, a.y - b.y); + + test("outwardNormal points away from a square's interior (screen coords)", () => { + const square = [ + { x: 0, y: 0 }, + { x: 10, y: 0 }, + { x: 10, y: 10 }, + { x: 0, y: 10 }, + ]; + const center = { x: 5, y: 5 }; + for (let i = 0; i < 4; i++) { + const a = square[i]!; + const b = square[(i + 1) % 4]!; + const mid = { x: (a.x + b.x) / 2, y: (a.y + b.y) / 2 }; + const n = outwardNormal(a, b); + // The normal translated from the edge midpoint moves AWAY from the center. + const moved = { x: mid.x + n.x, y: mid.y + n.y }; + assert.ok(dist(moved, center) > dist(mid, center), `edge ${i}: normal points inward`); + } + }); + + test("offsetPolygon moves every vertex AWAY from the centroid", () => { + const hull = convexHull([ + { x: 0, y: 0 }, { x: 40, y: 3 }, { x: 55, y: 30 }, { x: 20, y: 44 }, { x: 2, y: 25 }, + ]); + const centroid = { + x: hull.reduce((s, p) => s + p.x, 0) / hull.length, + y: hull.reduce((s, p) => s + p.y, 0) / hull.length, + }; + for (const [i, v] of hull.entries()) { + const moved = offsetPolygon(hull, 14)[i]!; + assert.ok( + dist(moved, centroid) > dist(v, centroid), + `vertex ${i} moved inward (was ${dist(v, centroid).toFixed(1)}, now ${dist(moved, centroid).toFixed(1)})`, + ); + } + }); + + test("offsetPolygon stays sane when the hull is SMALLER than the pad (the zoom-out case)", () => { + // A 10px cluster padded by 14px: must still be a simple polygon + // (no vertex crossing the centroid, no inversion). + const hull = convexHull([ + { x: 0, y: 0 }, { x: 10, y: 1 }, { x: 9, y: 9 }, { x: 1, y: 8 }, + ]); + const centroid = { + x: hull.reduce((s, p) => s + p.x, 0) / hull.length, + y: hull.reduce((s, p) => s + p.y, 0) / hull.length, + }; + const moved = offsetPolygon(hull, 14); + for (const v of moved) { + assert.ok(dist(v, centroid) >= 10, `vertex collapsed past the centroid: ${JSON.stringify(v)}`); + } + // Vertex ORDER stays angularly monotone around the centroid (no self-intersection). + const angles = moved.map((v) => Math.atan2(v.y - centroid.y, v.x - centroid.x)); + let prev = angles[0]!; + for (const a of angles.slice(1)) { + let d = a - prev; + while (d > Math.PI) d -= 2 * Math.PI; + while (d < -Math.PI) d += 2 * Math.PI; + assert.ok(d !== 0, "duplicate angle — degenerate offset"); + prev = a; + } + }); + + test("convexHull returns a clockwise-on-screen ring (the orientation the normal math assumes)", () => { + const hull = convexHull([ + { x: 0, y: 0 }, { x: 10, y: 0 }, { x: 10, y: 10 }, { x: 0, y: 10 }, + ]); + // Signed area in code coords: clockwise-on-screen (y-down) = positive. + let area = 0; + for (let i = 0; i < hull.length; i++) { + const a = hull[i]!; + const b = hull[(i + 1) % hull.length]!; + area += a.x * b.y - b.x * a.y; + } + assert.ok(area / 2 > 0, `expected positive (clockwise-on-screen) area, got ${area / 2}`); + }); + + test("clusterHue is stable per id and spreads", () => { + assert.equal(clusterHue("clu_abc123def456"), clusterHue("clu_abc123def456")); + const hues = new Set(Array.from({ length: 40 }, (_, i) => clusterHue(`clu_${i}`))); + assert.ok(hues.size > 25, `expected spread hues, got ${hues.size}`); + }); +}); + describe("detectCommunities", () => { test("two dense cliques joined by a single bridge split into two communities", () => { const left = ids(4, "L"); From 0513b1555cd6eed9577abca870b0ac67cba34f72 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 04:59:56 +0300 Subject: [PATCH 11/34] Research: graph rendering & cluster-first layout (decision basis) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four research passes (two library fact-checkers, two prior-art explorers) with all claims verified against shipped package code (npm tarballs of sigma@3.0.3 and @sigma/layer-webgl@3.0.0), the sigma.js storybook source, and primary docs. Nothing implemented yet — this is the decision document. Headline verified facts: - sigma v3.0.3 ships a PUBLIC createCanvas/createLayer API; our DOM injection was improvising a worse version of it. Sigma's own cluster-label storybook demonstrates our exact minimal-fix pattern (custom layer + afterRender + graphToViewport). - @sigma/layer-webgl contours render METABALL fields, not hull polygons; no dashed strokes; one WebGL context per community (~16 browser cap). Our visual is not achievable with shipped sigma layer programs. - Nobody in the graph-tool world ships hull regions (Gephi, Gephi Lite, Obsidian: node coloring only). Aesthetic references: ggforce geom_mark_hull, Bostock's thick-stroke hull trick, Excalidraw frames. - Canvas 2D at 338→low-thousands of elements is comfortably within demonstrated 60fps territory; the cost driver is repaint count x pixel area, not element count. - Cluster-first layout: two-level pattern (quotient anchors -> per-cluster layout -> compose) is well-trodden; LinLog FA2 is the one-afternoon A/B candidate; no turnkey JS package composes it (~150 lines of glue). Options scored: sigma + public 2D canvas layer (7/10, ~1d minimal fix), hand-rolled d3-zoom + single Canvas 2D (8/10, recommended target), PixiJS v8 (7/10). Full matrix, facts-not-to-rely-on list, and open items in the document. --- docs/research/graph-rendering.md | 227 +++++++++++++++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 docs/research/graph-rendering.md diff --git a/docs/research/graph-rendering.md b/docs/research/graph-rendering.md new file mode 100644 index 0000000..78273e6 --- /dev/null +++ b/docs/research/graph-rendering.md @@ -0,0 +1,227 @@ +# Graph rendering & cluster-first layout — research findings + +*Compiled 2026-09-13 from four research passes (library fact-check ×2, prior-art ×2), all +claims source-verified by subagents against shipped package code (`npm pack` tarballs of +`sigma@3.0.3`, `@sigma/layer-webgl@3.0.0`), the sigma.js GitHub storybook source, and +primary docs. Speculation is marked where it appears. This document is the decision +basis for the rendering rebuild; nothing has been implemented yet.* + +--- + +## 1. Requirements (as of this research) + +Current symptoms to eliminate: +- Hull layers lag behind nodes during pan (desync). +- Settings changes shift hulls up-left until the next interaction (canvas-sizing race). +- GPU ~50–100% during pan/zoom (redundant full-canvas 2D repaints per camera event + + WebGL re-render per frame; the earlier `ctx.filter: blur` per frame caused coil whine). +- Stale/garbage boundary shapes at extreme zoom-out. + +Feature set: +- Circles + directed arrows; precomputed layout (no runtime physics). +- Cluster regions: hull boundary around each community, dashed stroke, soft translucent + fill, stable color per region, region name label. +- Inverse zoom gates: node labels when zoomed IN, region labels when zoomed OUT, + cross-fading over a shared span. +- Pan, zoom-to-cursor, hover glow, click → detail card, edge-type filtering, camera + survives UI changes. 60fps, minimal GPU, ordinary laptop. + +New direction (user, 2026-09-13): +- **Cluster-first placement**: layout depends on clustering — clusters placed as units, + nodes within a cluster close together (compact, natural hull shapes as a consequence). +- **Freeform experimentation**: a dev-only playground to try layouts/shapes/styles + against real and synthetic data; renderer must be dumb, data-driven, swappable. + +Scale: 338 nodes / 211 edges today; low thousands expected (notes imports). + +## 2. Verified facts about the current stack (sigma.js v3) + +- **v3.0.3 ships a PUBLIC custom-layer API** — `createLayer` / `createCanvas` / + `createCanvasContext` / `killLayer` (typed in shipped `.d.ts`). Our current + `firstChild` DOM injection was improvising a worse version of a supported API. +- **Sigma sizes and DPR-scales ALL registered canvases** in its own `resize()` — + using the public API eliminates the canvas-sizing race by construction. +- **`afterRender` fires once per rendered frame** (rAF-debounced `scheduleRender()`). + Camera motion → full re-render each rAF. **Hover is cheap**: it only redraws the + `hoverNodes` WebGL layer + `edgeLabels` canvas via a separate rAF; no `afterRender`, + no full re-render. Sigma is strictly event-driven: no continuous loop, no rendering + when idle (verified: zero `setInterval`, all rAF sites are schedulers). +- **`renderer.graphToViewport()` is public and typed** — the sanctioned graph→pixel + path. `renderer.matrix` is **private**: building a `ctx.setTransform()` from it would + be guessing at internals; per-vertex `graphToViewport()` is the supported route. +- **The official cluster-label storybook example is exactly our pattern**: + custom DOM layer + `afterRender` + per-cluster `graphToViewport()` re-projection, + maintained in the sigma repo. Our minimal-change fix path is a demonstrated usage. +- Real v3 layer order (verified in source, NOT in docs): + `edges` (WebGL) → `edgeLabels` → `nodes` (WebGL) → `labels` → `hovers` → + `hoverNodes` (WebGL) → `mouse`. + +### `@sigma/layer-webgl@3.0.0` (official background-layer package) +- `bindWebGLLayer` inserts a background WebGL layer (`beforeLayer: "edges"`), redrawn + on `afterRender` with the same camera matrix — fully in-loop, no desync. +- **`createContoursProgram` renders METABALL-style implicit fields, not hull polygons** + (verified from the fragment shader: fullscreen quad, summed radial falloffs, + analytic contour lines via `dFdx/dFdy`). Translucent fill: yes (arbitrary CSS color + with alpha). **Dashed stroke: NO** — solid anti-aliased line only; no dash parameter + exists. Per-community color: yes, but one program + one WebGL context per community, + and each runs a fullscreen fragment pass per frame (browsers cap ~16 WebGL contexts — + a problem for dozens of regions). +- **Decisive fact**: our exact visual (hull polygon + translucent fill + dashed + stroke) is **NOT achievable with shipped layer-webgl programs**. It would require a + custom `WebGLLayerProgram` (supported extension point, moderate effort, dashes DIY in + GLSL) — whereas Canvas 2D gives it for free with `ctx.setLineDash()`. +- v4-era programs (heatmap, density-splat, dashed/fill layer factories) are NOT in the + v3 package. + +### sigma v4 +- Real but **beta** (4.0.0-beta.5; docs site self-describes as alpha). Single `stage` + canvas + public `createLayer`, GPU picking with downsizing, SDF labels, depth + layers. Breaking migration. Not a production bet today. + +### Pan GPU cost at our scale +- Structural facts verified (event-driven idle; full re-render per camera frame; + synchronous `gl.readPixels` picking per mousemove pre-#1546, since fixed). +- **Absolute GPU cost of panning 338–3k nodes in v3: UNVERIFIED** — no public + benchmark exists; nobody has measured it. If it matters, we measure it ourselves. + +## 3. Verified facts about the candidates + +| Candidate | Renderer | True hull fill+dash | Picking | Camera | Maintenance | License | Fit | +|---|---|---|---|---|---|---|---| +| sigma v3 + public 2D canvas layer | WebGL (+our 2D) | yes (our canvas, free) | GPU color-pick | built-in | active (3.0.3, 2026-04) | MIT | 7/10 | +| sigma v3 + layer-webgl contours | WebGL | **no** (metaballs, solid border) | — | built-in | active | MIT | 6/10 | +| sigma v4 + layers | WebGL, SDF | via custom program (beta risk) | GPU picking | built-in | **beta** | MIT | 6 (risky) | +| d3-zoom + Canvas 2D (hand-rolled loop) | Canvas 2D | yes, trivially | distance check | d3-zoom (verified math) | d3 very active | ISC | **8/10** | +| PixiJS v8 + pixi-viewport | WebGL/WebGPU/Canvas | yes (Graphics; dashes via shim `pixi-dashed-line`) | built-in events | pixi-viewport 6.0.3 | very active | MIT | 7/10 | +| cytoscape.js 3.34 | Canvas 2D | via bubblesets/layers extensions | built-in | built-in | active | MIT | 5/10 | +| vis-network 10.1 | Canvas 2D | **no** — clustering = supernodes (collapse, opposite of our regions) | built-in | built-in | sporadic | MIT | 2/10 | +| @cosmos.gl/graph (Cosmograph engine) | WebGL (luma.gl) | no (cluster = layout force only) | GPU | built-in | active | MIT | 3/10 | + +Fact nuances we must not over-trust: +- "Canvas 2D is GPU-rasterized": **half-true**. Skia may raster on CPU or GPU depending + on browser/OS config. Either way the cost driver is **repaint count × pixel area + (DPR²)**, not "Skia = GPU". +- Canvas 2D at 60fps for hundreds–low-thousands of elements: **supported by practitioner + evidence** (cytoscape is fully canvas-based into the thousands; 10k animated canvas + elements ≈ 55fps in one 2025/26 benchmark). Pixel area matters more than node count + at our scale. +- Obsidian's graph view renderer: **UNVERIFIED** — no primary source. Do not cite. +- PixiJS exact bundle size: unmeasured. yFiles: commercial, unverified details. + +## 4. Prior art for our specific features + +### Region/hull visuals +- Nobody in the graph-tool world ships our region aesthetic: Gephi desktop, Gephi Lite + (the flagship sigma app — zero hull code in its tree), Obsidian — all partition by + **node coloring only**. The region feature is a differentiator, not a copied pattern. +- The aesthetic reference: ggforce `geom_mark_hull` (expand + round the hull, low-alpha + fill, integrated label placement). +- **Bostock's thick-stroke hull trick**: one path, wide stroke (e.g. 32px) + + `linejoin: round` = padded, rounded hull with zero polygon-offset math. Replaces our + offset-normal code outright. +- Named-region UX validation: Excalidraw frames / tldraw regions (dashed titled + container around content). +- Closest feature twin: **Cosmograph** — documents "cluster labels when zoomed out, + point labels when zoomed in" as a first-class use case (our exact inverse gate). + +### Zoom-gated labels +- Consensus pattern = map-style per-layer zoom ranges (Mapbox/MapLibre + `minzoom`/`maxzoom` on symbol layers). State labels at far zoom, city labels at near + zoom is the standard cartographic solution; our inverse gates are the same mechanism + with opposite ranges. Sigma's own `labelRenderedSizeThreshold` is the zoom-in gate. +- Cross-fade + hysteresis over a shared span: no single canonical writeup (medium + confidence); nearest established practice is map label collision systems. + +### The sync pattern (fixes our desync class structurally) +- Canonical and repeatedly documented: **one rAF loop with a dirty flag; input/DOM + events only mutate state.** All layers drawn in the same frame, under the same + camera transform — either same canvas (`setTransform` once, draw back-to-front) or + stacked canvases all redrawn by the same loop. + Sources: browser.engineering, MDN "Optimizing canvas", web.dev canvas performance. + +### Picking +- Canvas 2D at our scale: nearest-node distance check per pointer event is the norm, + trivially fast (µs for 3k nodes; spatial grid only past ~10k). + +## 5. Cluster-first layout (the new direction) + +- **Two-level pattern is well-trodden**: place cluster anchors first (quotient graph — + one super-node per community, Gephi's "Groups by Partition" plugin is the desktop + prior art), layout nodes within each cluster, compose by translating local layouts + to cluster anchors. Bostock's Clustered Force Layout lineage; cosmos.gl's cluster + force verified from source (per-frame center-of-mass pull, or user-pinned positions — + cluster anchors as data independent of member layout). +- **graphology FA2 has NO cluster-aware settings** (full settings list read; gravity is + global). **LinLog mode** (`linLogMode: true`) is the documented "communities separate + tightly" switch — a one-afternoon A/B candidate before any custom pipeline. +- **No turnkey JS package does the composed two-level pipeline** — assemble from + graphology primitives (~150 lines of glue: quotient build, per-cluster FA2 with + circular init, compose transform, `noverlap` cleanup). The combination is our + synthesis from verified primitives. +- Deterministic given fixed init positions (no RNG dependency in FA2). +- **Server-side at extraction time is the better fit** for our architecture: positions + become data, rendering stays dumb. Browser worker fallback for live re-layout. +- Harness precedents: yFiles layout playground (polished), Observable/bl.ocks lineage + (notebook = graph + sliders + canvas), react-sigma layouts example. No canonical OSS + playground for our stack — copy the config-driven pattern (algorithm dropdown + + parameter sliders + seed input + live canvas). + +## 6. Options (the decision) + +**Option A — Stay on sigma v3; mount the hull canvas via the public +`createCanvas` API; redraw on `afterRender` via `graphToViewport()`.** +Supported, minimal-change, demonstrated by sigma's own cluster-label storybook. Kills +the desync, the sizing race, and the multi-event redraws (once per frame, sigma-sized). +Our exact visuals stay (dashes free in 2D). Keeps sigma's camera/labels/picking +machinery. Residual: full-canvas 2D repaint per interaction frame remains (raster cost, +mitigable with move-gating like sigma's own `hideLabelsOnMove`); WebGL + 2D raster +still both run per frame. Est: ~1 day. + +**Option A′ — sigma + custom `WebGLLayerProgram` for hulls.** Supported extension +point, but dashes are DIY GLSL and we'd rebuild in shaders what Canvas 2D gives free. +No reason given A exists. Est: several days, for a worse iteration loop. + +**Option B — d3-zoom + hand-rolled single Canvas 2D renderer.** +One canvas, one rAF + dirty flag, world-space geometry + one transform, all layers +back-to-front in the same frame. Lowest GPU/CPU, total control, perfectly matches the +new requirements: renderer is a dumb draw function over data (layout experiments can't +fight it), playground is trivial, cluster-first layout slots in as pure data. Costs +~600 lines + camera feel + picking + label gating owned by us. Est: ~1 focused day. + +**Option C — PixiJS v8 + pixi-viewport.** Cleanest scene-graph answer (one ticker, +sync by construction), WebGPU headroom. But: dashes need a third-party shim, no graph +semantics, bundle, more machinery than Canvas 2D needs at our scale. Est: 1–2 days. + +**Recommendation**: **Option B as the target** — it is the only option that structurally +matches all three requirement sets (bugs, cluster-first placement, freeform +experimentation), scored highest (8/10), and removes the vendor-internals class of +problems permanently. **Option A is a legitimate ~1-day stabilization** if we want the +current bugs dead immediately while deferring the rebuild; it reuses all our existing +visual code and is the officially sanctioned pattern, but it keeps us on sigma's +opinionated machinery and the experimentation direction will keep pushing against it. + +**Layout work rides along independent of renderer choice**: first A/B +LinLog-only FA2 (one afternoon), then the two-level pipeline +(quotient → per-cluster FA2 → compose → noverlap) if needed, developed in the playground. + +## 7. Facts we should NOT rely on + +- "Obsidian's graph view uses renderer X" — unverified. +- "Sigma's documented canvas DOM order" — source-only, implementation detail, changed + in v4. +- "Canvas 2D is always GPU-rasterized" — false as stated. +- "Sigma v4 is ready" — beta. +- "Sigma pan cost at our scale is X%" — nobody has measured it publicly. +- "@cosmograph/cosmos is the maintained Cosmograph engine" — stale npm channel (Dec + 2024); current engine is `@cosmos.gl/graph` v3. +- Any PixiJS bundle-size number — unmeasured. + +## 8. Open items for the implementation phase + +- Measure pan GPU/CPU ourselves (Chrome profiler) before/after whichever path is chosen. +- Playground design: `/playground` dev-only route, config-driven (layout algorithm + dropdown, parameter sliders, seed input, synthetic + real datasets), same renderer. +- Hull styling decision inside Option B: Bostock thick-stroke trick vs offset polygons + (geometry already unit-tested in `components/cluster-geometry.ts`). +- Label-gate hysteresis: small dead-band in log-zoom to avoid threshold flicker. +- Whether layout runs server-side at extraction (persisted as data) or in-browser. From 4e48d13fac934cb4d4ddde11f736fb5fcc592ef0 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 05:19:15 +0300 Subject: [PATCH 12/34] Renderer swap: drop sigma.js for a hand-rolled single-canvas 2D renderer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements Option B from docs/research/graph-rendering.md. sigma.js v3 (WebGL) is replaced by one canvas, one rAF loop with a dirty flag, and d3-zoom for the camera — eliminating the entire bug class from the sigma-era bolt-on architecture: - Hull/node desync during pan (two independent render loops) is now structurally impossible: hull fills, dashed hull strokes, edges with arrowheads, node discs, region labels, node labels, and hover glow are all drawn in ONE frame under ONE camera. - The settings-change canvas-sizing race (everything shifting up-left) is gone: settings changes only redraw; the canvas and camera persist (verified live: camera identical to 15 decimals across a slider drag). - The GPU floor is gone: idle schedules NO frames at all (verified via pixel-identity over 600ms idle); pan redraws exactly once per frame. components/graph-canvas.ts: GraphCanvasRenderer class plus pure helpers (zoomRatioOf, labelAlpha, fitTransform, arrowGeometry, pickNearest). The sigma camera-ratio convention (1 = fit, smaller = zoomed in) is kept verbatim so persisted labelZoom settings survive the swap; node labels and cluster region labels cross-fade over the same LABELS_FADE_SPAN (0.5, unchanged). Picking is a nearest-node distance check with the 10px pad that replaces the old padded-picking vertex shader. Hull math stays in components/cluster-geometry.ts, reused as-is. No ctx.filter, ever. graph-view.tsx keeps every feature: fetch/refetch on runs-empty, FA2 run-then-freeze layout, localStorage render settings + sanitizers, search filtering with edge drops and card-close, detail card with flipped anchoring and per-source provenance, settings panel, import button, pipeline status chips. Only the renderer plumbing changed. tests/graph-canvas.test.ts pins the pure math: ratio direction, label gate + inverse complement (alphas sum to 1), fit transform padding, arrowhead tip-on-disc-edge and symmetry, picking pad/ties/null cases. Live-verified end-to-end on the real 338-node graph (dev server port 3002): single canvas, paint at fit, hover cursor+glow+fade, click card open/anchored/close, stage click, search filter to zero with hint, label gate opens via slider and zoom, cluster-labels toggle, zoom-to- cursor 0.56px anchor drift, camera persistence across settings, zero idle repaints. sigma dependency removed from package.json; d3-zoom and d3-selection added. Note: the settings feed effect needed isEmpty in its deps — without it the renderer mounted with settings=null and every draw early-returned (blank canvas); caught live, not by typecheck. --- components/graph-canvas.ts | 677 ++++++++++++++++++++++++++++++++ components/graph-view.tsx | 768 +++++++------------------------------ package-lock.json | 13 +- package.json | 3 +- tests/graph-canvas.test.ts | 139 +++++++ 5 files changed, 965 insertions(+), 635 deletions(-) create mode 100644 components/graph-canvas.ts create mode 100644 tests/graph-canvas.test.ts diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts new file mode 100644 index 0000000..741587a --- /dev/null +++ b/components/graph-canvas.ts @@ -0,0 +1,677 @@ +"use client"; + +/** + * The hand-rolled single-canvas graph renderer (decided 2026-09-13 after + * the research pass — docs/research/graph-rendering.md, Option B). + * + * Replaces sigma.js. Why: sigma gave us WebGL node/edge rendering, + * picking, and a camera, but our cluster regions had to live on foreign + * canvases injected next to its private DOM stack, drawn from a separate + * event loop — so hulls lagged behind nodes during pan, raced sigma's + * canvas sizing on renderer rebuilds, and every camera event re-rasterized + * the full hull canvas (GPU spikes + audible coil whine). At our scale + * (338 → low thousands of nodes) one Canvas 2D is comfortably inside + * 60fps territory, and a single loop makes the desync class of bugs + * structurally impossible. + * + * Architecture (the canonical pattern — browser.engineering, MDN + * "Optimizing canvas", web.dev canvas performance): + * - The renderer is a DUMB DRAW FUNCTION over plain data: nodes with + * world-space positions, edges, clusters, colors, settings. Layout + * (forceAtlas2) and filtering happen OUTSIDE, in the React component; + * this module never computes a position. + * - ONE canvas, ONE requestAnimationFrame loop with a dirty flag. Input + * and DOM events only mutate state and mark dirty; the loop draws at + * most once per frame with every layer under the same camera — + * hull fills → hull strokes → edges → nodes → region labels → node + * labels → hover. Nothing can ever lag anything else. + * - Camera: d3-zoom supplies the verified pan / zoom-to-cursor / pinch + * math (screen = world·k + [x,y]). The camera lives in the renderer + * instance, so data refetches and settings changes never move it. + * - Picking: nearest-node distance check (µs at our scale), with a + * PICKING_PAD so the whole hover halo is hit-testable. + * + * Zoom-gated labels (the Obsidian text-fade feel): sigma's camera "ratio" + * convention is kept for the settings' persisted semantics — ratio 1 = + * fit zoom, SMALLER = zoomed in (ratio = kFit / k). Node labels are + * fully visible at ratio ≤ labelZoom and fade out across LABELS_FADE_SPAN + * of ratio; cluster labels fade in over the exact same span (sums to 1) + * so the two never coexist. All labels gate together — no per-node + * popping. + * + * The hover glow is unchanged in look: a soft halo in the node's own + * color at a constant radius, opacity cross-faded over ~180ms, with the + * name brightening on the SAME factor. Only opacity animates; the only + * per-frame work when idle is a dirty-flag check (no rAF is scheduled at + * all until something changes). + * + * Unit tests cover the pure helpers (tests/graph-canvas.test.ts); the + * class itself is DOM-bound and exercised live. Module scope stays + * DOM-free so the file imports cleanly under Node. + */ + +import { select } from "d3-selection"; +import { zoom, zoomIdentity, type ZoomBehavior, type ZoomTransform } from "d3-zoom"; +import { clusterHue, convexHull, offsetPolygon, traceRoundedPath, type Pt } from "./cluster-geometry.ts"; + +// ─── Public data contract ────────────────────────────────────────────────── + +/** One node to draw: world-space position + the short glanceable label. */ +export interface CanvasNode { + id: string; + x: number; + y: number; + label: string; +} + +/** One directed edge (follows | supports); arrow points prerequisite → dependent. */ +export interface CanvasEdge { + from: string; + to: string; +} + +/** One cluster region: member node ids + the LLM-coined label (or null). */ +export interface CanvasCluster { + id: string; + label: string | null; + members: string[]; +} + +/** Theme colors, resolved to #rrggbb by the caller (canvas wants hex, + * and the renderer must not read CSS itself). */ +export interface CanvasColors { + node: string; + edge: string; + label: string; + hoverText: string; +} + +/** The rendering controls (persisted by the caller in localStorage). */ +export interface GraphRenderSettings { + nodeSize: number; + edgeWidth: number; + arrowLength: number; + /** Camera-ratio threshold for node labels (1 = fit zoom, smaller = + * zoomed in); cluster labels fade in over the same span, inversely. */ + labelZoom: number; + /** Toggle for the cluster region labels (hulls always draw). */ + clusterLabels: boolean; +} + +export interface CanvasData { + nodes: CanvasNode[]; + edges: CanvasEdge[]; + clusters: CanvasCluster[]; + colors: CanvasColors; + /** Body font family (labels must match the app typography). */ + fontFamily: string; +} + +export interface RendererCallbacks { + /** A node was clicked (screen position included for card anchoring). */ + onSelect: (nodeId: string, screenPos: Pt) => void; + /** The empty stage was clicked. */ + onStageClick: () => void; + /** Any camera move (pan/zoom) — the caller dismisses anchored UI. */ + onCameraMove: () => void; +} + +// ─── Constants (ported verbatim from the sigma view) ──────────────────────── + +const LABEL_SIZE = 12; // px (sigma's labelSize) +const LABEL_WEIGHT = "500"; +const LABEL_MAX_WIDTH = 170; // px — wraps long names instead of a wide strip +const LABEL_GAP = 6; // px between node edge and first label line +const LABEL_LINE_HEIGHT = 1.3; +/** Zoom span (camera-ratio units) over which labels fade together. */ +export const LABELS_FADE_SPAN = 0.5; + +const GLOW_PAD = 9; // px the halo extends beyond the node disc (constant) +const GLOW_ALPHA = 0.45; // halo opacity at the disc edge, fading outward +const HOVER_FADE_MS = 180; // hover halo opacity fade length + +/** Extra pickable radius (px) beyond the disc — the whole halo is + * hit-testable (the old flicker fix, minus the picking shader). */ +const PICKING_PAD = 10; + +/** Click vs drag threshold: pointer travel under this many px is a click. */ +const CLICK_SLOP = 4; + +/** Cluster hull styling: layered low-alpha fills approximate the soft + * translucent tint (a canvas blur filter rasterizes on the GPU per call + * — the original coil-whine driver; never bring ctx.filter back). */ +const HULL_FILL_LAYERS = [ + [24, 0.025], + [19, 0.04], + [14, 0.06], +] as const; +const HULL_STROKE_PAD = 14; +const HULL_CORNER_RADIUS = 9; +const HULL_DASH = [7, 7] as const; +const REGION_LABEL_SIZE = 18; + +const clamp01 = (v: number) => Math.min(Math.max(v, 0), 1); + +// ─── Pure helpers (unit-tested) ───────────────────────────────────────────── + +/** + * The sigma-compatible "camera ratio": 1 at fit zoom, SMALLER when zoomed + * in (ratio = kFit / k). The persisted labelZoom setting is denominated in + * these units, so the gate semantics survive the renderer swap unchanged. + */ +export function zoomRatioOf(kFit: number, k: number): number { + return kFit / k; +} + +/** + * Node-label opacity at a given camera ratio: 1 at/below the labelZoom + * threshold, fading to 0 across LABELS_FADE_SPAN of ratio. Cluster labels + * use `1 - this` over the same span — the two cross-fade and never + * coexist at full opacity. + */ +export function labelAlpha(ratio: number, labelZoom: number, span = LABELS_FADE_SPAN): number { + return clamp01(1 - (ratio - labelZoom) / span); +} + +/** The camera transform that frames `bbox` in a w×h viewport with padding. */ +export function fitTransform( + bbox: { minX: number; maxX: number; minY: number; maxY: number }, + w: number, + h: number, + padding: number, +): { x: number; y: number; k: number } { + const bw = Math.max(bbox.maxX - bbox.minX, 1e-6); + const bh = Math.max(bbox.maxY - bbox.minY, 1e-6); + const k = Math.min((w - 2 * padding) / bw, (h - 2 * padding) / bh); + const cx = (bbox.minX + bbox.maxX) / 2; + const cy = (bbox.minY + bbox.maxY) / 2; + return { x: w / 2 - cx * k, y: h / 2 - cy * k, k }; +} + +/** The arrowhead geometry for a directed edge a→b: shaft endpoints + * (clear of both node discs) and the head triangle (tip on the target + * disc edge, wideness = 0.8 × head length — the sigma arrow shape). */ +export function arrowGeometry( + a: Pt, + b: Pt, + fromRadius: number, + toRadius: number, + headLength: number, +): { shaftFrom: Pt; shaftTo: Pt; tip: Pt; left: Pt; right: Pt } | null { + const dx = b.x - a.x; + const dy = b.y - a.y; + const len = Math.hypot(dx, dy); + if (len < 1e-6) return null; // degenerate (coincident) — skip + const ux = dx / len; + const uy = dy / len; + const tip = { x: b.x - ux * toRadius, y: b.y - uy * toRadius }; + const base = { x: tip.x - ux * headLength, y: tip.y - uy * headLength }; + const halfW = 0.4 * headLength; + return { + shaftFrom: { x: a.x + ux * fromRadius, y: a.y + uy * fromRadius }, + shaftTo: base, + tip, + left: { x: base.x - uy * halfW, y: base.y + ux * halfW }, + right: { x: base.x + uy * halfW, y: base.y - ux * halfW }, + }; +} + +/** Nearest node within maxDist of a screen point (linear scan — µs at + * our scale; a spatial grid only earns its keep past ~10k nodes). */ +export function pickNearest( + nodes: T[], + px: number, + py: number, + maxDist: number, +): T | null { + let best: T | null = null; + let bestD = maxDist * maxDist; // compare squared distances + for (const n of nodes) { + const d = (n.x - px) ** 2 + (n.y - py) ** 2; + if (d < bestD) { + bestD = d; + best = n; + } + } + return best; +} + +// ─── Canvas text helpers (ported verbatim) ───────────────────────────────── + +/** "#rrggbb" + alpha → "rgba(...)". */ +function rgbaFromHex(hex: string, alpha: number): string { + const r = Number.parseInt(hex.slice(1, 3), 16); + const g = Number.parseInt(hex.slice(3, 5), 16); + const b = Number.parseInt(hex.slice(5, 7), 16); + return `rgba(${r}, ${g}, ${b}, ${alpha})`; +} + +/** Word-wrap for canvas text. Never truncates: overlong single words are + * hard-broken by character so no line ever exceeds maxWidth. */ +function wrapLabel(context: CanvasRenderingContext2D, text: string, maxWidth: number): string[] { + const lines: string[] = []; + for (const word of text.split(/\s+/).filter(Boolean)) { + const last = lines.length > 0 ? lines[lines.length - 1]! : null; + if (last !== null) { + const joined = `${last} ${word}`; + if (context.measureText(joined).width <= maxWidth) { + lines[lines.length - 1] = joined; + continue; + } + } + let chunk = ""; + for (const char of word) { + if (chunk && context.measureText(`${chunk}${char}`).width > maxWidth) { + lines.push(chunk); + chunk = char; + } else { + chunk += char; + } + } + if (chunk) lines.push(chunk); + } + return lines; +} + +/** The wrapped node name, centered below the node — identical geometry at + * rest and on hover, so hover only changes the color, nothing jumps. */ +function drawName( + context: CanvasRenderingContext2D, + x: number, + y: number, + size: number, + label: string, + font: string, + color: string, +): void { + context.save(); + context.font = `${LABEL_WEIGHT} ${LABEL_SIZE}px ${font}`; + const lines = wrapLabel(context, label, LABEL_MAX_WIDTH); + context.fillStyle = color; + context.textAlign = "center"; + context.textBaseline = "top"; + const firstTop = y + size + LABEL_GAP; + for (let i = 0; i < lines.length; i++) { + context.fillText(lines[i]!, x, firstTop + i * LABEL_SIZE * LABEL_LINE_HEIGHT); + } + context.restore(); +} + +// ─── The renderer ─────────────────────────────────────────────────────────── + +export class GraphCanvasRenderer { + private canvas: HTMLCanvasElement; + private ctx: CanvasRenderingContext2D; + private callbacks: RendererCallbacks; + private zoomBehavior: ZoomBehavior; + private transform: ZoomTransform = zoomIdentity; + /** Set once the first fit happens; the camera then survives setData. */ + private hasFit = false; + /** The k that frames the whole graph — the label-gate's "1×". */ + private kFit = 1; + + private data: CanvasData | null = null; + /** id -> node for the current data (avoids O(n^2) scans). */ + private nodeById = new Map(); + private settings: GraphRenderSettings | null = null; + private visible: ReadonlySet | null = null; + + private frameId: number | null = null; + private dirty = true; + + // Hover transition: fade 0 = resting, 1 = fully hovered; the animating + // flag keeps the loop scheduled until the fade settles (then rAF stops). + private hoverId: string | null = null; + private hoverFade = 0; + private hoverFrom = 0; + private hoverTarget = 0; + private hoverStart = 0; + private animating = false; + + private resizeObserver: ResizeObserver; + private downPos: Pt | null = null; + + constructor(canvas: HTMLCanvasElement, callbacks: RendererCallbacks) { + this.canvas = canvas; + const ctx = canvas.getContext("2d"); + if (!ctx) throw new Error("2d context unavailable"); + this.ctx = ctx; + this.callbacks = callbacks; + + // d3-zoom owns pan / wheel zoom-to-cursor / touch pinch; every + // transform change just marks dirty (draws happen in OUR loop). + this.zoomBehavior = zoom() + .scaleExtent([0.02, 40]) + .on("zoom", ({ transform }) => { + this.transform = transform; + this.markDirty(); + this.callbacks.onCameraMove(); + }); + select(canvas) + .call(this.zoomBehavior) + // No surprise dblclick-zoom (sigma had none). + .on("dblclick.zoom", null); + + canvas.addEventListener("pointermove", this.onPointerMove); + canvas.addEventListener("pointerdown", this.onPointerDown); + canvas.addEventListener("pointerup", this.onPointerUp); + canvas.addEventListener("pointerleave", this.onPointerLeave); + + this.resizeObserver = new ResizeObserver(() => { + this.syncSize(); + if (!this.hasFit) this.fit(); // first real size → fit once + this.markDirty(); + }); + this.resizeObserver.observe(canvas); + this.syncSize(); + } + + // ── Public API ── + + setData(data: CanvasData): void { + this.data = data; + this.nodeById = new Map(data.nodes.map((n) => [n.id, n])); + if (!this.hasFit && data.nodes.length > 0) this.fit(); + this.markDirty(); + } + + setSettings(settings: GraphRenderSettings): void { + this.settings = settings; + this.markDirty(); + } + + setVisible(visible: ReadonlySet | null): void { + this.visible = visible; + this.markDirty(); + } + + /** Frame the whole graph (padding included). Keeps the camera on later + * setData calls — call fit() only for an explicit re-frame. */ + fit(): void { + if (!this.data || this.data.nodes.length === 0) return; + const w = this.canvas.clientWidth; + const h = this.canvas.clientHeight; + if (w < 10 || h < 10) return; // layout not ready yet; the ResizeObserver retries + let minX = Infinity; + let minY = Infinity; + let maxX = -Infinity; + let maxY = -Infinity; + for (const n of this.data.nodes) { + if (n.x < minX) minX = n.x; + if (n.y < minY) minY = n.y; + if (n.x > maxX) maxX = n.x; + if (n.y > maxY) maxY = n.y; + } + const t = fitTransform({ minX, maxX, minY, maxY }, w, h, 40); + this.kFit = t.k; + this.hasFit = true; + select(this.canvas).call( + this.zoomBehavior.transform, + zoomIdentity.translate(t.x, t.y).scale(t.k), + ); + } + + destroy(): void { + if (this.frameId !== null) cancelAnimationFrame(this.frameId); + this.resizeObserver.disconnect(); + select(this.canvas).on(".zoom", null); + this.canvas.removeEventListener("pointermove", this.onPointerMove); + this.canvas.removeEventListener("pointerdown", this.onPointerDown); + this.canvas.removeEventListener("pointerup", this.onPointerUp); + this.canvas.removeEventListener("pointerleave", this.onPointerLeave); + this.canvas.style.cursor = ""; + } + + // ── Event handlers (state mutation only — never draw) ── + + private onPointerMove = (e: PointerEvent) => { + const { x, y } = this.eventPos(e); + const node = this.pickAt(x, y); + if ((node?.id ?? null) !== this.hoverId) this.setHover(node?.id ?? null); + this.canvas.style.cursor = node ? "pointer" : ""; + }; + + private onPointerDown = (e: PointerEvent) => { + this.downPos = this.eventPos(e); + }; + + private onPointerUp = (e: PointerEvent) => { + if (!this.downPos) return; + const pos = this.eventPos(e); + const moved = Math.hypot(pos.x - this.downPos.x, pos.y - this.downPos.y); + this.downPos = null; + if (moved > CLICK_SLOP) return; // that was a drag (pan), not a click + const node = this.pickAt(pos.x, pos.y); + if (node) this.callbacks.onSelect(node.id, pos); + else this.callbacks.onStageClick(); + }; + + private onPointerLeave = () => { + this.downPos = null; + this.setHover(null); + }; + + // ── Internals ── + + private eventPos(e: PointerEvent): Pt { + const rect = this.canvas.getBoundingClientRect(); + return { x: e.clientX - rect.left, y: e.clientY - rect.top }; + } + + private pickAt(px: number, py: number): CanvasNode | null { + if (!this.data || !this.settings) return null; + // Screen positions, filtered to visible nodes. + const { x: tx, y: ty, k } = this.transform; + const maxDist = this.settings.nodeSize + PICKING_PAD; + const projected: { id: string; x: number; y: number }[] = []; + for (const n of this.data.nodes) { + if (this.visible && !this.visible.has(n.id)) continue; + projected.push({ id: n.id, x: n.x * k + tx, y: n.y * k + ty }); + } + const hit = pickNearest(projected, px, py, maxDist); + if (!hit) return null; + return this.nodeById.get(hit.id) ?? null; + } + + private setHover(nodeId: string | null): void { + if (nodeId === this.hoverId && this.hoverTarget === (nodeId ? 1 : 0)) return; + this.hoverId = nodeId; + // Rebase the animation from the CURRENT fade: rapid hover toggles + // glide instead of snapping (ported from the sigma-era behavior). + this.hoverFrom = this.hoverFade; + this.hoverTarget = nodeId ? 1 : 0; + this.hoverStart = performance.now(); + this.animating = true; + this.markDirty(); + } + + private markDirty(): void { + this.dirty = true; + if (this.frameId === null) { + this.frameId = requestAnimationFrame(() => { + this.frameId = null; + this.tick(); + }); + } + } + + private tick(): void { + this.advanceHover(); + this.draw(); + // Keep frames coming ONLY while the hover fade is in motion; an + // idle graph schedules no rAF at all. + if (this.animating) { + this.frameId = requestAnimationFrame(() => { + this.frameId = null; + this.tick(); + }); + } + } + + private advanceHover(): void { + if (!this.animating) return; + const t = Math.min((performance.now() - this.hoverStart) / HOVER_FADE_MS, 1); + this.hoverFade = this.hoverFrom + (this.hoverTarget - this.hoverFrom) * t; + if (t >= 1) this.animating = false; + } + + /** Resize the backing store to the CSS box × devicePixelRatio; all + * drawing then happens in CSS pixels through the DPR transform. */ + private syncSize(): void { + const w = this.canvas.clientWidth; + const h = this.canvas.clientHeight; + const dpr = window.devicePixelRatio || 1; + const bw = Math.max(1, Math.round(w * dpr)); + const bh = Math.max(1, Math.round(h * dpr)); + if (this.canvas.width !== bw || this.canvas.height !== bh) { + this.canvas.width = bw; + this.canvas.height = bh; + } + this.ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + } + + /** THE draw function: every layer, one frame, one camera. */ + private draw(): void { + this.dirty = false; + const data = this.data; + const settings = this.settings; + if (!data || !settings) return; + const w = this.canvas.clientWidth; + const h = this.canvas.clientHeight; + this.ctx.clearRect(0, 0, w, h); + + const { x: tx, y: ty, k } = this.transform; + const nodeById = new Map(); + const ratio = zoomRatioOf(this.kFit, k); + const nodeAlpha = labelAlpha(ratio, settings.labelZoom); + const regionAlpha = 1 - nodeAlpha; + + // Screen positions for every VISIBLE node (one pass), plus the + // byId lookup so the label/hover passes never go O(n^2). + const pos = new Map(); + for (const n of data.nodes) { + if (this.visible && !this.visible.has(n.id)) continue; + pos.set(n.id, { x: n.x * k + tx, y: n.y * k + ty }); + nodeById.set(n.id, n); + } + + // 1–2. Cluster hulls: soft layered fills, then the dashed boundary. + for (const cluster of data.clusters) { + const pts: Pt[] = []; + for (const id of cluster.members) { + const p = pos.get(id); + if (p) pts.push(p); + } + const hull = convexHull(pts); + if (hull.length < 3) continue; + const hue = clusterHue(cluster.id); + for (const [pad, alpha] of HULL_FILL_LAYERS) { + this.ctx.beginPath(); + traceRoundedPath(this.ctx, offsetPolygon(hull, pad), HULL_CORNER_RADIUS); + this.ctx.fillStyle = `hsla(${hue}, 65%, 60%, ${alpha})`; + this.ctx.fill(); + } + this.ctx.beginPath(); + traceRoundedPath(this.ctx, offsetPolygon(hull, HULL_STROKE_PAD), HULL_CORNER_RADIUS); + this.ctx.setLineDash([...HULL_DASH]); + this.ctx.strokeStyle = `hsla(${hue}, 65%, 62%, 0.5)`; + this.ctx.lineWidth = 1.5; + this.ctx.stroke(); + this.ctx.setLineDash([]); + } + + // 3. Edges: shaft + arrowhead (tip lands on the target disc edge). + if (settings.edgeWidth > 0) { + this.ctx.strokeStyle = data.colors.edge; + this.ctx.fillStyle = data.colors.edge; + this.ctx.lineWidth = settings.edgeWidth; + this.ctx.lineCap = "round"; + for (const e of data.edges) { + const from = pos.get(e.from); + const to = pos.get(e.to); + if (!from || !to) continue; // edge drops when either endpoint is hidden + const g = arrowGeometry(from, to, settings.nodeSize, settings.nodeSize, settings.arrowLength); + if (!g) continue; + this.ctx.beginPath(); + this.ctx.moveTo(g.shaftFrom.x, g.shaftFrom.y); + this.ctx.lineTo(g.shaftTo.x, g.shaftTo.y); + this.ctx.stroke(); + this.ctx.beginPath(); + this.ctx.moveTo(g.tip.x, g.tip.y); + this.ctx.lineTo(g.left.x, g.left.y); + this.ctx.lineTo(g.right.x, g.right.y); + this.ctx.closePath(); + this.ctx.fill(); + } + } + + // 4. Nodes: plain discs. + this.ctx.fillStyle = data.colors.node; + for (const [, p] of pos) { + this.ctx.beginPath(); + this.ctx.arc(p.x, p.y, settings.nodeSize, 0, Math.PI * 2); + this.ctx.fill(); + } + + // 5. Cluster region labels — inverse-gated on THE zoom control: + // they fade in over the exact span node labels fade out. Drawn + // above the nodes, below the node labels (the old layer order). + if (settings.clusterLabels && regionAlpha > 0.01) { + this.ctx.save(); + this.ctx.globalAlpha = regionAlpha; + this.ctx.font = `${LABEL_WEIGHT} ${REGION_LABEL_SIZE}px ${data.fontFamily}`; + this.ctx.textAlign = "center"; + this.ctx.textBaseline = "middle"; + for (const cluster of data.clusters) { + if (!cluster.label) continue; + const pts: Pt[] = []; + for (const id of cluster.members) { + const p = pos.get(id); + if (p) pts.push(p); + } + if (pts.length === 0) continue; + const cx = pts.reduce((s, p) => s + p.x, 0) / pts.length; + const cy = pts.reduce((s, p) => s + p.y, 0) / pts.length; + const hue = clusterHue(cluster.id); + this.ctx.fillStyle = `hsla(${hue}, 55%, 78%, 0.85)`; + this.ctx.fillText(cluster.label, cx, cy); + } + this.ctx.restore(); + } + + // 6. Node labels — one global gate, faded together (no per-node pop). + if (nodeAlpha > 0.01) { + this.ctx.save(); + this.ctx.globalAlpha = nodeAlpha; + for (const [id, p] of pos) { + const n = nodeById.get(id); + if (n) drawName(this.ctx, p.x, p.y, settings.nodeSize, n.label, data.fontFamily, data.colors.label); + } + this.ctx.restore(); + } + + // 7. Hover glow (topmost): halo in the node's color + the brightened + // name, both on the SAME fade factor. Shows even when labels are + // zoomed out (hover is its own affordance, not a label). + if (this.hoverId && this.hoverFade > 0.02) { + const p = pos.get(this.hoverId); + const n = p ? nodeById.get(this.hoverId) : null; + if (p && n) { + const inner = settings.nodeSize; + const outer = settings.nodeSize + GLOW_PAD; + const glow = this.ctx.createRadialGradient(p.x, p.y, inner, p.x, p.y, outer); + glow.addColorStop(0, rgbaFromHex(data.colors.node, GLOW_ALPHA * this.hoverFade)); + glow.addColorStop(1, rgbaFromHex(data.colors.node, 0)); + this.ctx.fillStyle = glow; + this.ctx.beginPath(); + this.ctx.arc(p.x, p.y, outer, 0, Math.PI * 2); + this.ctx.arc(p.x, p.y, inner, 0, Math.PI * 2, true); + this.ctx.fill("evenodd"); + this.ctx.save(); + this.ctx.globalAlpha = this.hoverFade; + drawName(this.ctx, p.x, p.y, settings.nodeSize, n.label, data.fontFamily, data.colors.hoverText); + this.ctx.restore(); + } + } + } +} diff --git a/components/graph-view.tsx b/components/graph-view.tsx index ba185cd..d939bc8 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -3,73 +3,57 @@ /** * The home knowledge-graph view (W4; vision §1 "Rendering (DECIDED)"). * - * sigma.js v3 + graphology + forceAtlas2, run-then-freeze: ONE synchronous - * forceAtlas2.assign (100 iterations, inferSettings) inside a useEffect - * writes x/y. No layout animation ever starts; the ONLY animation is the - * short hover fade (enter/leave transitions), strictly bounded in time - * and cancelled on unmount — idle CPU stays near zero. Physics is - * placement only: pan/zoom, no dragging, no editing. + * ARCHITECTURE (2026-09-13, decided after the research pass — + * docs/research/graph-rendering.md, Option B): sigma.js is GONE. The + * renderer is now a hand-rolled single Canvas 2D (./graph-canvas.ts) — + * a DUMB DRAW FUNCTION over plain data with ONE requestAnimationFrame + * loop and a dirty flag. Everything that made the sigma era fragile is + * structurally impossible now: hulls cannot lag nodes (one loop, one + * camera, one frame), settings changes cannot shift anything (the + * camera lives in the renderer and no rebuild ever happens), and the + * per-camera-event full-canvas repaints (GPU spikes + coil whine) are + * gone (at most one draw per frame, and NO frame at all while idle). * - * Labels render the node `name` — the very short, glanceable label the - * extractor coins (a glossary-headword-style handle); the full-precision - * `statement` lives on the detail card. Custom canvas drawers replace - * sigma's defaults: - * - defaultDrawNodeLabel: plain wrapped text, centered below the node, - * word-wrapped at a max line width — no truncation, no one-line sprawl. - * Cluster hulls (vision §1) live on TWO custom canvas layers around - * sigma's own stack: a hull layer behind everything (blurred low-opacity - * fill + dashed boundary per cluster, stable hue from the cluster id) - * and a region-label layer above nodes but below node labels (the label - * at the centroid, inverse-gated on THE zoom control — node labels fade - * out over the span as cluster labels fade in, so they never coexist). - * Geometry is a pure function of member positions at draw time: every - * afterRender re-derives hulls through the current camera, so pan/zoom - * can never invalidate it. - * - defaultDrawNodeHover: NO background box (sigma's default is a - * hardcoded white one). Instead the hovered node gets a soft glow - * halo in the node's color at a constant radius, flush with the - * disc edge, and its name brightens — the "this is clickable, - * click for the full fact" affordance. Entering/leaving hover fades - * ONE transition factor shared by halo and text alike, so the - * halo's opacity and the text's muted→bright cross-fade stay in - * sync (no growth, no pulse; interpolating from the current value - * means rapid hover toggles glide instead of snapping). Hover - * always shows the name, even when labels are zoomed out. - * Node hit-testing is the disc PLUS a pad: sigma picks nodes from a - * half-resolution framebuffer (no v3 setting enlarges hitboxes), and the - * glow halo is bigger than the stock hitbox — touching the halo - * de-hovered the node, which oscillated (the hover flicker). A - * NodeCircleProgram subclass (PICKING_VERTEX_SHADER below) adds - * PICKING_PAD_PX of pickable radius under #ifdef PICKING_MODE only, per - * the maintainer-endorsed v3 pattern (#1493); the visible disc stays - * pixel-identical, and the whole halo becomes hoverable/clickable. - * Labels are zoom-gated GLOBALLY with a fade: all names appear at once - * when the camera is at fit zoom or closer (ratio <= 1), then fade out - * together across a short zoom span before label rendering switches off — - * no per-node size gating, so labels never pop in one by one. (The Obsidian - * text-fade feel, applied to the whole graph.) + * Layout (graphology + forceAtlas2) runs-then-freezes exactly as + * before: ONE synchronous 100-iteration pass inside a useEffect writes + * world-space x/y, then the positions are handed to the renderer as + * data. No layout animation ever starts; the only animation is the + * short hover fade, strictly bounded in time — idle CPU is zero (the + * loop schedules no rAF until something actually changes). * - * The search panel (top-right overlay): a search box matching name or - * statement text — the transitional node-state classes are gone (2026-09-08; - * nodes render uniformly until the state model ideation lands), so there - * are no state legend rows or toggles anymore. Filtering flows through - * sigma's official - * nodeReducer/edgeReducer settings (`hidden`), so rendering, hover, - * picking, and labels all respect it; edges drop when either endpoint is - * hidden, and an open detail card closes if its node is filtered out. - * Clicking a node opens the detail card anchored just below the node's - * viewport position (flipping above near the bottom edge); any camera move - * (pan/zoom) dismisses it. The `requires` edges render as arrows pointing - * prerequisite → dependent. + * The component's job is everything AROUND drawing: fetch + refetch the + * snapshot, run the layout, resolve theme colors to hex, filter via the + * search box, own the settings panel / detail card / import button / + * pipeline chips, and FEED the renderer (data, settings, visibility). + * The renderer's contract stays presentation-only — which is exactly + * what the upcoming playground and cluster-first layout work needs: a + * renderer that draws whatever data it is given, never questioning it. * - * Rendering settings (2026-09-13): a gear in the search panel reveals four - * sliders — node size, edge width, arrowhead length, and THE zoom-level - * control (label visibility threshold; 1× = fit zoom) — persisted per browser - * (localStorage) and applied by REBUILDING ONLY THE RENDERER: the layout - * effect (graph + forceAtlas2) is untouched by settings, and the camera - * position carries across rebuilds, so tweaking a slider never re-fits or - * re-lays-out the view. The arrowhead is a program option (head px = - * thickness × lengthToThicknessRatio × 2), solved from the two settings. + * Visual semantics, all preserved verbatim from the sigma era: + * - Node labels (the short extractor-coined names) wrap under the + * node; one GLOBAL zoom gate fades them together (no per-node pop). + * - Hover: a soft glow halo in the node's own color at a constant + * radius, opacity cross-faded ~180ms with the name brightening on + * the same factor; the whole halo is hit-testable (picking pad). + * - Cluster hulls (vision §1): soft layered translucent fills + a + * dashed boundary per cluster, stable hue from the cluster id; the + * region label at the centroid fades in over the exact zoom span + * node labels fade out (THE single zoom control) — and is + * toggleable in the settings panel (hulls always draw). + * - The persisted labelZoom setting keeps sigma's camera-ratio + * convention (1 = fit zoom, SMALLER = zoomed in): labels until + * zoom 0.5x means labels appear once zoomed in past 2x fit. + * - Click a node -> the detail card anchored just below it (flipping + * above near the bottom edge); any camera move dismisses it. + * Clicking the empty stage closes it. Search filtering hides nodes + * (and their edges) from rendering, picking, hover, AND labels; + * an open card closes if its node is filtered out. + * + * Rendering settings (2026-09-13): the gear in the search panel reveals + * four sliders (node size, edge width, arrowhead length, the zoom + * threshold) plus the cluster-labels checkbox, persisted per browser + * (localStorage). Applied by just redrawing — no rebuild, no re-layout, + * and the camera never moves. */ import { useDeferredValue, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; @@ -77,41 +61,22 @@ import Link from "next/link"; import { FileTextIcon, Loader2Icon, SearchIcon, SettingsIcon, UploadIcon, WaypointsIcon } from "lucide-react"; import Graph from "graphology"; import forceAtlas2 from "graphology-layout-forceatlas2"; -import type { NodeHoverDrawingFunction, NodeLabelDrawingFunction } from "sigma/rendering"; import { seededRandom, type GraphNode, type GraphSnapshot } from "@/src/core"; -import { clusterHue, convexHull, offsetPolygon, traceRoundedPath, type Pt } from "./cluster-geometry.ts"; +import { GraphCanvasRenderer, type GraphRenderSettings } from "./graph-canvas.ts"; -/** The uniform node look (no state classes — WebGL wants hex, not oklch): +/** The uniform node look (no state classes — canvas wants hex, not oklch): * the accent color, until the state model lands. */ const NODE_COLOR_TOKEN = "--primary"; // ---- Render settings (the settings-panel controls) ------------------------ // // Three basic controls (user-decided 2026-09-13): node disc size, edge line -// width, and arrowhead length (px). The arrowhead program couples head size -// to thickness via lengthToThicknessRatio — the renderer derives the ratio -// from the two settings, so the controls stay independent. Defaults reflect -// the pre-panel look (edge 1px + its stock 5px head) except nodeSize, which -// dropped 8 → 6 for dense graphs (user: "nodes are quite large"). Persisted -// per browser in localStorage. +// width, and arrowhead length (px). Defaults reflect the pre-panel look +// (edge 1px + its stock 5px head) except nodeSize, which dropped 8 → 6 for +// dense graphs (user: "nodes are quite large"). Persisted per browser in +// localStorage. -interface RenderSettings { - nodeSize: number; - edgeWidth: number; - arrowLength: number; - /** - * The single zoom-level control (camera ratio at/below which node labels - * appear; 1 = fit zoom, higher = labels visible further zoomed out). It - * will ALSO drive cluster labels (vision §1, DECIDED 2026-09-13) — - * inversely: cluster/domain labels fade in over the same span as node - * labels fade out — so the two never coexist. Kept in the persisted - * settings for exactly that reuse. - */ - labelZoom: number; - /** Toggle for the cluster region labels (the hulls stay) — they can - * crowd the screen at fit zoom with many regions (user, 2026-09-13). */ - clusterLabels: boolean; -} +type RenderSettings = GraphRenderSettings; const DEFAULT_RENDER_SETTINGS: RenderSettings = { nodeSize: 6, edgeWidth: 1, arrowLength: 5, labelZoom: 0.5, clusterLabels: true }; const RENDER_SETTINGS_KEY = "graph-render-settings"; const clamp = (v: number, min: number, max: number) => Math.min(Math.max(v, min), max); @@ -130,7 +95,7 @@ function sanitizeRenderSettings(raw: unknown): RenderSettings { /** * Pure filter predicate for the search panel: a node is visible when the * query is empty or matches its name or statement (case-insensitive). - * Exported for tests; the sigma reducers just consume its verdict. + * Exported for tests; the visibility feed just consumes its verdict. */ export function isNodeVisible(node: Pick, query: string): boolean { const q = query.trim().toLowerCase(); @@ -138,223 +103,7 @@ export function isNodeVisible(node: Pick, query return node.name.toLowerCase().includes(q) || node.statement.toLowerCase().includes(q); } -// ---- Label drawing (canvas) --------------------------------------------- -// -// sigma's defaults draw the label on ONE line to the right of the node with -// a white box on hover. We replace both: names wrap under the node at a max -// line width, muted at rest; on hover the node pulses a soft glow in its -// own state color and the name brightens (no background box). - -const LABEL_MAX_WIDTH = 170; // px — wraps long names instead of a wide strip -const LABEL_GAP = 6; // px between node edge and first label line -const LABEL_LINE_HEIGHT = 1.3; -/** Zoom span (camera-ratio units) over which labels fade out together. */ -const LABELS_FADE_SPAN = 0.5; - -// Hover glow: a soft halo ring in the node's own color at a CONSTANT -// radius, starting flush at the disc edge (no gap — a gap reads as a -// false outline/border between node and glow). Only the halo's OPACITY -// animates (hover enter/leave); it never grows and never pulses. -const GLOW_PAD = 9; // px the halo extends beyond the node disc (constant) -const GLOW_ALPHA = 0.45; // halo opacity at the disc edge, fading outward -const HOVER_FADE_MS = 180; // hover halo opacity fade length - -// ---- Padded picking (WebGL) ---------------------------------------------- -// -// sigma picks nodes by reading a single pixel from a picking framebuffer -// rendered at HALF the CSS resolution: small nodes get tiny, ragged hit -// edges, and our glow halo (GLOW_PAD beyond the disc) is NOT pickable, so -// moving the cursor from the disc onto its own halo de-hovered it — an -// oscillation loop (the hover flicker). sigma v3 has no setting for hit -// padding; the maintainer-endorsed pattern (#1493, fixed officially in v4 -// as `nodePickingPadding`) is a node program whose vertex shader enlarges -// the disc ONLY in the picking shader pass. - -/** Extra radius (px) added to each node's PICKABLE disc (picking layer - * only — the visible disc is untouched). >= GLOW_PAD so the entire hover - * halo is hit-testable. */ -const PICKING_PAD_PX = 10; - -/** - * The node-circle vertex shader, verbatim from sigma 3.0.3 - * (dist/index-fad77a13.esm.js, SHADER_SOURCE$5) with ONE change: under - * `#ifdef PICKING_MODE` (sigma compiles this source twice — visible render - * vs picking readback — so the visible pass never takes this branch) the - * size gains PICKING_PAD_PX of radius, which is the same on-screen unit as - * `data.size` (the px radius the canvas drawers use for the halo). - */ -const PICKING_VERTEX_SHADER = /* glsl */ ` -attribute vec4 a_id; -attribute vec4 a_color; -attribute vec2 a_position; -attribute float a_size; -attribute float a_angle; - -uniform mat3 u_matrix; -uniform float u_sizeRatio; -uniform float u_correctionRatio; - -varying vec4 v_color; -varying vec2 v_diffVector; -varying float v_radius; -varying float v_border; - -const float bias = 255.0 / 254.0; - -void main() { - float pickedSize = a_size; -#ifdef PICKING_MODE - // Picking pass only: enlarge the hitbox by PICKING_PAD_PX of radius. - pickedSize += ${PICKING_PAD_PX.toFixed(1)}; -#endif - float size = pickedSize * u_correctionRatio / u_sizeRatio * 4.0; - vec2 diffVector = size * vec2(cos(a_angle), sin(a_angle)); - vec2 position = a_position + diffVector; - gl_Position = vec4( - (u_matrix * vec3(position, 1)).xy, - 0, - 1 - ); - - v_diffVector = diffVector; - v_radius = size / 2.0; - -#ifdef PICKING_MODE - // For picking mode, we use the ID as the color: - v_color = a_id; -#else - // For normal mode, we use the color: - v_color = a_color; -#endif - - v_color.a *= bias; -} -`; - -/** "#rrggbb" + alpha → "rgba(...)" (tokenHex only ever emits #rrggbb). */ -function rgbaFromHex(hex: string, alpha: number): string { - const r = Number.parseInt(hex.slice(1, 3), 16); - const g = Number.parseInt(hex.slice(3, 5), 16); - const b = Number.parseInt(hex.slice(5, 7), 16); - return `rgba(${r}, ${g}, ${b}, ${alpha})`; -} - -/** Word-wrap for canvas text. Never truncates: overlong single words are - * hard-broken by character so no line ever exceeds maxWidth. */ -function wrapLabel(context: CanvasRenderingContext2D, text: string, maxWidth: number): string[] { - const lines: string[] = []; - for (const word of text.split(/\s+/).filter(Boolean)) { - const last = lines.length > 0 ? lines[lines.length - 1]! : null; - if (last !== null) { - const joined = `${last} ${word}`; - if (context.measureText(joined).width <= maxWidth) { - lines[lines.length - 1] = joined; - continue; - } - } - let chunk = ""; - for (const char of word) { - if (chunk && context.measureText(`${chunk}${char}`).width > maxWidth) { - lines.push(chunk); - chunk = char; - } else { - chunk += char; - } - } - if (chunk) lines.push(chunk); - } - return lines; -} - -/** The wrapped node name (the short label), centered below the node — used by both drawers - * with identical geometry so hover only changes the color, nothing jumps. */ -function drawName( - context: CanvasRenderingContext2D, - data: { label?: string | null; x: number; y: number; size: number }, - settings: { labelSize: number; labelWeight: string; labelFont: string }, - color: string, -): void { - if (!data.label) return; - const size = settings.labelSize; - context.save(); - context.font = `${settings.labelWeight} ${size}px ${settings.labelFont}`; - const lines = wrapLabel(context, data.label, LABEL_MAX_WIDTH); - context.fillStyle = color; - context.textAlign = "center"; - context.textBaseline = "top"; - const firstTop = data.y + data.size + LABEL_GAP; - for (let i = 0; i < lines.length; i++) { - context.fillText(lines[i]!, data.x, firstTop + i * size * LABEL_LINE_HEIGHT); - } - context.restore(); -} - -/** The hovered-state paint at a given transition factor (0 = resting, - * 1 = fully hovered): a glow halo in the node's own state color at a - * CONSTANT radius (opacity fades in/out; the halo never grows) starting - * flush at the disc edge (the evenodd punch-out keeps the disc itself - * unpainted). The name is drawn at partial alpha in the bright hover - * color — the muted resting label on the layer beneath shows through, - * so text and halo cross-fade on the SAME factor and stay in sync. */ -function paintHover( - context: CanvasRenderingContext2D, - data: { label?: string | null; x: number; y: number; size: number; color: string }, - settings: { labelSize: number; labelWeight: string; labelFont: string }, - fade: number, - hoverTextColor: string, -): void { - context.save(); - if (fade > 0.02) { - const inner = data.size; - const outer = data.size + GLOW_PAD; - const glow = context.createRadialGradient(data.x, data.y, inner, data.x, data.y, outer); - glow.addColorStop(0, rgbaFromHex(data.color, GLOW_ALPHA * fade)); - glow.addColorStop(1, rgbaFromHex(data.color, 0)); - context.fillStyle = glow; - context.beginPath(); - context.arc(data.x, data.y, outer, 0, Math.PI * 2); - context.arc(data.x, data.y, inner, 0, Math.PI * 2, true); - context.fill("evenodd"); - } - // Same fade factor as the halo: text and glow rise together. The - // bright name at partial alpha over the muted resting label beneath is - // a smooth muted→bright cross-fade rather than an instant color swap. - context.globalAlpha = fade; - drawName(context, data, settings, hoverTextColor); - context.restore(); -} - -/** Mutable hover-transition state shared between the drawers and the - * effect's enter/leave animation. */ -interface HoverState { - fade: number; - /** Set on every sigma-driven hover paint; the leave animation draws - * through it directly, since sigma stops calling drawHover once the - * mouse has left the node. */ - repaint: ((fade: number) => void) | null; -} - -/** Make the pair of canvas label drawers bound to the theme's colors. */ -function makeLabelDrawers(labelColor: string, hoverTextColor: string, hover: HoverState) { - const drawLabel: NodeLabelDrawingFunction = (context, data, settings) => { - drawName(context, data, settings, labelColor); - }; - - const drawHover: NodeHoverDrawingFunction = (context, data, settings) => { - hover.repaint = (fade) => { - context.save(); - context.setTransform(1, 0, 0, 1, 0, 0); - context.clearRect(0, 0, context.canvas.width, context.canvas.height); - context.restore(); - if (fade > 0) paintHover(context, data, settings, fade, hoverTextColor); - }; - paintHover(context, data, settings, hover.fade, hoverTextColor); - }; - - return { drawLabel, drawHover }; -} - -// ---- Color helpers (WebGL/canvas want hex, not oklch) -------------------- +// ---- Color helpers (canvas wants hex, not oklch) --------------------------- /** Resolve a CSS variable (oklch in this app) to #rrggbb via the canvas. */ function cssColorToHex(css: string): string { @@ -383,7 +132,8 @@ interface Selection { /** The sources that touched this node: chat ids (deep-linkable sessions) * and notes-source ids ("src_…" — imported files, static entries). */ sourceIds: string[]; - /** Node position in container pixels (from graphToViewport at click time). */ + /** Node position in container pixels (at click time; camera moves + * dismiss the card, so it never needs re-anchoring). */ anchor: { x: number; y: number }; } @@ -425,18 +175,17 @@ function SettingSlider({ ); } -// ─── cluster hulls (vision §1, DECIDED 2026-09-13) ──────────────────────────── -// Boundary geometry is a PURE FUNCTION OF POSITIONS AT DRAW TIME: every -// redraw re-derives the hull from current member positions mapped through -// the camera, so pan/zoom/relayout can never invalidate it. The math -// lives in ./cluster-geometry.ts (unit-tested — see that file's header -// for the inward-normal bug this extraction is named after). - export function GraphView() { const containerRef = useRef(null); + const canvasRef = useRef(null); const cardRef = useRef(null); const [snapshot, setSnapshot] = useState(null); const [selected, setSelected] = useState(null); + // The latest snapshot for the renderer's click callbacks (they are + // constructed once, so they read through this ref, never a closure). + const snapshotRef = useRef(null); + snapshotRef.current = snapshot; + // Search panel state: the query text + the settings section toggle. const [query, setQuery] = useState(""); const [settingsOpen, setSettingsOpen] = useState(false); @@ -460,20 +209,14 @@ export function GraphView() { } }, [settings]); - // The sigma instance and its graphology graph live inside the mount - // effects (data: graph+layout; render: renderer); refs expose them to - // the filters effect, and `rendererEpoch` re-triggers it when a new - // renderer is constructed. `cameraRef` carries the camera position - // across renderer rebuilds (settings changes must never re-fit the - // view), and `hoverStateRef` lets the layout effect's hover repainter - // reach the CURRENT renderer's drawers. - const rendererRef = useRef(null); - const graphRef = useRef(null); - const [rendererEpoch, setRendererEpoch] = useState(0); - const cameraRef = useRef<{ x: number; y: number; ratio: number; angle: number } | null>(null); + // The renderer instance lives across snapshot refetches and settings + // changes (its camera is never reset by either); `posRef` carries the + // frozen layout out of the layout effect. + const rendererRef = useRef(null); + const posRef = useRef | null>(null); - // The id set that survives the current search — consumed by the sigma - // reducers and the "no matches" hint. + // The id set that survives the current search — consumed by the + // visibility feed and the "no matches" hint. const visibleIds = useMemo(() => { const set = new Set(); if (snapshot) @@ -545,9 +288,8 @@ export function GraphView() { // Build the graphology graph and lay it out ONCE per snapshot: seeded // start positions, one synchronous forceAtlas2 pass, then freeze. The - // renderer is a SEPARATE effect — render settings change only the - // renderer; the layout (positions live in this graph instance) is - // never recomputed for a settings tweak. + // renderer only ever receives the frozen positions as data — layout is + // never recomputed for a settings tweak or a refetch of identical data. useEffect(() => { if (!snapshot || snapshot.nodes.length === 0) return; const graph = new Graph(); @@ -556,9 +298,6 @@ export function GraphView() { // frozen layout stable for a given node set across remounts. const rand = seededRandom(node.id); graph.addNode(node.id, { - label: node.name, - size: DEFAULT_RENDER_SETTINGS.nodeSize, - color: tokenHex(NODE_COLOR_TOKEN), x: (rand() - 0.5) * 100, y: (rand() - 0.5) * 100, }); @@ -567,11 +306,7 @@ export function GraphView() { // Snapshot edges are already merged by (from,to,type); the guard // is just belt-and-braces for the simple graphology graph. if (!graph.hasEdge(edge.from, edge.to)) { - graph.addEdge(edge.from, edge.to, { - type: "arrow", - color: tokenHex("--ring"), - size: DEFAULT_RENDER_SETTINGS.edgeWidth, - }); + graph.addEdge(edge.from, edge.to); } } @@ -584,300 +319,85 @@ export function GraphView() { }; forceAtlas2.assign(graph, { iterations: 100, settings: layoutSettings }); - // The render effect (declared next) builds its Sigma over THIS - // instance in the same commit; on snapshot change it re-runs too - // (snapshot is in its deps) and picks the new graph up. - graphRef.current = graph; + const positions = new Map(); + graph.forEachNode((id, attrs) => positions.set(id, { x: attrs.x, y: attrs.y })); + posRef.current = positions; return () => { - graphRef.current = null; + posRef.current = null; }; }, [snapshot]); - // Mount sigma over the laid-out graph. Re-runs when the snapshot changes - // (new graph) or the (deferred) render settings change — nothing else. - // Sizes are stamped onto the graph here, and the arrowhead geometry is - // a PROGRAM option (head px = thickness × lengthToThicknessRatio × 2), - // so both flow through a renderer rebuild while the layout and the - // camera position (carried via cameraRef) stay put. - const deferredSettings = useDeferredValue(settings); + // Construct the renderer once the canvas exists; destroy when it goes + // away (the empty-graph placeholder replaces it). The callbacks are + // created once and read through refs, so they never go stale. + const isEmpty = snapshot === null || snapshot.nodes.length === 0; useEffect(() => { - if (!snapshot || snapshot.nodes.length === 0) return; - const container = containerRef.current; - const graph = graphRef.current; - if (!container || !graph) return; - - const s = deferredSettings; - // Stamp the size-controlled attributes (idempotent; the old renderer - // died in cleanup — this graph is renderer-less right now). - graph.updateEachNodeAttributes((_node, attrs) => ({ ...attrs, size: s.nodeSize })); - graph.updateEachEdgeAttributes((_edge, attrs) => ({ ...attrs, size: s.edgeWidth })); - - const labelColor = tokenHex("--muted-foreground"); - const hoverTextColor = tokenHex("--foreground"); - // Mutable hover-transition state: 0 = resting, 1 = fully hovered. - const hover: HoverState = { fade: 0, repaint: null }; - const { drawLabel, drawHover } = makeLabelDrawers(labelColor, hoverTextColor, hover); - - let killed = false; - let renderer: import("sigma").default | null = null; - // The custom cluster layers (hull + region label canvases) — created - // inside the dynamic-import callback, removed in this effect's cleanup. - let clusterCanvas: HTMLCanvasElement | null = null; - let clusterLabelCanvas: HTMLCanvasElement | null = null; - // rAF id of the hover fade transition — the ONLY animation (see the - // file header). Lives out here so the effect cleanup can cancel it. - let hoverAnimId: number | null = null; - // Both sigma and its rendering package touch WebGL globals at module - // scope, so both load dynamically (SSR-safe; the effect is browser-only). - void Promise.all([import("sigma"), import("sigma/rendering")]).then( - ([{ default: Sigma }, { NodeCircleProgram, createEdgeArrowProgram }]) => { - if (killed) return; - // The arrowhead program with our ratios: solve - // lengthToThicknessRatio for the desired head length - // (head px = thickness × ratio × 2), keeping the stock head - // SHAPE (wideness = 0.8 × length, as the 2.5/2 defaults). - const thicknessRatio = s.arrowLength / (2 * s.edgeWidth); - const arrowProgram = createEdgeArrowProgram({ - lengthToThicknessRatio: thicknessRatio, - widenessToThicknessRatio: thicknessRatio * 0.8, + if (isEmpty) return; + const canvas = canvasRef.current; + if (!canvas) return; + const renderer = new GraphCanvasRenderer(canvas, { + onSelect: (nodeId, pos) => { + const snap = snapshotRef.current; + const node = snap?.nodes.find((n) => n.id === nodeId); + if (!snap || !node) return; + setSelected({ + node, + sourceIds: snap.nodeSessions[node.id] ?? [], + anchor: pos, }); - // Padded picking (see PICKING_VERTEX_SHADER): the visible disc is - // pixel-identical to the stock circle program; the PICKABLE disc - // gains PICKING_PAD_PX of radius, so the whole hover halo is - // hit-testable and small nodes are comfortable to hover/click. - // (Maintainer-endorsed pattern for sigma v3; v4 ships this as - // the `nodePickingPadding` setting.) - const PaddedNodeCircleProgram = class extends NodeCircleProgram { - getDefinition() { - return { - ...super.getDefinition(), - VERTEX_SHADER_SOURCE: PICKING_VERTEX_SHADER, - }; - } - }; - renderer = new Sigma(graph, container, { - nodeProgramClasses: { circle: PaddedNodeCircleProgram }, - edgeProgramClasses: { arrow: arrowProgram }, - defaultEdgeType: "arrow", - // All labels appear/disappear together: no per-node size gating - // (threshold 0) and NO sigma label-grid declutter (density → - // ∞) — clustered nodes otherwise lose their cells' label - // quota and pop in late, un-faded, mid-zoom. The camera-ratio - // gate below is the ONLY label culling: every label appears at - // the same zoom with the same fade. - labelRenderedSizeThreshold: 0, - labelDensity: Number.MAX_SAFE_INTEGER, - // Match the app typography (Geist via the body's stack). - labelFont: getComputedStyle(document.body).fontFamily, - labelWeight: "500", - labelSize: 12, - labelColor: { color: labelColor }, - renderEdgeLabels: false, - defaultDrawNodeLabel: drawLabel, - defaultDrawNodeHover: drawHover, - }); - graphRef.current = graph; - rendererRef.current = renderer; - setRendererEpoch((e) => e + 1); - // A renderer rebuild (settings tweak) must never re-fit the - // view: restore the carried camera position, if any. - if (cameraRef.current) renderer.getCamera().setState(cameraRef.current); - - // ── Cluster hulls (vision §1): two custom canvas layers around - // sigma's own stack (DOM order: edges, nodes, [labels], edgeLabels, - // labels, ...). The HULL layer sits behind everything; the region - // LABEL layer sits above nodes but below node labels — the two - // label systems are inverse-gated on the same zoom control, so - // they never coexist (see syncRenderLabels below). - const sigmaCanvases = renderer.getCanvases(); - // Local consts so the draw closure below stays narrowed. - const hullCanvas = (clusterCanvas = document.createElement("canvas")); - hullCanvas.style.position = "absolute"; - container.insertBefore(hullCanvas, container.firstChild ?? null); - const regionCanvas = (clusterLabelCanvas = document.createElement("canvas")); - regionCanvas.style.position = "absolute"; - // Actual DOM order (sigma v3): edges, edgeLabels, nodes, labels, … - // so "before labels" = above nodes, below node labels — the two - // label systems are inverse-gated and never coexist anyway. - sigmaCanvases.labels?.parentNode?.insertBefore(regionCanvas, sigmaCanvases.labels); - // All hull geometry is derived per draw — positions through the - // CURRENT camera — so any pan/zoom/render stays correct. - const drawClusters = () => { - const r = renderer; - if (!r || snapshot.clusters.length === 0) return; - const ref = sigmaCanvases.nodes; - if (!ref) return; - const cssW = ref.clientWidth; - const cssH = ref.clientHeight; - if (cssW === 0 || cssH === 0) return; - const dpr = ref.width / cssW; - const hullCtx = hullCanvas.getContext("2d"); - const labelCtx = regionCanvas.getContext("2d"); - if (!hullCtx || !labelCtx) return; - for (const canvas of [hullCanvas, regionCanvas]) { - if (canvas.width !== ref.width || canvas.height !== ref.height) { - canvas.width = ref.width; - canvas.height = ref.height; - } - canvas.style.width = ref.style.width || `${cssW}px`; - canvas.style.height = ref.style.height || `${cssH}px`; - } - hullCtx.setTransform(dpr, 0, 0, dpr, 0, 0); - labelCtx.setTransform(dpr, 0, 0, dpr, 0, 0); - hullCtx.clearRect(0, 0, cssW, cssH); - labelCtx.clearRect(0, 0, cssW, cssH); - labelCtx.textAlign = "center"; - labelCtx.textBaseline = "middle"; - labelCtx.font = `500 18px ${getComputedStyle(document.body).fontFamily}`; - for (const cluster of snapshot.clusters) { - const pts: Pt[] = []; - for (const id of cluster.members) { - if (!graph.hasNode(id)) continue; - const a = graph.getNodeAttributes(id); - const vp = r.graphToViewport({ x: a.x ?? 0, y: a.y ?? 0 }); - if (vp) pts.push(vp); - } - const hull = convexHull(pts); - if (hull.length < 3) continue; - const hue = clusterHue(cluster.id); - // Soft translucent tint WITHOUT ctx.filter: a canvas blur - // filter rasterizes on the GPU per call — hooked to every - // render that was ~32 blurred fills per frame (sustained - // ~100% GPU rasterization + audible coil whine; reported - // 2026-09-13). Layered low-alpha fills approximate the - // same soft edge for a fraction of the cost. - for (const [pad, alpha] of [[24, 0.025], [19, 0.04], [14, 0.06]] as const) { - hullCtx.beginPath(); - traceRoundedPath(hullCtx, offsetPolygon(hull, pad), 9); - hullCtx.fillStyle = `hsla(${hue}, 65%, 60%, ${alpha})`; - hullCtx.fill(); - } - // Dashed boundary on the innermost offset. - hullCtx.beginPath(); - traceRoundedPath(hullCtx, offsetPolygon(hull, 14), 9); - hullCtx.setLineDash([7, 7]); - hullCtx.strokeStyle = `hsla(${hue}, 65%, 62%, 0.5)`; - hullCtx.lineWidth = 1.5; - hullCtx.stroke(); - hullCtx.setLineDash([]); - // The region label at the centroid (inverse-gated layer) — - // skippable entirely via the settings toggle. - if (!settings.clusterLabels || !cluster.label) continue; - const centroid = hull.reduce((acc, p) => ({ x: acc.x + p.x / hull.length, y: acc.y + p.y / hull.length }), { x: 0, y: 0 }); - labelCtx.fillStyle = `hsla(${hue}, 55%, 78%, 0.85)`; - labelCtx.fillText(cluster.label, centroid.x, centroid.y); - } - }; - // Redraw ONLY when the view actually moves: camera updates and - // resizes. NOT on sigma's afterRender — hover animations re-render - // sigma many frames per interaction, and hulls don't care. - renderer.on("resize", drawClusters); - - renderer.on("clickNode", ({ node }) => { - const graphNode = snapshot.nodes.find((n) => n.id === node); - if (!graphNode) return; - const pos = graph.getNodeAttributes(node); - const anchor = renderer?.graphToViewport({ x: pos.x ?? 0, y: pos.y ?? 0 }); - setSelected({ - node: graphNode, - sourceIds: snapshot.nodeSessions[node] ?? [], - anchor: anchor ?? { x: 0, y: 0 }, - }); - }); - // Clicking the empty stage closes the node card. - renderer.on("clickStage", () => setSelected(null)); - // One global label gate with a fade: names are fully visible at fit - // zoom or closer, then fade out together across LABELS_FADE_SPAN of - // zoom (the labels canvas layer's opacity), and only once fully - // faded does label rendering switch off. Any pan/zoom also detaches - // the card from its node — close it. - const labelsCanvas = renderer.getCanvases().labels; - let labelsRendering: boolean | null = null; - const syncRenderLabels = (ratio: number) => { - // 0 at the labelZoom threshold → 1 at the end of the fade span - const fade = (ratio - s.labelZoom) / LABELS_FADE_SPAN; - const alpha = Math.min(Math.max(1 - fade, 0), 1); - const shouldRender = alpha > 0; - if (shouldRender !== labelsRendering) { - labelsRendering = shouldRender; - renderer?.setSetting("renderLabels", shouldRender); - } - if (labelsCanvas) labelsCanvas.style.opacity = alpha.toFixed(3); - // The inverse gate (vision §1): cluster labels fade in over - // the exact span node labels fade out — THE one zoom control. - regionCanvas.style.opacity = (1 - alpha).toFixed(3); - }; - drawClusters(); - syncRenderLabels(renderer.getCamera().getState().ratio); - renderer.getCamera().on("updated", (state) => { - syncRenderLabels(state.ratio); - drawClusters(); - setSelected(null); - }); - // The hover fade: entering schedules renders (sigma re-invokes - // drawHover with the new fade); leaving draws through the captured - // repainter, since sigma stops calling drawHover once unhovered. - // Both directions are bounded (~HOVER_FADE_MS) and cancel each - // other, so idle CPU returns to zero after every transition. - const animateHoverTo = (target: number) => { - if (hoverAnimId !== null) { - cancelAnimationFrame(hoverAnimId); - hoverAnimId = null; - } - const from = hover.fade; - if (from === target) { - if (target === 0) hover.repaint?.(0); - return; - } - const start = performance.now(); - const tick = (now: number) => { - const t = Math.min((now - start) / HOVER_FADE_MS, 1); - hover.fade = from + (target - from) * t; - if (target === 0) hover.repaint?.(hover.fade); - else renderer?.scheduleRefresh(); - hoverAnimId = t < 1 ? requestAnimationFrame(tick) : null; - }; - hoverAnimId = requestAnimationFrame(tick); - }; - renderer.on("enterNode", () => animateHoverTo(1)); - renderer.on("leaveNode", () => animateHoverTo(0)); - renderer.on("leaveStage", () => animateHoverTo(0)); - }); - + }, + onStageClick: () => setSelected(null), + // Any camera move detaches the card from its node — close it. + onCameraMove: () => setSelected(null), + }); + rendererRef.current = renderer; return () => { - killed = true; - if (hoverAnimId !== null) cancelAnimationFrame(hoverAnimId); - // Carry the camera across rebuilds — a settings tweak keeps the - // user's pan/zoom (and the detail-card anchors stay valid). - if (renderer) cameraRef.current = renderer.getCamera().getState(); rendererRef.current = null; - renderer?.kill(); - // The custom cluster layers are ours — sigma's kill() removes only - // its own canvases. - clusterCanvas?.remove(); - clusterLabelCanvas?.remove(); + renderer.destroy(); }; - }, [snapshot, deferredSettings]); + }, [isEmpty]); - // Filtering through sigma's official reducers: hidden nodes drop out of - // rendering, picking, hover, AND labels; edges drop when either endpoint - // is hidden. Re-runs whenever filters change or a renderer is (re)built - // (epoch), so the reducers always close over current values. + // Feed #1 — data: positions + edges + clusters + resolved colors. Runs + // after the layout effect (same commit, declaration order) and after + // the construct effect, so the renderer exists and positions are fresh. useEffect(() => { const renderer = rendererRef.current; - const graph = graphRef.current; - if (!renderer || !graph) return; - renderer.setSetting("nodeReducer", (key, attrs) => - visibleIds.has(key) ? attrs : { ...attrs, hidden: true }, - ); - renderer.setSetting("edgeReducer", (key, attrs) => { - const source = graph.source(key); - const target = graph.target(key); - return visibleIds.has(source) && visibleIds.has(target) ? attrs : { ...attrs, hidden: true }; + const positions = posRef.current; + if (!renderer || !snapshot || snapshot.nodes.length === 0 || !positions) return; + renderer.setData({ + nodes: snapshot.nodes.map((n) => ({ + id: n.id, + x: positions.get(n.id)?.x ?? 0, + y: positions.get(n.id)?.y ?? 0, + label: n.name, + })), + edges: snapshot.edges.map((e) => ({ from: e.from, to: e.to })), + clusters: snapshot.clusters.map((c) => ({ id: c.id, label: c.label, members: c.members })), + colors: { + node: tokenHex(NODE_COLOR_TOKEN), + edge: tokenHex("--ring"), + label: tokenHex("--muted-foreground"), + hoverText: tokenHex("--foreground"), + }, + // Match the app typography (Geist via the body's stack). + fontFamily: getComputedStyle(document.body).fontFamily, }); + }, [snapshot, isEmpty]); + + // Feed #2 — settings: just marks the frame dirty. No rebuild, no + // re-layout, no camera touch (the whole point of the new renderer). + const deferredSettings = useDeferredValue(settings); + useEffect(() => { + rendererRef.current?.setSettings(deferredSettings); + }, [deferredSettings, isEmpty]); + + // Feed #3 — visibility: hidden nodes drop out of rendering, picking, + // hover, AND labels; edges drop when either endpoint is hidden. + useEffect(() => { + rendererRef.current?.setVisible(query.trim() ? visibleIds : null); // The open detail card cannot outlive its node's visibility. setSelected((sel) => (sel && visibleIds.has(sel.node.id) ? sel : null)); - }, [rendererEpoch, visibleIds]); + }, [visibleIds, query]); // Position the detail card: below the clicked node, flipped above near // the bottom edge, clamped horizontally. Imperative (ref + layout effect, @@ -903,8 +423,6 @@ export function GraphView() { card.style.visibility = "visible"; }, [selected]); - const isEmpty = snapshot === null || snapshot.nodes.length === 0; - return (
{isEmpty ? ( @@ -920,7 +438,11 @@ export function GraphView() {
) : ( -
+
+ {/* THE single canvas: every layer (hulls, edges, nodes, both + label systems, hover) draws into it, once per frame. */} + +
)} {/* Legend + filters: search box and per-state toggle rows. */} @@ -1076,7 +598,7 @@ export function GraphView() {
{snapshot?.sessions[sourceId] ?? sourceId} diff --git a/package-lock.json b/package-lock.json index e095625..3f1e3c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,8 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", + "d3-selection": "^3.0.0", + "d3-zoom": "^3.0.0", "embla-carousel-react": "^8.6.0", "graphology": "^0.26.0", "graphology-communities-louvain": "^2.0.2", @@ -43,7 +45,6 @@ "react-dom": "^19.2.8", "react-jsx-parser": "^2.4.1", "remark-breaks": "^4.0.0", - "sigma": "^3.0.3", "streamdown": "^2.5.0", "tailwind-merge": "^3.6.0", "tokenlens": "^1.3.1", @@ -13342,16 +13343,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/sigma": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/sigma/-/sigma-3.0.3.tgz", - "integrity": "sha512-5H0zFlx6/NTQpqBg4Rm569ZOpnBOXMaS25UQThIWMU3XyzI5AhmorK/gnl87BvJBLhQd0tW4C0LIp3enWzMoNw==", - "license": "MIT", - "dependencies": { - "events": "^3.3.0", - "graphology-utils": "^2.5.2" - } - }, "node_modules/source-map-js": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", diff --git a/package.json b/package.json index 32a7514..3f784f2 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,8 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", + "d3-selection": "^3.0.0", + "d3-zoom": "^3.0.0", "embla-carousel-react": "^8.6.0", "graphology": "^0.26.0", "graphology-communities-louvain": "^2.0.2", @@ -48,7 +50,6 @@ "react-dom": "^19.2.8", "react-jsx-parser": "^2.4.1", "remark-breaks": "^4.0.0", - "sigma": "^3.0.3", "streamdown": "^2.5.0", "tailwind-merge": "^3.6.0", "tokenlens": "^1.3.1", diff --git a/tests/graph-canvas.test.ts b/tests/graph-canvas.test.ts new file mode 100644 index 0000000..cc6b4be --- /dev/null +++ b/tests/graph-canvas.test.ts @@ -0,0 +1,139 @@ +/** + * Pure-helper tests for the hand-rolled canvas renderer + * (components/graph-canvas.ts). The renderer class itself is DOM-bound + * (canvas, rAF, ResizeObserver) and is verified live in the browser; the + * math it runs on every frame must be exact, so it is pinned here. + * + * The camera-ratio convention is the load-bearing contract with the + * persisted settings: sigma's ratio (1 = fit, SMALLER = zoomed in) is kept + * verbatim so stored labelZoom values survive the renderer swap. + */ + +import assert from "node:assert/strict"; +import { describe, it as test } from "node:test"; +import { + arrowGeometry, + fitTransform, + labelAlpha, + pickNearest, + zoomRatioOf, + LABELS_FADE_SPAN, +} from "../components/graph-canvas.ts"; + +describe("zoomRatioOf (the sigma-compatible camera ratio)", () => { + test("1 at fit zoom, smaller when zoomed in, larger when zoomed out", () => { + assert.equal(zoomRatioOf(2, 2), 1); // at fit + assert.equal(zoomRatioOf(2, 4), 0.5); // zoomed in 2x -> ratio halves + assert.equal(zoomRatioOf(2, 1), 2); // zoomed out 2x -> ratio doubles + }); +}); + +describe("labelAlpha (the single zoom gate)", () => { + test("fully visible at/below the threshold, faded out after the span", () => { + assert.equal(labelAlpha(0.5, 0.5), 1); // at the threshold + assert.equal(labelAlpha(0.25, 0.5), 1); // zoomed further in: stays 1 + assert.equal(labelAlpha(0.75, 0.5), 0.5); // mid-fade + assert.equal(labelAlpha(1.0, 0.5), 0); // one span out: gone + assert.equal(labelAlpha(2.0, 0.5), 0); // far out: clamped + }); + + test("node labels and cluster labels cross-fade over the exact same span", () => { + // The renderer draws cluster labels at 1 - labelAlpha; the two must + // always sum to 1 within the fade and never coexist at full opacity. + for (const ratio of [0, 0.25, 0.5, 0.6, 0.75, 0.9, 1, 2, 4]) { + const node = labelAlpha(ratio, 0.5); + const cluster = 1 - node; + assert.ok(node + cluster > 0.999 && node + cluster < 1.001, `sum at ratio ${ratio}`); + assert.ok(node <= 1 && cluster <= 1, `clamped at ratio ${ratio}`); + } + }); + + test("the span constant is unchanged from the sigma era (stored settings depend on it)", () => { + assert.equal(LABELS_FADE_SPAN, 0.5); + }); +}); + +describe("fitTransform", () => { + test("centers the bbox and respects padding", () => { + // A 100x50 graph in a 500x300 viewport, 40px padding: + // available 420x220 -> k = min(4.2, 4.4) = 4.2 + const t = fitTransform({ minX: -50, maxX: 50, minY: -25, maxY: 25 }, 500, 300, 40); + assert.ok(Math.abs(t.k - 4.2) < 1e-9, `k=${t.k}`); + assert.ok(Math.abs(t.x - 250) < 1e-9, `x=${t.x}`); // centered + assert.ok(Math.abs(t.y - 150) < 1e-9, `y=${t.y}`); + // World center maps to viewport center: + assert.ok(Math.abs(0 * t.k + t.x - 250) < 1e-9); + }); + + test("never returns a non-positive scale for degenerate bboxes", () => { + const t = fitTransform({ minX: 5, maxX: 5, minY: 5, maxY: 5 }, 100, 100, 10); + assert.ok(t.k > 0 && Number.isFinite(t.k)); + }); +}); + +describe("arrowGeometry", () => { + test("the tip lands exactly on the target disc edge", () => { + const g = arrowGeometry({ x: 0, y: 0 }, { x: 100, y: 0 }, 6, 6, 5); + assert.ok(g, "non-degenerate edge"); + assert.ok(Math.abs(g!.tip.x - 94) < 1e-9 && Math.abs(g!.tip.y) < 1e-9); + }); + + test("the shaft starts outside the source disc", () => { + const g = arrowGeometry({ x: 0, y: 0 }, { x: 100, y: 0 }, 6, 6, 5)!; + assert.ok(Math.abs(g.shaftFrom.x - 6) < 1e-9 && Math.abs(g.shaftFrom.y) < 1e-9); + // Shaft ends at the head base; head is drawn from base to tip. + assert.ok(Math.abs(g.shaftTo.x - 89) < 1e-9); + }); + + test("the head triangle is symmetric around the shaft axis (0.8x wideness)", () => { + const g = arrowGeometry({ x: 0, y: 0 }, { x: 100, y: 0 }, 6, 6, 5)!; + // halfW = 0.4 x headLength = 2; both base corners share x, mirrored y. + assert.ok(Math.abs(Math.abs(g.left.y) - 2) < 1e-9 && Math.abs(Math.abs(g.right.y) - 2) < 1e-9); + assert.ok(Math.abs(g.left.y + g.right.y) < 1e-9); // mirror + assert.ok(Math.abs(g.left.x - g.right.x) < 1e-9); + }); + + test("works at arbitrary angles (diagonal stays normalized)", () => { + const a = { x: 0, y: 0 }; + const b = { x: 60, y: 80 }; // len 100, dir (0.6, 0.8) + const g = arrowGeometry(a, b, 10, 10, 8)!; + assert.ok(Math.abs(g.tip.x - (60 - 0.6 * 10)) < 1e-9 && Math.abs(g.tip.y - (80 - 0.8 * 10)) < 1e-9); + }); + + test("null for coincident endpoints (never a NaN arrow)", () => { + assert.equal(arrowGeometry({ x: 5, y: 5 }, { x: 5, y: 5 }, 6, 6, 5), null); + }); +}); + +describe("pickNearest (distance-check picking)", () => { + const nodes = [ + { id: "a", x: 10, y: 10 }, + { id: "b", x: 50, y: 50 }, + { id: "c", x: 90, y: 10 }, + ]; + + test("picks the nearest node within the pad", () => { + assert.equal(pickNearest(nodes, 12, 12, 16)?.id, "a"); + assert.equal(pickNearest(nodes, 48, 52, 16)?.id, "b"); + assert.equal(pickNearest(nodes, 88, 8, 16)?.id, "c"); + }); + + test("the picking pad extends the hitbox beyond the disc", () => { + // 20px away, only reachable because the pad covers it. + assert.equal(pickNearest(nodes, 28, 18, 25)?.id, "a"); + // Beyond the pad: nothing. + assert.equal(pickNearest(nodes, 30, 40, 3), null); + }); + + test("ties resolve deterministically (first scanned wins)", () => { + const pair = [ + { id: "first", x: 0, y: 0 }, + { id: "second", x: 10, y: 0 }, + ]; + assert.equal(pickNearest(pair, 5, 0, 10)?.id, "first"); + }); + + test("empty node list picks nothing", () => { + assert.equal(pickNearest([], 0, 0, 100), null); + }); +}); From 8e2d2118a6f3c7113041b644d3faabae27ffc54c Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 05:56:27 +0300 Subject: [PATCH 13/34] Graph view refinements: server-persisted settings, label size, eased zoom, resize-blank fix Five refinements to the hand-rolled renderer, all user-directed: 1. Settings are SERVER-persisted (data/render-settings.json via new GET/PUT /api/graph/render-settings) - one source of truth for every browser, replacing localStorage. The client loads once after mount, saves debounced on user changes, and never lets a slow GET clobber a faster user (or mounting PUT defaults over the server copy). The shape/defaults/sanitizer moved to src/core/render-settings.ts - pure, shared by client, server store, and tests; every disk/network value passes through sanitizeRenderSettings. 2. New "Label size" control (default 12px - the previous hardcoded size). Font, wrap width, and line height all scale; verified live: +76k px with the label gate open. 3. "Labels until zoom" slider reworked: log-scaled over 0.0625-4 so the useful band (0.25-0.5 camera-ratio units) sits at 33%/50% of the track instead of crammed at the far left of the old linear 0.25-4 slider. Display units are normalized: the default 0.5 reads as "1x", 0.25 as "0.5x". The underlying labelZoom semantics are unchanged, so existing values survive. 4. Smooth wheel zoom: d3-zoom instant per-notch jumps (x1.2, bang) are replaced by an eased glide - wheel events accumulate a cursor-anchored target scale and each tick eases the camera toward it (exponential, 120ms constant, frame-rate independent). Pan/pinch stay d3-owned; a pan cancels a glide in progress. Verified live: 8-frame glide. 5. Resize blanking fixed (graph vanished for the duration of a window drag, reappearing ~300ms after). Cause: every resize event reallocated the canvas backing store - which CLEARS it - and the redraw was deferred to rAF, which live resizes starve via relayout, so the canvas sat wiped for the whole drag. Fix: the ResizeObserver callback now redraws SYNCHRONOUSLY after reallocating (verified via a probe RO firing in the same delivery batch: canvas fully painted). kFit (the label gate 1x reference) is also recomputed on resize from the now-cached world bbox - the camera itself still persists. tests/render-settings.test.ts: 10 tests pinning the sanitizer (garbage/ partial/clamps, legacy no-labelSize records) and the slider mapping (default dead-center at 500, monotonic, exact round-trips, display units). 402/402 green; typecheck, lint, build pass. --- app/api/graph/render-settings/route.ts | 31 +++++ components/graph-canvas.ts | 155 ++++++++++++++++++------- components/graph-view.tsx | 134 +++++++++++++-------- config.ts | 6 + src/core/render-settings.ts | 93 +++++++++++++++ src/store/render-settings.ts | 40 +++++++ tests/render-settings.test.ts | 99 ++++++++++++++++ 7 files changed, 473 insertions(+), 85 deletions(-) create mode 100644 app/api/graph/render-settings/route.ts create mode 100644 src/core/render-settings.ts create mode 100644 src/store/render-settings.ts create mode 100644 tests/render-settings.test.ts diff --git a/app/api/graph/render-settings/route.ts b/app/api/graph/render-settings/route.ts new file mode 100644 index 0000000..3ee8719 --- /dev/null +++ b/app/api/graph/render-settings/route.ts @@ -0,0 +1,31 @@ +/** + * GET/PUT /api/graph/render-settings — server persistence for the graph + * view's rendering controls (node/edge/arrow/label sizes, the label-zoom + * threshold, the cluster-labels toggle). The file lives at + * data/render-settings.json (config.RENDER_SETTINGS_FILE). + * + * GET → 200 {settings} once saved, 200 null on first boot (client keeps + * in-memory defaults until the user changes something). + * PUT → 200 {settings} (the sanitized copy actually persisted — the + * client may echo it to stay honest with clamped values). + * + * Every body is passed through sanitizeRenderSettings (src/core) before + * touching disk: garbage clamps or falls back to defaults, never throws. + */ +import { readRenderSettings, writeRenderSettings } from "@/src/store/render-settings.ts"; + +export const dynamic = "force-dynamic"; + +export async function GET() { + return Response.json(readRenderSettings()); +} + +export async function PUT(request: Request) { + let body: unknown; + try { + body = await request.json(); + } catch { + return Response.json({ error: "invalid JSON body" }, { status: 400 }); + } + return Response.json(writeRenderSettings(body)); +} diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts index 741587a..3c57853 100644 --- a/components/graph-canvas.ts +++ b/components/graph-canvas.ts @@ -53,6 +53,9 @@ import { select } from "d3-selection"; import { zoom, zoomIdentity, type ZoomBehavior, type ZoomTransform } from "d3-zoom"; import { clusterHue, convexHull, offsetPolygon, traceRoundedPath, type Pt } from "./cluster-geometry.ts"; +import type { GraphRenderSettings } from "../src/core/render-settings.ts"; + +export type { GraphRenderSettings }; // ─── Public data contract ────────────────────────────────────────────────── @@ -86,18 +89,6 @@ export interface CanvasColors { hoverText: string; } -/** The rendering controls (persisted by the caller in localStorage). */ -export interface GraphRenderSettings { - nodeSize: number; - edgeWidth: number; - arrowLength: number; - /** Camera-ratio threshold for node labels (1 = fit zoom, smaller = - * zoomed in); cluster labels fade in over the same span, inversely. */ - labelZoom: number; - /** Toggle for the cluster region labels (hulls always draw). */ - clusterLabels: boolean; -} - export interface CanvasData { nodes: CanvasNode[]; edges: CanvasEdge[]; @@ -118,7 +109,7 @@ export interface RendererCallbacks { // ─── Constants (ported verbatim from the sigma view) ──────────────────────── -const LABEL_SIZE = 12; // px (sigma's labelSize) +const LABEL_SIZE = 12; // px — the default label size (settings.labelSize scales it) const LABEL_WEIGHT = "500"; const LABEL_MAX_WIDTH = 170; // px — wraps long names instead of a wide strip const LABEL_GAP = 6; // px between node edge and first label line @@ -129,6 +120,7 @@ export const LABELS_FADE_SPAN = 0.5; const GLOW_PAD = 9; // px the halo extends beyond the node disc (constant) const GLOW_ALPHA = 0.45; // halo opacity at the disc edge, fading outward const HOVER_FADE_MS = 180; // hover halo opacity fade length +const ZOOM_EASE_MS = 120; // wheel-zoom glide: exponential-ease time constant /** Extra pickable radius (px) beyond the disc — the whole halo is * hit-testable (the old flicker fix, minus the picking shader). */ @@ -280,19 +272,22 @@ function drawName( x: number, y: number, size: number, + labelSize: number, label: string, font: string, color: string, ): void { context.save(); - context.font = `${LABEL_WEIGHT} ${LABEL_SIZE}px ${font}`; - const lines = wrapLabel(context, label, LABEL_MAX_WIDTH); + context.font = `${LABEL_WEIGHT} ${labelSize}px ${font}`; + // Wrap width scales with the font so bigger labels wrap at the same + // word count, not earlier. + const lines = wrapLabel(context, label, LABEL_MAX_WIDTH * (labelSize / LABEL_SIZE)); context.fillStyle = color; context.textAlign = "center"; context.textBaseline = "top"; const firstTop = y + size + LABEL_GAP; for (let i = 0; i < lines.length; i++) { - context.fillText(lines[i]!, x, firstTop + i * LABEL_SIZE * LABEL_LINE_HEIGHT); + context.fillText(lines[i]!, x, firstTop + i * labelSize * LABEL_LINE_HEIGHT); } context.restore(); } @@ -309,6 +304,9 @@ export class GraphCanvasRenderer { private hasFit = false; /** The k that frames the whole graph — the label-gate's "1×". */ private kFit = 1; + /** World-space bounds of the current nodes — fit() and the resize-time + * kFit refresh share it (computed once per setData, not per resize). */ + private worldBBox: { minX: number; maxX: number; minY: number; maxY: number } | null = null; private data: CanvasData | null = null; /** id -> node for the current data (avoids O(n^2) scans). */ @@ -326,6 +324,11 @@ export class GraphCanvasRenderer { private hoverFrom = 0; private hoverTarget = 0; private hoverStart = 0; + // Smooth wheel zoom: wheel events accumulate a target scale (cursor- + // anchored); each tick eases the camera toward it instead of jumping. + private zoomTarget: { k: number; x: number; y: number } | null = null; + /** Clock for frame-rate-independent easing (updated every tick). */ + private lastTickAt = 0; private animating = false; private resizeObserver: ResizeObserver; @@ -350,8 +353,12 @@ export class GraphCanvasRenderer { select(canvas) .call(this.zoomBehavior) // No surprise dblclick-zoom (sigma had none). - .on("dblclick.zoom", null); + .on("dblclick.zoom", null) + // Wheel is OURS now: d3's instant per-notch jump felt janky; the + // eased glide below replaces it (pan and pinch stay d3's). + .on("wheel.zoom", null); + canvas.addEventListener("wheel", this.onWheel, { passive: false }); canvas.addEventListener("pointermove", this.onPointerMove); canvas.addEventListener("pointerdown", this.onPointerDown); canvas.addEventListener("pointerup", this.onPointerUp); @@ -360,7 +367,18 @@ export class GraphCanvasRenderer { this.resizeObserver = new ResizeObserver(() => { this.syncSize(); if (!this.hasFit) this.fit(); // first real size → fit once - this.markDirty(); + else if (this.worldBBox) { + // Keep the label-gate's "1×" honest: kFit is viewport-bound, + // so a resize changes it even though the camera persists. + const w = this.canvas.clientWidth; + const h = this.canvas.clientHeight; + if (w >= 10 && h >= 10) this.kFit = fitTransform(this.worldBBox, w, h, 40).k; + } + // Redraw SYNCHRONOUSLY: syncSize may have reallocated the backing + // store (which clears the canvas), and during live window resizes + // rAF is starved by relayout — a deferred draw would leave the + // canvas blank for the whole drag (the resize-blanking bug). + this.draw(); }); this.resizeObserver.observe(canvas); this.syncSize(); @@ -371,6 +389,17 @@ export class GraphCanvasRenderer { setData(data: CanvasData): void { this.data = data; this.nodeById = new Map(data.nodes.map((n) => [n.id, n])); + let minX = Infinity; + let minY = Infinity; + let maxX = -Infinity; + let maxY = -Infinity; + for (const n of data.nodes) { + if (n.x < minX) minX = n.x; + if (n.y < minY) minY = n.y; + if (n.x > maxX) maxX = n.x; + if (n.y > maxY) maxY = n.y; + } + this.worldBBox = { minX, maxX, minY, maxY }; if (!this.hasFit && data.nodes.length > 0) this.fit(); this.markDirty(); } @@ -388,21 +417,11 @@ export class GraphCanvasRenderer { /** Frame the whole graph (padding included). Keeps the camera on later * setData calls — call fit() only for an explicit re-frame. */ fit(): void { - if (!this.data || this.data.nodes.length === 0) return; + if (!this.data || this.data.nodes.length === 0 || !this.worldBBox) return; const w = this.canvas.clientWidth; const h = this.canvas.clientHeight; if (w < 10 || h < 10) return; // layout not ready yet; the ResizeObserver retries - let minX = Infinity; - let minY = Infinity; - let maxX = -Infinity; - let maxY = -Infinity; - for (const n of this.data.nodes) { - if (n.x < minX) minX = n.x; - if (n.y < minY) minY = n.y; - if (n.x > maxX) maxX = n.x; - if (n.y > maxY) maxY = n.y; - } - const t = fitTransform({ minX, maxX, minY, maxY }, w, h, 40); + const t = fitTransform(this.worldBBox, w, h, 40); this.kFit = t.k; this.hasFit = true; select(this.canvas).call( @@ -415,6 +434,7 @@ export class GraphCanvasRenderer { if (this.frameId !== null) cancelAnimationFrame(this.frameId); this.resizeObserver.disconnect(); select(this.canvas).on(".zoom", null); + this.canvas.removeEventListener("wheel", this.onWheel); this.canvas.removeEventListener("pointermove", this.onPointerMove); this.canvas.removeEventListener("pointerdown", this.onPointerDown); this.canvas.removeEventListener("pointerup", this.onPointerUp); @@ -424,6 +444,21 @@ export class GraphCanvasRenderer { // ── Event handlers (state mutation only — never draw) ── + /** Wheel: accumulate the zoom target; the camera GLIDES toward it in the + * tick loop (cursor-anchored). Sensitivity formula is d3-zoom's own. */ + private onWheel = (e: WheelEvent) => { + e.preventDefault(); + const delta = -e.deltaY * (e.deltaMode === 1 ? 0.05 : e.deltaMode ? 1 : 0.002); + const { x, y } = this.eventPos(e); + const current = this.zoomTarget?.k ?? this.transform.k; + // Must stay inside the zoomBehavior's scaleExtent [0.02, 40]. + const k = Math.min(Math.max(current * Math.exp(delta), 0.02), 40); + this.zoomTarget = { k, x, y }; + // Fresh easing clock so the glide starts gentle, never half-done. + this.lastTickAt = performance.now(); + this.markDirty(); + }; + private onPointerMove = (e: PointerEvent) => { const { x, y } = this.eventPos(e); const node = this.pickAt(x, y); @@ -433,6 +468,7 @@ export class GraphCanvasRenderer { private onPointerDown = (e: PointerEvent) => { this.downPos = this.eventPos(e); + this.zoomTarget = null; // a pan takes over — cancel any zoom glide }; private onPointerUp = (e: PointerEvent) => { @@ -453,7 +489,7 @@ export class GraphCanvasRenderer { // ── Internals ── - private eventPos(e: PointerEvent): Pt { + private eventPos(e: { clientX: number; clientY: number }): Pt { const rect = this.canvas.getBoundingClientRect(); return { x: e.clientX - rect.left, y: e.clientY - rect.top }; } @@ -496,10 +532,16 @@ export class GraphCanvasRenderer { } private tick(): void { - this.advanceHover(); + const now = performance.now(); + // Clamp the dt: a long idle gap must not teleport the ease forward. + const dt = Math.min(Math.max(now - this.lastTickAt, 8), 100); + this.lastTickAt = now; + const hoverAnim = this.advanceHover(); + const zoomAnim = this.advanceZoom(dt); this.draw(); - // Keep frames coming ONLY while the hover fade is in motion; an - // idle graph schedules no rAF at all. + // Keep frames coming ONLY while an animation is in motion (hover + // fade or zoom glide); an idle graph schedules no rAF at all. + this.animating = hoverAnim || zoomAnim; if (this.animating) { this.frameId = requestAnimationFrame(() => { this.frameId = null; @@ -508,11 +550,39 @@ export class GraphCanvasRenderer { } } - private advanceHover(): void { - if (!this.animating) return; + /** Hover fade step; returns true while still moving. */ + private advanceHover(): boolean { + if (this.hoverFade === this.hoverTarget) return false; const t = Math.min((performance.now() - this.hoverStart) / HOVER_FADE_MS, 1); this.hoverFade = this.hoverFrom + (this.hoverTarget - this.hoverFrom) * t; - if (t >= 1) this.animating = false; + return t < 1; + } + + /** Zoom glide step: ease k toward the wheel target, keeping the world + * point under the cursor pinned. Frame-rate independent (exponential + * approach, time constant ZOOM_EASE_MS). Returns true while gliding. */ + private advanceZoom(dt: number): boolean { + const target = this.zoomTarget; + if (!target) return false; + const t = this.transform; + const blend = 1 - Math.exp(-dt / ZOOM_EASE_MS); + let k = t.k + (target.k - t.k) * blend; + let done = false; + if (Math.abs(target.k - k) / target.k < 0.001) { + k = target.k; + done = true; + } + const scale = k / t.k; + const x = target.x - (target.x - t.x) * scale; + const y = target.y - (target.y - t.y) * scale; + if (done) this.zoomTarget = null; + // Applied through d3-zoom so constraints + the zoom event (markDirty, + // onCameraMove) fire exactly as with any other camera change. + select(this.canvas).call( + this.zoomBehavior.transform, + zoomIdentity.translate(x, y).scale(k), + ); + return !done; } /** Resize the backing store to the CSS box × devicePixelRatio; all @@ -645,7 +715,11 @@ export class GraphCanvasRenderer { this.ctx.globalAlpha = nodeAlpha; for (const [id, p] of pos) { const n = nodeById.get(id); - if (n) drawName(this.ctx, p.x, p.y, settings.nodeSize, n.label, data.fontFamily, data.colors.label); + if (n) + drawName( + this.ctx, p.x, p.y, settings.nodeSize, settings.labelSize, n.label, + data.fontFamily, data.colors.label, + ); } this.ctx.restore(); } @@ -669,7 +743,10 @@ export class GraphCanvasRenderer { this.ctx.fill("evenodd"); this.ctx.save(); this.ctx.globalAlpha = this.hoverFade; - drawName(this.ctx, p.x, p.y, settings.nodeSize, n.label, data.fontFamily, data.colors.hoverText); + drawName( + this.ctx, p.x, p.y, settings.nodeSize, settings.labelSize, n.label, + data.fontFamily, data.colors.hoverText, + ); this.ctx.restore(); } } diff --git a/components/graph-view.tsx b/components/graph-view.tsx index d939bc8..0ad7cf6 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -50,10 +50,15 @@ * an open card closes if its node is filtered out. * * Rendering settings (2026-09-13): the gear in the search panel reveals - * four sliders (node size, edge width, arrowhead length, the zoom - * threshold) plus the cluster-labels checkbox, persisted per browser - * (localStorage). Applied by just redrawing — no rebuild, no re-layout, - * and the camera never moves. + * five sliders (node size, edge width, arrowhead length, label size, the + * zoom threshold) plus the cluster-labels checkbox. SERVER-persisted + * (data/render-settings.json via /api/graph/render-settings): loaded + * once after mount, saved debounced on every user change — one source + * of truth shared by every browser. The label-zoom slider is log-scaled + * (its useful band 0.25–0.5 sits mid-slider, not crammed at the left) + * and displays normalized units where the default 0.5 reads as "1×". + * Settings apply by just redrawing — no rebuild, no re-layout, and the + * camera never moves. */ import { useDeferredValue, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; @@ -63,6 +68,13 @@ import Graph from "graphology"; import forceAtlas2 from "graphology-layout-forceatlas2"; import { seededRandom, type GraphNode, type GraphSnapshot } from "@/src/core"; import { GraphCanvasRenderer, type GraphRenderSettings } from "./graph-canvas.ts"; +import { + DEFAULT_RENDER_SETTINGS, + labelZoomDisplay, + labelZoomToSlider, + sanitizeRenderSettings, + sliderToLabelZoom, +} from "../src/core/render-settings.ts"; /** The uniform node look (no state classes — canvas wants hex, not oklch): * the accent color, until the state model lands. */ @@ -70,27 +82,10 @@ const NODE_COLOR_TOKEN = "--primary"; // ---- Render settings (the settings-panel controls) ------------------------ // -// Three basic controls (user-decided 2026-09-13): node disc size, edge line -// width, and arrowhead length (px). Defaults reflect the pre-panel look -// (edge 1px + its stock 5px head) except nodeSize, which dropped 8 → 6 for -// dense graphs (user: "nodes are quite large"). Persisted per browser in -// localStorage. +// The shape, defaults, sanitizer, and slider mappings live in +// src/core/render-settings.ts (shared with the server store + tests). type RenderSettings = GraphRenderSettings; -const DEFAULT_RENDER_SETTINGS: RenderSettings = { nodeSize: 6, edgeWidth: 1, arrowLength: 5, labelZoom: 0.5, clusterLabels: true }; -const RENDER_SETTINGS_KEY = "graph-render-settings"; -const clamp = (v: number, min: number, max: number) => Math.min(Math.max(v, min), max); -/** Defensively merge stored settings (stale/partial/corrupt → defaults). */ -function sanitizeRenderSettings(raw: unknown): RenderSettings { - const r = raw as Partial | null; - return { - nodeSize: clamp(Number(r?.nodeSize ?? DEFAULT_RENDER_SETTINGS.nodeSize), 2, 14), - edgeWidth: clamp(Number(r?.edgeWidth ?? DEFAULT_RENDER_SETTINGS.edgeWidth), 0.5, 4), - arrowLength: clamp(Number(r?.arrowLength ?? DEFAULT_RENDER_SETTINGS.arrowLength), 2, 15), - labelZoom: clamp(Number(r?.labelZoom ?? DEFAULT_RENDER_SETTINGS.labelZoom), 0.25, 4), - clusterLabels: typeof r?.clusterLabels === "boolean" ? r.clusterLabels : DEFAULT_RENDER_SETTINGS.clusterLabels, - }; -} /** * Pure filter predicate for the search panel: a node is visible when the @@ -189,24 +184,59 @@ export function GraphView() { // Search panel state: the query text + the settings section toggle. const [query, setQuery] = useState(""); const [settingsOpen, setSettingsOpen] = useState(false); - // Render settings (node size / edge width / arrowhead): loaded from - // localStorage AFTER mount (an initial-state read would mismatch the - // server-rendered HTML), saved on every change. + // Render settings: SERVER-persisted (data/render-settings.json via + // /api/graph/render-settings). Loaded once after mount; every user + // change saves back, debounced. Defaults render until the load + // resolves — a fraction of a second at worst. const [settings, setSettings] = useState(DEFAULT_RENDER_SETTINGS); + const settingsRef = useRef(settings); + settingsRef.current = settings; + // True once the user has moved a control this mount: the server copy + // must never clobber a (possibly newer) local change. + const userTouched = useRef(false); + // True once the initial GET settled — saves are held back until then, + // so merely mounting never PUTs defaults over the server copy. + const serverLoaded = useRef(false); + const savePending = useRef(false); + const changeSettings = (update: (cur: RenderSettings) => RenderSettings) => { + userTouched.current = true; + setSettings(update); + }; + const saveToServer = (s: RenderSettings) => { + fetch("/api/graph/render-settings", { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify(s), + }).catch((err) => console.error("[graph] settings save failed:", err)); + }; useEffect(() => { - try { - const stored = localStorage.getItem(RENDER_SETTINGS_KEY); - if (stored) setSettings(sanitizeRenderSettings(JSON.parse(stored))); - } catch { - // corrupt storage → defaults, never crash the view - } + fetch("/api/graph/render-settings") + .then((res) => (res.ok ? (res.json() as Promise) : null)) + .then((raw) => { + // Apply only if the user hasn't already moved a slider — + // if they outran the network, their change wins. + if (raw !== null && !userTouched.current) setSettings(sanitizeRenderSettings(raw)); + }) + .catch(() => { + // server unreachable → in-memory defaults keep working + }) + .finally(() => { + serverLoaded.current = true; + // A change made during the fetch was held back — send now. + if (savePending.current) { + savePending.current = false; + saveToServer(settingsRef.current); + } + }); }, []); useEffect(() => { - try { - localStorage.setItem(RENDER_SETTINGS_KEY, JSON.stringify(settings)); - } catch { - // storage full/blocked → in-memory settings still work + if (!userTouched.current) return; // mount-run / server-apply: no save + if (!serverLoaded.current) { + savePending.current = true; // hold until the GET settles + return; } + const id = setTimeout(() => saveToServer(settings), 400); + return () => clearTimeout(id); }, [settings]); // The renderer instance lives across snapshot refetches and settings @@ -474,7 +504,7 @@ export function GraphView() { max={14} step={1} value={settings.nodeSize} - onChange={(v) => setSettings((cur) => ({ ...cur, nodeSize: v }))} + onChange={(v) => changeSettings((cur) => ({ ...cur, nodeSize: v }))} /> setSettings((cur) => ({ ...cur, edgeWidth: v }))} + onChange={(v) => changeSettings((cur) => ({ ...cur, edgeWidth: v }))} /> setSettings((cur) => ({ ...cur, arrowLength: v }))} + onChange={(v) => changeSettings((cur) => ({ ...cur, arrowLength: v }))} /> + changeSettings((cur) => ({ ...cur, labelSize: v }))} + /> + {/* Log-scaled: the useful band (0.25–0.5 in camera-ratio + units) sits mid-slider; displayed units normalize the + default (0.5) to "1×". Slider position ↔ labelZoom via + src/core/render-settings.ts. */} setSettings((cur) => ({ ...cur, labelZoom: v }))} + min={0} + max={1000} + step={5} + value={labelZoomToSlider(settings.labelZoom)} + display={labelZoomDisplay(settings.labelZoom)} + onChange={(v) => changeSettings((cur) => ({ ...cur, labelZoom: sliderToLabelZoom(v) }))} /> diff --git a/config.ts b/config.ts index 3472ad8..d6039bf 100644 --- a/config.ts +++ b/config.ts @@ -84,3 +84,9 @@ export const LEDGER_FILES = { * log. Read-modify-write like a chat file, not append-only like the ledgers. */ export const SOURCES_FILE = path.join(PATHS.graph, "sources.json"); + +/** + * Graph render settings (the settings-panel controls): one small JSON blob, + * read-modify-write. Shared shape/sanitizer: src/core/render-settings.ts. + */ +export const RENDER_SETTINGS_FILE = path.join(PATHS.graph, "render-settings.json"); diff --git a/src/core/render-settings.ts b/src/core/render-settings.ts new file mode 100644 index 0000000..8946890 --- /dev/null +++ b/src/core/render-settings.ts @@ -0,0 +1,93 @@ +/** + * Graph render settings: the shape, the defaults, the sanitizer, and the + * label-zoom slider mapping — shared by the client (components/graph-canvas.ts, + * graph-view.tsx), the server store (src/store/render-settings.ts, which + * persists them to data/render-settings.json), and the API route. + * + * This module is pure: no DOM, no d3, no node APIs — safe everywhere. + * + * Settings are server-persisted (render-settings.json); the localStorage era + * is over. Whatever arrives from disk or the network is passed through + * sanitizeRenderSettings before use — never trusted raw. + * + * The labelZoom semantics: the sigma-compatible "camera ratio" threshold + * (1 = fit zoom, SMALLER = zoomed in). Node labels show at ratio <= labelZoom + * and fade over LABELS_FADE_SPAN; cluster region labels fade in inversely. + * Persisted values keep these units across the renderer swap. + */ + +export interface GraphRenderSettings { + nodeSize: number; + edgeWidth: number; + arrowLength: number; + /** Camera-ratio threshold for node labels (see module doc). */ + labelZoom: number; + /** Node + hover label font size, px. */ + labelSize: number; + /** Toggle for the cluster region labels (hulls always draw). */ + clusterLabels: boolean; +} + +export const DEFAULT_RENDER_SETTINGS: GraphRenderSettings = { + nodeSize: 6, + edgeWidth: 1, + arrowLength: 5, + labelZoom: 0.5, + labelSize: 12, + clusterLabels: true, +}; + +const clamp = (v: number, min: number, max: number) => Math.min(Math.max(v, min), max); + +/** + * Coerce arbitrary JSON (server file, network body, localStorage leftovers) + * into valid settings: missing keys fall back to defaults, out-of-range + * values clamp, wrong types reset. Never throws. + */ +export function sanitizeRenderSettings(raw: unknown): GraphRenderSettings { + const r = (raw ?? {}) as Partial; + const num = (v: unknown, fallback: number, min: number, max: number) => { + const n = typeof v === "number" && Number.isFinite(v) ? v : fallback; + return clamp(n, min, max); + }; + return { + nodeSize: num(r.nodeSize, DEFAULT_RENDER_SETTINGS.nodeSize, 2, 14), + edgeWidth: num(r.edgeWidth, DEFAULT_RENDER_SETTINGS.edgeWidth, 0.5, 4), + arrowLength: num(r.arrowLength, DEFAULT_RENDER_SETTINGS.arrowLength, 2, 15), + // Range matches the log slider's span [LABEL_ZOOM_MIN, LABEL_ZOOM_MAX]. + labelZoom: num(r.labelZoom, DEFAULT_RENDER_SETTINGS.labelZoom, 0.0625, 4), + labelSize: num(r.labelSize, DEFAULT_RENDER_SETTINGS.labelSize, 6, 24), + clusterLabels: + typeof r.clusterLabels === "boolean" ? r.clusterLabels : DEFAULT_RENDER_SETTINGS.clusterLabels, + }; +} + +// ─── The "Labels until zoom" slider mapping ────────────────────────────────── +// +// The slider is LOG-scaled over the labelZoom range so the useful band +// (0.25–0.5) sits mid-slider instead of crammed at the far left of a linear +// 0.25–4 track. Display units are normalized to the default: labelZoom 0.5 +// shows as "1×", 0.25 as "0.5×" — the default reads as 1, per spec. + +export const LABEL_ZOOM_MIN = 0.0625; // slider far left: labels only very deep in +export const LABEL_ZOOM_MAX = 4; // slider far right: labels visible at fit zoom + +/** labelZoom → slider position [0, 1000] (log scale). 0.5 lands at 500. */ +export function labelZoomToSlider(labelZoom: number): number { + const clamped = clamp(labelZoom, LABEL_ZOOM_MIN, LABEL_ZOOM_MAX); + const logSpan = Math.log(LABEL_ZOOM_MAX / LABEL_ZOOM_MIN); + return (1000 * Math.log(clamped / LABEL_ZOOM_MIN)) / logSpan; +} + +/** Slider position [0, 1000] → labelZoom (inverse of labelZoomToSlider). */ +export function sliderToLabelZoom(position: number): number { + const p = clamp(position, 0, 1000) / 1000; + const logSpan = Math.log(LABEL_ZOOM_MAX / LABEL_ZOOM_MIN); + return LABEL_ZOOM_MIN * Math.exp(p * logSpan); +} + +/** Display units: normalized so the default (0.5) reads as 1. */ +export function labelZoomDisplay(labelZoom: number): string { + const v = Math.round((labelZoom / DEFAULT_RENDER_SETTINGS.labelZoom) * 100) / 100; + return `${v}×`; +} diff --git a/src/store/render-settings.ts b/src/store/render-settings.ts new file mode 100644 index 0000000..754135c --- /dev/null +++ b/src/store/render-settings.ts @@ -0,0 +1,40 @@ +/** + * Server-side persistence for the graph render settings (the settings-panel + * controls): data/render-settings.json — read-modify-write, NOT a ledger. + * + * The shape, defaults, and sanitizer live in src/core/render-settings.ts + * (pure, shared with the client); this module only moves bytes. Whatever + * comes off disk is sanitized by the caller on load — and we sanitize on + * write too, so the file can never hold an out-of-range value that would + * silently shift a slider. + */ + +import fs from "node:fs"; +import path from "node:path"; +import { RENDER_SETTINGS_FILE } from "../../config.ts"; +import { DEFAULT_RENDER_SETTINGS, sanitizeRenderSettings, type GraphRenderSettings } from "../core/render-settings.ts"; + +/** The persisted settings, or null when no file exists yet (first boot). */ +export function readRenderSettings(): GraphRenderSettings | null { + let raw: string; + try { + raw = fs.readFileSync(RENDER_SETTINGS_FILE, "utf-8"); + } catch { + return null; // never booted / unreadable → caller uses defaults + } + try { + return sanitizeRenderSettings(JSON.parse(raw)); + } catch { + return null; // corrupt → defaults, never crash the view + } +} + +/** Persist settings (sanitized). Returns what was written. */ +export function writeRenderSettings(raw: unknown): GraphRenderSettings { + const settings = sanitizeRenderSettings(raw); + fs.mkdirSync(path.dirname(RENDER_SETTINGS_FILE), { recursive: true }); + fs.writeFileSync(RENDER_SETTINGS_FILE, JSON.stringify(settings, null, "\t"), "utf-8"); + return settings; +} + +export { DEFAULT_RENDER_SETTINGS }; diff --git a/tests/render-settings.test.ts b/tests/render-settings.test.ts new file mode 100644 index 0000000..55d2b5c --- /dev/null +++ b/tests/render-settings.test.ts @@ -0,0 +1,99 @@ +/** + * Tests for src/core/render-settings.ts — the shared settings contract + * (shape, defaults, sanitizer, label-zoom slider mapping). Pure module, + * pure tests: whatever JSON arrives from disk or the network goes through + * sanitizeRenderSettings and must come out valid, never throw. + */ + +import assert from "node:assert/strict"; +import { describe, it as test } from "node:test"; +import { + DEFAULT_RENDER_SETTINGS, + LABEL_ZOOM_MAX, + LABEL_ZOOM_MIN, + labelZoomDisplay, + labelZoomToSlider, + sanitizeRenderSettings, + sliderToLabelZoom, +} from "../src/core/render-settings.ts"; + +describe("sanitizeRenderSettings", () => { + test("garbage in → full defaults out", () => { + assert.deepEqual(sanitizeRenderSettings(null), DEFAULT_RENDER_SETTINGS); + assert.deepEqual(sanitizeRenderSettings(undefined), DEFAULT_RENDER_SETTINGS); + assert.deepEqual(sanitizeRenderSettings("junk"), DEFAULT_RENDER_SETTINGS); + assert.deepEqual(sanitizeRenderSettings(42), DEFAULT_RENDER_SETTINGS); + }); + + test("partial input → defaults fill the gaps", () => { + const s = sanitizeRenderSettings({ nodeSize: 9 }); + assert.equal(s.nodeSize, 9); + assert.equal(s.edgeWidth, DEFAULT_RENDER_SETTINGS.edgeWidth); + assert.equal(s.labelSize, DEFAULT_RENDER_SETTINGS.labelSize); + assert.equal(s.labelZoom, DEFAULT_RENDER_SETTINGS.labelZoom); + assert.equal(s.clusterLabels, DEFAULT_RENDER_SETTINGS.clusterLabels); + }); + + test("out-of-range values clamp; wrong types reset", () => { + const s = sanitizeRenderSettings({ + nodeSize: 999, + edgeWidth: 0, + arrowLength: -5, + labelZoom: 100, + labelSize: "big" as unknown as number, + clusterLabels: "yes" as unknown as boolean, + }); + assert.equal(s.nodeSize, 14); // clamped to max + assert.equal(s.edgeWidth, 0.5); // clamped to min + assert.equal(s.arrowLength, 2); + assert.equal(s.labelZoom, LABEL_ZOOM_MAX); + assert.equal(s.labelSize, DEFAULT_RENDER_SETTINGS.labelSize); // NaN-safe reset + assert.equal(s.clusterLabels, DEFAULT_RENDER_SETTINGS.clusterLabels); + }); + + test("a fully valid object passes through unchanged", () => { + const valid = { nodeSize: 8, edgeWidth: 2, arrowLength: 7, labelZoom: 0.25, labelSize: 16, clusterLabels: false }; + assert.deepEqual(sanitizeRenderSettings(valid), valid); + }); + + test("the pre-wave stored shape (no labelSize) still loads", () => { + // localStorage-era records lacked labelSize — must not shift anything. + const s = sanitizeRenderSettings({ nodeSize: 4, edgeWidth: 1, arrowLength: 5, labelZoom: 0.5, clusterLabels: true }); + assert.equal(s.labelSize, DEFAULT_RENDER_SETTINGS.labelSize); + assert.equal(s.nodeSize, 4); + }); +}); + +describe("label-zoom slider mapping (log scale)", () => { + test("the default (0.5) sits dead-center at position 500", () => { + assert.equal(Math.round(labelZoomToSlider(DEFAULT_RENDER_SETTINGS.labelZoom)), 500); + }); + + test("the useful band sits mid-slider, not crammed at the left", () => { + // 0.25 → a third of the way in; the old linear 0.25–4 slider had it at 0%. + const p025 = labelZoomToSlider(0.25); + assert.ok(p025 > 250 && p025 < 400, `0.25 at ${p025}`); + assert.ok(labelZoomToSlider(0.5) > p025, "monotonic in labelZoom"); + }); + + test("round-trips both directions across the whole range", () => { + for (const lz of [LABEL_ZOOM_MIN, 0.0625, 0.1, 0.25, 0.5, 1, 2, LABEL_ZOOM_MAX]) { + const back = sliderToLabelZoom(labelZoomToSlider(lz)); + assert.ok(Math.abs(back - lz) < 1e-9, `${lz} -> ${back}`); + const fwd = labelZoomToSlider(sliderToLabelZoom(labelZoomToSlider(lz))); + assert.ok(Math.abs(fwd - labelZoomToSlider(lz)) < 1e-9); + } + }); + + test("slider extremes map to the range ends", () => { + assert.equal(Math.round(sliderToLabelZoom(0) * 1e6) / 1e6, LABEL_ZOOM_MIN); + assert.ok(Math.abs(sliderToLabelZoom(1000) - LABEL_ZOOM_MAX) < 1e-9); + }); + + test("display units normalize the default to 1", () => { + assert.equal(labelZoomDisplay(0.5), "1×"); + assert.equal(labelZoomDisplay(0.25), "0.5×"); + assert.equal(labelZoomDisplay(4), "8×"); + assert.equal(labelZoomDisplay(0.0625), "0.13×"); // rounded to 2 decimals + }); +}); From ecadededd913e6c19db29baeffb391f2a7b2c9df Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 07:18:37 +0300 Subject: [PATCH 14/34] World-space rendering: geometry cached once, zoom pays only rasterization Fixes the zoom frame-rate collapse and rebuilds draw() around a single camera transform, per the user's Obsidian-style sizing proposal. Bug fix first: the eased zoom leaked a queued rAF callback per frame -- advanceZoom applies each step through d3-zoom, whose zoom event called markDirty() from inside the tick loop, scheduling an extra draw while the loop scheduled its own. Draws multiplied per displayed frame; sustained wheel ran at ~20fps against a 120Hz display. markDirty now skips scheduling while inTick (verified: 62 frames/500ms during a 28x zoom burst, matching pan). World-space model: every draw call now runs in graph coordinates under one setTransform(k). Sizes are defined as screen-px-at-fit and converted once per geometry build (/ kFit), so nodes, node labels, arrowheads and the hover halo scale with the graph as you zoom (0.25x painted area at half zoom, exactly k^2) while edge widths and hull borders stay viewer-relative hairlines (/ k). Cluster hull geometry (convex hull + three offset fills + solid boundary -- dashes dropped for now) is built as cached Path2Ds per data/visibility/settings change; camera moves do zero geometry math, making zoom per-frame cost identical to pan. Label wrapping runs once per build, not per frame. Region labels stay viewer-relative (18px at any zoom) as UI chrome. Picking radius scales with the world-sized discs. Glide feel: ZOOM_EASE_MS 120 -> 45, convergence 0.1% -> 0.2%; a notch now settles in ~260ms instead of ~750ms. Resize still paints synchronously. Also fixes barrel-law lint violations from the settings wave (import from @/src/core and @/src/server, not deep module paths). --- app/api/graph/render-settings/route.ts | 2 +- components/cluster-geometry.ts | 2 +- components/graph-canvas.ts | 396 +++++++++++++++---------- components/graph-view.tsx | 2 +- src/core/index.ts | 1 + src/server/index.ts | 1 + 6 files changed, 252 insertions(+), 152 deletions(-) diff --git a/app/api/graph/render-settings/route.ts b/app/api/graph/render-settings/route.ts index 3ee8719..bc82f1b 100644 --- a/app/api/graph/render-settings/route.ts +++ b/app/api/graph/render-settings/route.ts @@ -12,7 +12,7 @@ * Every body is passed through sanitizeRenderSettings (src/core) before * touching disk: garbage clamps or falls back to defaults, never throws. */ -import { readRenderSettings, writeRenderSettings } from "@/src/store/render-settings.ts"; +import { readRenderSettings, writeRenderSettings } from "@/src/server"; export const dynamic = "force-dynamic"; diff --git a/components/cluster-geometry.ts b/components/cluster-geometry.ts index 15d9a80..b122323 100644 --- a/components/cluster-geometry.ts +++ b/components/cluster-geometry.ts @@ -68,7 +68,7 @@ export function offsetPolygon(points: Pt[], pad: number): Pt[] { } /** Path through the polygon with rounded corners (arcTo). */ -export function traceRoundedPath(ctx: CanvasRenderingContext2D, points: Pt[], radius: number): void { +export function traceRoundedPath(ctx: CanvasRenderingContext2D | Path2D, points: Pt[], radius: number): void { const first = points[0]!; ctx.moveTo(first.x, first.y); for (let i = 1; i <= points.length; i++) { diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts index 3c57853..285437b 100644 --- a/components/graph-canvas.ts +++ b/components/graph-canvas.ts @@ -53,7 +53,7 @@ import { select } from "d3-selection"; import { zoom, zoomIdentity, type ZoomBehavior, type ZoomTransform } from "d3-zoom"; import { clusterHue, convexHull, offsetPolygon, traceRoundedPath, type Pt } from "./cluster-geometry.ts"; -import type { GraphRenderSettings } from "../src/core/render-settings.ts"; +import type { GraphRenderSettings } from "@/src/core"; export type { GraphRenderSettings }; @@ -120,7 +120,7 @@ export const LABELS_FADE_SPAN = 0.5; const GLOW_PAD = 9; // px the halo extends beyond the node disc (constant) const GLOW_ALPHA = 0.45; // halo opacity at the disc edge, fading outward const HOVER_FADE_MS = 180; // hover halo opacity fade length -const ZOOM_EASE_MS = 120; // wheel-zoom glide: exponential-ease time constant +const ZOOM_EASE_MS = 45; // wheel-zoom glide: exponential-ease time constant (~250ms per notch) /** Extra pickable radius (px) beyond the disc — the whole halo is * hit-testable (the old flicker fix, minus the picking shader). */ @@ -139,9 +139,13 @@ const HULL_FILL_LAYERS = [ ] as const; const HULL_STROKE_PAD = 14; const HULL_CORNER_RADIUS = 9; -const HULL_DASH = [7, 7] as const; const REGION_LABEL_SIZE = 18; +/** All "px" constants above are screen px AT FIT ZOOM. The renderer scales + * them into world units (÷ kFit) once per geometry build; camera zoom then + * scales nodes/labels/arrowheads with the graph (Obsidian-style), while + * edge widths and hull borders stay viewer-relative hairlines (÷ k). */ + const clamp01 = (v: number) => Math.min(Math.max(v, 0), 1); // ─── Pure helpers (unit-tested) ───────────────────────────────────────────── @@ -265,31 +269,33 @@ function wrapLabel(context: CanvasRenderingContext2D, text: string, maxWidth: nu return lines; } -/** The wrapped node name, centered below the node — identical geometry at - * rest and on hover, so hover only changes the color, nothing jumps. */ -function drawName( +/** World-unit sizes for the label pass, derived once per geometry build. */ +interface WorldSizes { + nodeR: number; // node disc radius, world units + glowOuter: number; // hover halo outer radius, world units + labelSize: number; // node label font, world units + labelGap: number; // node edge → first label line, world units +} + +/** The pre-wrapped node name, centered below the node in WORLD units — + * identical geometry at rest and on hover, so hover only changes the + * color, nothing jumps. Lines are wrapped once per geometry build, never + * per frame. */ +function drawNameLines( context: CanvasRenderingContext2D, x: number, y: number, - size: number, - labelSize: number, - label: string, - font: string, + lines: string[], + world: WorldSizes, color: string, ): void { - context.save(); - context.font = `${LABEL_WEIGHT} ${labelSize}px ${font}`; - // Wrap width scales with the font so bigger labels wrap at the same - // word count, not earlier. - const lines = wrapLabel(context, label, LABEL_MAX_WIDTH * (labelSize / LABEL_SIZE)); context.fillStyle = color; context.textAlign = "center"; context.textBaseline = "top"; - const firstTop = y + size + LABEL_GAP; + const firstTop = y + world.nodeR + world.labelGap; for (let i = 0; i < lines.length; i++) { - context.fillText(lines[i]!, x, firstTop + i * labelSize * LABEL_LINE_HEIGHT); + context.fillText(lines[i]!, x, firstTop + i * world.labelSize * LABEL_LINE_HEIGHT); } - context.restore(); } // ─── The renderer ─────────────────────────────────────────────────────────── @@ -316,6 +322,11 @@ export class GraphCanvasRenderer { private frameId: number | null = null; private dirty = true; + /** True while tick() runs: markDirty inside the loop (d3's zoom event + * fires from advanceZoom) must NOT schedule — the loop schedules + * itself below. Un-guarded, each glide frame leaked an extra queued + * tick and draws multiplied per displayed frame (20fps zoom bug). */ + private inTick = false; // Hover transition: fade 0 = resting, 1 = fully hovered; the animating // flag keeps the loop scheduled until the fade settles (then rAF stops). @@ -334,6 +345,27 @@ export class GraphCanvasRenderer { private resizeObserver: ResizeObserver; private downPos: Pt | null = null; + // ── Cached world geometry ───────────────────────────────────────────── + // Everything above is drawn per frame; everything below is derived from + // data+visibility+settings+kFit and rebuilt ONLY when one of those + // changes (geoVersion). Camera moves just re-rasterize the same paths. + private geoVersion = 0; // bump = geometry inputs changed + private builtVersion = -1; // version the cache was built at + private nodesPath: Path2D | null = null; + private shaftsPath: Path2D = new Path2D(); + private headsPath: Path2D = new Path2D(); + private hullShapes: { + fill: Path2D[]; + stroke: Path2D; + label: string | null; + cx: number; // world centroid + cy: number; + hue: number; + }[] = []; + private labelLines = new Map(); + private worldLabelFont = ""; + private world: WorldSizes = { nodeR: 0, glowOuter: 0, labelSize: 0, labelGap: 0 }; + constructor(canvas: HTMLCanvasElement, callbacks: RendererCallbacks) { this.canvas = canvas; const ctx = canvas.getContext("2d"); @@ -372,7 +404,10 @@ export class GraphCanvasRenderer { // so a resize changes it even though the camera persists. const w = this.canvas.clientWidth; const h = this.canvas.clientHeight; - if (w >= 10 && h >= 10) this.kFit = fitTransform(this.worldBBox, w, h, 40).k; + if (w >= 10 && h >= 10) { + this.kFit = fitTransform(this.worldBBox, w, h, 40).k; + this.geoVersion++; // resize redefines the fit-zoom scale + } } // Redraw SYNCHRONOUSLY: syncSize may have reallocated the backing // store (which clears the canvas), and during live window resizes @@ -401,16 +436,19 @@ export class GraphCanvasRenderer { } this.worldBBox = { minX, maxX, minY, maxY }; if (!this.hasFit && data.nodes.length > 0) this.fit(); + this.geoVersion++; // new data → rebuild world geometry this.markDirty(); } setSettings(settings: GraphRenderSettings): void { this.settings = settings; + this.geoVersion++; // sizes/wrap width changed → rebuild geometry this.markDirty(); } setVisible(visible: ReadonlySet | null): void { this.visible = visible; + this.geoVersion++; // member visibility changes hulls + paths this.markDirty(); } @@ -424,6 +462,7 @@ export class GraphCanvasRenderer { const t = fitTransform(this.worldBBox, w, h, 40); this.kFit = t.k; this.hasFit = true; + this.geoVersion++; // kFit defines the world-unit scale select(this.canvas).call( this.zoomBehavior.transform, zoomIdentity.translate(t.x, t.y).scale(t.k), @@ -498,7 +537,9 @@ export class GraphCanvasRenderer { if (!this.data || !this.settings) return null; // Screen positions, filtered to visible nodes. const { x: tx, y: ty, k } = this.transform; - const maxDist = this.settings.nodeSize + PICKING_PAD; + // Discs scale with the camera now: their screen radius is + // nodeSize × k/kFit; the halo pad stays a constant screen pad. + const maxDist = this.settings.nodeSize * (k / this.kFit) + PICKING_PAD; const projected: { id: string; x: number; y: number }[] = []; for (const n of this.data.nodes) { if (this.visible && !this.visible.has(n.id)) continue; @@ -523,7 +564,7 @@ export class GraphCanvasRenderer { private markDirty(): void { this.dirty = true; - if (this.frameId === null) { + if (this.frameId === null && !this.inTick) { this.frameId = requestAnimationFrame(() => { this.frameId = null; this.tick(); @@ -532,16 +573,21 @@ export class GraphCanvasRenderer { } private tick(): void { - const now = performance.now(); - // Clamp the dt: a long idle gap must not teleport the ease forward. - const dt = Math.min(Math.max(now - this.lastTickAt, 8), 100); - this.lastTickAt = now; - const hoverAnim = this.advanceHover(); - const zoomAnim = this.advanceZoom(dt); - this.draw(); - // Keep frames coming ONLY while an animation is in motion (hover - // fade or zoom glide); an idle graph schedules no rAF at all. - this.animating = hoverAnim || zoomAnim; + this.inTick = true; + try { + const now = performance.now(); + // Clamp the dt: a long idle gap must not teleport the ease forward. + const dt = Math.min(Math.max(now - this.lastTickAt, 8), 100); + this.lastTickAt = now; + const hoverAnim = this.advanceHover(); + const zoomAnim = this.advanceZoom(dt); + this.draw(); + // Keep frames coming ONLY while an animation is in motion (hover + // fade or zoom glide); an idle graph schedules no rAF at all. + this.animating = hoverAnim || zoomAnim; + } finally { + this.inTick = false; + } if (this.animating) { this.frameId = requestAnimationFrame(() => { this.frameId = null; @@ -568,7 +614,7 @@ export class GraphCanvasRenderer { const blend = 1 - Math.exp(-dt / ZOOM_EASE_MS); let k = t.k + (target.k - t.k) * blend; let done = false; - if (Math.abs(target.k - k) / target.k < 0.001) { + if (Math.abs(target.k - k) / target.k < 0.002) { k = target.k; done = true; } @@ -600,7 +646,100 @@ export class GraphCanvasRenderer { this.ctx.setTransform(dpr, 0, 0, dpr, 0, 0); } - /** THE draw function: every layer, one frame, one camera. */ + /** Derive ALL world geometry from data + visibility + settings + kFit. + * Runs only when geoVersion moves — never per frame, never per camera + * change. Zoom/pan pay zero geometry math, only rasterization. */ + private buildGeometry(): void { + this.builtVersion = this.geoVersion; + const data = this.data; + const settings = this.settings; + if (!data || !settings) return; + + // Screen px at fit zoom → world units. + const s = 1 / this.kFit; + const nodeR = settings.nodeSize * s; + this.world = { + nodeR, + glowOuter: (settings.nodeSize + GLOW_PAD) * s, + labelSize: settings.labelSize * s, + labelGap: LABEL_GAP * s, + }; + this.worldLabelFont = `${LABEL_WEIGHT} ${settings.labelSize * s}px ${data.fontFamily}`; + const arrowLen = settings.arrowLength * s; + const visible = this.visible; + const isVisible = (id: string) => !visible || visible.has(id); + + // Node discs: one Path2D, one fill() per frame. + const nodesPath = new Path2D(); + for (const n of data.nodes) { + if (!isVisible(n.id)) continue; + nodesPath.moveTo(n.x + nodeR, n.y); + nodesPath.arc(n.x, n.y, nodeR, 0, Math.PI * 2); + } + this.nodesPath = nodesPath; + + // Edge shafts + arrowheads: tip lands on the target disc edge. + const shafts = new Path2D(); + const heads = new Path2D(); + for (const e of data.edges) { + if (!isVisible(e.from) || !isVisible(e.to)) continue; + const a = this.nodeById.get(e.from); + const b = this.nodeById.get(e.to); + if (!a || !b) continue; + const g = arrowGeometry(a, b, nodeR, nodeR, arrowLen); + if (!g) continue; + shafts.moveTo(g.shaftFrom.x, g.shaftFrom.y); + shafts.lineTo(g.shaftTo.x, g.shaftTo.y); + heads.moveTo(g.tip.x, g.tip.y); + heads.lineTo(g.left.x, g.left.y); + heads.lineTo(g.right.x, g.right.y); + heads.closePath(); + } + this.shaftsPath = shafts; + this.headsPath = heads; + + // Cluster hulls: convex hull of member positions (world = raw data + // coordinates — no projection at all), three offset fill layers, + // and a SOLID boundary path. + this.hullShapes = []; + for (const cluster of data.clusters) { + const pts: Pt[] = []; + for (const id of cluster.members) { + if (!isVisible(id)) continue; + const n = this.nodeById.get(id); + if (n) pts.push({ x: n.x, y: n.y }); + } + const hull = convexHull(pts); + if (hull.length < 3) continue; + const fills = HULL_FILL_LAYERS.map(([pad]) => { + const path = new Path2D(); + traceRoundedPath(path, offsetPolygon(hull, pad * s), HULL_CORNER_RADIUS * s); + return path; + }); + const stroke = new Path2D(); + traceRoundedPath(stroke, offsetPolygon(hull, HULL_STROKE_PAD * s), HULL_CORNER_RADIUS * s); + const cx = pts.reduce((sum, p) => sum + p.x, 0) / pts.length; + const cy = pts.reduce((sum, p) => sum + p.y, 0) / pts.length; + this.hullShapes.push({ fill: fills, stroke, label: cluster.label || null, cx, cy, hue: clusterHue(cluster.id) }); + } + + // Label wrapping: measureText once per node per geometry build, + // never per frame. Wrap width keeps the bigger-label/bigger-wrap + // behavior from the sigma era. + this.labelLines = new Map(); + const ctx = this.ctx; + const prevFont = ctx.font; + ctx.font = this.worldLabelFont; + const maxW = LABEL_MAX_WIDTH * (settings.labelSize / LABEL_SIZE) * s; + for (const n of data.nodes) { + if (isVisible(n.id)) this.labelLines.set(n.id, wrapLabel(ctx, n.label, maxW)); + } + ctx.font = prevFont; + } + + /** THE draw function: every layer, one frame, one camera. All drawing + * happens in WORLD coordinates under the camera transform; nothing is + * recomputed per frame except layer styling. */ private draw(): void { this.dirty = false; const data = this.data; @@ -608,146 +747,105 @@ export class GraphCanvasRenderer { if (!data || !settings) return; const w = this.canvas.clientWidth; const h = this.canvas.clientHeight; - this.ctx.clearRect(0, 0, w, h); + const dpr = window.devicePixelRatio || 1; + const ctx = this.ctx; + + ctx.setTransform(dpr, 0, 0, dpr, 0, 0); + ctx.clearRect(0, 0, w, h); + + if (this.builtVersion !== this.geoVersion) this.buildGeometry(); + if (!this.nodesPath) return; const { x: tx, y: ty, k } = this.transform; - const nodeById = new Map(); + // THE camera: one transform, then everything is world coordinates. + ctx.setTransform(dpr * k, 0, 0, dpr * k, dpr * tx, dpr * ty); + const ratio = zoomRatioOf(this.kFit, k); const nodeAlpha = labelAlpha(ratio, settings.labelZoom); const regionAlpha = 1 - nodeAlpha; + const { nodeR, glowOuter } = this.world; - // Screen positions for every VISIBLE node (one pass), plus the - // byId lookup so the label/hover passes never go O(n^2). - const pos = new Map(); - for (const n of data.nodes) { - if (this.visible && !this.visible.has(n.id)) continue; - pos.set(n.id, { x: n.x * k + tx, y: n.y * k + ty }); - nodeById.set(n.id, n); + // 1–2. Cluster hulls: cached world paths — soft layered fills, then + // a SOLID viewer-relative hairline border (dashes dropped for now). + for (const hs of this.hullShapes) { + for (let i = 0; i < hs.fill.length; i++) { + ctx.fillStyle = `hsla(${hs.hue}, 65%, 60%, ${HULL_FILL_LAYERS[i]![1]})`; + ctx.fill(hs.fill[i]!); + } + ctx.strokeStyle = `hsla(${hs.hue}, 65%, 62%, 0.5)`; + ctx.lineWidth = 1.5 / k; // constant screen px + ctx.stroke(hs.stroke); } - // 1–2. Cluster hulls: soft layered fills, then the dashed boundary. - for (const cluster of data.clusters) { - const pts: Pt[] = []; - for (const id of cluster.members) { - const p = pos.get(id); - if (p) pts.push(p); - } - const hull = convexHull(pts); - if (hull.length < 3) continue; - const hue = clusterHue(cluster.id); - for (const [pad, alpha] of HULL_FILL_LAYERS) { - this.ctx.beginPath(); - traceRoundedPath(this.ctx, offsetPolygon(hull, pad), HULL_CORNER_RADIUS); - this.ctx.fillStyle = `hsla(${hue}, 65%, 60%, ${alpha})`; - this.ctx.fill(); - } - this.ctx.beginPath(); - traceRoundedPath(this.ctx, offsetPolygon(hull, HULL_STROKE_PAD), HULL_CORNER_RADIUS); - this.ctx.setLineDash([...HULL_DASH]); - this.ctx.strokeStyle = `hsla(${hue}, 65%, 62%, 0.5)`; - this.ctx.lineWidth = 1.5; - this.ctx.stroke(); - this.ctx.setLineDash([]); - } - - // 3. Edges: shaft + arrowhead (tip lands on the target disc edge). + // 3. Edges: shafts + arrowheads from cached paths; width constant on + // screen (÷ k), geometry in world units (scales with the graph). if (settings.edgeWidth > 0) { - this.ctx.strokeStyle = data.colors.edge; - this.ctx.fillStyle = data.colors.edge; - this.ctx.lineWidth = settings.edgeWidth; - this.ctx.lineCap = "round"; - for (const e of data.edges) { - const from = pos.get(e.from); - const to = pos.get(e.to); - if (!from || !to) continue; // edge drops when either endpoint is hidden - const g = arrowGeometry(from, to, settings.nodeSize, settings.nodeSize, settings.arrowLength); - if (!g) continue; - this.ctx.beginPath(); - this.ctx.moveTo(g.shaftFrom.x, g.shaftFrom.y); - this.ctx.lineTo(g.shaftTo.x, g.shaftTo.y); - this.ctx.stroke(); - this.ctx.beginPath(); - this.ctx.moveTo(g.tip.x, g.tip.y); - this.ctx.lineTo(g.left.x, g.left.y); - this.ctx.lineTo(g.right.x, g.right.y); - this.ctx.closePath(); - this.ctx.fill(); - } + ctx.strokeStyle = data.colors.edge; + ctx.fillStyle = data.colors.edge; + ctx.lineWidth = settings.edgeWidth / k; + ctx.lineCap = "round"; + ctx.stroke(this.shaftsPath); + ctx.fill(this.headsPath); } - // 4. Nodes: plain discs. - this.ctx.fillStyle = data.colors.node; - for (const [, p] of pos) { - this.ctx.beginPath(); - this.ctx.arc(p.x, p.y, settings.nodeSize, 0, Math.PI * 2); - this.ctx.fill(); - } + // 4. Nodes: one cached Path2D, one fill. + ctx.fillStyle = data.colors.node; + ctx.fill(this.nodesPath); - // 5. Cluster region labels — inverse-gated on THE zoom control: - // they fade in over the exact span node labels fade out. Drawn - // above the nodes, below the node labels (the old layer order). + // 5. Cluster region labels — inverse-gated on THE zoom control. + // Viewer-relative by design (UI, not graph content): drawn at the + // world centroid with a font of REGION_LABEL_SIZE / k, which is a + // constant 18px on screen at any zoom. if (settings.clusterLabels && regionAlpha > 0.01) { - this.ctx.save(); - this.ctx.globalAlpha = regionAlpha; - this.ctx.font = `${LABEL_WEIGHT} ${REGION_LABEL_SIZE}px ${data.fontFamily}`; - this.ctx.textAlign = "center"; - this.ctx.textBaseline = "middle"; - for (const cluster of data.clusters) { - if (!cluster.label) continue; - const pts: Pt[] = []; - for (const id of cluster.members) { - const p = pos.get(id); - if (p) pts.push(p); - } - if (pts.length === 0) continue; - const cx = pts.reduce((s, p) => s + p.x, 0) / pts.length; - const cy = pts.reduce((s, p) => s + p.y, 0) / pts.length; - const hue = clusterHue(cluster.id); - this.ctx.fillStyle = `hsla(${hue}, 55%, 78%, 0.85)`; - this.ctx.fillText(cluster.label, cx, cy); + ctx.save(); + ctx.globalAlpha = regionAlpha; + ctx.font = `${LABEL_WEIGHT} ${REGION_LABEL_SIZE / k}px ${data.fontFamily}`; + ctx.textAlign = "center"; + ctx.textBaseline = "middle"; + for (const hs of this.hullShapes) { + if (!hs.label) continue; + ctx.fillStyle = `hsla(${hs.hue}, 55%, 78%, 0.85)`; + ctx.fillText(hs.label, hs.cx, hs.cy); } - this.ctx.restore(); + ctx.restore(); } - // 6. Node labels — one global gate, faded together (no per-node pop). + // 6. Node labels — one global gate, faded together. World-scaled + // with the graph (they grow as you zoom in, Obsidian-style). if (nodeAlpha > 0.01) { - this.ctx.save(); - this.ctx.globalAlpha = nodeAlpha; - for (const [id, p] of pos) { - const n = nodeById.get(id); - if (n) - drawName( - this.ctx, p.x, p.y, settings.nodeSize, settings.labelSize, n.label, - data.fontFamily, data.colors.label, - ); + ctx.save(); + ctx.globalAlpha = nodeAlpha; + ctx.font = this.worldLabelFont; + for (const [id, lines] of this.labelLines) { + const n = this.nodeById.get(id); + if (!n) continue; + drawNameLines(ctx, n.x, n.y, lines, this.world, data.colors.label); } - this.ctx.restore(); + ctx.restore(); } // 7. Hover glow (topmost): halo in the node's color + the brightened - // name, both on the SAME fade factor. Shows even when labels are - // zoomed out (hover is its own affordance, not a label). + // name, both on the SAME fade factor. World units like everything + // else. Shows even when labels are zoomed out (hover is its own + // affordance, not a label). if (this.hoverId && this.hoverFade > 0.02) { - const p = pos.get(this.hoverId); - const n = p ? nodeById.get(this.hoverId) : null; - if (p && n) { - const inner = settings.nodeSize; - const outer = settings.nodeSize + GLOW_PAD; - const glow = this.ctx.createRadialGradient(p.x, p.y, inner, p.x, p.y, outer); + const n = this.visible && !this.visible.has(this.hoverId) + ? null // filtered out by search — no glow on hidden nodes + : this.nodeById.get(this.hoverId); + if (n) { + const glow = ctx.createRadialGradient(n.x, n.y, nodeR, n.x, n.y, glowOuter); glow.addColorStop(0, rgbaFromHex(data.colors.node, GLOW_ALPHA * this.hoverFade)); glow.addColorStop(1, rgbaFromHex(data.colors.node, 0)); - this.ctx.fillStyle = glow; - this.ctx.beginPath(); - this.ctx.arc(p.x, p.y, outer, 0, Math.PI * 2); - this.ctx.arc(p.x, p.y, inner, 0, Math.PI * 2, true); - this.ctx.fill("evenodd"); - this.ctx.save(); - this.ctx.globalAlpha = this.hoverFade; - drawName( - this.ctx, p.x, p.y, settings.nodeSize, settings.labelSize, n.label, - data.fontFamily, data.colors.hoverText, - ); - this.ctx.restore(); + ctx.fillStyle = glow; + ctx.beginPath(); + ctx.arc(n.x, n.y, glowOuter, 0, Math.PI * 2); + ctx.arc(n.x, n.y, nodeR, 0, Math.PI * 2, true); + ctx.fill("evenodd"); + ctx.save(); + ctx.globalAlpha = this.hoverFade; + ctx.font = this.worldLabelFont; + drawNameLines(ctx, n.x, n.y, this.labelLines.get(n.id) ?? [], this.world, data.colors.hoverText); + ctx.restore(); } } } diff --git a/components/graph-view.tsx b/components/graph-view.tsx index 0ad7cf6..b27c485 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -74,7 +74,7 @@ import { labelZoomToSlider, sanitizeRenderSettings, sliderToLabelZoom, -} from "../src/core/render-settings.ts"; +} from "@/src/core"; /** The uniform node look (no state classes — canvas wants hex, not oklch): * the accent color, until the state model lands. */ diff --git a/src/core/index.ts b/src/core/index.ts index a611727..39c97a8 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -17,6 +17,7 @@ export * from "./contracts.ts"; export * from "./ports.ts"; +export * from "./render-settings.ts"; export * from "./question-core.ts"; export * from "./clusters.ts"; diff --git a/src/server/index.ts b/src/server/index.ts index bf5b3b6..7163d12 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -9,6 +9,7 @@ export * from "./chats.ts"; export * from "./graph.ts"; export * from "./jobs.ts"; export * from "./stores.ts"; +export { readRenderSettings, writeRenderSettings } from "../store/render-settings.ts"; export * from "./turn-liveness.ts"; export * from "./observability.ts"; export * from "./model-catalog.ts"; From c096854e87d9432d60d40ec45cb3a53dca0826fc Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 07:26:22 +0300 Subject: [PATCH 15/34] Re-tune render sizes for the world-space model: smaller defaults, wide low ends The defaults were calibrated in the screen-fixed era; world-scaling renders sizes at setting x k/kFit, so at the default label gate (2x fit) everything read ~2x too big ("labels way too large even at the minimum"). Fit-zoom normalizes layout scale away, so the fix belongs in sizes, not layout: spreading nodes further apart in world units changes nothing at fit. - Defaults: nodeSize 6 -> 4, arrowLength 5 -> 4, labelSize 12 -> 7. - Clamps/slider ranges widened at the low end: nodeSize 1-14 (step 0.5), edgeWidth 0.25-4 (step 0.25), arrowLength 1-15 (step 0.5), labelSize 3-24 (step 0.5). - GLOW_PAD 9 -> 6 (halo dominated small discs), REGION_LABEL_SIZE 18 -> 14. - Tests updated for the new clamp bounds plus a low-end pass-through case (403/403). The saved render-settings.json still shadows defaults by design; the local copy was reset so the new defaults showed immediately. --- components/graph-canvas.ts | 4 ++-- components/graph-view.tsx | 16 ++++++++-------- src/core/render-settings.ts | 21 +++++++++++++-------- tests/render-settings.test.ts | 12 ++++++++++-- 4 files changed, 33 insertions(+), 20 deletions(-) diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts index 285437b..5007412 100644 --- a/components/graph-canvas.ts +++ b/components/graph-canvas.ts @@ -117,7 +117,7 @@ const LABEL_LINE_HEIGHT = 1.3; /** Zoom span (camera-ratio units) over which labels fade together. */ export const LABELS_FADE_SPAN = 0.5; -const GLOW_PAD = 9; // px the halo extends beyond the node disc (constant) +const GLOW_PAD = 6; // px the halo extends beyond the node disc (constant) const GLOW_ALPHA = 0.45; // halo opacity at the disc edge, fading outward const HOVER_FADE_MS = 180; // hover halo opacity fade length const ZOOM_EASE_MS = 45; // wheel-zoom glide: exponential-ease time constant (~250ms per notch) @@ -139,7 +139,7 @@ const HULL_FILL_LAYERS = [ ] as const; const HULL_STROKE_PAD = 14; const HULL_CORNER_RADIUS = 9; -const REGION_LABEL_SIZE = 18; +const REGION_LABEL_SIZE = 14; /** All "px" constants above are screen px AT FIT ZOOM. The renderer scales * them into world units (÷ kFit) once per geometry build; camera zoom then diff --git a/components/graph-view.tsx b/components/graph-view.tsx index b27c485..54d4a4b 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -500,33 +500,33 @@ export function GraphView() {
changeSettings((cur) => ({ ...cur, nodeSize: v }))} /> changeSettings((cur) => ({ ...cur, edgeWidth: v }))} /> changeSettings((cur) => ({ ...cur, arrowLength: v }))} /> changeSettings((cur) => ({ ...cur, labelSize: v }))} /> diff --git a/src/core/render-settings.ts b/src/core/render-settings.ts index 8946890..0ca1e2b 100644 --- a/src/core/render-settings.ts +++ b/src/core/render-settings.ts @@ -22,18 +22,23 @@ export interface GraphRenderSettings { arrowLength: number; /** Camera-ratio threshold for node labels (see module doc). */ labelZoom: number; - /** Node + hover label font size, px. */ + /** Node + hover label font size, px at fit zoom (scales with camera). */ labelSize: number; /** Toggle for the cluster region labels (hulls always draw). */ clusterLabels: boolean; } +// Sizes are screen px AT FIT ZOOM (the world-space renderer scales them +// with the camera, so a label reads at labelSize x k/kFit on screen). +// Defaults were re-tuned for world-scaling on 2026-09-13: the old values +// were calibrated in the screen-fixed era, so everything rendered ~2x +// too big once sizes started scaling with the graph. export const DEFAULT_RENDER_SETTINGS: GraphRenderSettings = { - nodeSize: 6, + nodeSize: 4, edgeWidth: 1, - arrowLength: 5, + arrowLength: 4, labelZoom: 0.5, - labelSize: 12, + labelSize: 7, clusterLabels: true, }; @@ -51,12 +56,12 @@ export function sanitizeRenderSettings(raw: unknown): GraphRenderSettings { return clamp(n, min, max); }; return { - nodeSize: num(r.nodeSize, DEFAULT_RENDER_SETTINGS.nodeSize, 2, 14), - edgeWidth: num(r.edgeWidth, DEFAULT_RENDER_SETTINGS.edgeWidth, 0.5, 4), - arrowLength: num(r.arrowLength, DEFAULT_RENDER_SETTINGS.arrowLength, 2, 15), + nodeSize: num(r.nodeSize, DEFAULT_RENDER_SETTINGS.nodeSize, 1, 14), + edgeWidth: num(r.edgeWidth, DEFAULT_RENDER_SETTINGS.edgeWidth, 0.25, 4), + arrowLength: num(r.arrowLength, DEFAULT_RENDER_SETTINGS.arrowLength, 1, 15), // Range matches the log slider's span [LABEL_ZOOM_MIN, LABEL_ZOOM_MAX]. labelZoom: num(r.labelZoom, DEFAULT_RENDER_SETTINGS.labelZoom, 0.0625, 4), - labelSize: num(r.labelSize, DEFAULT_RENDER_SETTINGS.labelSize, 6, 24), + labelSize: num(r.labelSize, DEFAULT_RENDER_SETTINGS.labelSize, 3, 24), clusterLabels: typeof r.clusterLabels === "boolean" ? r.clusterLabels : DEFAULT_RENDER_SETTINGS.clusterLabels, }; diff --git a/tests/render-settings.test.ts b/tests/render-settings.test.ts index 55d2b5c..7cddeef 100644 --- a/tests/render-settings.test.ts +++ b/tests/render-settings.test.ts @@ -44,13 +44,21 @@ describe("sanitizeRenderSettings", () => { clusterLabels: "yes" as unknown as boolean, }); assert.equal(s.nodeSize, 14); // clamped to max - assert.equal(s.edgeWidth, 0.5); // clamped to min - assert.equal(s.arrowLength, 2); + assert.equal(s.edgeWidth, 0.25); // clamped to min + assert.equal(s.arrowLength, 1); assert.equal(s.labelZoom, LABEL_ZOOM_MAX); assert.equal(s.labelSize, DEFAULT_RENDER_SETTINGS.labelSize); // NaN-safe reset assert.equal(s.clusterLabels, DEFAULT_RENDER_SETTINGS.clusterLabels); }); + test("the widened low ends accept small sizes", () => { + const s = sanitizeRenderSettings({ nodeSize: 1, edgeWidth: 0.25, arrowLength: 1, labelSize: 3 }); + assert.deepEqual( + { nodeSize: s.nodeSize, edgeWidth: s.edgeWidth, arrowLength: s.arrowLength, labelSize: s.labelSize }, + { nodeSize: 1, edgeWidth: 0.25, arrowLength: 1, labelSize: 3 }, + ); + }); + test("a fully valid object passes through unchanged", () => { const valid = { nodeSize: 8, edgeWidth: 2, arrowLength: 7, labelZoom: 0.25, labelSize: 16, clusterLabels: false }; assert.deepEqual(sanitizeRenderSettings(valid), valid); From 6692b19a530ba015c0ed02355bb38e834e20e12f Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 07:40:58 +0300 Subject: [PATCH 16/34] User-tuned defaults, label-size floor 1, cluster glow removed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Defaults per user: nodeSize 2, edgeWidth 1, arrowLength 2, labelSize 2 (labelZoom 0.5 stays — reads as the "1x" they confirmed good). - Label size slider floor 3 -> 1 so the default 2 is not the smallest value; sanitize clamp widened to match. - Cluster hull glow (layered low-alpha fills) removed for now: it read as jagged up close (straight-line hull approximation at high zoom) and does not belong until the layout settles. The solid viewer-relative border and the inverse-gated region labels stay. HULL_FILL_LAYERS constant deleted; hullShapes cache carries only the stroke path now. - Saved render-settings.json reset so the new defaults apply on next load. EOF --- components/graph-canvas.ts | 31 ++++++++----------------------- components/graph-view.tsx | 2 +- src/core/render-settings.ts | 8 ++++---- 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts index 5007412..c213266 100644 --- a/components/graph-canvas.ts +++ b/components/graph-canvas.ts @@ -129,14 +129,10 @@ const PICKING_PAD = 10; /** Click vs drag threshold: pointer travel under this many px is a click. */ const CLICK_SLOP = 4; -/** Cluster hull styling: layered low-alpha fills approximate the soft - * translucent tint (a canvas blur filter rasterizes on the GPU per call - * — the original coil-whine driver; never bring ctx.filter back). */ -const HULL_FILL_LAYERS = [ - [24, 0.025], - [19, 0.04], - [14, 0.06], -] as const; +/** Cluster hull styling: a SOLID viewer-relative border only. The layered + * low-alpha fills (the "glow") were removed on user request — they read + * as jagged up close (straight-line hull approximation) and are better + * revisited once the layout settles. */ const HULL_STROKE_PAD = 14; const HULL_CORNER_RADIUS = 9; const REGION_LABEL_SIZE = 14; @@ -355,7 +351,6 @@ export class GraphCanvasRenderer { private shaftsPath: Path2D = new Path2D(); private headsPath: Path2D = new Path2D(); private hullShapes: { - fill: Path2D[]; stroke: Path2D; label: string | null; cx: number; // world centroid @@ -699,8 +694,7 @@ export class GraphCanvasRenderer { this.headsPath = heads; // Cluster hulls: convex hull of member positions (world = raw data - // coordinates — no projection at all), three offset fill layers, - // and a SOLID boundary path. + // coordinates — no projection at all) and a SOLID boundary path. this.hullShapes = []; for (const cluster of data.clusters) { const pts: Pt[] = []; @@ -711,16 +705,11 @@ export class GraphCanvasRenderer { } const hull = convexHull(pts); if (hull.length < 3) continue; - const fills = HULL_FILL_LAYERS.map(([pad]) => { - const path = new Path2D(); - traceRoundedPath(path, offsetPolygon(hull, pad * s), HULL_CORNER_RADIUS * s); - return path; - }); const stroke = new Path2D(); traceRoundedPath(stroke, offsetPolygon(hull, HULL_STROKE_PAD * s), HULL_CORNER_RADIUS * s); const cx = pts.reduce((sum, p) => sum + p.x, 0) / pts.length; const cy = pts.reduce((sum, p) => sum + p.y, 0) / pts.length; - this.hullShapes.push({ fill: fills, stroke, label: cluster.label || null, cx, cy, hue: clusterHue(cluster.id) }); + this.hullShapes.push({ stroke, label: cluster.label || null, cx, cy, hue: clusterHue(cluster.id) }); } // Label wrapping: measureText once per node per geometry build, @@ -765,13 +754,9 @@ export class GraphCanvasRenderer { const regionAlpha = 1 - nodeAlpha; const { nodeR, glowOuter } = this.world; - // 1–2. Cluster hulls: cached world paths — soft layered fills, then - // a SOLID viewer-relative hairline border (dashes dropped for now). + // 1. Cluster hulls: cached world paths — SOLID viewer-relative + // hairline border only (no fill glow, removed for now). for (const hs of this.hullShapes) { - for (let i = 0; i < hs.fill.length; i++) { - ctx.fillStyle = `hsla(${hs.hue}, 65%, 60%, ${HULL_FILL_LAYERS[i]![1]})`; - ctx.fill(hs.fill[i]!); - } ctx.strokeStyle = `hsla(${hs.hue}, 65%, 62%, 0.5)`; ctx.lineWidth = 1.5 / k; // constant screen px ctx.stroke(hs.stroke); diff --git a/components/graph-view.tsx b/components/graph-view.tsx index 54d4a4b..7be2d28 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -524,7 +524,7 @@ export function GraphView() { /> Date: Sun, 13 Sep 2026 07:49:00 +0300 Subject: [PATCH 17/34] Label gap, proportional glow, hover-out fade, centered coarse sliders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - LABEL_GAP 6 -> 2 (user: labels sit way too far below their nodes). - Hover glow is now PROPORTIONAL to the disc — outer radius = node radius x 1.7 (GLOW_SCALE), matching the blur-a-copy-of-the-disc mental model, instead of the old constant +6px pad that read huge on small nodes and thin on big ones. GLOW_ALPHA 0.45 -> 0.3. - Hover-OUT fade was broken by construction: draw() gated the glow on hoverId, which nulls instantly on leave, so the 180ms fade-out ran invisibly and the glow popped off in one frame. New hoverNode field outlives hoverId through the outgoing fade (cleared when the fade completes); verified live: 8 distinct frames over ~200ms then stable. - Sliders rescaled per spec: ALL five are now log-scaled 0-100 with the default dead-center at 50 (ranges chosen as default/4..default*4: node/arrow/label size 0.5-8, edge width 0.25-4, labelZoom 0.0625-4), coarse step 5 (21 stops; labels-until-zoom was "way too granular" at 201). Generic toSliderPosition/fromSliderPosition replace the labelZoom-only pair; sanitize clamps follow the same range constants; size sliders display compact values ("2", "1.6", "0.75"). EOF --- components/graph-canvas.ts | 24 +++++++---- components/graph-view.tsx | 77 ++++++++++++++++++++++------------- src/core/render-settings.ts | 48 ++++++++++++---------- tests/render-settings.test.ts | 70 +++++++++++++++++-------------- 4 files changed, 131 insertions(+), 88 deletions(-) diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts index c213266..9692c1f 100644 --- a/components/graph-canvas.ts +++ b/components/graph-canvas.ts @@ -112,13 +112,13 @@ export interface RendererCallbacks { const LABEL_SIZE = 12; // px — the default label size (settings.labelSize scales it) const LABEL_WEIGHT = "500"; const LABEL_MAX_WIDTH = 170; // px — wraps long names instead of a wide strip -const LABEL_GAP = 6; // px between node edge and first label line +const LABEL_GAP = 2; // px between node edge and first label line (was 6 — user: way too far) const LABEL_LINE_HEIGHT = 1.3; /** Zoom span (camera-ratio units) over which labels fade together. */ export const LABELS_FADE_SPAN = 0.5; -const GLOW_PAD = 6; // px the halo extends beyond the node disc (constant) -const GLOW_ALPHA = 0.45; // halo opacity at the disc edge, fading outward +const GLOW_SCALE = 1.7; // halo outer radius = node radius x this (proportional, like blurring a copy of the disc) +const GLOW_ALPHA = 0.3; // halo opacity at the disc edge, fading outward const HOVER_FADE_MS = 180; // hover halo opacity fade length const ZOOM_EASE_MS = 45; // wheel-zoom glide: exponential-ease time constant (~250ms per notch) @@ -327,6 +327,11 @@ export class GraphCanvasRenderer { // Hover transition: fade 0 = resting, 1 = fully hovered; the animating // flag keeps the loop scheduled until the fade settles (then rAF stops). private hoverId: string | null = null; + /** The node the glow is drawn on. OUTLIVES hoverId on hover-out so the + * fade has something to draw (hoverId nulls instantly; without this + * the glow popped off in one frame instead of fading). Cleared when + * the outgoing fade completes. */ + private hoverNode: CanvasNode | null = null; private hoverFade = 0; private hoverFrom = 0; private hoverTarget = 0; @@ -548,6 +553,9 @@ export class GraphCanvasRenderer { private setHover(nodeId: string | null): void { if (nodeId === this.hoverId && this.hoverTarget === (nodeId ? 1 : 0)) return; this.hoverId = nodeId; + // Hover-out deliberately KEEPS hoverNode: the glow must fade out + // from the last hovered node, not vanish with the id. + if (nodeId) this.hoverNode = this.nodeById.get(nodeId) ?? null; // Rebase the animation from the CURRENT fade: rapid hover toggles // glide instead of snapping (ported from the sigma-era behavior). this.hoverFrom = this.hoverFade; @@ -596,6 +604,8 @@ export class GraphCanvasRenderer { if (this.hoverFade === this.hoverTarget) return false; const t = Math.min((performance.now() - this.hoverStart) / HOVER_FADE_MS, 1); this.hoverFade = this.hoverFrom + (this.hoverTarget - this.hoverFrom) * t; + // Outgoing fade finished — release the glow node. + if (this.hoverTarget === 0 && this.hoverFade === 0) this.hoverNode = null; return t < 1; } @@ -655,7 +665,7 @@ export class GraphCanvasRenderer { const nodeR = settings.nodeSize * s; this.world = { nodeR, - glowOuter: (settings.nodeSize + GLOW_PAD) * s, + glowOuter: nodeR * GLOW_SCALE, labelSize: settings.labelSize * s, labelGap: LABEL_GAP * s, }; @@ -813,10 +823,10 @@ export class GraphCanvasRenderer { // name, both on the SAME fade factor. World units like everything // else. Shows even when labels are zoomed out (hover is its own // affordance, not a label). - if (this.hoverId && this.hoverFade > 0.02) { - const n = this.visible && !this.visible.has(this.hoverId) + if (this.hoverNode && this.hoverFade > 0.02) { + const n = this.visible && !this.visible.has(this.hoverNode.id) ? null // filtered out by search — no glow on hidden nodes - : this.nodeById.get(this.hoverId); + : this.hoverNode; if (n) { const glow = ctx.createRadialGradient(n.x, n.y, nodeR, n.x, n.y, glowOuter); glow.addColorStop(0, rgbaFromHex(data.colors.node, GLOW_ALPHA * this.hoverFade)); diff --git a/components/graph-view.tsx b/components/graph-view.tsx index 7be2d28..6e58b61 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -69,11 +69,17 @@ import forceAtlas2 from "graphology-layout-forceatlas2"; import { seededRandom, type GraphNode, type GraphSnapshot } from "@/src/core"; import { GraphCanvasRenderer, type GraphRenderSettings } from "./graph-canvas.ts"; import { + ARROW_LENGTH_RANGE, DEFAULT_RENDER_SETTINGS, + EDGE_WIDTH_RANGE, + LABEL_SIZE_RANGE, + LABEL_ZOOM_MAX, + LABEL_ZOOM_MIN, + NODE_SIZE_RANGE, + fromSliderPosition, labelZoomDisplay, - labelZoomToSlider, sanitizeRenderSettings, - sliderToLabelZoom, + toSliderPosition, } from "@/src/core"; /** The uniform node look (no state classes — canvas wants hex, not oklch): @@ -133,6 +139,11 @@ interface Selection { } /** One labeled range row for the rendering-settings panel. */ +/** Compact size display for the log sliders: "2", "1.6", "0.75". */ +function sizeDisplay(v: number): string { + return `${v >= 1 ? Math.round(v * 10) / 10 : Math.round(v * 100) / 100}`; +} + function SettingSlider({ label, min, @@ -498,50 +509,58 @@ export function GraphView() {
{settingsOpen && (
+ {/* All sliders: log-scaled 0-100, step 5, so every + DEFAULT sits at 50 (src/core/render-settings.ts). */} changeSettings((cur) => ({ ...cur, nodeSize: v }))} + min={0} + max={100} + step={5} + value={toSliderPosition(settings.nodeSize, NODE_SIZE_RANGE.min, NODE_SIZE_RANGE.max)} + display={sizeDisplay(settings.nodeSize)} + onChange={(p) => + changeSettings((cur) => ({ ...cur, nodeSize: fromSliderPosition(p, NODE_SIZE_RANGE.min, NODE_SIZE_RANGE.max) }))} /> changeSettings((cur) => ({ ...cur, edgeWidth: v }))} + min={0} + max={100} + step={5} + value={toSliderPosition(settings.edgeWidth, EDGE_WIDTH_RANGE.min, EDGE_WIDTH_RANGE.max)} + display={sizeDisplay(settings.edgeWidth)} + onChange={(p) => + changeSettings((cur) => ({ ...cur, edgeWidth: fromSliderPosition(p, EDGE_WIDTH_RANGE.min, EDGE_WIDTH_RANGE.max) }))} /> changeSettings((cur) => ({ ...cur, arrowLength: v }))} + min={0} + max={100} + step={5} + value={toSliderPosition(settings.arrowLength, ARROW_LENGTH_RANGE.min, ARROW_LENGTH_RANGE.max)} + display={sizeDisplay(settings.arrowLength)} + onChange={(p) => + changeSettings((cur) => ({ ...cur, arrowLength: fromSliderPosition(p, ARROW_LENGTH_RANGE.min, ARROW_LENGTH_RANGE.max) }))} /> changeSettings((cur) => ({ ...cur, labelSize: v }))} + min={0} + max={100} + step={5} + value={toSliderPosition(settings.labelSize, LABEL_SIZE_RANGE.min, LABEL_SIZE_RANGE.max)} + display={sizeDisplay(settings.labelSize)} + onChange={(p) => + changeSettings((cur) => ({ ...cur, labelSize: fromSliderPosition(p, LABEL_SIZE_RANGE.min, LABEL_SIZE_RANGE.max) }))} /> - {/* Log-scaled: the useful band (0.25–0.5 in camera-ratio - units) sits mid-slider; displayed units normalize the - default (0.5) to "1×". Slider position ↔ labelZoom via - src/core/render-settings.ts. */} + {/* Displayed units normalize the default (0.5) to "1×". */} changeSettings((cur) => ({ ...cur, labelZoom: sliderToLabelZoom(v) }))} + onChange={(p) => + changeSettings((cur) => ({ ...cur, labelZoom: fromSliderPosition(p, LABEL_ZOOM_MIN, LABEL_ZOOM_MAX) }))} /> + {/* Regions (dev): quick-tuning number inputs. Session state + only - never persisted. Bad/empty values fall back to + the last sane tuning inside the renderer. */} +
+ + Regions (dev) + + setRegionTuning((t) => ({ ...t, dilation: v }))} + /> + setRegionTuning((t) => ({ ...t, blur: v }))} + /> + setRegionTuning((t) => ({ ...t, bitmap: v }))} + /> + setRegionTuning((t) => ({ ...t, edgeAlpha: v }))} + /> +
)} {snapshot !== null && visibleIds.size === 0 && query.trim() !== "" && ( diff --git a/tests/graph-canvas.test.ts b/tests/graph-canvas.test.ts index cc6b4be..6c57911 100644 --- a/tests/graph-canvas.test.ts +++ b/tests/graph-canvas.test.ts @@ -13,10 +13,13 @@ import assert from "node:assert/strict"; import { describe, it as test } from "node:test"; import { arrowGeometry, + boxBlurRound, fitTransform, labelAlpha, pickNearest, + sanitizeRegionTuning, zoomRatioOf, + DEFAULT_REGION_TUNING, LABELS_FADE_SPAN, } from "../components/graph-canvas.ts"; @@ -137,3 +140,57 @@ describe("pickNearest (distance-check picking)", () => { assert.equal(pickNearest([], 0, 0, 100), null); }); }); + +describe("sanitizeRegionTuning (dev number inputs)", () => { + test("defaults survive untouched; partial input keeps the rest", () => { + const t = sanitizeRegionTuning(undefined); + assert.deepEqual(t, DEFAULT_REGION_TUNING); + const half = sanitizeRegionTuning({ blur: 0 }); + assert.equal(half.blur, 0); + assert.equal(half.dilation, DEFAULT_REGION_TUNING.dilation); + }); + + test("garbage clamps: NaN falls back, out-of-range clamps", () => { + const t = sanitizeRegionTuning({ dilation: Number.NaN, blur: -5, bitmap: 99999, edgeAlpha: 42 }); + assert.equal(t.dilation, DEFAULT_REGION_TUNING.dilation); // NaN -> default + assert.equal(t.blur, 0); + assert.equal(t.bitmap, 1024); + assert.equal(t.edgeAlpha, 1); + }); + + test("empty string / null / undefined values fall back per-field", () => { + const t = sanitizeRegionTuning({ edgeAlpha: Number.NaN }); + assert.equal(t.edgeAlpha, DEFAULT_REGION_TUNING.edgeAlpha); + }); +}); + +describe("boxBlurRound (premultiplied region blur)", () => { + test("spreads a single opaque px into its neighborhood and conserves sum", () => { + // 5x5 RGBA, one white opaque pixel at (2,2) + const w = 5, h = 5; + const a = new Float32Array(w * h * 4); + const center = (2 * w + 2) * 4; + a[center] = 255; a[center + 1] = 255; a[center + 2] = 255; a[center + 3] = 255; + const tmp = new Float32Array(a.length); + const b = new Float32Array(a.length); + boxBlurRound(a, tmp, b, w, h, 1); + // Alpha conservation: total alpha unchanged. + let sumA = 0; + for (let i = 3; i < b.length; i += 4) sumA += b[i]!; + assert.ok(Math.abs(sumA - 255) < 1e-3, `alpha sum ${sumA}`); + // The center dimmed, the 8 neighbors picked some up. + assert.ok(b[center + 3]! < 255 && b[center + 3]! > 0); + const up = ((1 * w) + 2) * 4 + 3; + assert.ok(b[up]! > 0, "neighbor got alpha"); + // Far corner (4,4) untouched. + assert.equal(b[(4 * 5 + 4) * 4 + 3], 0); + }); + + test("radius 0 is a plain copy", () => { + const a = new Float32Array([1, 2, 3, 4, 5, 6, 7, 8]); + const tmp = new Float32Array(8); + const b = new Float32Array(8); + boxBlurRound(a, tmp, b, 1, 2, 0); + assert.deepEqual([...b], [1, 2, 3, 4, 5, 6, 7, 8]); + }); +}); From 975a78a30495bae5699b18753dad16f001d66749 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 09:52:12 +0300 Subject: [PATCH 22/34] Cluster gravity: per-cluster centroid pull force in the engine The 'virtual node' realized as a centroid force: two allocation-free O(n) passes per tick (membership resolved once at sim creation; Float64Array sums). Measured indistinguishable from force-off at 338 nodes (271ms vs 272ms per 400-tick settle). - createGraphSimulation: SimOptions.clusterPull + linkStrength + velocityDecay (physics controls plumbing); degree-scaled link stiffness so linkStrength=1 is exactly d3's default - settleGraphLayout split into layout-settle.ts (it computes the Louvain partition for the pull); layout.ts stays browser-barrel-pure - barrel no longer re-exports clusters.ts: graphology + louvain are fully out of the client bundle (verified against built chunks) - replay passes the per-batch partition; CLUSTER_PULL_STRENGTH=0.15 tuned by sweep: cluster spread 63->36 world units, centroid separation and total span unchanged (no blob merge, no collapse) --- scripts/replay-layout.ts | 11 +-- src/agent/subagents/extraction.ts | 2 +- src/core/index.ts | 6 +- src/core/layout-settle.ts | 43 +++++++++++ src/core/layout.ts | 114 ++++++++++++++++++++++-------- src/server/commit-positions.ts | 2 +- tests/layout.test.ts | 46 +++++++++++- 7 files changed, 183 insertions(+), 41 deletions(-) create mode 100644 src/core/layout-settle.ts diff --git a/scripts/replay-layout.ts b/scripts/replay-layout.ts index 7c56d84..eb7f862 100644 --- a/scripts/replay-layout.ts +++ b/scripts/replay-layout.ts @@ -10,6 +10,7 @@ * npx tsx scripts/replay-layout.ts */ import { graphStore } from "../src/store/graph.ts"; +import { detectCommunities } from "../src/core/clusters.ts"; import { incrementalLayout } from "../src/core/layout.ts"; import type { GraphNode, NodeMapping, Position } from "../src/core/contracts.ts"; @@ -69,11 +70,11 @@ function main() { const activeEdges = edges.filter((e) => delivered.has(e.from) && delivered.has(e.to)); const before = new Map(positions); - positions = incrementalLayout( - nodes.filter((n) => delivered.has(n.id)), - activeEdges, - positions, - ); + const batchNodes = nodes.filter((n) => delivered.has(n.id)); + positions = incrementalLayout(batchNodes, activeEdges, positions, { + // The same partition the real settle would see for this batch. + clusters: detectCommunities(batchNodes, activeEdges).communities, + }); for (const [id, prev] of before) { const now = positions.get(id)!; const shift = Math.hypot(prev.x - now.x, prev.y - now.y); diff --git a/src/agent/subagents/extraction.ts b/src/agent/subagents/extraction.ts index ce388a7..f0e2030 100644 --- a/src/agent/subagents/extraction.ts +++ b/src/agent/subagents/extraction.ts @@ -16,7 +16,7 @@ import type { SessionFindings, } from "../../core/contracts.ts"; import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts"; -import { settleGraphLayout } from "../../core/layout.ts"; +import { settleGraphLayout } from "../../core/layout-settle.ts"; import { refreshClusterLabels } from "./cluster-namer.ts"; /** diff --git a/src/core/index.ts b/src/core/index.ts index 792689b..70c2ebb 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -20,7 +20,11 @@ export * from "./ports.ts"; export * from "./render-settings.ts"; export * from "./question-core.ts"; -export * from "./clusters.ts"; +// clusters.ts (Louvain) is deliberately NOT re-exported: nothing in the +// shell runs community detection — the snapshot carries the partition +// from the server. Keeping it out keeps graphology-louvain out of the +// client bundle (verified 2026-09-13: chunks contained it while this +// line existed). Same logic as layout-settle.ts below. export * from "./layout.ts"; export * from "./plan-tree.ts"; export * from "./phases.ts"; diff --git a/src/core/layout-settle.ts b/src/core/layout-settle.ts new file mode 100644 index 0000000..eb23aa7 --- /dev/null +++ b/src/core/layout-settle.ts @@ -0,0 +1,43 @@ +import type { GraphEdge, GraphNode } from "./contracts.ts"; +import { detectCommunities } from "./clusters.ts"; +import { incrementalLayout } from "./layout.ts"; +import type { Position } from "./contracts.ts"; + +/** + * The merge-time settle: recompute the resting layout from the CURRENT + * graph + the positions ledger, and replace the ledger. Called by the + * extraction pipeline after every merge (the sole writer) and by the + * positions-commit endpoint when the graph grew under a live-physics + * commit (client positions become the anchor set). Synchronous — it must + * finish before the next /api/graph fetch reflects the change. + * + * SPLIT FROM layout.ts deliberately (2026-09-13): layout.ts is in the + * browser barrel, and the settle needs the Louvain partition for the + * cluster centroid pull — graphology-louvain must not ship in the client + * bundle. Server code imports this module DIRECTLY; it is never + * re-exported through src/core/index.ts. + */ +export function settleGraphLayout(graphStore: { + readNodes(): GraphNode[]; + readEdges(): GraphEdge[]; + readPositions(): Record; + writePositions(positions: Record): void; +}): void { + const nodes = graphStore.readNodes(); + if (nodes.length === 0) return; + // Node LINES may repeat an id (link-appends) — dedupe, first defines. + const seen = new Set(); + const unique = nodes.filter((n) => { + if (seen.has(n.id)) return false; + seen.add(n.id); + return true; + }); + const edges = graphStore.readEdges(); + const previous = new Map(Object.entries(graphStore.readPositions())); + // The cluster partition for the centroid pull: same Louvain the + // snapshot uses (same nodes/edges = same communities). 338 nodes: + // ~1-2ms, noise next to the settle itself. + const { communities } = detectCommunities(unique, edges); + const next = incrementalLayout(unique, edges, previous, { clusters: communities }); + graphStore.writePositions(Object.fromEntries(next)); +} \ No newline at end of file diff --git a/src/core/layout.ts b/src/core/layout.ts index b1a7d54..e2da57e 100644 --- a/src/core/layout.ts +++ b/src/core/layout.ts @@ -46,6 +46,9 @@ type SimLink = SimulationLinkDatum; export interface SimOptions { /** Link spring rest length (world units). */ linkDistance?: number; + /** Link spring stiffness MULTIPLIER (1 = d3's default + * 1/min(degree) stiffness). */ + linkStrength?: number; /** Many-body repulsion strength (negative). */ chargeStrength?: number; /** Center-of-mass gravity: pulls loose components inward. Weak — @@ -54,6 +57,18 @@ export interface SimOptions { /** Alpha decay per tick — higher = snappier settle. The merge settle * uses d3's default 0.0228 (~300 ticks); the live sim passes ~0.05. */ alphaDecay?: number; + /** Velocity damping per tick (d3 default 0.4). Higher = more friction. */ + velocityDecay?: number; + /** Cluster centroid pull (the "virtual node" without the node): + * members of each cluster accelerate toward their own cluster's + * centroid. Two O(n) passes per tick — cheaper than a real virtual + * node (which would need one extra link spring per member plus a + * maintained position). Strength 0 or absent = force off. + * Callers supply the partition (Louvain output shape). */ + clusterPull?: { + clusters: ReadonlyArray<{ members: ReadonlyArray }>; + strength: number; + }; } /** Link input: edges are undirected for layout purposes. */ @@ -62,8 +77,8 @@ export interface LayoutEdge { to: string; } -const LINK_DISTANCE = 40; -const CHARGE_STRENGTH = -120; +export const LINK_DISTANCE = 40; +export const CHARGE_STRENGTH = -120; /** Per-tick anchor re-blend: fraction of an anchored node's displacement * from its previous position that SURVIVES each tick. 0.85 = gentle * evolution where forces are strong, stillness where they are not — @@ -81,6 +96,10 @@ const BOOTSTRAP_GRAVITY = 0.2; /** Exported: the live sim reuses this so its rest state matches the * server settle's equilibrium (one engine, one rest). */ export const INCREMENTAL_GRAVITY = 0.12; +/** Exported: default cluster-centroid pull strength (tuned by replay — + * compact enough to read as blobs, weak enough that cluster shape is + * still the edge network's). */ +export const CLUSTER_PULL_STRENGTH = 0.15; /** * The one force set. Returns a STOPPED simulation — callers drive it @@ -93,6 +112,15 @@ export function createGraphSimulation( edges: ReadonlyArray, options: SimOptions = {}, ): Simulation { + // Link stiffness: d3's default is 1/min(deg) (degree-scaled springs). + // linkStrength multiplies it, so 1 = unchanged default behavior. + const degree = new Map(); + for (const e of edges) { + degree.set(e.from, (degree.get(e.from) ?? 0) + 1); + degree.set(e.to, (degree.get(e.to) ?? 0) + 1); + } + const linkStrength = options.linkStrength ?? 1; + const sim = forceSimulation(nodes) .force( "link", @@ -100,10 +128,16 @@ export function createGraphSimulation( edges.map((e) => ({ source: e.from, target: e.to })), ) .id((d) => d.id) - .distance(options.linkDistance ?? LINK_DISTANCE), + .distance(options.linkDistance ?? LINK_DISTANCE) + .strength( + (l) => + (linkStrength * 1) / + Math.min(degree.get((l.source as LayoutNode).id) ?? 1, degree.get((l.target as LayoutNode).id) ?? 1), + ), ) .force("charge", forceManyBody().strength(options.chargeStrength ?? CHARGE_STRENGTH)) .alphaDecay(options.alphaDecay ?? 0.0228) + .velocityDecay(options.velocityDecay ?? 0.4) .stop(); const g = options.gravity; @@ -111,6 +145,43 @@ export function createGraphSimulation( sim.force("x", forceX(g.x).strength(g.strength)); sim.force("y", forceY(g.y).strength(g.strength)); } + + // Cluster centroid pull: the membership index is resolved ONCE (nodes + // don't change clusters mid-sim); each tick is two allocation-free + // O(n) passes (sum, then pull). Singletons (<2 members) skip themselves. + const cp = options.clusterPull; + if (cp && cp.strength > 0 && cp.clusters.length > 0) { + const clusterIdx = new Map(); + cp.clusters.forEach((c, i) => { + for (const id of c.members) clusterIdx.set(id, i); + }); + const nodeCluster = nodes.map((n) => clusterIdx.get(n.id) ?? -1); + const k = cp.clusters.length; + const sumX = new Float64Array(k); + const sumY = new Float64Array(k); + const cnt = new Float64Array(k); + const strength = cp.strength; + sim.force("clusterPull", (alpha: number) => { + sumX.fill(0); + sumY.fill(0); + cnt.fill(0); + for (let i = 0; i < nodes.length; i++) { + const ci = nodeCluster[i]!; + if (ci < 0) continue; + const n = nodes[i]!; + sumX[ci] = sumX[ci]! + (n.x ?? 0); + sumY[ci] = sumY[ci]! + (n.y ?? 0); + cnt[ci] = cnt[ci]! + 1; + } + for (let i = 0; i < nodes.length; i++) { + const ci = nodeCluster[i]!; + if (ci < 0 || (cnt[ci] ?? 0) < 2) continue; + const n = nodes[i]!; + n.vx = (n.vx ?? 0) + (sumX[ci]! / cnt[ci]! - (n.x ?? 0)) * strength * alpha; + n.vy = (n.vy ?? 0) + (sumY[ci]! / cnt[ci]! - (n.y ?? 0)) * strength * alpha; + } + }); + } return sim; } @@ -122,6 +193,12 @@ export interface IncrementalLayoutOptions { /** Center-of-mass gravity strength override (tuning/A-B experiments; * defaults to the internal bootstrap/incremental constants). */ gravityStrength?: number; + /** Cluster partition (Louvain output shape). Supplied by callers so + * this module stays louvain-free (the browser bundle must not carry + * graphology-louvain); absent = no cluster pull. */ + clusters?: ReadonlyArray<{ members: ReadonlyArray }>; + /** Cluster centroid pull strength (default CLUSTER_PULL_STRENGTH). */ + clusterPullStrength?: number; } /** @@ -253,6 +330,9 @@ export function incrementalLayout( y: cy, strength: options.gravityStrength ?? (hasCore ? INCREMENTAL_GRAVITY : BOOTSTRAP_GRAVITY), }, + clusterPull: options.clusters + ? { clusters: options.clusters, strength: options.clusterPullStrength ?? CLUSTER_PULL_STRENGTH } + : undefined, }); const byId = new Map(simNodes.map((n) => [n.id, n])); for (let i = 0; i < ticks; i++) { @@ -275,31 +355,3 @@ export function incrementalLayout( for (const n of simNodes) out.set(n.id, { x: n.x ?? 0, y: n.y ?? 0 }); return out; } - -/** - * The merge-time settle: recompute the resting layout from the CURRENT - * graph + the positions ledger, and replace the ledger. Called by the - * extraction pipeline after every merge (the sole writer) and by the - * positions-commit endpoint when the graph grew under a live-physics - * commit (client positions become the anchor set). Synchronous — it must - * finish before the next /api/graph fetch reflects the change. - */ -export function settleGraphLayout(graphStore: { - readNodes(): GraphNode[]; - readEdges(): GraphEdge[]; - readPositions(): Record; - writePositions(positions: Record): void; -}): void { - const nodes = graphStore.readNodes(); - if (nodes.length === 0) return; - // Node LINES may repeat an id (link-appends) — dedupe, first defines. - const seen = new Set(); - const unique = nodes.filter((n) => { - if (seen.has(n.id)) return false; - seen.add(n.id); - return true; - }); - const previous = new Map(Object.entries(graphStore.readPositions())); - const next = incrementalLayout(unique, graphStore.readEdges(), previous); - graphStore.writePositions(Object.fromEntries(next)); -} \ No newline at end of file diff --git a/src/server/commit-positions.ts b/src/server/commit-positions.ts index 8178cf7..a31232e 100644 --- a/src/server/commit-positions.ts +++ b/src/server/commit-positions.ts @@ -12,7 +12,7 @@ */ import type { Position } from "../core/contracts.ts"; import { getGraph } from "../core/graph.ts"; -import { settleGraphLayout } from "../core/layout.ts"; +import { settleGraphLayout } from "../core/layout-settle.ts"; import { stores } from "./stores.ts"; /** Max |x| or |y| accepted — NaN/Infinity clamp here, absurd too. */ diff --git a/tests/layout.test.ts b/tests/layout.test.ts index adabea6..15e6fc0 100644 --- a/tests/layout.test.ts +++ b/tests/layout.test.ts @@ -6,7 +6,8 @@ */ import assert from "node:assert/strict"; import { describe, it } from "node:test"; -import { incrementalLayout, settleGraphLayout } from "../src/core/layout.ts"; +import { createGraphSimulation, incrementalLayout } from "../src/core/layout.ts"; +import { settleGraphLayout } from "../src/core/layout-settle.ts"; import type { GraphNode, GraphEdge, Position } from "../src/core/contracts.ts"; const node = (id: string): GraphNode => ({ @@ -125,4 +126,45 @@ describe("settleGraphLayout", () => { }); assert.equal(written.length, 0); }); -}); \ No newline at end of file +}); +describe("clusterPull (the centroid force)", () => { + it("compacts each cluster toward its OWN centroid; strength 0 = off", () => { + // Two loose chains, far apart, edges only WITHIN each chain. + const mk = (p: string, baseX: number) => + Array.from({ length: 8 }, (_, i) => ({ id: `${p}${i}`, x: baseX + i * 40, y: i * 15 })); + const a = mk("a", -400); + const b = mk("b", 400); + const nodes = [...a, ...b]; + const edges = [ + ...a.slice(0, -1).map((n, i) => ({ from: n.id, to: a[i + 1]!.id })), + ...b.slice(0, -1).map((n, i) => ({ from: n.id, to: b[i + 1]!.id })), + ]; + const clusters = [{ members: a.map((n) => n.id) }, { members: b.map((n) => n.id) }]; + + const spreadOf = (ns: { id: string; x: number; y: number }[]): number => { + let sum = 0; + for (const group of [ns.filter((n) => n.id[0] === "a"), ns.filter((n) => n.id[0] === "b")]) { + const cx = group.reduce((s, n) => s + n.x, 0) / group.length; + const cy = group.reduce((s, n) => s + n.y, 0) / group.length; + sum += group.reduce((s, n) => s + Math.hypot(n.x - cx, n.y - cy), 0) / group.length; + } + return sum; + }; + + const run = (strength: number) => { + const sim = createGraphSimulation( + nodes.map((n) => ({ ...n })), + edges, + { clusterPull: { clusters, strength } }, + ); + for (let i = 0; i < 150; i++) sim.tick(); + return spreadOf(sim.nodes() as unknown as { id: string; x: number; y: number }[]); + }; + const pulled = run(0.15); + const unpulled = run(0); + // The force compacts (well under the unpulled baseline)... + assert.ok(pulled < unpulled * 0.8, `pulled spread ${pulled.toFixed(1)} should be well under ${unpulled.toFixed(1)}`); + // ...and both groups survive as separate blobs (no merge). + assert.ok(pulled > 0, "spread is finite/positive"); + }); +}); From 7ebdf065c8475e8e4d80b43af31467fbcb08790c Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 09:52:14 +0300 Subject: [PATCH 23/34] Physics controls + FA2 fallback retirement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User-facing live-sim controls (settings panel, Physics section): node repulsion, edge attraction, edge length, central pull, cluster pull, friction — session-only number inputs feeding createGraphSimulation (the SAME engine the server settle runs, so the feel IS the physics). Changing a value rebuilds the sim from current positions; drag pins (fx/fy live on the node objects) survive; a mid-settle change re-anneals. The graph-view FA2 bootstrap fallback is rewritten on incrementalLayout (d3-force): the last graphology import in the shell is gone. --- components/graph-canvas.ts | 100 ++++++++++++++++++++++++++++++++----- components/graph-view.tsx | 96 +++++++++++++++++++++++------------ tests/graph-canvas.test.ts | 28 +++++++++++ 3 files changed, 179 insertions(+), 45 deletions(-) diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts index 58ea245..5b3c71a 100644 --- a/components/graph-canvas.ts +++ b/components/graph-canvas.ts @@ -55,8 +55,11 @@ import { zoom, zoomIdentity, type ZoomBehavior, type ZoomTransform } from "d3-zo import type { Simulation, SimulationLinkDatum } from "d3-force"; import { clusterHue, type Pt } from "./cluster-geometry.ts"; import { + CHARGE_STRENGTH, + CLUSTER_PULL_STRENGTH, createGraphSimulation, INCREMENTAL_GRAVITY, + LINK_DISTANCE, type GraphRenderSettings, type LayoutNode, type Position, @@ -196,6 +199,51 @@ export function sanitizeRegionTuning(input: Partial | null | undef }; } +/** User-facing live-sim physics controls (2026-09-13, user request): + * every value feeds createGraphSimulation — the SAME engine the server + * settle runs — so tuning the feel changes exactly what a drag relaxes + * toward. Session state, never persisted. The cluster pull is the + * "virtual node" realized as a centroid force (O(n)/tick, measured + * indistinguishable from off). */ +export interface PhysicsTuning { + /** Node repulsion (d3 charge, negative = push apart). */ + chargeStrength: number; + /** Edge attraction: link-spring stiffness multiplier (1 = d3 default). */ + linkStrength: number; + /** Link spring rest length (world units). */ + linkDistance: number; + /** Central pull toward the graph centroid. */ + gravity: number; + /** Per-cluster pull toward the cluster centroid. */ + clusterPull: number; + /** Per-tick velocity damping (d3 default 0.4). */ + velocityDecay: number; +} + +export const DEFAULT_PHYSICS_TUNING: PhysicsTuning = { + chargeStrength: CHARGE_STRENGTH, + linkStrength: 1, + linkDistance: LINK_DISTANCE, + gravity: INCREMENTAL_GRAVITY, + clusterPull: CLUSTER_PULL_STRENGTH, + velocityDecay: 0.4, +}; + +/** Clamp anything into a usable tuning (number inputs can be empty/NaN). */ +export function sanitizePhysicsTuning(input: Partial | null | undefined): PhysicsTuning { + const d = DEFAULT_PHYSICS_TUNING; + const num = (v: unknown, fallback: number): number => + typeof v === "number" && Number.isFinite(v) ? v : fallback; + return { + chargeStrength: Math.min(Math.max(num(input?.chargeStrength, d.chargeStrength), -1000), -10), + linkStrength: Math.min(Math.max(num(input?.linkStrength, d.linkStrength), 0), 5), + linkDistance: Math.min(Math.max(num(input?.linkDistance, d.linkDistance), 1), 300), + gravity: Math.min(Math.max(num(input?.gravity, d.gravity), 0), 1), + clusterPull: Math.min(Math.max(num(input?.clusterPull, d.clusterPull), 0), 2), + velocityDecay: Math.min(Math.max(num(input?.velocityDecay, d.velocityDecay), 0), 0.99), + }; +} + /** One horizontal+vertical box-blur round over PREMULTIPLIED RGBA * (Float32, stride 4, length w*h*4). Premultiplying first is what keeps * the blur halo-free: blurring non-premultiplied RGB over varying alpha @@ -504,6 +552,9 @@ export class GraphCanvasRenderer { /** Region labels: cluster centroid + hue, computed at splat build. */ private regionMeta: { label: string | null; cx: number; cy: number; hue: number }[] = []; private regionTuning: RegionTuning = DEFAULT_REGION_TUNING; + private physicsTuning: PhysicsTuning = DEFAULT_PHYSICS_TUNING; + /** Gravity center for the live sim (graph bbox centroid at setData). */ + private simCenter: Pt = { x: 0, y: 0 }; private regionBuiltVersion = -1; private frameCount = 0; // draw counter for the live-sim splat throttle private labelLines = new Map(); @@ -590,18 +641,8 @@ export class GraphCanvasRenderer { this.simActive = false; this.dragId = null; this.simMoved = false; - if (data.nodes.length > 0) { - const cx = (minX + maxX) / 2; - const cy = (minY + maxY) / 2; - this.sim = createGraphSimulation(data.nodes, data.edges, { - gravity: { x: cx, y: cy, strength: INCREMENTAL_GRAVITY }, - alphaDecay: LIVE_ALPHA_DECAY, - }); - this.simById = new Map(data.nodes.map((n) => [n.id, n as LayoutNode])); - } else { - this.sim = null; - this.simById = new Map(); - } + this.simCenter = { x: (minX + maxX) / 2, y: (minY + maxY) / 2 }; + this.buildSim(); this.geoVersion++; // new data → rebuild world geometry this.markDirty(); } @@ -621,6 +662,17 @@ export class GraphCanvasRenderer { this.markDirty(); } + /** Live-sim physics controls: rebuilds the sim from the CURRENT + * node positions (same node objects — x/y and any drag pin fx/fy + * survive; velocities restart clean, which reads as "re-relax with + * the new feel"). A mid-settle change re-anneals, which is the + * honest response to new physics. */ + setPhysicsTuning(input: Partial): void { + this.physicsTuning = sanitizePhysicsTuning(input); + this.buildSim(); + this.markDirty(); + } + setVisible(visible: ReadonlySet | null): void { this.visible = visible; this.geoVersion++; // member visibility changes hulls + paths @@ -1013,6 +1065,30 @@ export class GraphCanvasRenderer { ctx.font = prevFont; } + /** (Re)create the live sim from data.nodes at their current + * positions + the current physics tuning. Called by setData and + * setPhysicsTuning. */ + private buildSim(): void { + const data = this.data; + if (!data || data.nodes.length === 0) { + this.sim = null; + this.simById = new Map(); + return; + } + const t = this.physicsTuning; + this.sim = createGraphSimulation(data.nodes, data.edges, { + gravity: { x: this.simCenter.x, y: this.simCenter.y, strength: t.gravity }, + alphaDecay: LIVE_ALPHA_DECAY, + velocityDecay: t.velocityDecay, + chargeStrength: t.chargeStrength, + linkStrength: t.linkStrength, + linkDistance: t.linkDistance, + clusterPull: + data.clusters.length > 0 ? { clusters: data.clusters, strength: t.clusterPull } : undefined, + }); + this.simById = new Map(data.nodes.map((n) => [n.id, n as LayoutNode])); + } + /** Rasterize cluster regions into the low-res world-space bitmap. * Runs on data/visibility/settings changes at rest, and every * REGION_EVERY-th frame while the live sim moves nodes (the in-between diff --git a/components/graph-view.tsx b/components/graph-view.tsx index ac81d15..c0f64fb 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -64,13 +64,13 @@ import { useDeferredValue, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import Link from "next/link"; import { FileTextIcon, Loader2Icon, SearchIcon, SettingsIcon, UploadIcon, WaypointsIcon } from "lucide-react"; -import Graph from "graphology"; -import forceAtlas2 from "graphology-layout-forceatlas2"; -import { seededRandom, type GraphNode, type GraphSnapshot } from "@/src/core"; +import { incrementalLayout, type GraphNode, type GraphSnapshot } from "@/src/core"; import { + DEFAULT_PHYSICS_TUNING, DEFAULT_REGION_TUNING, GraphCanvasRenderer, type GraphRenderSettings, + type PhysicsTuning, type RegionTuning, } from "./graph-canvas.ts"; import { @@ -231,6 +231,10 @@ export function GraphView() { // render settings, which live on the server). Number inputs so any // value can be tried quickly. const [regionTuning, setRegionTuning] = useState(DEFAULT_REGION_TUNING); + // Live-sim physics controls: session state, never persisted. Changes + // rebuild the renderer's sim from the current node positions — the + // same engine the server settle runs, so the feel IS the physics. + const [physicsTuning, setPhysicsTuning] = useState(DEFAULT_PHYSICS_TUNING); // Render settings: SERVER-persisted (data/render-settings.json via // /api/graph/render-settings). Loaded once after mount; every user // change saves back, debounced. Defaults render until the load @@ -387,36 +391,11 @@ export function GraphView() { }; } } - const graph = new Graph(); - for (const node of snapshot.nodes) { - // Seeded start positions (FA2 requires non-zero x/y) keep the - // frozen layout stable for a given node set across remounts. - const rand = seededRandom(node.id); - graph.addNode(node.id, { - x: (rand() - 0.5) * 100, - y: (rand() - 0.5) * 100, - }); - } - for (const edge of snapshot.edges) { - // Snapshot edges are already merged by (from,to,type); the guard - // is just belt-and-braces for the simple graphology graph. - if (!graph.hasEdge(edge.from, edge.to)) { - graph.addEdge(edge.from, edge.to); - } - } - - // Run-then-freeze: one synchronous pass, writes x/y, then done. - // Barnes-Hut comes on automatically above ~500 nodes (the vision's - // first escalation rung — see scripts/bench-graph.ts for the numbers). - const layoutSettings = { - ...forceAtlas2.inferSettings(graph), - ...(graph.order > 500 ? { barnesHutOptimize: true } : {}), - }; - forceAtlas2.assign(graph, { iterations: 100, settings: layoutSettings }); - - const positions = new Map(); - graph.forEachNode((id, attrs) => positions.set(id, { x: attrs.x, y: attrs.y })); - posRef.current = positions; + // Bootstrap fallback: the ledger is absent (fresh graph) — run + // the SAME engine the server settle uses (d3-force via + // incrementalLayout, seeded deterministic) and freeze the result. + // Deterministic for a given node set across remounts. + posRef.current = incrementalLayout(snapshot.nodes, snapshot.edges, new Map()); return () => { posRef.current = null; }; @@ -509,6 +488,12 @@ export function GraphView() { rendererRef.current?.setRegionTuning(regionTuning); }, [regionTuning]); + // Feed #5 — physics tuning: rebuilds the live sim in place (drag pins + // survive; a mid-settle change re-anneals, the honest response). + useEffect(() => { + rendererRef.current?.setPhysicsTuning(physicsTuning); + }, [physicsTuning]); + // Position the detail card: below the clicked node, flipped above near // the bottom edge, clamped horizontally. Imperative (ref + layout effect, // before paint) because the flip needs the card's measured height. @@ -672,6 +657,51 @@ export function GraphView() { onChange={(v) => setRegionTuning((t) => ({ ...t, edgeAlpha: v }))} />
+ {/* Physics (user-facing): the live sim's forces. Every value feeds + the SAME engine the server settle runs (createGraphSimulation); + changing one rebuilds the sim from the current positions — + drag mid-settle re-anneals, pins survive. Session-only. */} +
+ + Physics + + setPhysicsTuning((t) => ({ ...t, chargeStrength: v }))} + /> + setPhysicsTuning((t) => ({ ...t, linkStrength: v }))} + /> + setPhysicsTuning((t) => ({ ...t, linkDistance: v }))} + /> + setPhysicsTuning((t) => ({ ...t, gravity: v }))} + /> + setPhysicsTuning((t) => ({ ...t, clusterPull: v }))} + /> + setPhysicsTuning((t) => ({ ...t, velocityDecay: v }))} + /> +
)} {snapshot !== null && visibleIds.size === 0 && query.trim() !== "" && ( diff --git a/tests/graph-canvas.test.ts b/tests/graph-canvas.test.ts index 6c57911..87aaa71 100644 --- a/tests/graph-canvas.test.ts +++ b/tests/graph-canvas.test.ts @@ -17,8 +17,10 @@ import { fitTransform, labelAlpha, pickNearest, + sanitizePhysicsTuning, sanitizeRegionTuning, zoomRatioOf, + DEFAULT_PHYSICS_TUNING, DEFAULT_REGION_TUNING, LABELS_FADE_SPAN, } from "../components/graph-canvas.ts"; @@ -194,3 +196,29 @@ describe("boxBlurRound (premultiplied region blur)", () => { assert.deepEqual([...b], [1, 2, 3, 4, 5, 6, 7, 8]); }); }); + +describe("sanitizePhysicsTuning (physics controls)", () => { + test("defaults survive untouched; partial input keeps the rest", () => { + assert.deepEqual(sanitizePhysicsTuning(undefined), DEFAULT_PHYSICS_TUNING); + const half = sanitizePhysicsTuning({ clusterPull: 0.6 }); + assert.equal(half.clusterPull, 0.6); + assert.equal(half.chargeStrength, DEFAULT_PHYSICS_TUNING.chargeStrength); + }); + + test("garbage clamps: NaN falls back, out-of-range clamps", () => { + const t = sanitizePhysicsTuning({ + chargeStrength: Number.NaN, + linkStrength: 99, + linkDistance: -5, + gravity: 7, + clusterPull: -1, + velocityDecay: 3, + }); + assert.equal(t.chargeStrength, DEFAULT_PHYSICS_TUNING.chargeStrength); + assert.equal(t.linkStrength, 5); + assert.equal(t.linkDistance, 1); + assert.equal(t.gravity, 1); + assert.equal(t.clusterPull, 0); + assert.equal(t.velocityDecay, 0.99); + }); +}); From 3a1c18b9de9a6ef24612f7b2481fd02c3408cef3 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 10:20:50 +0300 Subject: [PATCH 24/34] Inference-depth anchoring playground: vertical + radial modes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The graph's follows/supports edges encode derivation (premise -> conclusion); the layout can now express that rank spatially, as a session-only playground toggle (nothing promoted yet): - inferenceDepths: longest chain from a base node, CYCLE-SAFE by SCC condensation (Tarjan, iterative) — the DAG property is typical, not guaranteed; cycle members share their component's depth (mutual support = equal footing). O(V+E), pure, deterministic. - DepthAnchor force: per-node target scalar — vertical mode rows (baseline at the anchor centroid, arrows up), radial mode rings (foundations center, outer boundary = edge of knowledge). The rank is computed, not simulated: pairwise up-forces can't give baselines. - Mode projections: gravity x-only in vertical / dropped in radial; cluster pull x-only in vertical / tangential-only in radial; the anchor re-blend preserves user arrangement in the FREE dimension only (x / angle) — meaning owns the constrained axis, memory owns the free one. - setPhysicsTuning reheats (alpha 1): toggling a mode re-forms the graph live and commits the new rest state; drag release is springy along the constrained axis (node returns toward its level/ring). - Sweep on the real graph: strength 0.8 = 215/216 arrows correct with organic sag; 1.5 = crisp rows. 174 base nodes, max chain depth 7, zero cycles in current data. replay-layout --depth= bakes a mode into the ledger for rest-state experiments. --- components/graph-canvas.ts | 38 ++++++ components/graph-view.tsx | 35 ++++++ scripts/replay-layout.ts | 19 ++- src/core/layout.ts | 235 ++++++++++++++++++++++++++++++++++++- tests/layout.test.ts | 103 +++++++++++++++- 5 files changed, 422 insertions(+), 8 deletions(-) diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts index 5b3c71a..37d368e 100644 --- a/components/graph-canvas.ts +++ b/components/graph-canvas.ts @@ -60,6 +60,7 @@ import { createGraphSimulation, INCREMENTAL_GRAVITY, LINK_DISTANCE, + type DepthAnchorMode, type GraphRenderSettings, type LayoutNode, type Position, @@ -218,6 +219,14 @@ export interface PhysicsTuning { clusterPull: number; /** Per-tick velocity damping (d3 default 0.4). */ velocityDecay: number; + /** Inference-depth anchoring mode (see DepthAnchor in core). */ + depthMode: DepthAnchorMode; + /** Depth-anchor pull strength. */ + depthStrength: number; + /** Spacing between inference levels (world units). */ + depthSpacing: number; + /** Inner ring radius, radial mode (world units). */ + depthBaseRadius: number; } export const DEFAULT_PHYSICS_TUNING: PhysicsTuning = { @@ -227,6 +236,13 @@ export const DEFAULT_PHYSICS_TUNING: PhysicsTuning = { gravity: INCREMENTAL_GRAVITY, clusterPull: CLUSTER_PULL_STRENGTH, velocityDecay: 0.4, + depthMode: "off", + // 0.8 by sweep on the real graph: 215/216 arrows read upward with + // organic sag (mean |dev| from level ~34u at spacing 60); 1.5 gives + // crisp rows (dev ~18) if the promoted default wants them. + depthStrength: 0.8, + depthSpacing: 60, + depthBaseRadius: 220, }; /** Clamp anything into a usable tuning (number inputs can be empty/NaN). */ @@ -241,6 +257,10 @@ export function sanitizePhysicsTuning(input: Partial | null | und gravity: Math.min(Math.max(num(input?.gravity, d.gravity), 0), 1), clusterPull: Math.min(Math.max(num(input?.clusterPull, d.clusterPull), 0), 2), velocityDecay: Math.min(Math.max(num(input?.velocityDecay, d.velocityDecay), 0), 0.99), + depthMode: input?.depthMode === "vertical" || input?.depthMode === "radial" ? input.depthMode : "off", + depthStrength: Math.min(Math.max(num(input?.depthStrength, d.depthStrength), 0), 2), + depthSpacing: Math.min(Math.max(num(input?.depthSpacing, d.depthSpacing), 1), 500), + depthBaseRadius: Math.min(Math.max(num(input?.depthBaseRadius, d.depthBaseRadius), 0), 2000), }; } @@ -670,6 +690,14 @@ export class GraphCanvasRenderer { setPhysicsTuning(input: Partial): void { this.physicsTuning = sanitizePhysicsTuning(input); this.buildSim(); + // Reheat: a tuning change re-anneals the layout from its current + // state, so mode toggles (off/vertical/radial) visibly RE-FORM the + // graph instead of waiting for the next drag. Mid-drag is safe — + // buildSim preserved the pinned node's fx/fy. + if (this.sim) { + this.sim.alpha(1); + this.simActive = true; + } this.markDirty(); } @@ -1085,6 +1113,16 @@ export class GraphCanvasRenderer { linkDistance: t.linkDistance, clusterPull: data.clusters.length > 0 ? { clusters: data.clusters, strength: t.clusterPull } : undefined, + depthAnchor: + t.depthMode === "off" + ? undefined + : { + mode: t.depthMode, + strength: t.depthStrength, + spacing: t.depthSpacing, + baseRadius: t.depthBaseRadius, + center: this.simCenter, + }, }); this.simById = new Map(data.nodes.map((n) => [n.id, n as LayoutNode])); } diff --git a/components/graph-view.tsx b/components/graph-view.tsx index c0f64fb..822a6bf 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -665,6 +665,41 @@ export function GraphView() { Physics + + setPhysicsTuning((t) => ({ ...t, depthStrength: v }))} + /> + setPhysicsTuning((t) => ({ ...t, depthSpacing: v }))} + /> + setPhysicsTuning((t) => ({ ...t, depthBaseRadius: v }))} + /> a.startsWith("--depth=")); +const depthMode: "off" | "vertical" | "radial" = + depthArg === "--depth=vertical" ? "vertical" : depthArg === "--depth=radial" ? "radial" : "off"; import { incrementalLayout } from "../src/core/layout.ts"; import type { GraphNode, NodeMapping, Position } from "../src/core/contracts.ts"; @@ -74,6 +81,16 @@ function main() { positions = incrementalLayout(batchNodes, activeEdges, positions, { // The same partition the real settle would see for this batch. clusters: detectCommunities(batchNodes, activeEdges).communities, + ...(depthMode === "off" + ? {} + : { + depthAnchor: { + mode: depthMode, + strength: 0.8, + spacing: 60, + ...(depthMode === "radial" ? { baseRadius: 220 } : {}), + }, + }), }); for (const [id, prev] of before) { const now = positions.get(id)!; diff --git a/src/core/layout.ts b/src/core/layout.ts index e2da57e..26fad48 100644 --- a/src/core/layout.ts +++ b/src/core/layout.ts @@ -43,6 +43,122 @@ export interface LayoutNode extends SimulationNodeDatum { /** A link as d3 sees it (id strings resolve to node refs on init). */ type SimLink = SimulationLinkDatum; +/** How the inference rank is expressed spatially (playground, 2026-09-13): + * the graph's `follows`/`supports` edges encode derivation (premise → + * conclusion), so depth-0 nodes (no incoming edges — the fundamentals) + * can anchor a shared baseline and every derivation level rises from it. + * `vertical`: levels are horizontal rows, arrows point up. + * `radial`: levels are concentric rings, foundations at the center, the + * outer boundary IS the edge of the user's knowledge. + * `off`: today's free layout. */ +export type DepthAnchorMode = "off" | "vertical" | "radial"; + +export interface DepthAnchor { + mode: DepthAnchorMode; + /** Anchor pull toward the level (same units as clusterPull). */ + strength: number; + /** Distance between consecutive inference levels (world units). */ + spacing: number; + /** Baseline point: vertical mode uses .y as the depth-0 line; radial + * mode uses the full point as the ring center. Default (0,0). */ + center?: { x: number; y: number }; + /** Inner ring radius (radial only; depth-0 nodes sit at this ring). */ + baseRadius?: number; +} + +/** Inference depth per node: the longest chain of directed edges from a + * base node. CYCLE-SAFE (2026-09-13, user requirement — the DAG property + * is typical, not guaranteed): Tarjan SCC condensation collapses each + * strongly connected component to one super-node, ranks the (always + * acyclic) condensation, and every cycle member shares its component's + * depth — mutual support reads as equal footing. Iterative Tarjan (no + * recursion depth limits), O(V+E), deterministic. */ +export function inferenceDepths( + nodes: ReadonlyArray>, + edges: ReadonlyArray, +): Map { + const valid = new Set(nodes.map((n) => n.id)); + const adj = new Map(); + for (const e of edges) { + if (e.from === e.to || !valid.has(e.from) || !valid.has(e.to)) continue; + let list = adj.get(e.from); + if (!list) adj.set(e.from, (list = [])); + list.push(e.to); + } + + // Iterative Tarjan SCC. + const num = new Map(); + const low = new Map(); + const onStack = new Set(); + const stack: string[] = []; + const comp = new Map(); + let counter = 0; + let compCount = 0; + for (const start of valid) { + if (num.has(start)) continue; + num.set(start, counter); + low.set(start, counter); + counter++; + stack.push(start); + onStack.add(start); + const work: [string, number][] = [[start, 0]]; + while (work.length > 0) { + const frame = work[work.length - 1]!; + const v = frame[0]; + const outs = adj.get(v) ?? []; + if (frame[1] < outs.length) { + const i = frame[1]; + frame[1] = i + 1; + const w = outs[i]!; + if (!num.has(w)) { + num.set(w, counter); + low.set(w, counter); + counter++; + stack.push(w); + onStack.add(w); + work.push([w, 0]); + } else if (onStack.has(w)) { + low.set(v, Math.min(low.get(v)!, num.get(w)!)); + } + } else { + work.pop(); + if (low.get(v) === num.get(v)) { + let w: string; + do { + w = stack.pop()!; + onStack.delete(w); + comp.set(w, compCount); + } while (w !== v); + compCount++; + } + if (work.length > 0) { + const parent = work[work.length - 1]![0]; + low.set(parent, Math.min(low.get(parent)!, low.get(v)!)); + } + } + } + } + + // Condensation predecessors, then longest path. Tarjan emits sinks + // first, so descending comp index = topological order (sources first). + const preds: Set[] = Array.from({ length: compCount }, () => new Set()); + for (const e of edges) { + const a = comp.get(e.from); + const b = comp.get(e.to); + if (a == null || b == null || a === b) continue; + preds[b]!.add(a); + } + const depth = new Array(compCount).fill(0); + for (let c = compCount - 1; c >= 0; c--) { + let d = 0; + for (const p of preds[c]!) d = Math.max(d, depth[p]! + 1); + depth[c] = d; + } + const out = new Map(); + for (const n of nodes) out.set(n.id, depth[comp.get(n.id)!] ?? 0); + return out; +} + export interface SimOptions { /** Link spring rest length (world units). */ linkDistance?: number; @@ -59,6 +175,8 @@ export interface SimOptions { alphaDecay?: number; /** Velocity damping per tick (d3 default 0.4). Higher = more friction. */ velocityDecay?: number; + /** Inference-depth anchoring (see DepthAnchor). Off by default. */ + depthAnchor?: DepthAnchor; /** Cluster centroid pull (the "virtual node" without the node): * members of each cluster accelerate toward their own cluster's * centroid. Two O(n) passes per tick — cheaper than a real virtual @@ -140,10 +258,15 @@ export function createGraphSimulation( .velocityDecay(options.velocityDecay ?? 0.4) .stop(); + // Gravity, mode-projected: in vertical mode the depth force OWNS the + // y axis, so gravity goes x-only (it would fight the level lines); in + // radial mode a center pull is purely radial and would fight the + // rings, so it is dropped entirely (rings already bound the radius). + const mode = options.depthAnchor?.mode ?? "off"; const g = options.gravity; - if (g) { + if (g && mode !== "radial") { sim.force("x", forceX(g.x).strength(g.strength)); - sim.force("y", forceY(g.y).strength(g.strength)); + if (mode !== "vertical") sim.force("y", forceY(g.y).strength(g.strength)); } // Cluster centroid pull: the membership index is resolved ONCE (nodes @@ -161,6 +284,13 @@ export function createGraphSimulation( const sumY = new Float64Array(k); const cnt = new Float64Array(k); const strength = cp.strength; + // The cluster centroid pull also respects the depth mode: vertical + // → x-only (the depth force owns y); radial → tangential only + // (the radial component would drag members off their rings; the + // tangential component still tightens each cluster's angular + // spread, which is all clustering needs to do there). + const ringCx = options.depthAnchor?.center?.x ?? 0; + const ringCy = options.depthAnchor?.center?.y ?? 0; sim.force("clusterPull", (alpha: number) => { sumX.fill(0); sumY.fill(0); @@ -177,11 +307,72 @@ export function createGraphSimulation( const ci = nodeCluster[i]!; if (ci < 0 || (cnt[ci] ?? 0) < 2) continue; const n = nodes[i]!; - n.vx = (n.vx ?? 0) + (sumX[ci]! / cnt[ci]! - (n.x ?? 0)) * strength * alpha; - n.vy = (n.vy ?? 0) + (sumY[ci]! / cnt[ci]! - (n.y ?? 0)) * strength * alpha; + const px = n.x ?? 0; + const py = n.y ?? 0; + const ux = sumX[ci]! / cnt[ci]! - px; + const uy = sumY[ci]! / cnt[ci]! - py; + if (mode === "vertical") { + n.vx = (n.vx ?? 0) + ux * strength * alpha; + } else if (mode === "radial") { + const rx = px - ringCx; + const ry = py - ringCy; + const r = Math.hypot(rx, ry); + if (r > 1e-9) { + // Tangential component: u − (u·r̂)r̂ + const dot = (ux * rx + uy * ry) / (r * r); + n.vx = (n.vx ?? 0) + (ux - dot * rx) * strength * alpha; + n.vy = (n.vy ?? 0) + (uy - dot * ry) * strength * alpha; + } + } else { + n.vx = (n.vx ?? 0) + ux * strength * alpha; + n.vy = (n.vy ?? 0) + uy * strength * alpha; + } } }); } + + // ── Inference-depth anchoring ──────────────────────────────────────── + // The rank is COMPUTED (not simulated): per-node target scalar derived + // from inferenceDepths, then one O(n) pull pass per tick. Soft knob — + // high strength approaches pinned rows/rings while low strength leaves + // organic sag. + const da = options.depthAnchor; + if (da && da.mode !== "off" && da.strength > 0) { + const depths = inferenceDepths(nodes, edges); + const strength = da.strength; + const spacing = da.spacing; + const c = da.center ?? { x: 0, y: 0 }; + if (da.mode === "vertical") { + const baseY = c.y; + sim.force("depth", (alpha: number) => { + for (let i = 0; i < nodes.length; i++) { + const n = nodes[i]!; + const targetY = baseY - (depths.get(n.id) ?? 0) * spacing; + n.vy = (n.vy ?? 0) + (targetY - (n.y ?? 0)) * strength * alpha; + } + }); + } else { + const baseR = da.baseRadius ?? 0; + const cx0 = c.x; + const cy0 = c.y; + sim.force("depth", (alpha: number) => { + for (let i = 0; i < nodes.length; i++) { + const n = nodes[i]!; + const dx = (n.x ?? 0) - cx0; + const dy = (n.y ?? 0) - cy0; + // Degenerate direction at the exact ring center: nudge + // outward on a deterministic axis. + const r = Math.hypot(dx, dy); + const ux = r > 1e-9 ? dx / r : 1; + const uy = r > 1e-9 ? dy / r : 0; + const target = baseR + (depths.get(n.id) ?? 0) * spacing; + const mag = (target - r) * strength * alpha; + n.vx = (n.vx ?? 0) + ux * mag; + n.vy = (n.vy ?? 0) + uy * mag; + } + }); + } + } return sim; } @@ -199,6 +390,9 @@ export interface IncrementalLayoutOptions { clusters?: ReadonlyArray<{ members: ReadonlyArray }>; /** Cluster centroid pull strength (default CLUSTER_PULL_STRENGTH). */ clusterPullStrength?: number; + /** Inference-depth anchoring (see DepthAnchor). `center` defaults to + * the anchored core's centroid. */ + depthAnchor?: DepthAnchor; } /** @@ -333,20 +527,49 @@ export function incrementalLayout( clusterPull: options.clusters ? { clusters: options.clusters, strength: options.clusterPullStrength ?? CLUSTER_PULL_STRENGTH } : undefined, + depthAnchor: + options.depthAnchor && options.depthAnchor.mode !== "off" + ? { ...options.depthAnchor, center: options.depthAnchor.center ?? { x: cx, y: cy } } + : options.depthAnchor, }); + const dMode = options.depthAnchor?.mode ?? "off"; + const dCenter = options.depthAnchor?.center ?? { x: cx, y: cy }; const byId = new Map(simNodes.map((n) => [n.id, n])); for (let i = 0; i < ticks; i++) { sim.tick(); // Anchor re-blend (position space, alpha-independent — see the // ANCHOR_RETENTION note). Pinned nodes (fx/fy) are left to d3. + // DEPTH-MODE PROJECTION: the re-blend preserves the user's + // arrangement in the FREE dimension only — x in vertical mode, the + // angle in radial mode — because the depth force owns the + // constrained dimension (a node's level/ring is meaning, not + // memory). Off-mode: full 2D blend as before. if (hasCore && retention < 1) { for (const [id, a] of anchors) { const n = byId.get(id); if (!n || n.fx != null || n.fy != null) continue; const x = n.x ?? 0; const y = n.y ?? 0; - n.x = a.x + (x - a.x) * retention; - n.y = a.y + (y - a.y) * retention; + if (dMode === "vertical") { + n.x = a.x + (x - a.x) * retention; + } else if (dMode === "radial") { + const rx = x - dCenter.x; + const ry = y - dCenter.y; + const r = Math.hypot(rx, ry); + const ax = a.x - dCenter.x; + const ay = a.y - dCenter.y; + const ar = Math.hypot(ax, ay); + if (r > 1e-9 && ar > 1e-9) { + const theta = Math.atan2(ry, rx); + const thetaA = Math.atan2(ay, ax); + const blended = thetaA + Math.atan2(Math.sin(theta - thetaA), Math.cos(theta - thetaA)) * retention; + n.x = dCenter.x + r * Math.cos(blended); + n.y = dCenter.y + r * Math.sin(blended); + } + } else { + n.x = a.x + (x - a.x) * retention; + n.y = a.y + (y - a.y) * retention; + } } } } diff --git a/tests/layout.test.ts b/tests/layout.test.ts index 15e6fc0..7e71f44 100644 --- a/tests/layout.test.ts +++ b/tests/layout.test.ts @@ -6,7 +6,7 @@ */ import assert from "node:assert/strict"; import { describe, it } from "node:test"; -import { createGraphSimulation, incrementalLayout } from "../src/core/layout.ts"; +import { createGraphSimulation, incrementalLayout, inferenceDepths } from "../src/core/layout.ts"; import { settleGraphLayout } from "../src/core/layout-settle.ts"; import type { GraphNode, GraphEdge, Position } from "../src/core/contracts.ts"; @@ -168,3 +168,104 @@ describe("clusterPull (the centroid force)", () => { assert.ok(pulled > 0, "spread is finite/positive"); }); }); + +describe("inferenceDepths (SCC-safe ranking)", () => { + const nodes = (ids: string[]) => ids.map((id) => ({ id })); + + it("ranks a chain, a diamond takes the longest path, orphans are depth 0", () => { + // chain a->b->c; diamond a->d, b->d, c->d (d reachable at 1 and 2 and 3) + const ns = nodes(["a", "b", "c", "d", "loner"]); + const edges = [ + { from: "a", to: "b" }, + { from: "b", to: "c" }, + { from: "a", to: "d" }, + { from: "b", to: "d" }, + { from: "c", to: "d" }, + ]; + const d = inferenceDepths(ns, edges); + assert.equal(d.get("a"), 0); + assert.equal(d.get("b"), 1); + assert.equal(d.get("c"), 2); + assert.equal(d.get("d"), 3); // longest path, not shortest + assert.equal(d.get("loner"), 0); + }); + + it("cycles share a depth (mutual support = equal footing)", () => { + // x <-> y (a cycle), fed by s; z follows x. + const ns = nodes(["s", "x", "y", "z"]); + const edges = [ + { from: "s", to: "x" }, + { from: "x", to: "y" }, + { from: "y", to: "x" }, + { from: "x", to: "z" }, + ]; + const d = inferenceDepths(ns, edges); + assert.equal(d.get("s"), 0); + assert.equal(d.get("x"), d.get("y"), "cycle members share a depth"); + assert.equal(d.get("x"), 1); + assert.equal(d.get("z"), 2); + // Deterministic across calls. + assert.deepEqual([...inferenceDepths(ns, edges).entries()], [...d.entries()]); + }); +}); + +describe("depthAnchor force", () => { + const nodes = (ids: string[]) => ids.map((id) => ({ id, x: 0, y: 0 })); + + it("vertical: the premise lands below its conclusion", () => { + const ns = nodes(["base", "derived"]); + const edges = [{ from: "base", to: "derived" }]; + const sim = createGraphSimulation(ns, edges, { + depthAnchor: { mode: "vertical", strength: 1.2, spacing: 100, center: { x: 0, y: 0 } }, + }); + for (let i = 0; i < 300; i++) sim.tick(); + const byId = new Map(sim.nodes().map((n) => [n.id, n])); + const baseY = byId.get("base")!.y!; + const derivedY = byId.get("derived")!.y!; + assert.ok(baseY > derivedY, `base (${baseY.toFixed(0)}) should sit BELOW derived (${derivedY.toFixed(0)})`); + assert.ok( + baseY - derivedY > 60, + `level separation should be substantial (got ${(baseY - derivedY).toFixed(0)})`, + ); + }); + + it("radial: inference depth becomes radius", () => { + const ns = nodes(["base", "mid", "top"]); + const edges = [ + { from: "base", to: "mid" }, + { from: "mid", to: "top" }, + ]; + const sim = createGraphSimulation(ns, edges, { + depthAnchor: { mode: "radial", strength: 1.2, spacing: 80, baseRadius: 100, center: { x: 0, y: 0 } }, + }); + for (let i = 0; i < 300; i++) sim.tick(); + const r = (id: string) => { + const n = sim.nodes().find((x) => x.id === id)!; + return Math.hypot(n.x!, n.y!); + }; + assert.ok(r("base") < r("mid"), `base ring (${r("base").toFixed(0)}) inside mid (${r("mid").toFixed(0)})`); + assert.ok(r("mid") < r("top"), `mid ring (${r("mid").toFixed(0)}) inside top (${r("top").toFixed(0)})`); + }); + + it("settle with depthAnchor: old nodes keep x memory but snap to their level", () => { + // Node "old" was previously placed at y=300 (off-level). With + // vertical anchoring a re-settle must move it to the baseline + // while preserving (roughly) its x arrangement. + const previous = new Map([["old", { x: 250, y: 300 }], ["peer", { x: 260, y: 290 }]]); + const next = incrementalLayout( + [{ id: "old" }, { id: "peer" }, { id: "child" }], + [ + { from: "old", to: "child" }, + { from: "peer", to: "child" }, + ], + previous, + { depthAnchor: { mode: "vertical", strength: 0.6, spacing: 80 } }, + ); + const old = next.get("old")!; + // The baseline defaults to the anchor centroid: (300+290)/2 = 295. + assert.ok(Math.abs(old.y - 295) < 20, `old node snaps to the baseline (y=${old.y.toFixed(1)})`); + assert.ok(old.x > 150, `x memory preserved (x=${old.x.toFixed(1)})`); + const child = next.get("child")!; + assert.ok(child.y < old.y - 40, `child rises above its premises (child=${child.y.toFixed(1)}, old=${old.y.toFixed(1)})`); + }); +}); From f5edc7f0a58dc81714f6d84ea1388b72b3b2fa81 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 14:29:16 +0300 Subject: [PATCH 25/34] Semantic topic clusters: local embeddings replace Louvain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Louvain only ever found the reasoning chains (the graph is disconnected by nature — every connected component became its own cluster) and edge-orphans could never join anything. Clusters are now RELATED TOPICS: propositions grouped by meaning. - Embedder (src/server/embeddings.ts): in-process transformers.js v4 running EmbeddingGemma-300M q8 (MTEB 68.4, ~300MB one-time download), MRL-truncated to 512d, "title: none | text:" prefix on 100% of inputs, L2-normalized so cosine = dot. Model/dim mismatch voids the ledger (full re-embed). Injected at the composition root — the agent layer never imports the server layer (architecture law). - Ledger (data/graph/embeddings.json): model + dim + vectors + the persisted cluster partition. Reclustering runs at the END of every embedding batch (post-merge hook, both front doors) — never per read (the agglomerative pass is ~1.5s at 338 nodes). - Clustering (src/core/semantic-clusters.ts): hand-rolled average-linkage agglomerative over cosine (npm HDBSCAN is an abandoned alpha). Stragglers stay singletons by default; attachStragglers is the all-in toggle. Threshold 0.60 swept on the real graph: 26 topics, 74/90 edge-orphans land in real topics, 10 stragglers. - Consumers (snapshot, settle, namer, replay) read the persisted partition — one cluster concept everywhere; core-survival label matching unchanged (20/26 topics kept labels). - Louvain retired: detectCommunities + graphology* deps deleted (graphology-louvain had leaked into the client bundle since the clusters wave); scripts/bench-graph.ts (dead FA2 bench) removed. - scripts/embed-graph.ts: backfill/cold-start (338 nodes in 150s including the one-time model download). --- components/graph-view.tsx | 8 +- config.ts | 6 + next.config.ts | 8 +- package-lock.json | 882 +++++++++++++++++++++++---- package.json | 4 +- scripts/backfill-graph.ts | 3 +- scripts/bench-graph.ts | 70 --- scripts/embed-graph.ts | 23 + scripts/import-notes.ts | 3 +- scripts/replay-layout.ts | 15 +- src/agent/subagents/cluster-namer.ts | 22 +- src/agent/subagents/extraction.ts | 24 +- src/core/clusters.ts | 120 +--- src/core/contracts/graph.ts | 20 + src/core/graph.ts | 22 +- src/core/index.ts | 2 +- src/core/layout-settle.ts | 15 +- src/core/layout.ts | 2 +- src/core/ports.ts | 8 + src/core/semantic-clusters.ts | 141 +++++ src/server/chat-handler.ts | 8 +- src/server/embeddings.ts | 125 ++++ src/store/graph.ts | 60 +- tests/clusters.test.ts | 89 +-- tests/extraction.test.ts | 52 ++ tests/graph.test.ts | 22 +- tests/layout.test.ts | 2 + tests/semantic-clusters.test.ts | 118 ++++ 28 files changed, 1443 insertions(+), 431 deletions(-) delete mode 100644 scripts/bench-graph.ts create mode 100644 scripts/embed-graph.ts create mode 100644 src/core/semantic-clusters.ts create mode 100644 src/server/embeddings.ts create mode 100644 tests/semantic-clusters.test.ts diff --git a/components/graph-view.tsx b/components/graph-view.tsx index 822a6bf..a139ddf 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -14,10 +14,10 @@ * per-camera-event full-canvas repaints (GPU spikes + coil whine) are * gone (at most one draw per frame, and NO frame at all while idle). * - * Layout (graphology + forceAtlas2) runs-then-freezes exactly as - * before: ONE synchronous 100-iteration pass inside a useEffect writes - * world-space x/y, then the positions are handed to the renderer as - * data. No layout animation ever starts; the only animation is the + * Layout fallback (ledger absent): ONE synchronous incrementalLayout + * pass (d3-force, core/layout.ts) inside a useEffect writes world-space + * x/y, then the positions are handed to the renderer as data. No layout + * animation ever starts; the only animation is the * short hover fade, strictly bounded in time — idle CPU is zero (the * loop schedules no rAF until something actually changes). * diff --git a/config.ts b/config.ts index 8985557..8162991 100644 --- a/config.ts +++ b/config.ts @@ -89,6 +89,12 @@ export const SOURCES_FILE = path.join(PATHS.graph, "sources.json"); * Carries an updatedAt stamp for future live-physics conflict handling. */ export const POSITIONS_FILE = path.join(PATHS.graph, "positions.json"); +/** The embedding ledger: one L2-normalized vector per node id, all from + * ONE model (swapping models re-embeds everything — vectors from + * different models live in incompatible spaces). Read-modify-write + * JSON like positions/sources. */ +export const EMBEDDINGS_FILE = path.join(PATHS.graph, "embeddings.json"); + /** * Graph render settings (the settings-panel controls): one small JSON blob, * read-modify-write. Shared shape/sanitizer: src/core/render-settings.ts. diff --git a/next.config.ts b/next.config.ts index 5d5d02e..c3236a4 100644 --- a/next.config.ts +++ b/next.config.ts @@ -4,7 +4,13 @@ const nextConfig: NextConfig = { // Framework-free core lives in src/ (plain TS, imported by routes and CLI alike). // The pi engine must stay external: its providers are runtime-loaded Node // code (undici streams, catalog data) that breaks when bundled into routes. - serverExternalPackages: ["@earendil-works/pi-ai", "@earendil-works/pi-agent-core"], + // transformers.js (embeddings) is runtime-loaded ONNX — same reason as + // the pi engine: bundling onnxruntime-node breaks route builds. + serverExternalPackages: [ + "@earendil-works/pi-ai", + "@earendil-works/pi-agent-core", + "@huggingface/transformers", + ], }; export default nextConfig; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 862e963..d6c41f0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "@ai-sdk/react": "^4.0.79", "@earendil-works/pi-agent-core": "^0.85.1", "@earendil-works/pi-ai": "^0.85.1", + "@huggingface/transformers": "^4.2.0", "@mozilla/readability": "^0.6.0", "@radix-ui/react-use-controllable-state": "^1.2.6", "@rive-app/react-webgl2": "^4.32.1", @@ -28,9 +29,6 @@ "d3-selection": "^3.0.0", "d3-zoom": "^3.0.0", "embla-carousel-react": "^8.6.0", - "graphology": "^0.26.0", - "graphology-communities-louvain": "^2.0.2", - "graphology-layout-forceatlas2": "^0.10.1", "katex": "^0.16.47", "linkedom": "^0.18.13", "lucide-react": "^1.33.0", @@ -1913,6 +1911,582 @@ } } }, + "node_modules/@huggingface/jinja": { + "version": "0.5.10", + "resolved": "https://registry.npmjs.org/@huggingface/jinja/-/jinja-0.5.10.tgz", + "integrity": "sha512-SgS1D1bglQ94ceD4ZCL6eayUDy9uV1xuyk61OjgSxU5GJh7upqZCKII0JoTYMc6wWsg3JUgaPRX0ElQzXPk3Cw==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/tokenizers": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@huggingface/tokenizers/-/tokenizers-0.1.3.tgz", + "integrity": "sha512-8rF/RRT10u+kn7YuUbUg0OF30K8rjTc78aHpxT+qJ1uWSqxT1MHi8+9ltwYfkFYJzT/oS+qw3JVfHtNMGAdqyA==", + "license": "Apache-2.0" + }, + "node_modules/@huggingface/transformers": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/@huggingface/transformers/-/transformers-4.2.0.tgz", + "integrity": "sha512-8BRCoBMH0XsWaEIamuR0LrJGAfftgHAfb2Vrffy0VKlSAE/MnUJ5/h/zTfEP3fDIft+nk7TqB8xXEyABGitBjQ==", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/jinja": "^0.5.6", + "@huggingface/tokenizers": "^0.1.3", + "onnxruntime-node": "1.24.3", + "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c", + "sharp": "^0.34.5" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-darwin-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.34.5.tgz", + "integrity": "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-darwin-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.34.5.tgz", + "integrity": "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.2.4.tgz", + "integrity": "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==", + "cpu": [ + "arm64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.2.4.tgz", + "integrity": "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==", + "cpu": [ + "x64" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.2.4.tgz", + "integrity": "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.2.4.tgz", + "integrity": "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linux-ppc64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.2.4.tgz", + "integrity": "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linux-riscv64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.2.4.tgz", + "integrity": "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.2.4.tgz", + "integrity": "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.2.4.tgz", + "integrity": "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.2.4.tgz", + "integrity": "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.2.4.tgz", + "integrity": "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "LGPL-3.0-or-later", + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linux-arm": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.34.5.tgz", + "integrity": "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==", + "cpu": [ + "arm" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linux-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.34.5.tgz", + "integrity": "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==", + "cpu": [ + "arm64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linux-ppc64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.34.5.tgz", + "integrity": "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==", + "cpu": [ + "ppc64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-ppc64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linux-riscv64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.34.5.tgz", + "integrity": "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==", + "cpu": [ + "riscv64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-riscv64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linux-s390x": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.34.5.tgz", + "integrity": "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==", + "cpu": [ + "s390x" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linux-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.34.5.tgz", + "integrity": "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==", + "cpu": [ + "x64" + ], + "libc": [ + "glibc" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.34.5.tgz", + "integrity": "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==", + "cpu": [ + "arm64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.34.5.tgz", + "integrity": "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==", + "cpu": [ + "x64" + ], + "libc": [ + "musl" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.2.4" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-wasm32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.34.5.tgz", + "integrity": "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==", + "cpu": [ + "wasm32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.7.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-win32-arm64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.34.5.tgz", + "integrity": "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-win32-ia32": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.34.5.tgz", + "integrity": "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==", + "cpu": [ + "ia32" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/@img/sharp-win32-x64": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.34.5.tgz", + "integrity": "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0 AND LGPL-3.0-or-later", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@huggingface/transformers/node_modules/sharp": { + "version": "0.34.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", + "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@img/colour": "^1.0.0", + "detect-libc": "^2.1.2", + "semver": "^7.7.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.34.5", + "@img/sharp-darwin-x64": "0.34.5", + "@img/sharp-libvips-darwin-arm64": "1.2.4", + "@img/sharp-libvips-darwin-x64": "1.2.4", + "@img/sharp-libvips-linux-arm": "1.2.4", + "@img/sharp-libvips-linux-arm64": "1.2.4", + "@img/sharp-libvips-linux-ppc64": "1.2.4", + "@img/sharp-libvips-linux-riscv64": "1.2.4", + "@img/sharp-libvips-linux-s390x": "1.2.4", + "@img/sharp-libvips-linux-x64": "1.2.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", + "@img/sharp-libvips-linuxmusl-x64": "1.2.4", + "@img/sharp-linux-arm": "0.34.5", + "@img/sharp-linux-arm64": "0.34.5", + "@img/sharp-linux-ppc64": "0.34.5", + "@img/sharp-linux-riscv64": "0.34.5", + "@img/sharp-linux-s390x": "0.34.5", + "@img/sharp-linux-x64": "0.34.5", + "@img/sharp-linuxmusl-arm64": "0.34.5", + "@img/sharp-linuxmusl-x64": "0.34.5", + "@img/sharp-wasm32": "0.34.5", + "@img/sharp-win32-arm64": "0.34.5", + "@img/sharp-win32-ia32": "0.34.5", + "@img/sharp-win32-x64": "0.34.5" + } + }, "node_modules/@humanfs/core": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", @@ -2001,7 +2575,6 @@ "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", "license": "MIT", - "optional": true, "engines": { "node": ">=18" } @@ -5965,6 +6538,15 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/adm-zip": { + "version": "0.5.18", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.18.tgz", + "integrity": "sha512-ufJnssQGbxzLNS1Ho9bCtX4rQKCCvoVuDLHoJyc3F9dOGDB4BkWs2Ci0kv53lqocAEQ/Cbi+I2XCsNYGqVYqng==", + "license": "MIT", + "engines": { + "node": ">=12.0" + } + }, "node_modules/agent-base": { "version": "7.1.4", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", @@ -6294,6 +6876,13 @@ "url": "https://github.com/sponsors/fb55" } }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "MIT" + }, "node_modules/bowser": { "version": "2.14.1", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", @@ -7343,7 +7932,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", - "dev": true, "license": "MIT", "dependencies": { "es-define-property": "^1.0.0", @@ -7361,7 +7949,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", - "dev": true, "license": "MIT", "dependencies": { "define-data-property": "^1.0.1", @@ -7397,12 +7984,17 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "devOptional": true, "license": "Apache-2.0", "engines": { "node": ">=8" } }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "license": "MIT" + }, "node_modules/detect-node-es": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", @@ -7714,7 +8306,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -7724,7 +8315,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -7805,6 +8395,12 @@ "tests/browser-compat" ] }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "license": "MIT" + }, "node_modules/esbuild": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", @@ -8304,15 +8900,6 @@ "node": ">=0.10.0" } }, - "node_modules/events": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", - "license": "MIT", - "engines": { - "node": ">=0.8.x" - } - }, "node_modules/eventsource-parser": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.1.tgz", @@ -8484,6 +9071,12 @@ "node": ">=16" } }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, "node_modules/flatted": { "version": "3.4.4", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", @@ -8752,6 +9345,23 @@ "node": ">=10.13.0" } }, + "node_modules/global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, "node_modules/globals": { "version": "14.0.0", "resolved": "https://registry.npmjs.org/globals/-/globals-14.0.0.tgz", @@ -8769,7 +9379,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", - "dev": true, "license": "MIT", "dependencies": { "define-properties": "^1.2.1", @@ -8812,7 +9421,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -8828,73 +9436,11 @@ "dev": true, "license": "ISC" }, - "node_modules/graphology": { - "version": "0.26.0", - "resolved": "https://registry.npmjs.org/graphology/-/graphology-0.26.0.tgz", - "integrity": "sha512-8SSImzgUUYC89Z042s+0r/vMibY7GX/Emz4LDO5e7jYXhuoWfHISPFJYjpRLUSJGq6UQ6xlenvX1p/hJdfXuXg==", - "license": "MIT", - "dependencies": { - "events": "^3.3.0" - }, - "peerDependencies": { - "graphology-types": ">=0.24.0" - } - }, - "node_modules/graphology-communities-louvain": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/graphology-communities-louvain/-/graphology-communities-louvain-2.0.2.tgz", - "integrity": "sha512-zt+2hHVPYxjEquyecxWXoUoIuN/UvYzsvI7boDdMNz0rRvpESQ7+e+Ejv6wK7AThycbZXuQ6DkG8NPMCq6XwoA==", - "license": "MIT", - "dependencies": { - "graphology-indices": "^0.17.0", - "graphology-utils": "^2.4.4", - "mnemonist": "^0.39.0", - "pandemonium": "^2.4.1" - }, - "peerDependencies": { - "graphology-types": ">=0.19.0" - } - }, - "node_modules/graphology-indices": { - "version": "0.17.0", - "resolved": "https://registry.npmjs.org/graphology-indices/-/graphology-indices-0.17.0.tgz", - "integrity": "sha512-A7RXuKQvdqSWOpn7ZVQo4S33O0vCfPBnUSf7FwE0zNCasqwZVUaCXePuWo5HBpWw68KJcwObZDHpFk6HKH6MYQ==", - "license": "MIT", - "dependencies": { - "graphology-utils": "^2.4.2", - "mnemonist": "^0.39.0" - }, - "peerDependencies": { - "graphology-types": ">=0.20.0" - } - }, - "node_modules/graphology-layout-forceatlas2": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/graphology-layout-forceatlas2/-/graphology-layout-forceatlas2-0.10.1.tgz", - "integrity": "sha512-ogzBeF1FvWzjkikrIFwxhlZXvD2+wlY54lqhsrWprcdPjopM2J9HoMweUmIgwaTvY4bUYVimpSsOdvDv1gPRFQ==", - "license": "MIT", - "dependencies": { - "graphology-utils": "^2.1.0" - }, - "peerDependencies": { - "graphology-types": ">=0.19.0" - } - }, - "node_modules/graphology-types": { - "version": "0.24.8", - "resolved": "https://registry.npmjs.org/graphology-types/-/graphology-types-0.24.8.tgz", - "integrity": "sha512-hDRKYXa8TsoZHjgEaysSRyPdT6uB78Ci8WnjgbStlQysz7xR52PInxNsmnB7IBOM1BhikxkNyCVEFgmPKnpx3Q==", - "license": "MIT", - "peer": true - }, - "node_modules/graphology-utils": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/graphology-utils/-/graphology-utils-2.5.2.tgz", - "integrity": "sha512-ckHg8MXrXJkOARk56ZaSCM1g1Wihe2d6iTmz1enGOz4W/l831MBCKSayeFQfowgF8wd+PQ4rlch/56Vs/VZLDQ==", - "license": "MIT", - "peerDependencies": { - "graphology-types": ">=0.23.0" - } + "node_modules/guid-typescript": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/guid-typescript/-/guid-typescript-1.0.9.tgz", + "integrity": "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==", + "license": "ISC" }, "node_modules/hachure-fill": { "version": "0.5.2", @@ -8929,7 +9475,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", - "dev": true, "license": "MIT", "dependencies": { "es-define-property": "^1.0.0" @@ -10055,6 +10600,12 @@ "dev": true, "license": "MIT" }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "license": "ISC" + }, "node_modules/json5": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", @@ -10520,6 +11071,30 @@ "node": ">= 20" } }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/matcher/node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -11626,15 +12201,6 @@ "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", "license": "MIT" }, - "node_modules/mnemonist": { - "version": "0.39.8", - "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.8.tgz", - "integrity": "sha512-vyWo2K3fjrUw8YeeZ1zF0fy6Mu59RHokURlld8ymdUPjMlD9EC9ov1/YPqTgqRvUN9nTr3Gqfz29LYAmu0PHPQ==", - "license": "MIT", - "dependencies": { - "obliterator": "^2.0.1" - } - }, "node_modules/modern-tar": { "version": "0.8.4", "resolved": "https://registry.npmjs.org/modern-tar/-/modern-tar-0.8.4.tgz", @@ -11937,7 +12503,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -12033,12 +12598,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/obliterator": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.5.tgz", - "integrity": "sha512-42CPE9AhahZRsMNslczq0ctAEtqk8Eka26QofnqC346BZdHDySk3LWka23LI7ULIw11NmltpiLagIq8gBozxTw==", - "license": "MIT" - }, "node_modules/oniguruma-parser": { "version": "0.12.2", "resolved": "https://registry.npmjs.org/oniguruma-parser/-/oniguruma-parser-0.12.2.tgz", @@ -12056,6 +12615,49 @@ "regex-recursion": "^6.0.2" } }, + "node_modules/onnxruntime-common": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.3.tgz", + "integrity": "sha512-GeuPZO6U/LBJXvwdaqHbuUmoXiEdeCjWi/EG7Y1HNnDwJYuk6WUbNXpF6luSUY8yASul3cmUlLGrCCL1ZgVXqA==", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.24.3", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.24.3.tgz", + "integrity": "sha512-JH7+czbc8ALA819vlTgcV+Q214/+VjGeBHDjX81+ZCD0PCVCIFGFNtT0V4sXG/1JXypKPgScQcB3ij/hk3YnTg==", + "hasInstallScript": true, + "license": "MIT", + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "adm-zip": "^0.5.16", + "global-agent": "^3.0.0", + "onnxruntime-common": "1.24.3" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.26.0-dev.20260416-b7804b056c", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.26.0-dev.20260416-b7804b056c.tgz", + "integrity": "sha512-MD6Ss4GSpQBo6zqoJzyT9LRbKYs7x/JVN23FT24EcEvlqF4VuzPOeH6X38orZPKHQDbprn7K+SBpu0/mj2CQiw==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^25.1.24", + "guid-typescript": "^1.0.9", + "long": "^5.2.3", + "onnxruntime-common": "1.24.0-dev.20251116-b39e144322", + "platform": "^1.3.6", + "protobufjs": "^7.2.4" + } + }, + "node_modules/onnxruntime-web/node_modules/onnxruntime-common": { + "version": "1.24.0-dev.20251116-b39e144322", + "resolved": "https://registry.npmjs.org/onnxruntime-common/-/onnxruntime-common-1.24.0-dev.20251116-b39e144322.tgz", + "integrity": "sha512-BOoomdHYmNRL5r4iQ4bMvsl2t0/hzVQ3OM3PHD0gxeXu1PmggqBv3puZicEUVOA3AtHHYmqZtjMj9FOfGrATTw==", + "license": "MIT" + }, "node_modules/openai": { "version": "6.40.0", "resolved": "https://registry.npmjs.org/openai/-/openai-6.40.0.tgz", @@ -12162,15 +12764,6 @@ "integrity": "sha512-yQA4H19AmPEoMUeavPMDIe1higySl/gH/yaQrkT/s07Qp+7pp2hYz30N3z2l5BkjVkF9Ow6o0wjJamm2y7Sn0A==", "license": "MIT" }, - "node_modules/pandemonium": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/pandemonium/-/pandemonium-2.4.1.tgz", - "integrity": "sha512-wRqjisUyiUfXowgm7MFH2rwJzKIr20rca5FsHXCMNm1W5YPP1hCtrZfgmQ62kP7OZ7Xt+cR858aB28lu5NX55g==", - "license": "MIT", - "dependencies": { - "mnemonist": "^0.39.2" - } - }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -12288,6 +12881,12 @@ "node": ">=16.20.0" } }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, "node_modules/points-on-curve": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/points-on-curve/-/points-on-curve-0.2.0.tgz", @@ -12998,6 +13597,23 @@ "node": ">=0.10.0" } }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, "node_modules/robust-predicates": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/robust-predicates/-/robust-predicates-3.0.3.tgz", @@ -13137,7 +13753,6 @@ "version": "7.8.5", "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "devOptional": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -13146,6 +13761,27 @@ "node": ">=10" } }, + "node_modules/semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", + "license": "MIT" + }, + "node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "license": "MIT", + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", @@ -13363,6 +13999,12 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "license": "BSD-3-Clause" + }, "node_modules/stable-hash-x": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/stable-hash-x/-/stable-hash-x-0.2.0.tgz", @@ -13894,6 +14536,18 @@ "node": ">= 0.8.0" } }, + "node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/typebox": { "version": "1.3.7", "resolved": "https://registry.npmjs.org/typebox/-/typebox-1.3.7.tgz", diff --git a/package.json b/package.json index 1e1f895..6969412 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "@ai-sdk/react": "^4.0.79", "@earendil-works/pi-agent-core": "^0.85.1", "@earendil-works/pi-ai": "^0.85.1", + "@huggingface/transformers": "^4.2.0", "@mozilla/readability": "^0.6.0", "@radix-ui/react-use-controllable-state": "^1.2.6", "@rive-app/react-webgl2": "^4.32.1", @@ -33,9 +34,6 @@ "d3-selection": "^3.0.0", "d3-zoom": "^3.0.0", "embla-carousel-react": "^8.6.0", - "graphology": "^0.26.0", - "graphology-communities-louvain": "^2.0.2", - "graphology-layout-forceatlas2": "^0.10.1", "katex": "^0.16.47", "linkedom": "^0.18.13", "lucide-react": "^1.33.0", diff --git a/scripts/backfill-graph.ts b/scripts/backfill-graph.ts index 194addc..9eefef0 100644 --- a/scripts/backfill-graph.ts +++ b/scripts/backfill-graph.ts @@ -8,13 +8,14 @@ import { chatStore } from "../src/store/chats.ts"; import { graphStore } from "../src/store/graph.ts"; import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts"; +import { embedNewNodes } from "../src/server/embeddings.ts"; import { getGraph } from "../src/core/graph.ts"; async function main() { const only = process.argv[2]; // optional: extract just one chat id const chats = only ? chatStore.listChats().filter((c) => c.id === only) : chatStore.listChats(); console.log(`[backfill] ${chats.length} chats to extract`); - const pipeline = createExtractionPipeline({ chatStore, graphStore }); + const pipeline = createExtractionPipeline({ chatStore, graphStore, embedder: embedNewNodes }); for (const chat of chats) { const t0 = Date.now(); try { diff --git a/scripts/bench-graph.ts b/scripts/bench-graph.ts deleted file mode 100644 index a3a2d12..0000000 --- a/scripts/bench-graph.ts +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Layout benchmark (W4; vision §1 "Rendering (DECIDED)" — "Pre-implementation - * validation: 2k-node synthetic benchmark to confirm layout time"). - * - * npx tsx scripts/bench-graph.ts - * - * Two scales: the vision's target regime (hundreds of nodes, <150 ms budget - * with a synchronous run-then-freeze pass) and the stress case (2,000 nodes, - * ~4,000 edges), the latter timed with and without barnesHutOptimize. - * - * VERDICT (2026-08-22, Apple silicon, Node 24): hundreds of nodes are - * comfortably inside the budget (see the measured lines — ~25 ms at 100, - * ~90 ms at 300). At 2k nodes the naive O(n²) pass takes seconds, and - * barnesHutOptimize cuts it roughly in half — still over budget, which is - * exactly the vision's escalation ladder: Barnes-Hut is already switched on - * automatically above ~500 nodes in components/graph-view.tsx, and the - * bundled Web Worker / server-cached positions stay available as later - * rungs if real graphs ever get there. v1 learner graphs are dozens of - * nodes; the stress case is preparation, not expectation. - */ - -import Graph from "graphology"; -import forceAtlas2 from "graphology-layout-forceatlas2"; - -const ITERATIONS = 100; - -/** A plausible learning graph: mostly-local chains plus long-range cross-links. */ -function synthesize(nodeCount: number, edgeCount: number): Graph { - const graph = new Graph(); - for (let i = 0; i < nodeCount; i++) { - graph.addNode(`n${i}`, { - label: `node ${i}`, - size: 6, - x: Math.random() * 100, - y: Math.random() * 100, - }); - } - let added = 0; - for (let i = 0; added < edgeCount && i < edgeCount * 3; i++) { - const from = Math.floor(Math.random() * nodeCount); - const span = Math.random() < 0.8 ? 1 + Math.floor(Math.random() * 10) : Math.floor(Math.random() * nodeCount); - const to = (from + span) % nodeCount; - if (from === to || graph.hasEdge(`n${from}`, `n${to}`)) continue; - graph.addEdge(`n${from}`, `n${to}`, { type: "arrow", size: 1 }); - added++; - } - return graph; -} - -function bench(label: string, nodeCount: number, edgeCount: number, settings: Record): void { - const graph = synthesize(nodeCount, edgeCount); - const start = performance.now(); - forceAtlas2.assign(graph, { - iterations: ITERATIONS, - settings: { ...forceAtlas2.inferSettings(graph), ...settings }, - }); - const elapsed = performance.now() - start; - console.log( - `${label}: ${elapsed.toFixed(1)} ms (${graph.order} nodes, ${graph.size} edges, ${ITERATIONS} iterations)`, - ); -} - -console.log(`Target regime — synchronous run-then-freeze (budget: <150 ms):\n`); -bench(" 100 nodes, ~200 edges", 100, 200, {}); -bench(" 300 nodes, ~600 edges", 300, 600, {}); -bench(" 500 nodes, ~1000 edges", 500, 1000, {}); -console.log(`\nStress case — 2k nodes (~4000 edges), the vision's escalation ladder:\n`); -bench(" naive repulsion (O(n²)) ", 2000, 4000, {}); -bench(" barnesHutOptimize ", 2000, 4000, { barnesHutOptimize: true }); -bench(" barnesHutOptimize (warmed JIT)", 2000, 4000, { barnesHutOptimize: true }); diff --git a/scripts/embed-graph.ts b/scripts/embed-graph.ts new file mode 100644 index 0000000..04514a0 --- /dev/null +++ b/scripts/embed-graph.ts @@ -0,0 +1,23 @@ +/** + * Embed the whole graph (backfill / cold start): one-time pass that + * fills the embedding ledger for every node — the same embedNewNodes the + * post-merge hook runs, just with an empty (or stale-model) ledger so + * every node is "new". First run downloads the model weights (~300MB) + * to the local HF cache; subsequent runs embed only what's missing. + * + * npx tsx scripts/embed-graph.ts + */ +import { graphStore } from "../src/store/graph.ts"; +import { embedNewNodes, EMBEDDING_MODEL, EMBEDDING_DIM } from "../src/server/embeddings.ts"; + +async function main() { + const t0 = Date.now(); + console.log(`[embed] model: ${EMBEDDING_MODEL} (dim ${EMBEDDING_DIM}, q8)`); + const { embedded, total } = await embedNewNodes(graphStore); + console.log(`[embed] embedded ${embedded} nodes → ledger holds ${total} vectors (${((Date.now() - t0) / 1000).toFixed(1)}s)`); +} + +main().catch((err) => { + console.error("[embed] failed:", err); + process.exit(1); +}); diff --git a/scripts/import-notes.ts b/scripts/import-notes.ts index 11cf144..f1c7acf 100644 --- a/scripts/import-notes.ts +++ b/scripts/import-notes.ts @@ -15,6 +15,7 @@ import * as fs from "node:fs"; import * as path from "node:path"; import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts"; +import { embedNewNodes } from "../src/server/embeddings.ts"; import { clusterNamingSettled } from "../src/agent/subagents/cluster-namer.ts"; import { getGraph } from "../src/core/graph.ts"; import { stores } from "../src/server/stores.ts"; @@ -28,7 +29,7 @@ async function main() { .sort(); console.log(`[import-notes] ${files.length} note files from ${dir} (concurrency ${limit})`); - const pipeline = createExtractionPipeline(stores); + const pipeline = createExtractionPipeline({ ...stores, embedder: embedNewNodes }); const queue = [...files]; const totals = { nodes: 0, edges: 0, events: 0, failed: 0 }; const workers = Array.from({ length: Math.min(limit, files.length) }, async () => { diff --git a/scripts/replay-layout.ts b/scripts/replay-layout.ts index 0de4d64..ef71041 100644 --- a/scripts/replay-layout.ts +++ b/scripts/replay-layout.ts @@ -13,7 +13,6 @@ * playground mode baked into the rest state); omit it for the free layout. */ import { graphStore } from "../src/store/graph.ts"; -import { detectCommunities } from "../src/core/clusters.ts"; // --depth=vertical|radial: bake the playground depth mode into the ledger. const depthArg = process.argv.find((a) => a.startsWith("--depth=")); const depthMode: "off" | "vertical" | "radial" = @@ -33,6 +32,12 @@ function main() { nodes.push(line); } const edges = graphStore.readEdges(); + // The semantic-topic source: the embedding ledger (run + // scripts/embed-graph.ts first if missing/stale). + const embeddings = graphStore.readEmbeddings(); + if (Object.keys(embeddings.vectors).length === 0) { + console.warn("[replay] embedding ledger is empty — clusters will be absent (run scripts/embed-graph.ts)"); + } // Batches: each imported file, in real import order (sources.json // importedAt). A node belongs to the file whose mapping first claims it @@ -79,8 +84,12 @@ function main() { const before = new Map(positions); const batchNodes = nodes.filter((n) => delivered.has(n.id)); positions = incrementalLayout(batchNodes, activeEdges, positions, { - // The same partition the real settle would see for this batch. - clusters: detectCommunities(batchNodes, activeEdges).communities, + // The same partition the real settle would see for this batch: + // the persisted semantic topics, filtered to delivered nodes + // (never recomputed — the agglomerative pass is ~1.5s at scale). + clusters: embeddings.clusters + .map((c) => ({ members: c.members.filter((id) => delivered.has(id)) })) + .filter((c) => c.members.length > 0), ...(depthMode === "off" ? {} : { diff --git a/src/agent/subagents/cluster-namer.ts b/src/agent/subagents/cluster-namer.ts index 4c21f5c..ba4ef8c 100644 --- a/src/agent/subagents/cluster-namer.ts +++ b/src/agent/subagents/cluster-namer.ts @@ -21,12 +21,7 @@ import type { PiEngine } from "../models.ts"; import { runSubagent } from "./pi-run.ts"; import type { GraphNode } from "../../core/contracts.ts"; import { getGraph, clusterContentId } from "../../core/graph.ts"; -import { - detectCommunities, - matchClusterLabelings, - MIN_CLUSTER_SIZE, - type Community, -} from "../../core/clusters.ts"; +import { matchClusterLabelings, MIN_CLUSTER_SIZE, type Community } from "../../core/clusters.ts"; import type { GraphStore } from "../../core/ports.ts"; /** How many member propositions the namer sees per cluster — enough to @@ -138,14 +133,17 @@ async function doRefresh(deps: { signal?: AbortSignal; }): Promise { const { graphStore, engine, signal } = deps; - const { nodes, edges } = getGraph(graphStore); - const detection = detectCommunities(nodes, edges); + const { nodes } = getGraph(graphStore); + // Semantic topics: the partition the embedder persisted in the + // embedding ledger (same source the snapshot renders — never + // recomputed here; the agglomerative pass is ~1.5s at scale). + const communities = graphStore.readEmbeddings().clusters; const labelings = graphStore.readClusterLabels(); - const claims = matchClusterLabelings(detection.communities, labelings); + const claims = matchClusterLabelings(communities, labelings); const byId = new Map(nodes.map((n) => [n.id, n])); const claimed = claims.filter((c) => c !== null).length; - const unnamed = detection.communities + const unnamed = communities .filter((c, i) => !claims[i] && c.members.length >= MIN_CLUSTER_SIZE) .map((c: Community, i) => { const nodesFor = c.members @@ -156,7 +154,7 @@ async function doRefresh(deps: { return { key: `c${i}`, members: c.members, nodes: nodesFor }; }); - if (unnamed.length === 0) return { detected: detection.communities.length, claimed, named: 0, appended: 0 }; + if (unnamed.length === 0) return { detected: communities.length, claimed, named: 0, appended: 0 }; const labels = await nameClusters(engine, signal, unnamed); let appended = 0; @@ -174,7 +172,7 @@ async function doRefresh(deps: { }); appended++; } - return { detected: detection.communities.length, claimed, named: unnamed.length, appended }; + return { detected: communities.length, claimed, named: unnamed.length, appended }; } let namingTail: Promise = Promise.resolve(); diff --git a/src/agent/subagents/extraction.ts b/src/agent/subagents/extraction.ts index f0e2030..67157f6 100644 --- a/src/agent/subagents/extraction.ts +++ b/src/agent/subagents/extraction.ts @@ -16,6 +16,7 @@ import type { SessionFindings, } from "../../core/contracts.ts"; import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts"; +import type { GraphStore } from "../../core/ports.ts"; import { settleGraphLayout } from "../../core/layout-settle.ts"; import { refreshClusterLabels } from "./cluster-namer.ts"; @@ -554,7 +555,14 @@ export interface ExtractionModels { labeler?: PiEngine; } -export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {} +export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels { + /** The post-merge embedding batch (semantic clusters run on it). + * INJECTED by the composition root (the agent layer never imports + * the server layer — architecture law): production passes the real + * local-model embedder (server/embeddings.ts embedNewNodes), tests + * fake it. Failure inside it is non-fatal by contract. */ + embedder: (graphStore: GraphStore) => Promise; +} export interface NotesImportInput { /** The source's file path — its identity (deterministic src id) AND its @@ -576,7 +584,7 @@ export interface ExtractionPipeline { } export function createExtractionPipeline(deps: ExtractionPipelineDeps): ExtractionPipeline { - const { chatStore, graphStore } = deps; + const { chatStore, graphStore, embedder: embed } = deps; const inFlight = new Set(); function setPhase(chatId: string, phase: ExtractionPhase, kind: "chat" | "notes", label?: string): void { @@ -715,6 +723,13 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti messages: record.messages as UIMessage[], foldedAt: record.updatedAt, }); + // Semantic embeddings (DECIDED 2026-09-13): topics come from + // MEANING, so embed the new propositions BEFORE the settle and + // the namer look at clusters. Failure is non-fatal — clustering + // runs without the new nodes until the next merge retries. + await embed(graphStore).catch((err) => { + console.error(`[extraction ${chatId}] embedding failed (non-fatal):`, err); + }); // Option A: the layout ledger evolves with the merge (sync — the // next snapshot fetch must already reflect the new nodes). settleGraphLayout(graphStore); @@ -828,6 +843,11 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti // (messages: null) — nothing to attribute, nothing to build. const auditResult: AuditResult = { ...notesResult, attribution: { quiz: {} } }; const stats = mergeFindings(id, auditResult, { chatStore, graphStore }, { messages: null, foldedAt: input.importedAt }); + // Same post-merge chain as the chat path: embeddings first + // (topics from meaning), then the settle, then the namer. + await embed(graphStore).catch((err) => { + console.error(`[extraction ${id}] embedding failed (non-fatal):`, err); + }); // Option A: same post-merge settle as the chat path. settleGraphLayout(graphStore); persistRunUsage(id, Math.round(performance.now() - t0), stats); diff --git a/src/core/clusters.ts b/src/core/clusters.ts index a762a73..7bc17b2 100644 --- a/src/core/clusters.ts +++ b/src/core/clusters.ts @@ -1,31 +1,19 @@ /** - * COMMUNITY DETECTION for the knowledge graph (vision §1, DECIDED - * 2026-09-13): Louvain modularity over the undirected, unweighted - * follows+supports structure — purely structural, so it never depends on - * layout, and the extraction pipeline never knows clusters exist. + * CLUSTER LABEL MATCHING (vision §1). DETECTION moved to semantic topic + * clustering over embeddings (core/semantic-clusters.ts, DECIDED + * 2026-09-13) — Louvain over the edge structure is retired: it only ever + * found the reasoning chains (the graph is disconnected by nature), and + * edge-orphans could never join anything. What REMAINS here is the + * cluster contract + core-survival label matching, which is + * detection-agnostic: it consumes the same { members } shape semantic + * clustering emits. * - * Determinism: Louvain's random-walk traversal is driven by a SEEDED - * mulberry32 RNG (default seed constant below), so the same graph always - * yields the same partition. Community INDICES are still not stable - * across graph edits — which is why labelings (Phase B) key to sorted - * MEMBER SETS, never to indices. + * Labelings key to sorted MEMBER SETS, never to detection indices — + * a labeling record outlives any reshuffle of the partition. * - * Undirected + unweighted (user-decided 2026-09-13): both edge types are - * equally real relations between propositions — direction is - * derivational-vs-grounding, not stronger-vs-weaker — and a follows AND - * a supports between the same pair collapse to one edge. - * - * This module is browser-safe pure logic (no node builtins, no fs): the - * server runs it for the snapshot payload, the renderer for nothing - * (clusters arrive precomputed), and tests run it directly. - * - * NOTE: no minimum-size filter here — detection returns every community, - * singletons included. The ≥3-node render threshold (decided) is a - * PRESENTATION rule; it lives in the payload derivation and the - * renderer, not in the algorithm. + * Browser-safe pure logic (no node builtins, no fs): the server runs + * matching for the snapshot payload, tests run it directly. */ -import Graph from "graphology"; -import louvain from "graphology-communities-louvain"; import type { ClusterLabeling } from "./contracts.ts"; /** A detected community. Members are SORTED node ids — the member set IS @@ -39,90 +27,6 @@ export interface Community { * least three propositions to be worth a hull and a label. */ export const MIN_CLUSTER_SIZE = 3; -export interface CommunityDetection { - communities: Community[]; - /** Modularity of the found partition — the one quality number. */ - modularity: number; -} - -export interface DetectOptions { - /** Louvain resolution (higher → more, smaller communities). Default 1. - * Parked for the multi-scale work; exposed now so that dial has a home. */ - resolution?: number; - /** RNG seed. Default "lh-clusters". Same seed + same graph → same - * partition, always. */ - seed?: string; -} - -/** mulberry32 seeded by a string (FNV-1a) — the same recipe as - * question-core's seededRandom, inlined so this module has no - * cross-domain dependency. */ -function seededRng(seed: string): () => number { - let h = 2166136261; - for (let i = 0; i < seed.length; i++) { - h ^= seed.charCodeAt(i); - h = Math.imul(h, 16777619); - } - let a = h >>> 0; - return () => { - a |= 0; - a = (a + 0x6d2b79f5) | 0; - let t = Math.imul(a ^ (a >>> 15), 1 | a); - t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; - return ((t ^ (t >>> 14)) >>> 0) / 4294967296; - }; -} - -/** - * Detect communities over a graph's structure. Inputs are the minimal - * structural shapes of the snapshot contracts (GraphNode.id, - * GraphEdge.from/to), so both the real snapshot and test fixtures feed it. - * - * Edges referencing unknown node ids are skipped defensively (the ledgers - * are append-only; a mid-merge read should never crash detection), and - * duplicate pairs (follows+supports, or reverse direction) collapse — - * mergeEdge keeps the first. - */ -export function detectCommunities( - nodes: readonly { id: string }[], - edges: readonly { from: string; to: string }[], - opts?: DetectOptions, -): CommunityDetection { - const graph = new Graph({ type: "undirected" }); - for (const node of nodes) graph.addNode(node.id); - for (const edge of edges) { - if (!graph.hasNode(edge.from) || !graph.hasNode(edge.to)) continue; - if (edge.from === edge.to) continue; // self-loops: no community signal - graph.mergeEdge(edge.from, edge.to); - } - // Louvain on a 0-edge graph is degenerate; every node is trivially its - // own community (and louvain on an empty graph returns nothing useful). - if (graph.size === 0) { - return { - communities: nodes.map((n) => ({ members: [n.id] })).sort((a, b) => (a.members[0] ?? "").localeCompare(b.members[0] ?? "")), - modularity: 0, - }; - } - const details = louvain.detailed(graph, { - rng: seededRng(opts?.seed ?? "lh-clusters"), - resolution: opts?.resolution ?? 1, - }); - const byIndex = new Map(); - for (const [id, index] of Object.entries(details.communities)) { - const bucket = byIndex.get(index) ?? []; - bucket.push(id); - byIndex.set(index, bucket); - } - const communities = [...byIndex.values()] - .map((ids) => ({ members: ids.sort() })) - .sort( - (a, b) => - b.members.length - a.members.length || - (a.members[0] ?? "").localeCompare(b.members[0] ?? ""), - ); - return { communities, modularity: details.modularity }; -} - /** * Claim labeling records for detected communities — CORE-SURVIVAL (DECIDED * 2026-09-13): a community keeps a record's label while AT LEAST HALF of diff --git a/src/core/contracts/graph.ts b/src/core/contracts/graph.ts index cb56644..3a69258 100644 --- a/src/core/contracts/graph.ts +++ b/src/core/contracts/graph.ts @@ -278,6 +278,26 @@ export interface AuditResult { * least half its core survives in some single community (core-survival; * under half → the community is renamed, splits can keep only one * successor). */ +/** The embedding ledger shape (config.EMBEDDINGS_FILE): vectors for + * EVERY embedded node, all produced by one model at one dimensionality. */ +export interface EmbeddingsLedger { + /** The embedding model id (e.g. the HF repo) — mismatch = stale + * vectors, the writer re-embeds everything. */ + model: string; + /** Vector dimensionality (MRL-truncated on write). */ + dim: number; + /** nodeId → L2-normalized vector (cosine similarity = dot product). */ + vectors: Record; + /** The topic partition over those vectors, computed by the SAME + * embedder run (the user-decided trigger: recluster at the end of + * every embedding batch, never per read — the agglomerative pass is + * ~1.5s at current scale). Consumers (snapshot, settle, namer) read + * this instead of recomputing. Empty when nothing is embedded. */ + clusters: Array<{ members: string[] }>; + /** ISO timestamp of the last write. */ + updatedAt: string; +} + export interface ClusterLabeling { /** "clu_" + content hash of the sorted core. */ id: string; diff --git a/src/core/graph.ts b/src/core/graph.ts index c1aa7a5..1c4863c 100644 --- a/src/core/graph.ts +++ b/src/core/graph.ts @@ -51,7 +51,7 @@ import type { LedgerEvent, LedgerEventKind, } from "./contracts.ts"; -import { detectCommunities, matchClusterLabelings, MIN_CLUSTER_SIZE } from "./clusters.ts"; +import { matchClusterLabelings, MIN_CLUSTER_SIZE } from "./clusters.ts"; import type { QuestionAnswerData } from "./question-core.ts"; import type { ChatStore, GraphStore } from "./ports.ts"; @@ -497,14 +497,18 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot { (nodeObservations[nodeId] ??= {})[chatId] = note; } - // Structural communities (vision §1): Louvain over the undirected - // follows+supports structure, joined with persisted labelings — - // core-survival matching, size-filtered at the presentation threshold. - // Detection is DETERMINISTIC (seeded), so no memo in v1: a snapshot - // read recomputes it (ms at current scale; revisit at vault scale). - const detection = detectCommunities(nodes, edges); - const claims = matchClusterLabelings(detection.communities, deps.graphStore.readClusterLabels()); - const clusters: Cluster[] = detection.communities + // Semantic topic communities (DECIDED 2026-09-13, replacing structural + // Louvain): average-linkage clustering over embeddings — related + // topics, not reasoning chains. Edge-orphans join topics by meaning; + // stragglers stay loose (filtered at the presentation threshold + // below). The partition is computed ONCE per embedding batch by the + // embedder (the agglomerative pass is ~1.5s at current scale) and + // PERSISTED in the embedding ledger — reads just load it. Nodes whose + // embedding hasn't landed yet simply sit unclustered until the next + // read after the embedder runs. + const communities = deps.graphStore.readEmbeddings().clusters; + const claims = matchClusterLabelings(communities, deps.graphStore.readClusterLabels()); + const clusters: Cluster[] = communities .map((community, index) => ({ community, claim: claims[index] })) .filter(({ community }) => community.members.length >= MIN_CLUSTER_SIZE) .map(({ community, claim }) => ({ diff --git a/src/core/index.ts b/src/core/index.ts index 70c2ebb..5e1a4bf 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -22,7 +22,7 @@ export * from "./render-settings.ts"; export * from "./question-core.ts"; // clusters.ts (Louvain) is deliberately NOT re-exported: nothing in the // shell runs community detection — the snapshot carries the partition -// from the server. Keeping it out keeps graphology-louvain out of the +// from the server. Keeping it out keeps the heavy clustering deps out of the // client bundle (verified 2026-09-13: chunks contained it while this // line existed). Same logic as layout-settle.ts below. export * from "./layout.ts"; diff --git a/src/core/layout-settle.ts b/src/core/layout-settle.ts index eb23aa7..8a405d6 100644 --- a/src/core/layout-settle.ts +++ b/src/core/layout-settle.ts @@ -1,5 +1,4 @@ -import type { GraphEdge, GraphNode } from "./contracts.ts"; -import { detectCommunities } from "./clusters.ts"; +import type { EmbeddingsLedger, GraphEdge, GraphNode } from "./contracts.ts"; import { incrementalLayout } from "./layout.ts"; import type { Position } from "./contracts.ts"; @@ -13,7 +12,7 @@ import type { Position } from "./contracts.ts"; * * SPLIT FROM layout.ts deliberately (2026-09-13): layout.ts is in the * browser barrel, and the settle needs the Louvain partition for the - * cluster centroid pull — graphology-louvain must not ship in the client + * cluster centroid pull — the heavy deps must not ship in the client * bundle. Server code imports this module DIRECTLY; it is never * re-exported through src/core/index.ts. */ @@ -22,6 +21,7 @@ export function settleGraphLayout(graphStore: { readEdges(): GraphEdge[]; readPositions(): Record; writePositions(positions: Record): void; + readEmbeddings(): EmbeddingsLedger; }): void { const nodes = graphStore.readNodes(); if (nodes.length === 0) return; @@ -34,10 +34,11 @@ export function settleGraphLayout(graphStore: { }); const edges = graphStore.readEdges(); const previous = new Map(Object.entries(graphStore.readPositions())); - // The cluster partition for the centroid pull: same Louvain the - // snapshot uses (same nodes/edges = same communities). 338 nodes: - // ~1-2ms, noise next to the settle itself. - const { communities } = detectCommunities(unique, edges); + // The cluster partition for the centroid pull: the SAME persisted + // semantic topics the snapshot renders (computed by the embedder + // once per batch, stored in the embedding ledger) — one cluster + // concept everywhere. + const communities = graphStore.readEmbeddings().clusters; const next = incrementalLayout(unique, edges, previous, { clusters: communities }); graphStore.writePositions(Object.fromEntries(next)); } \ No newline at end of file diff --git a/src/core/layout.ts b/src/core/layout.ts index 26fad48..a555240 100644 --- a/src/core/layout.ts +++ b/src/core/layout.ts @@ -386,7 +386,7 @@ export interface IncrementalLayoutOptions { gravityStrength?: number; /** Cluster partition (Louvain output shape). Supplied by callers so * this module stays louvain-free (the browser bundle must not carry - * graphology-louvain); absent = no cluster pull. */ + * embedding ledger; absent = no cluster pull. */ clusters?: ReadonlyArray<{ members: ReadonlyArray }>; /** Cluster centroid pull strength (default CLUSTER_PULL_STRENGTH). */ clusterPullStrength?: number; diff --git a/src/core/ports.ts b/src/core/ports.ts index 307280c..9f9d893 100644 --- a/src/core/ports.ts +++ b/src/core/ports.ts @@ -27,6 +27,7 @@ import type { TurnUsage, NotesSource, ClusterLabeling, + EmbeddingsLedger, } from "./contracts.ts"; // ─── ChatStore — the session record's home (data/chats) ───────────────────── @@ -104,6 +105,13 @@ export interface GraphStore { readPositions(): Record; /** Replace the whole ledger (the settle is a full-graph computation). */ writePositions(positions: Record): void; + + /** The embedding ledger (config.EMBEDDINGS_FILE). WRITTEN ONLY by + * the embedder (server/embeddings.ts) after a merge — read-only for + * clustering consumers (snapshot, settle, namer). */ + readEmbeddings(): EmbeddingsLedger; + /** Replace the whole ledger (embedding is a full-batch computation). */ + writeEmbeddings(ledger: EmbeddingsLedger): void; } // ─── JobStore — background sub-agent jobs (data/jobs) ───────────────────────── diff --git a/src/core/semantic-clusters.ts b/src/core/semantic-clusters.ts new file mode 100644 index 0000000..7772c60 --- /dev/null +++ b/src/core/semantic-clusters.ts @@ -0,0 +1,141 @@ +// Semantic topic clustering — the REPLACEMENT for Louvain community +// detection (DECIDED 2026-09-13): structural clusters were just the +// reasoning chains (the graph is disconnected by nature — every connected +// component became its own "cluster"), and edge-orphans could never join +// anything. Clusters are now RELATED TOPICS: nodes whose proposition +// MEANING is close, computed over the embedding ledger +// (config.EMBEDDINGS_FILE). This pulls orphans into topics by meaning +// and groups small chains with the large chains they discuss. +// +// Algorithm: threshold AGGLOMERATIVE (average linkage over cosine +// similarity). No JS HDBSCAN is trustworthy (the npm package is an +// abandoned alpha), and at proposition scale (~100s-1000s of short texts) +// brute-force pairwise cosine is sub-second — so the honest ~60-line +// hand-roll wins. "Stragglers" (nodes that fit nothing) fall out +// naturally: they simply never merge, and stay singletons. + +/** The default similarity threshold, SWEPT ON THE REAL GRAPH + * (2026-09-13): 0.60 → 26 topics of ~20 propositions, 74 of the 90 + * edge-orphans land in real topics, stragglers 10. Higher (0.65-0.68) + * gives tighter micro-topics with far more stragglers; lower (0.55) + * collapses into ~13 mega-topics. */ +export const SEMANTIC_SIM_THRESHOLD = 0.6; + +/** Cosine similarity of two vectors (defensively normalizes inputs; + * ledger vectors are already L2-normalized, so this is a dot product). */ +export function cosineSimilarity(a: readonly number[], b: readonly number[]): number { + let dot = 0; + let ma = 0; + let mb = 0; + for (let i = 0; i < a.length; i++) { + dot += a[i]! * b[i]!; + ma += a[i]! * a[i]!; + mb += b[i]! * b[i]!; + } + if (ma === 0 || mb === 0) return 0; + return dot / (Math.sqrt(ma) * Math.sqrt(mb)); +} + +export interface SemanticClusterOptions { + /** Merge while the best cluster pair's average-linkage similarity is + * >= this (default SEMANTIC_SIM_THRESHOLD). */ + similarityThreshold?: number; + /** After the cut, force every singleton into its nearest topic + * ("all-in" mode) instead of leaving stragglers loose. Default + * false: some propositions genuinely fit nothing. */ + attachStragglers?: boolean; +} + +/** Average-linkage agglomerative clustering over embedding vectors. + * Pure and deterministic (stable Map iteration, greedy by best + * similarity, first-found wins ties). Returns clusters size-desc — the + * same { members } shape Louvain produced, so label matching and the + * renderer contract are unchanged. An empty/missing ledger returns [] + * (nothing embedded yet: no clusters, every node loose — the snapshot + * degrades gracefully until the first embed). */ +export function semanticClusters( + vectors: Readonly>, + options: SemanticClusterOptions = {}, +): Array<{ members: string[] }> { + const ids = Object.keys(vectors); + if (ids.length === 0) return []; + const simThreshold = options.similarityThreshold ?? SEMANTIC_SIM_THRESHOLD; + const attach = options.attachStragglers ?? false; + + // Sum vectors per cluster: average-linkage similarity between clusters + // A and B is dot(sumA, sumB) / (|A|·|B|) — the exact mean of pairwise + // dots — so the merge loop never revisits the raw O(n²) matrix. + const sums = new Map(); + const members = new Map(); + ids.forEach((id, i) => { + sums.set(i, [...vectors[id]!]); + members.set(i, [id]); + }); + + // Greedy merge: repeatedly join the pair with the highest + // average-linkage similarity until it drops below the threshold. + // O(k²) pair scan per merge — trivial at proposition scale. + for (;;) { + let bestA = -1; + let bestB = -1; + let bestSim = -Infinity; + for (const [a, sumA] of sums) { + for (const [b, sumB] of sums) { + if (a >= b) continue; + let dot = 0; + for (let i = 0; i < sumA.length; i++) dot += sumA[i]! * sumB[i]!; + const sim = dot / (members.get(a)!.length * members.get(b)!.length); + if (sim > bestSim) { + bestSim = sim; + bestA = a; + bestB = b; + } + } + } + if (bestA < 0 || bestSim < simThreshold) break; + // Merge B into A (deterministic direction: lower cluster index). + const sumA = sums.get(bestA)!; + const sumB = sums.get(bestB)!; + for (let i = 0; i < sumA.length; i++) sumA[i] = sumA[i]! + sumB[i]!; + members.get(bestA)!.push(...members.get(bestB)!); + sums.delete(bestB); + members.delete(bestB); + } + + // Optional all-in pass: singletons join their nearest topic regardless + // of threshold — exactly the "forced membership" mode. The difference + // from stragglers mode is this one pass. Singleton-vs-cluster + // average-linkage similarity is dot(sumS, sumK) / |K|. + if (attach) { + const singletons = [...members.entries()].filter(([, m]) => m.length === 1).map(([k]) => k); + for (const s of singletons) { + if (!members.has(s)) continue; // absorbed by an earlier attach + let bestTarget = -1; + let bestSim = -Infinity; + for (const [k, m] of members) { + if (k === s || m.length < 2) continue; + let dot = 0; + const sumS = sums.get(s)!; + const sumK = sums.get(k)!; + for (let i = 0; i < sumS.length; i++) dot += sumS[i]! * sumK[i]!; + const sim = dot / m.length; + if (sim > bestSim) { + bestSim = sim; + bestTarget = k; + } + } + if (bestTarget >= 0) { + const sumT = sums.get(bestTarget)!; + const sumS = sums.get(s)!; + for (let i = 0; i < sumT.length; i++) sumT[i] = sumT[i]! + sumS[i]!; + members.get(bestTarget)!.push(...members.get(s)!); + sums.delete(s); + members.delete(s); + } + } + } + + return [...members.values()] + .map((m) => ({ members: [...m].sort() })) + .sort((a, b) => b.members.length - a.members.length || (a.members[0] ?? "").localeCompare(b.members[0] ?? "")); +} diff --git a/src/server/chat-handler.ts b/src/server/chat-handler.ts index 589516f..a160b9f 100644 --- a/src/server/chat-handler.ts +++ b/src/server/chat-handler.ts @@ -4,6 +4,7 @@ import { createProberAgent, createTeacherAgent, type TeachingAgent } from "../ag import { piEngineFor } from "../agent/models.ts"; import { createJobRunner } from "../agent/subagents/runner.ts"; import { createExtractionPipeline } from "../agent/subagents/extraction.ts"; +import { embedNewNodes } from "./embeddings.ts"; import type { ModelRole } from "../../config.ts"; import { claimSteers, deliverJobResult, setJobRespawn, setTurnRunner } from "./jobs.ts"; import { recordToPiMessages } from "./pi/record-adapter.ts"; @@ -40,7 +41,12 @@ const jobRunner = createJobRunner({ /** The app's ONE extraction pipeline instance — both front doors share it: * quiz-triggered session runs (turn-service) and notes imports * (/api/graph/import) hit the same single-flight registry. */ -export const extraction = createExtractionPipeline(stores); +export const extraction = createExtractionPipeline({ + ...stores, + // The REAL embedder (local transformer model): post-merge embedding + // batch — semantic clusters are computed and persisted here. + embedder: embedNewNodes, +}); // Wake-pipeline wiring (contracts.ts "THE ASYNC PROTOCOL" §4): the pipeline // fires its server-side teacher turns through THIS composition root, and diff --git a/src/server/embeddings.ts b/src/server/embeddings.ts new file mode 100644 index 0000000..0d9194f --- /dev/null +++ b/src/server/embeddings.ts @@ -0,0 +1,125 @@ +/** + * THE EMBEDDER (DECIDED 2026-09-13): semantic topic clustering needs + * vectors, and vectors need a model running ON the user's machine — no + * external services, no API bills, no network dependency for re-embedding. + * + * Runtime: in-process transformers.js (@huggingface/transformers v4 over + * onnxruntime-node — native Apple Silicon acceleration). The model + * weights download ONCE to the local HF cache on first run (~300MB). + * + * Model: EmbeddingGemma-300M (q8) — best quality-per-megabyte small + * embedding model as of late 2025 (MTEB-en 68.36), built for on-device + * use, with MRL truncation so we store 512 of its 768 dims. Every input + * uses the model card's document format ("title: none | text: …"), + * consistently — mixing prefixed and unprefixed inputs degrades quality. + * + * Model swaps re-embed EVERYTHING: vectors from different models live in + * incompatible spaces. The ledger records model + dim; a mismatch marks + * the whole ledger stale and this module re-embeds from scratch (minutes + * at current scale). + */ +import { pipeline, type FeatureExtractionPipeline } from "@huggingface/transformers"; +import type { EmbeddingsLedger, GraphNode } from "../core/contracts.ts"; +import { semanticClusters, SEMANTIC_SIM_THRESHOLD } from "../core/semantic-clusters.ts"; + +export const EMBEDDING_MODEL = "onnx-community/embeddinggemma-300m-ONNX"; +/** MRL truncation of the native 768 — negligible quality loss, half the + * ledger size (2026-09-13 researcher sweep). */ +export const EMBEDDING_DIM = 512; +/** Batch size bounds peak memory; propositions are tiny so 32 per call + * keeps the pipeline warm without thrashing. */ +const EMBED_BATCH = 32; + +let pipelinePromise: Promise | null = null; + +function getPipeline(): Promise { + pipelinePromise ??= pipeline("feature-extraction", EMBEDDING_MODEL, { dtype: "q8" }); + return pipelinePromise; +} + +/** EmbeddingGemma's required document format (model card). */ +function embedInput(statement: string): string { + return `title: none | text: ${statement}`; +} + +/** Truncate an L2-normalized vector to EMBEDDING_DIM (MRL: leading dims + * carry most of the signal) and re-normalize so cosine = dot product. */ +function truncateNormalize(vec: readonly number[]): number[] { + const out = vec.slice(0, EMBEDDING_DIM); + let mag = 0; + for (const x of out) mag += x * x; + mag = Math.sqrt(mag); + return mag > 0 ? out.map((x) => x / mag) : out; +} + +/** + * Embed every node the ledger is missing — the post-merge hook (new + * propositions only, typically a handful) AND the cold-start backfill + * (empty ledger = every node) in one function. Model/dim mismatch voids + * the ledger first (full re-embed). Also prunes vectors for nodes that + * no longer exist. Writes the whole ledger and returns counts for logs. + * Non-fatal by contract: the caller treats failure as "clustering runs + * without the new nodes until the next merge retries". + */ +export async function embedNewNodes( + graphStore: { + readNodes(): GraphNode[]; + readEmbeddings(): EmbeddingsLedger; + writeEmbeddings(ledger: EmbeddingsLedger): void; + }, +): Promise<{ embedded: number; total: number }> { + const ledger = graphStore.readEmbeddings(); + const stale = ledger.model !== EMBEDDING_MODEL || ledger.dim !== EMBEDDING_DIM; + const vectors: Record = stale ? {} : { ...ledger.vectors }; + + // Node LINES may repeat an id — dedupe, first defines (getGraph rule). + const seen = new Set(); + const todo: GraphNode[] = []; + for (const node of graphStore.readNodes()) { + if (seen.has(node.id)) continue; + seen.add(node.id); + if (!vectors[node.id]) todo.push(node); + } + + if (todo.length > 0) { + const extractor = await getPipeline(); + for (let i = 0; i < todo.length; i += EMBED_BATCH) { + const batch = todo.slice(i, i + EMBED_BATCH); + const output = await extractor(batch.map((n) => embedInput(n.statement)), { + pooling: "mean", + normalize: true, + }); + const rows = output.tolist() as number[][]; + for (let j = 0; j < batch.length; j++) { + vectors[batch[j]!.id] = truncateNormalize(rows[j]!); + } + } + } + + // Prune ghosts: vectors for node ids that no longer exist. + let pruned = 0; + for (const id of Object.keys(vectors)) { + if (!seen.has(id)) { + delete vectors[id]; + pruned++; + } + } + + // RECLUSTER AT THE END OF THE BATCH (user-decided 2026-09-13): the + // agglomerative pass is ~1.5s at current scale, so it runs HERE — + // once per embedding batch — and consumers (snapshot, settle, namer) + // read the persisted partition instead of recomputing per read. + const clusters = semanticClusters(vectors, { similarityThreshold: SEMANTIC_SIM_THRESHOLD }); + + const wrote = todo.length > 0 || pruned > 0 || stale || ledger.clusters.length !== clusters.length; + if (wrote) { + graphStore.writeEmbeddings({ + model: EMBEDDING_MODEL, + dim: EMBEDDING_DIM, + vectors, + clusters, + updatedAt: new Date().toISOString(), + }); + } + return { embedded: todo.length, total: Object.keys(vectors).length }; +} diff --git a/src/store/graph.ts b/src/store/graph.ts index d7a9545..8c72007 100644 --- a/src/store/graph.ts +++ b/src/store/graph.ts @@ -29,9 +29,10 @@ import * as fs from "node:fs"; import * as path from "node:path"; -import { LEDGER_FILES, POSITIONS_FILE, SOURCES_FILE } from "../../config.ts"; +import { EMBEDDINGS_FILE, LEDGER_FILES, POSITIONS_FILE, SOURCES_FILE } from "../../config.ts"; import type { ClusterLabeling, + EmbeddingsLedger, GraphEdge, GraphNode, ExtractionRunUsage, @@ -216,6 +217,12 @@ export const graphStore: GraphStore = { writePositions(positions: Record): void { writePositionMap(positions); }, + readEmbeddings(): EmbeddingsLedger { + return readEmbeddingsLedger(); + }, + writeEmbeddings(ledger: EmbeddingsLedger): void { + writeEmbeddingsLedger(ledger); + }, }; /** positions.json — { positions, updatedAt }. Read-modify-write, NOT @@ -252,6 +259,57 @@ function writePositionMap(positions: Record): void { ); } +/** embeddings.json — the whole EmbeddingsLedger at once. Missing/corrupt + * file reads as an EMPTY ledger (model "") so consumers see "nothing + * embedded yet" rather than an error; the embedder backfills. */ +function readEmbeddingsLedger(): EmbeddingsLedger { + let raw: string; + try { + raw = fs.readFileSync(EMBEDDINGS_FILE, "utf-8"); + } catch { + return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }; + } + try { + const parsed: unknown = JSON.parse(raw); + if (!isRecord(parsed)) return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }; + const model = typeof parsed.model === "string" ? parsed.model : ""; + const dim = typeof parsed.dim === "number" && Number.isFinite(parsed.dim) ? parsed.dim : 0; + const vectors: Record = {}; + if (isRecord(parsed.vectors)) { + for (const [id, value] of Object.entries(parsed.vectors)) { + if ( + Array.isArray(value) && + value.length > 0 && + value.every((x) => typeof x === "number" && Number.isFinite(x)) + ) { + vectors[id] = value as number[]; + } + } + } + const clusters: Array<{ members: string[] }> = []; + if (Array.isArray(parsed.clusters)) { + for (const c of parsed.clusters) { + if ( + isRecord(c) && + Array.isArray(c.members) && + c.members.length > 0 && + c.members.every((m) => typeof m === "string") + ) { + clusters.push({ members: [...(c.members as string[])] }); + } + } + } + return { model, dim, vectors, clusters, updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : "" }; + } catch { + return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }; + } +} + +function writeEmbeddingsLedger(ledger: EmbeddingsLedger): void { + fs.mkdirSync(path.dirname(EMBEDDINGS_FILE), { recursive: true }); + fs.writeFileSync(EMBEDDINGS_FILE, JSON.stringify(ledger, null, "\t"), "utf-8"); +} + const isPosition = (v: unknown): v is Position => isRecord(v) && typeof v.x === "number" && Number.isFinite(v.x) && typeof v.y === "number" && Number.isFinite(v.y); diff --git a/tests/clusters.test.ts b/tests/clusters.test.ts index 7436141..b7a4ffc 100644 --- a/tests/clusters.test.ts +++ b/tests/clusters.test.ts @@ -6,9 +6,8 @@ */ import { strict as assert } from "node:assert"; import { describe, test } from "node:test"; -import { detectCommunities, matchClusterLabelings, MIN_CLUSTER_SIZE } from "../src/core/clusters.ts"; +import { matchClusterLabelings, MIN_CLUSTER_SIZE, type Community } from "../src/core/clusters.ts"; import { clusterHue } from "../components/cluster-geometry.ts"; -import type { Community } from "../src/core/clusters.ts"; import type { ClusterLabeling } from "../src/core/contracts.ts"; const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${prefix}${i}`); @@ -84,89 +83,3 @@ describe("clusterHue", () => { }); }); -describe("detectCommunities", () => { - test("two dense cliques joined by a single bridge split into two communities", () => { - const left = ids(4, "L"); - const right = ids(4, "R"); - const bridge = { from: "L0", to: "R0" }; - const result = detectCommunities([...left, ...right].map((id) => ({ id })), [...clique(left), ...clique(right), bridge]); - assert.equal(result.communities.length, 2); - assert.equal(result.modularity > 0.3, true); - // Member sets, not order: the two big communities are exactly the cliques. - const big = result.communities.filter((c) => c.members.length === 4); - assert.deepEqual(big.map((c) => c.members.sort()).sort(), [left.slice().sort(), right.slice().sort()]); - }); - - test("deterministic: same graph, same seed → identical partition", () => { - const nodes = ids(10).map((id) => ({ id })); - const edges = [...clique(ids(5)), ...clique(ids(5, "n").slice().reverse().slice(0, 5)), { from: "n0", to: "n9" }]; - const a = detectCommunities(nodes, edges); - const b = detectCommunities(nodes, edges); - assert.deepEqual(a.communities, b.communities); - assert.equal(a.modularity, b.modularity); - }); - - test("no edges: every node is its own singleton community", () => { - const result = detectCommunities(ids(3).map((id) => ({ id })), []); - assert.equal(result.communities.length, 3); - for (const c of result.communities) assert.equal(c.members.length, 1); - assert.equal(result.modularity, 0); - }); - - test("empty graph: zero communities", () => { - assert.deepEqual(detectCommunities([], []), { communities: [], modularity: 0 }); - }); - - test("duplicate and reversed pairs collapse to one edge (undirected, unweighted)", () => { - const nodes = ["a", "b", "c"].map((id) => ({ id })); - // follows a→b AND supports b→a — same pair twice, both directions. - const result = detectCommunities(nodes, [ - { from: "a", to: "b" }, - { from: "b", to: "a" }, - { from: "b", to: "c" }, - ]); - assert.equal(result.communities.length, 1); - assert.deepEqual(result.communities[0]!.members.sort(), ["a", "b", "c"]); - }); - - test("self-loops and unknown-endpoint edges are skipped defensively", () => { - const nodes = ["a", "b"].map((id) => ({ id })); - const result = detectCommunities(nodes, [ - { from: "a", to: "a" }, // self-loop - { from: "a", to: "ghost" }, // dangling - { from: "ghost1", to: "ghost2" }, - { from: "a", to: "b" }, // the one real edge - ]); - assert.equal(result.communities.length, 1); - assert.deepEqual(result.communities[0]!.members.sort(), ["a", "b"]); - }); - - test("members are sorted; communities ordered by size then id", () => { - const left = ids(4, "L"); - const right = ids(4, "R"); - const result = detectCommunities( - [...left, ...right, "iso"].map((id) => ({ id })), - [...clique(left), ...clique(right), { from: "L0", to: "R0" }], - ); - for (const c of result.communities as Community[]) { - const sorted = c.members.slice().sort(); - assert.deepEqual(c.members, sorted); - } - // iso (isolated) is last: size 1 < 4. - const last = result.communities[result.communities.length - 1]!; - assert.deepEqual(last.members, ["iso"]); - }); - - test("seed override changes nothing structurally but stays deterministic", () => { - const nodes = ids(8).map((id) => ({ id })); - const edges = [...clique(["n0", "n1", "n2", "n3"]), ...clique(["n4", "n5", "n6", "n7"]), { from: "n0", to: "n4" }]; - const a = detectCommunities(nodes, edges, { seed: "one" }); - const b = detectCommunities(nodes, edges, { seed: "one" }); - const c = detectCommunities(nodes, edges, { seed: "two" }); - assert.deepEqual(a.communities, b.communities); - // A different seed partitions the same graph into member sets that - // may or may not coincide — but both runs are 2 communities here, - // because the structure dominates. - assert.equal(c.communities.length, 2); - }); -}); diff --git a/tests/extraction.test.ts b/tests/extraction.test.ts index 02ec278..24e90ca 100644 --- a/tests/extraction.test.ts +++ b/tests/extraction.test.ts @@ -40,6 +40,7 @@ import type { import type { GraphStore } from "../src/core/ports.ts"; import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts"; import { clusterNamingSettled, refreshClusterLabels } from "../src/agent/subagents/cluster-namer.ts"; +import { semanticClusters } from "../src/core/semantic-clusters.ts"; import { chatStore } from "../src/store/chats.ts"; import { PATHS } from "../config.ts"; @@ -67,6 +68,13 @@ function memoryLedgers() { const sources = new Map(); const clusterLabels: ClusterLabeling[] = []; const positions = new Map(); + const embeddings = { + model: "", + dim: 0, + updatedAt: "", + vectors: new Map(), + clusters: new Array>(), + }; return { appendNode: (n: GraphNode) => void nodes.push(n), readNodes: () => nodes.map((n) => ({ ...n })), @@ -102,6 +110,16 @@ function memoryLedgers() { positions.clear(); for (const [id, p] of Object.entries(next)) positions.set(id, { x: p.x, y: p.y }); }, + // The embedding ledger — the embedder writes it after a merge. + readEmbeddings: () => ({ model: embeddings.model, dim: embeddings.dim, vectors: Object.fromEntries([...embeddings.vectors].map(([id, v]) => [id, [...v]])), clusters: [...embeddings.clusters.map((c) => ({ members: [...c] }))], updatedAt: embeddings.updatedAt }), + writeEmbeddings: (next) => { + embeddings.model = next.model; + embeddings.dim = next.dim; + embeddings.updatedAt = next.updatedAt; + embeddings.vectors.clear(); + for (const [id, v] of Object.entries(next.vectors)) embeddings.vectors.set(id, [...v]); + embeddings.clusters = next.clusters.map((c) => [...c.members]); + }, } satisfies GraphStore; } @@ -319,6 +337,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: submitModel("submit_findings", FINDINGS), auditor: submitModel("submit_review", verdict), labeler: submitModel("submit_labels", { labels: [] }), @@ -360,6 +379,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: submitModel("submit_findings", FINDINGS), auditor: silentModel(), // never submits the review labeler: submitModel("submit_labels", { labels: [] }), @@ -371,6 +391,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const failingExtractor = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: silentModel(), auditor: submitModel("submit_review", { ...FINDINGS, dropped: [] }), labeler: submitModel("submit_labels", { labels: [] }), @@ -402,6 +423,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, // Each stage's first generate is the observation point: the // registry must already carry this chat at the right phase. extractor: submitModel("submit_findings", FINDINGS, () => seen.push(phaseOf())), @@ -435,6 +457,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, // During the auditor's generate, the registry must already carry // the extractor's fully accumulated live usage (2 steps × 10/5). extractor: submitModel("submit_findings", FINDINGS), @@ -476,6 +499,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: hangingModel(), // hangs until aborted — no step finishes auditor: submitModel("submit_review", { propositions: [], edges: [], attribution: { quiz: {} }, dropped: [] }), labeler: submitModel("submit_labels", { labels: [] }), @@ -527,6 +551,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: gatedExtractor, auditor: silentModel(), labeler: submitModel("submit_labels", { labels: [] }), @@ -545,6 +570,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: silentModel(), auditor: silentModel(), labeler: submitModel("submit_labels", { labels: [] }), @@ -603,6 +629,27 @@ const CLUSTER_VERDICT = { dropped: [], } as const; +/** Test seam for the embedder: fakes the post-merge embedding batch — + * one identical unit vector per node → semanticClusters merges them all + * into ONE topic — then persists the partition like the real embedder. */ +const fakeEmbedder = async (store: GraphStore) => { + const seen = new Set(); + const ids: string[] = []; + for (const n of store.readNodes()) { + if (seen.has(n.id)) continue; + seen.add(n.id); + ids.push(n.id); + } + const vectors = Object.fromEntries(ids.map((id) => [id, [1, 0, 0]])); + store.writeEmbeddings({ + model: "test/embedder", + dim: 3, + vectors, + clusters: semanticClusters(vectors), + updatedAt: "2026-09-13T00:00:00.000Z", + }); +}; + describe("cluster naming (the post-extraction garnish)", () => { it("names a new community after a notes merge — one record, claimed on the next refresh", async () => { const ledgers = memoryLedgers(); @@ -611,6 +658,7 @@ describe("cluster naming (the post-extraction garnish)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: submitModel("submit_findings", CLUSTER_FINDINGS), auditor: submitModel("submit_review", CLUSTER_VERDICT), labeler, @@ -638,6 +686,7 @@ describe("cluster naming (the post-extraction garnish)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: submitModel("submit_findings", CLUSTER_FINDINGS), auditor: submitModel("submit_review", CLUSTER_VERDICT), labeler: silentModel(), // never submits — the namer throws @@ -656,6 +705,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: submitModel("submit_findings", NOTES_FINDINGS), // During the auditor's call the registry must show the notes run. auditor: submitModel("submit_review", NOTES_VERDICT, () => { @@ -704,6 +754,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => { createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: submitModel("submit_findings", NOTES_FINDINGS), auditor: submitModel("submit_review", NOTES_VERDICT), labeler: submitModel("submit_labels", { labels: [] }), @@ -724,6 +775,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => { const pipeline = createExtractionPipeline({ chatStore, graphStore: ledgers, + embedder: fakeEmbedder, extractor: hangingModel(), auditor: submitModel("submit_review", NOTES_VERDICT), labeler: submitModel("submit_labels", { labels: [] }), diff --git a/tests/graph.test.ts b/tests/graph.test.ts index c695394..2bfc79c 100644 --- a/tests/graph.test.ts +++ b/tests/graph.test.ts @@ -23,7 +23,7 @@ import type { QuestionAnswerData } from "../src/core/question-core.ts"; import { clusterContentId, getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts"; import { graphStore } from "../src/store/graph.ts"; import { chatStore } from "../src/store/chats.ts"; -import { LEDGER_FILES, PATHS, SOURCES_FILE } from "../config.ts"; +import { LEDGER_FILES, PATHS, SOURCES_FILE, EMBEDDINGS_FILE } from "../config.ts"; /** The wired fs adapters, exactly what src/server/stores.ts passes in production. */ const graphDeps = { chatStore, graphStore } as const satisfies GraphDeps; @@ -38,7 +38,9 @@ function sessionSource(chatId: string): { messages: UIMessage[]; foldedAt: strin // ─── ledger isolation (backup-and-clear of data/graph/*.jsonl) ────────────── -const LEDGER_PATHS = [...Object.values(LEDGER_FILES), SOURCES_FILE]; +// EMBEDDINGS_FILE too: the semantic-cluster source (read by graphSnapshot) +// — isolated like the rest so this suite never touches the real vectors. +const LEDGER_PATHS = [...Object.values(LEDGER_FILES), SOURCES_FILE, EMBEDDINGS_FILE]; const backups: Array<{ file: string; bak: string }> = []; const createdChats: string[] = []; @@ -407,8 +409,10 @@ describe("the merger (mergeFindings)", () => { assert.equal(snap.nodeObservations[node.id]!["src_snapfixture"], "solid"); }); - it("derives structural clusters in graphSnapshot, joined with persisted labelings", () => { - // A self-contained 3-clique: three propositions, all mutually linked. + it("derives semantic clusters in graphSnapshot, joined with persisted labelings", () => { + // Three propositions; the cluster comes from the EMBEDDING ledger + // (semantic topics, DECIDED 2026-09-13) — the embedder persists the + // partition, the snapshot just reads it. Here we play the embedder. const tri = ["Alpha.", "Beta.", "Gamma cluster."] as const; const merge = mergeFindings( "src_clusterfixture", @@ -433,6 +437,16 @@ describe("the merger (mergeFindings)", () => { { messages: null, foldedAt: "2026-09-13T00:00:00.000Z" }, ); assert.equal(merge.nodes, 3); + // The embedder's contract: recluster at the end of the batch and + // PERSIST the partition in the embeddings ledger. + const ids = tri.map((s) => getGraph(graphStore).nodes.find((n) => n.statement === s)!.id); + graphStore.writeEmbeddings({ + model: "test/embedder", + dim: 2, + vectors: {}, + clusters: [{ members: ids }], + updatedAt: "2026-09-13T00:00:00.000Z", + }); const snap = graphSnapshot(graphDeps); // Presentation threshold: every cluster is >= 3 members. assert.ok(snap.clusters.every((c) => c.members.length >= 3)); diff --git a/tests/layout.test.ts b/tests/layout.test.ts index 7e71f44..9ae9fc9 100644 --- a/tests/layout.test.ts +++ b/tests/layout.test.ts @@ -110,6 +110,7 @@ describe("settleGraphLayout", () => { readEdges: () => [edge("a", "b")], readPositions: () => ({} as Record), writePositions: (p: Record) => void written.push(p), + readEmbeddings: () => ({ model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }), }; settleGraphLayout(store); assert.equal(written.length, 1); @@ -123,6 +124,7 @@ describe("settleGraphLayout", () => { readEdges: () => [], readPositions: () => ({} as Record), writePositions: (p) => void written.push(p), + readEmbeddings: () => ({ model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" }), }); assert.equal(written.length, 0); }); diff --git a/tests/semantic-clusters.test.ts b/tests/semantic-clusters.test.ts new file mode 100644 index 0000000..54f7fb2 --- /dev/null +++ b/tests/semantic-clusters.test.ts @@ -0,0 +1,118 @@ +/** + * Semantic topic clustering (the Louvain REPLACEMENT, DECIDED + * 2026-09-13): average-linkage agglomerative clustering over the + * embedding ledger — related TOPICS, not reasoning chains. Pure and + * deterministic, so these tests are exact. + */ +import { strict as assert } from "node:assert"; +import { describe, test } from "node:test"; +import { semanticClusters, SEMANTIC_SIM_THRESHOLD, cosineSimilarity } from "../src/core/semantic-clusters.ts"; + +/** Two clearly-related one-hot-ish vectors (high cosine similarity). */ +function near(vec: number[], target: number[], amount = 0.2): number[] { + let dot = 0; + for (let i = 0; i < vec.length; i++) dot += vec[i]! * target[i]!; + const mixed = vec.map((x, i) => x + amount * target[i]!); + let mag = 0; + for (const x of mixed) mag += x * x; + mag = Math.sqrt(mag); + return mixed.map((x) => x / mag); +} + +function axis(dim: number, k: number): number[] { + return Array.from({ length: dim }, (_, i) => (i === k ? 1 : 0)); +} + +describe("cosineSimilarity", () => { + test("identical unit vectors → 1; orthogonal → 0; defensive normalization", () => { + const a = axis(4, 0); + const b = axis(4, 1); + assert.ok(cosineSimilarity(a, a) > 0.999); + assert.ok(Math.abs(cosineSimilarity(a, b)) < 1e-9); + // Unnormalized twins still match. + assert.ok(cosineSimilarity([3, 0], [5, 0]) > 0.999); + }); +}); + +describe("semanticClusters", () => { + test("empty ledger: zero clusters (nothing embedded yet)", () => { + assert.deepEqual(semanticClusters({}), []); + }); + + test("related vectors merge; distant ones stay separate (threshold)", () => { + const dim = 8; + const topicA = axis(dim, 0); + const topicB = axis(dim, 1); + const vectors = { + a1: near(topicA, topicB, 0.1), + a2: near(topicA, topicB, 0.15), + a3: near(topicA, topicB, 0.05), + b1: near(topicB, topicA, 0.1), + b2: near(topicB, topicA, 0.12), + }; + const clusters = semanticClusters(vectors); + // Both topics form; average-linkage similarity within each is well + // above the default threshold, across is well below. + const big = clusters.filter((c) => c.members.length >= 2); + assert.equal(big.length, 2); + assert.deepEqual( + big.map((c) => c.members.sort()).sort((x, y) => x[0]!.localeCompare(y[0]!)), + [["a1", "a2", "a3"], ["b1", "b2"]], + ); + }); + + test("threshold is the granularity knob: higher splits, lower merges", () => { + const dim = 6; + const vectors = { + x: near(axis(dim, 0), axis(dim, 1), 0.3), + y: near(axis(dim, 1), axis(dim, 0), 0.3), + }; + // Similarity ~0.72: merges below that, splits above. + const sim = cosineSimilarity(vectors.x!, vectors.y!); + assert.equal(semanticClusters(vectors, { similarityThreshold: sim - 0.01 }).length, 1); + assert.equal(semanticClusters(vectors, { similarityThreshold: sim + 0.01 }).length, 2); + }); + + test("stragglers stay singletons by default; attachStragglers forces membership", () => { + const dim = 6; + const topic = axis(dim, 0); + const vectors = { + t1: near(topic, axis(dim, 1), 0.1), + t2: near(topic, axis(dim, 2), 0.1), + t3: near(topic, axis(dim, 3), 0.1), + loner: axis(dim, 5), // orthogonal: fits nothing + }; + const loose = semanticClusters(vectors); + const looseLoner = loose.find((c) => c.members.includes("loner")); + assert.equal(looseLoner?.members.length, 1, "default: loner is a straggler"); + assert.equal(loose.find((c) => c.members.length >= 3)?.members.length, 3); + + const allIn = semanticClusters(vectors, { attachStragglers: true }); + assert.equal(allIn.length, 1, "attach: the loner joins the nearest topic"); + assert.ok(allIn[0]!.members.includes("loner")); + }); + + test("members sorted; clusters ordered by size then id (deterministic)", () => { + const dim = 4; + const vectors = { + solo: axis(dim, 3), + p1: near(axis(dim, 0), axis(dim, 1), 0.05), + p2: near(axis(dim, 0), axis(dim, 2), 0.05), + q1: near(axis(dim, 1), axis(dim, 0), 0.05), + q2: near(axis(dim, 1), axis(dim, 2), 0.05), + }; + const a = semanticClusters(vectors); + const b = semanticClusters(vectors); + assert.deepEqual(a, b, "same input → identical partition"); + for (let i = 1; i < a.length; i++) { + assert.ok(a[i - 1]!.members.length >= a[i]!.members.length, "size-descending"); + } + for (const c of a) { + assert.deepEqual(c.members, [...c.members].sort(), "members sorted"); + } + }); + + test("default threshold is the swept real-graph value", () => { + assert.equal(SEMANTIC_SIM_THRESHOLD, 0.6); + }); +}); From 652a25e382be8b6fdc7d33d5564d7d22680da3bd Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 15:05:05 +0300 Subject: [PATCH 26/34] Orphan discs in regions + orphan exemption from depth anchoring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-ups on the semantic-cluster wave, both user-reported: 1. Region coloring: edge-only dilation (the anti-double-counting rule from the Louvain era) left 114 of 338 cluster members completely uncolored — semantic clusters contain members that share no intra-cluster edge with anyone (90 edge-orphans + isolated micro-topics), and 20 of 45 clusters painted nothing at all. Fix: a disc (radius = dilation, same as a round cap) for exactly the members no intra-cluster edge stroke reached. Every member is painted by its edges' caps OR one disc, never both — the no-double-counting rule survives. Verified live: all 74 orphans-in-clusters sit on tinted region pixels. 2. Depth modes (ORPHAN RULE): a node with no edges has no dependency structure, so depth-0 stamping put 90 unconnected propositions on the vertical baseline / radial base ring — meaning they don't have. Orphans are now exempt from the depth force and get the regular view's unprojected forces instead (full center gravity + full cluster pull). Structured-node projections unchanged (x-only gravity in vertical, none in radial; tangential-only cluster pull in radial). Gravity in depth modes is a custom force with EXACTLY d3's forceX/forceY math (strength × alpha — the first draft forgot the alpha and shifted the settle). Verified live: radial orphans follow their topics instead of the base ring; vertical orphans spread 1006 world units (6/90 near the baseline band vs 90/90 before). --- components/graph-canvas.ts | 29 ++++++++++++++++--- src/core/layout.ts | 57 +++++++++++++++++++++++++++++++++++--- tests/layout.test.ts | 42 ++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 8 deletions(-) diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts index 37d368e..b9bab57 100644 --- a/components/graph-canvas.ts +++ b/components/graph-canvas.ts @@ -1205,10 +1205,9 @@ export class GraphCanvasRenderer { }); // The region IS the dilated skeleton: intra-cluster edges as - // round-cap strokes (Minkowski dilation — points become discs at - // line ends via the caps). NO per-node discs: a cluster is its - // connected points, and discs double-counted where nodes sit on - // edges (user-identified hot spots). + // round-cap strokes (Minkowski dilation). NO blanket per-node discs + // — they double-counted where nodes sit on edges (user-identified + // hot spots, when clusters were edge-based). this.regionMeta = []; data.clusters.forEach((cluster, ci) => { const hue = clusterHue(cluster.id); @@ -1217,6 +1216,8 @@ export class GraphCanvasRenderer { let members = 0; let sumX = 0; let sumY = 0; + // Endpoints actually covered by a stroke's round caps. + const touched = new Set(); for (const e of data.edges) { if (clusterOf.get(e.from) !== ci) continue; if (clusterOf.get(e.to) !== ci) continue; @@ -1225,9 +1226,29 @@ export class GraphCanvasRenderer { if (!a || !b) continue; rctx.moveTo((a.x - minX) / cell, (a.y - minY) / cell); rctx.lineTo((b.x - minX) / cell, (b.y - minY) / cell); + touched.add(e.from); + touched.add(e.to); } rctx.stroke(); + // Member discs for the SEMANTIC era (DECIDED 2026-09-13): clusters + // are now topics by meaning, and many members (orphans, isolated + // micro-topics) share NO intra-cluster edge with anyone — edge + // strokes alone left them completely uncolored. A disc (same + // radius as a round cap = dilation) covers exactly the members no + // stroke reached: every member is painted by its edges' caps OR + // one disc, never both — the no-double-counting rule survives. + rctx.fillStyle = `hsla(${hue}, 65%, 60%, ${tuning.edgeAlpha})`; + const discR = (tuning.dilation * s) / cell; + for (const id of cluster.members) { + if (touched.has(id) || !isVisible(id)) continue; + const n = this.nodeById.get(id); + if (!n) continue; + rctx.beginPath(); + rctx.arc((n.x - minX) / cell, (n.y - minY) / cell, discR, 0, Math.PI * 2); + rctx.fill(); + } + // Label anchor: member centroid (world coords). for (const id of cluster.members) { if (!isVisible(id)) continue; diff --git a/src/core/layout.ts b/src/core/layout.ts index a555240..df388db 100644 --- a/src/core/layout.ts +++ b/src/core/layout.ts @@ -262,11 +262,53 @@ export function createGraphSimulation( // y axis, so gravity goes x-only (it would fight the level lines); in // radial mode a center pull is purely radial and would fight the // rings, so it is dropped entirely (rings already bound the radius). + // Depth mode projections (DECIDED 2026-09-13), with the ORPHAN RULE + // (user-decided, same day): a node with no edges at all has no + // dependency structure — depth is a property of inference chains, so + // stamping unconnected propositions onto the baseline / base ring + // assigns them meaning they don't have. Orphans are exempt from the + // depth force and keep the REGULAR view's forces instead (full center + // gravity + full cluster pull, unprojected). const mode = options.depthAnchor?.mode ?? "off"; + let orphan: Uint8Array | null = null; + if (mode !== "off") { + const connected = new Set(); + const valid = new Set(nodes.map((n) => n.id)); + for (const e of edges) { + // Self-loops and dangling endpoints are not a structure (same + // guards as inferenceDepths). + if (e.from === e.to || !valid.has(e.from) || !valid.has(e.to)) continue; + connected.add(e.from); + connected.add(e.to); + } + orphan = new Uint8Array(nodes.length); + nodes.forEach((n, i) => { + if (!connected.has(n.id)) orphan![i] = 1; + }); + } const g = options.gravity; - if (g && mode !== "radial") { + if (g && mode === "off") { sim.force("x", forceX(g.x).strength(g.strength)); - if (mode !== "vertical") sim.force("y", forceY(g.y).strength(g.strength)); + sim.force("y", forceY(g.y).strength(g.strength)); + } else if (g) { + // Mode-projected gravity, EXACTLY d3's forceX/forceY math (restoring + // force × strength × alpha): structured nodes get x-only in vertical + // mode (the depth force owns y) and none in radial mode (the rings + // own the radius); ORPHANS get the full regular-view pull in every + // mode. + const orphanMask = orphan!; + sim.force("gravity", (alpha: number) => { + for (let i = 0; i < nodes.length; i++) { + const n = nodes[i]!; + if (orphanMask[i]) { + n.vx = (n.vx ?? 0) + (g.x - (n.x ?? 0)) * g.strength * alpha; + n.vy = (n.vy ?? 0) + (g.y - (n.y ?? 0)) * g.strength * alpha; + } else if (mode === "vertical") { + n.vx = (n.vx ?? 0) + (g.x - (n.x ?? 0)) * g.strength * alpha; + } + // radial + structured: nothing — the rings own the radius. + } + }); } // Cluster centroid pull: the membership index is resolved ONCE (nodes @@ -311,7 +353,11 @@ export function createGraphSimulation( const py = n.y ?? 0; const ux = sumX[ci]! / cnt[ci]! - px; const uy = sumY[ci]! / cnt[ci]! - py; - if (mode === "vertical") { + if (orphan && orphan[i]) { + // Orphans: full regular-view pull (see the ORPHAN RULE above). + n.vx = (n.vx ?? 0) + ux * strength * alpha; + n.vy = (n.vy ?? 0) + uy * strength * alpha; + } else if (mode === "vertical") { n.vx = (n.vx ?? 0) + ux * strength * alpha; } else if (mode === "radial") { const rx = px - ringCx; @@ -342,10 +388,12 @@ export function createGraphSimulation( const strength = da.strength; const spacing = da.spacing; const c = da.center ?? { x: 0, y: 0 }; + const orphanMask = orphan; // ORPHAN RULE: depth skips unstructured nodes if (da.mode === "vertical") { const baseY = c.y; sim.force("depth", (alpha: number) => { for (let i = 0; i < nodes.length; i++) { + if (orphanMask?.[i]) continue; const n = nodes[i]!; const targetY = baseY - (depths.get(n.id) ?? 0) * spacing; n.vy = (n.vy ?? 0) + (targetY - (n.y ?? 0)) * strength * alpha; @@ -357,8 +405,9 @@ export function createGraphSimulation( const cy0 = c.y; sim.force("depth", (alpha: number) => { for (let i = 0; i < nodes.length; i++) { + if (orphanMask?.[i]) continue; const n = nodes[i]!; - const dx = (n.x ?? 0) - cx0; + const dx = (n.x ?? 0) - cx0; const dy = (n.y ?? 0) - cy0; // Degenerate direction at the exact ring center: nudge // outward on a deterministic axis. diff --git a/tests/layout.test.ts b/tests/layout.test.ts index 9ae9fc9..f9e1a11 100644 --- a/tests/layout.test.ts +++ b/tests/layout.test.ts @@ -270,4 +270,46 @@ describe("depthAnchor force", () => { const child = next.get("child")!; assert.ok(child.y < old.y - 40, `child rises above its premises (child=${child.y.toFixed(1)}, old=${old.y.toFixed(1)})`); }); + + it("orphans are exempt from depth anchoring — regular forces govern them (ORPHAN RULE)", () => { + // A premise→conclusion chain plus an unconnected loner. Gravity's + // center is FAR from the baseline, so the two are distinguishable: + // a depth-stamped orphan would sit on the baseline; an exempt one + // follows gravity instead. + const ns = [ + { id: "base", x: 0, y: 0 }, + { id: "derived", x: 10, y: 0 }, + { id: "loner", x: 0, y: 0 }, + ]; + const sim = createGraphSimulation(ns, [{ from: "base", to: "derived" }], { + gravity: { x: 0, y: 500, strength: 0.1 }, + depthAnchor: { mode: "vertical", strength: 1.5, spacing: 120, center: { x: 0, y: 0 } }, + }); + for (let i = 0; i < 400; i++) sim.tick(); + const byId = new Map(sim.nodes().map((n) => [n.id, n])); + assert.ok(Math.abs(byId.get("base")!.y!) < 40, "structured base sits on the baseline"); + assert.ok(byId.get("derived")!.y! < -60, "derived rises a level"); + const lonerY = byId.get("loner")!.y!; + assert.ok(lonerY > 250, `the orphan follows gravity, not the baseline (y=${lonerY.toFixed(0)})`); + assert.ok(Math.abs(lonerY - 500) < 120, `the orphan converges on gravity's center (y=${lonerY.toFixed(0)})`); + }); + + it("radial: orphans are exempt — they gather at the center, not on the base ring", () => { + const ns = [ + { id: "base", x: 0, y: 0 }, + { id: "mid", x: 10, y: 0 }, + { id: "loner", x: 300, y: 300 }, + ]; + const sim = createGraphSimulation(ns, [{ from: "base", to: "mid" }], { + gravity: { x: 0, y: 0, strength: 0.1 }, + depthAnchor: { mode: "radial", strength: 1.5, spacing: 80, baseRadius: 150, center: { x: 0, y: 0 } }, + }); + for (let i = 0; i < 400; i++) sim.tick(); + const r = (id: string) => { + const n = sim.nodes().find((x) => x.id === id)!; + return Math.hypot(n.x!, n.y!); + }; + assert.ok(Math.abs(r("base") - 150) < 40, `base sits on the base ring (r=${r("base").toFixed(0)})`); + assert.ok(r("loner") < 60, `the orphan follows gravity to the center, not the ring (r=${r("loner").toFixed(0)})`); + }); }); From c7c1148989180b969ba9cab329c35878c14005d0 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 16:00:28 +0300 Subject: [PATCH 27/34] Region boundaries: iso-line contours of the blurred region field MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Marching squares over each cluster's own contribution to the shared blurred bitmap, recovered by projecting premultiplied pixels onto the cluster's paint color (blends split proportionally between overlapping halos — the natural divider). Boundary level = fraction of the cluster's peak field, user-adjustable in the settings panel; 0 = off. One paint, one blur, one readback; contouring is cheap typed-array math over small per-cluster crops (stride-2 sampling: measured 1.2ms for all 26 clusters, vs 7.3ms naive). During the live sim boundaries recompute every 2nd region rebuild and stale paths ride along — drag cadence matches the pre-boundary baseline (avg 8.46ms vs 8.37ms). --- components/graph-canvas.ts | 113 +++++++++++++++++- components/graph-view.tsx | 6 + components/region-contour.ts | 223 +++++++++++++++++++++++++++++++++++ tests/region-contour.test.ts | 168 ++++++++++++++++++++++++++ 4 files changed, 507 insertions(+), 3 deletions(-) create mode 100644 components/region-contour.ts create mode 100644 tests/region-contour.test.ts diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts index b9bab57..7df06c1 100644 --- a/components/graph-canvas.ts +++ b/components/graph-canvas.ts @@ -54,6 +54,7 @@ import { select } from "d3-selection"; import { zoom, zoomIdentity, type ZoomBehavior, type ZoomTransform } from "d3-zoom"; import type { Simulation, SimulationLinkDatum } from "d3-force"; import { clusterHue, type Pt } from "./cluster-geometry.ts"; +import { hslToRgb, joinContourSegments, marchingSquares, projectCrop } from "./region-contour.ts"; import { CHARGE_STRENGTH, CLUSTER_PULL_STRENGTH, @@ -159,6 +160,15 @@ const COMMIT_DEBOUNCE_MS = 250; // settle → ledger PUT delay (new drags cancel * offset polygons re-rasterized at DPR 2 (the old GPU coil-whine). */ const REGION_EVERY = 3; // live-sim frames: rebuild every Nth, blit the rest const REGION_LABEL_SIZE = 14; +/** Region REBUILDS between boundary recomputes during the live sim: the + * boundary stage (projection + marching squares over ~26 crops) costs + * about as much as the whole fill pipeline, so it runs on every 2nd + * rebuild (~6th frame) and the stale paths ride along — at rest it + * always syncs with the fill. */ +const BOUNDARY_EVERY = 2; +/** Boundary stroke width, screen px at fit zoom (world units = this / kFit, + * so it zooms with the graph like every other stroke). */ +const REGION_BOUNDARY_WIDTH = 1.5; /** Dev-tunable region rendering (2026-09-13, user request): plain number * inputs in the settings panel — session-only, never persisted — so the @@ -178,6 +188,10 @@ export interface RegionTuning { bitmap: number; /** Per-stroke alpha; overlapping strokes stack toward the core. */ edgeAlpha: number; + /** Boundary iso-level as a FRACTION OF each cluster's peak field + * value (0.3 = the line where the blurred halo has faded to 30% of + * its densest point). 0 = no boundaries. */ + boundary: number; } export const DEFAULT_REGION_TUNING: RegionTuning = { @@ -185,6 +199,7 @@ export const DEFAULT_REGION_TUNING: RegionTuning = { blur: 10, bitmap: 384, edgeAlpha: 0.35, + boundary: 0.3, }; /** Clamp anything into a usable tuning (number inputs can be empty/NaN). */ @@ -197,6 +212,7 @@ export function sanitizeRegionTuning(input: Partial | null | undef blur: Math.min(Math.max(num(input?.blur, d.blur), 0), 40), bitmap: Math.round(Math.min(Math.max(num(input?.bitmap, d.bitmap), 128), 1024)), edgeAlpha: Math.min(Math.max(num(input?.edgeAlpha, d.edgeAlpha), 0.02), 1), + boundary: Math.min(Math.max(num(input?.boundary, d.boundary), 0), 0.95), }; } @@ -571,11 +587,19 @@ export class GraphCanvasRenderer { private regionRect: { minX: number; minY: number; w: number; h: number } | null = null; /** Region labels: cluster centroid + hue, computed at splat build. */ private regionMeta: { label: string | null; cx: number; cy: number; hue: number }[] = []; + /** Boundary paths (world-space Path2D per cluster), from the last + * region rebuild — stroked every frame under the camera transform. */ + private regionBoundaries: { path: Path2D; hue: number }[] = []; private regionTuning: RegionTuning = DEFAULT_REGION_TUNING; private physicsTuning: PhysicsTuning = DEFAULT_PHYSICS_TUNING; /** Gravity center for the live sim (graph bbox centroid at setData). */ private simCenter: Pt = { x: 0, y: 0 }; private regionBuiltVersion = -1; + /** geoVersion of the boundary paths (they lag one rebuild behind the + * region fill during the live sim — see BOUNDARY_EVERY). */ + private boundaryBuiltVersion = -1; + /** Rebuild counter for the boundary throttle (counts region rebuilds). */ + private regionRebuildCount = 0; private frameCount = 0; // draw counter for the live-sim splat throttle private labelLines = new Map(); private worldLabelFont = ""; @@ -1138,6 +1162,7 @@ export class GraphCanvasRenderer { this.regionCanvas = null; this.regionRect = null; this.regionMeta = []; + this.regionBoundaries = []; return; } const visible = this.visible; @@ -1147,23 +1172,32 @@ export class GraphCanvasRenderer { const s = 1 / this.kFit; // World rect over VISIBLE cluster members (regions cover members - // and their intra-cluster edges — nothing else). + // and their intra-cluster edges — nothing else), plus each cluster's + // OWN bbox for the boundary crops (the contour only needs to scan + // where a cluster's halo can reach). let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity; - for (const c of data.clusters) { + const boxes = data.clusters.map((c) => { + let a = Infinity, b = Infinity, e = -Infinity, f = -Infinity; for (const id of c.members) { if (!isVisible(id)) continue; const n = this.nodeById.get(id); if (!n) continue; + if (n.x < a) a = n.x; + if (n.x > e) e = n.x; + if (n.y < b) b = n.y; + if (n.y > f) f = n.y; if (n.x < minX) minX = n.x; if (n.x > maxX) maxX = n.x; if (n.y < minY) minY = n.y; if (n.y > maxY) maxY = n.y; - } } + return a > e ? null : { minX: a, minY: b, maxX: e, maxY: f }; + }); if (minX > maxX) { this.regionCanvas = null; this.regionRect = null; this.regionMeta = []; + this.regionBoundaries = []; return; } @@ -1266,6 +1300,7 @@ export class GraphCanvasRenderer { // THE blur: 3 box-blur rounds on the premultiplied bitmap. Bilinear // upscaling alone is no blur at all (tent ramps — the JPEG look); // this makes the field genuinely smooth before the blit. + let blurred: Float32Array | null = null; // premultiplied field, for boundaries if (tuning.blur > 0) { const img = rctx.getImageData(0, 0, bw, bh); const n = bw * bh * 4; @@ -1299,6 +1334,65 @@ export class GraphCanvasRenderer { } } rctx.putImageData(img, 0, 0); + blurred = b; + } + + // ── Boundaries (DECIDED 2026-09-13) ────────────────────────────── + // Iso-lines of the blurred field at a FRACTION OF each cluster's + // peak. Per-cluster fields are recovered from the ONE shared + // bitmap by projecting crops of the blurred premultiplied array + // onto each cluster's paint color (pure-i pixels → their alpha; + // overlaps split proportionally — the natural divider between + // neighboring halos). One paint, one blur, one readback — the + // per-cluster work is cheap typed-array math over small crops. + // During the live sim this doubles the rebuild cost, so it runs on + // every BOUNDARY_EVERY-th region rebuild (at rest it always syncs; + // the stale paths just ride along for ~3 frames). + if (tuning.boundary <= 0 || !blurred) { + // Off (or no blurred field): never keep stale paths around. + this.regionBoundaries = []; + } else if (!this.simActive || this.regionRebuildCount % BOUNDARY_EVERY === 0) { + this.regionBoundaries = []; + for (let ci = 0; ci < data.clusters.length; ci++) { + const box = boxes[ci]; + if (!box) continue; + const cluster = data.clusters[ci]!; + // Crop in bitmap coords: the cluster bbox + the halo reach, + // clamped to the bitmap. + const cx0 = Math.max(0, Math.floor((box.minX - padWorld - minX) / cell)); + const cy0 = Math.max(0, Math.floor((box.minY - padWorld - minY) / cell)); + const cx1 = Math.min(bw - 1, Math.ceil((box.maxX + padWorld - minX) / cell)); + const cy1 = Math.min(bh - 1, Math.ceil((box.maxY + padWorld - minY) / cell)); + const cw = cx1 - cx0; + const ch = cy1 - cy0; + if (cw < 3 || ch < 3) continue; + const hue = clusterHue(cluster.id); + const rgb = hslToRgb(hue, 0.65, 0.6); + // Stride 2: sample every other bitmap pixel — the blurred field + // varies on a ~blur-radius scale, so the half-res grid loses + // nothing and the contour work quarters (measured ~1ms total). + const { field, peak } = projectCrop(blurred, bw, { x: cx0, y: cy0, w: cw, h: ch }, rgb, 2); + const level = tuning.boundary * peak; + if (peak <= 0 || level <= 0) continue; + const fw = Math.ceil(cw / 2); + const fh = Math.ceil(ch / 2); + const segments = marchingSquares(field, fw, fh, level); + if (segments.length === 0) continue; + // Grid coords (stride-2 samples) → world coords → one stroked + // path per cluster. Sample (gx, gy) is bitmap pixel + // (cx0 + gx*2, cy0 + gy*2). + const path = new Path2D(); + for (const line of joinContourSegments(segments)) { + if (line.length < 4) continue; + path.moveTo(minX + (cx0 + line[0]! * 2) * cell, minY + (cy0 + line[1]! * 2) * cell); + for (let i = 2; i < line.length; i += 2) { + path.lineTo(minX + (cx0 + line[i]! * 2) * cell, minY + (cy0 + line[i + 1]! * 2) * cell); + } + // Closed loop (first point == last point) → close the path. + if (line[0] === line[line.length - 2] && line[1] === line[line.length - 1]) path.closePath(); + } + this.regionBoundaries.push({ path, hue }); + } } this.regionRect = { minX, minY, w: spanX, h: spanY }; } @@ -1326,6 +1420,7 @@ export class GraphCanvasRenderer { this.regionBuiltVersion !== this.geoVersion && (!this.simActive || this.frameCount % REGION_EVERY === 0) ) { + this.regionRebuildCount++; this.buildRegions(); this.regionBuiltVersion = this.geoVersion; } @@ -1351,6 +1446,18 @@ export class GraphCanvasRenderer { ctx.drawImage(this.regionCanvas, minX, minY, w, h); } + // 2. Region boundaries: the cached iso-line paths, one stroke per + // cluster (hue-tinted, zoom-scaling width — see REGION_BOUNDARY_WIDTH). + if (this.regionBoundaries.length > 0) { + ctx.lineWidth = REGION_BOUNDARY_WIDTH * (1 / this.kFit); + ctx.lineJoin = "round"; + ctx.lineCap = "round"; + for (const rb of this.regionBoundaries) { + ctx.strokeStyle = `hsla(${rb.hue}, 65%, 68%, 0.55)`; + ctx.stroke(rb.path); + } + } + // 3. Edges: shafts + arrowheads from cached paths; width constant on // screen (÷ k), geometry in world units (scales with the graph). if (settings.edgeWidth > 0) { diff --git a/components/graph-view.tsx b/components/graph-view.tsx index a139ddf..41a66cb 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -656,6 +656,12 @@ export function GraphView() { step={0.05} onChange={(v) => setRegionTuning((t) => ({ ...t, edgeAlpha: v }))} /> + setRegionTuning((t) => ({ ...t, boundary: v }))} + /> {/* Physics (user-facing): the live sim's forces. Every value feeds the SAME engine the server settle runs (createGraphSimulation); diff --git a/components/region-contour.ts b/components/region-contour.ts new file mode 100644 index 0000000..8294d3a --- /dev/null +++ b/components/region-contour.ts @@ -0,0 +1,223 @@ +/** + * REGION BOUNDARIES (DECIDED 2026-09-13): iso-lines of the blurred + * region field — "draw a line along a constant luminance of the blurred + * background color". Each cluster's boundary is the contour of ITS OWN + * contribution to the field, recovered by projecting the blurred + * PREMULTIPLIED bitmap onto the cluster's paint color (a pixel painted + * only by cluster i projects to its alpha; a blend between two clusters + * splits proportionally — the projection is the natural divider where + * neighboring halos overlap). ONE paint pass, ONE blur, ONE readback; + * the per-cluster work is pure typed-array math over small crops. + * + * Pure module (no DOM, no Path2D) so it is unit-testable in node. + */ + +/** HSL → RGB (0-255 each), the same conversion the canvas performs for + * hsla() paint colors — the projection must divide by exactly what was + * painted. h in degrees, s and l in 0-1. */ +export function hslToRgb(h: number, s: number, l: number): [number, number, number] { + const c = (1 - Math.abs(2 * l - 1)) * s; + const hp = (((h % 360) + 360) % 360) / 60; + const x = c * (1 - Math.abs((hp % 2) - 1)); + let r = 0; + let g = 0; + let b = 0; + if (hp < 1) [r, g, b] = [c, x, 0]; + else if (hp < 2) [r, g, b] = [x, c, 0]; + else if (hp < 3) [r, g, b] = [0, c, x]; + else if (hp < 4) [r, g, b] = [0, x, c]; + else if (hp < 5) [r, g, b] = [x, 0, c]; + else [r, g, b] = [c, 0, x]; + const m = l - c / 2; + return [Math.round((r + m) * 255), Math.round((g + m) * 255), Math.round((b + m) * 255)]; +} + +/** + * Project a crop of the blurred premultiplied bitmap onto a paint color: + * field(p) = ⟨pm(p), rgb⟩ / |rgb|² — the fraction of pixel p's + * premultiplied color that lies along the cluster's paint direction. + * Pure-i pixels → their alpha fraction (0-1); blends split between the + * contributors. Returns the crop field and its peak (for the relative + * threshold). + * + * `stride` samples every Nth bitmap pixel (default 1): the blurred field + * varies on a ~blur-radius scale, so a stride-2 grid loses nothing and + * quarters the contour work (2026-09-13: stride-1 doubled the rebuild + * cost and broke the 120fps drag cadence every 6th frame; stride-2 + * contouring is measured ~1ms for the whole boundary stage). + */ +export function projectCrop( + pm: Float32Array, + pmW: number, + crop: { x: number; y: number; w: number; h: number }, + rgb: readonly [number, number, number], + stride = 1, +): { field: Float32Array; peak: number } { + const [r, g, b] = rgb; + const denom = r * r + g * g + b * b; + const fw = Math.ceil(crop.w / stride); + const fh = Math.ceil(crop.h / stride); + if (denom <= 0) return { field: new Float32Array(fw * fh), peak: 0 }; + const field = new Float32Array(fw * fh); + let peak = 0; + for (let y = 0; y < fh; y++) { + const src = ((crop.y + y * stride) * pmW + crop.x) * 4; + const dst = y * fw; + for (let x = 0; x < fw; x++) { + const i = src + x * stride * 4; + const v = (pm[i]! * r + pm[i + 1]! * g + pm[i + 2]! * b) / denom; + field[dst + x] = v; + if (v > peak) peak = v; + } + } + return { field, peak }; +} + +/** + * Marching squares over a scalar field: emits iso-line SEGMENTS (grid + * coordinates, linear interpolation between samples) where the field + * crosses `level`. Flat output [x1,y1,x2,y2, ...] — join with + * joinContourSegments. Saddle cases (5, 10) are disambiguated by the + * cell-center average. Grid coords: sample (x, y) is field[y*w + x]; + * a segment point may be fractional (between samples). + */ +export function marchingSquares( + field: Float32Array, + w: number, + h: number, + level: number, +): number[] { + // Hot loop (measured ~7ms over ~210k cells when the crossing helpers + // were per-cell closures — allocation dominated). All crossings are + // computed inline; zero allocations per empty cell. + const segments: number[] = []; + for (let y = 0; y < h - 1; y++) { + const row = y * w; + const rowBelow = row + w; + for (let x = 0; x < w - 1; x++) { + const tl = field[row + x]!; + const tr = field[row + x + 1]!; + const br = field[rowBelow + x + 1]!; + const bl = field[rowBelow + x]!; + const idx = + (tl >= level ? 8 : 0) | (tr >= level ? 4 : 0) | (br >= level ? 2 : 0) | (bl >= level ? 1 : 0); + if (idx === 0 || idx === 15) continue; + // Crossing points on the four cell edges (grid coords, + // linear interpolation between the two corner samples). + const topX = x + (level - tl) / (tr - tl); + const rightY = y + (level - tr) / (br - tr); + const bottomX = x + (level - bl) / (br - bl); + const leftY = y + (level - tl) / (bl - tl); + switch (idx) { + case 1: + case 14: + segments.push(x, leftY, bottomX, y + 1); + break; + case 2: + case 13: + segments.push(bottomX, y + 1, x + 1, rightY); + break; + case 3: + case 12: + segments.push(x, leftY, x + 1, rightY); + break; + case 4: + case 11: + segments.push(topX, y, x + 1, rightY); + break; + case 6: + case 9: + segments.push(topX, y, bottomX, y + 1); + break; + case 7: + case 8: + segments.push(topX, y, x, leftY); + break; + case 5: + // Saddle: tr+bl inside. Center decides connected vs split. + if ((tl + tr + br + bl) / 4 >= level) { + segments.push(topX, y, x, leftY, bottomX, y + 1, x + 1, rightY); + } else { + segments.push(topX, y, x + 1, rightY, x, leftY, bottomX, y + 1); + } + break; + case 10: + // Saddle: tl+br inside. + if ((tl + tr + br + bl) / 4 >= level) { + segments.push(topX, y, x + 1, rightY, x, leftY, bottomX, y + 1); + } else { + segments.push(topX, y, x, leftY, bottomX, y + 1, x + 1, rightY); + } + break; + } + } + } + return segments; +} + +/** + * Join iso-line segments into POLYLINES: chains of connected segments, + * closed loops where the contour is closed (first point == last point). + * Endpoints are quantized — marching-squares crossings on a shared cell + * edge are bit-identical from both cells, so any deterministic + * quantization matches them exactly. Output: one flat [x,y,x,y,...] array + * per polyline. + */ +export function joinContourSegments(segments: readonly number[]): number[][] { + const key = (x: number, y: number): string => `${Math.round(x * 1024)},${Math.round(y * 1024)}`; + // point key -> [{seg index, which end}] + const at = new Map>(); + const n = segments.length / 4; + for (let s = 0; s < n; s++) { + const x1 = segments[s * 4]!; + const y1 = segments[s * 4 + 1]!; + const x2 = segments[s * 4 + 2]!; + const y2 = segments[s * 4 + 3]!; + const k1 = key(x1, y1); + const k2 = key(x2, y2); + let l1 = at.get(k1); + if (!l1) at.set(k1, (l1 = [])); + l1.push({ s, end: 0 }); + let l2 = at.get(k2); + if (!l2) at.set(k2, (l2 = [])); + l2.push({ s, end: 1 }); + } + const used = new Array(n).fill(false); + const polylines: number[][] = []; + const takeNeighbor = (x: number, y: number, seg: number): { s: number; x: number; y: number } | null => { + const list = at.get(key(x, y)); + if (!list) return null; + for (const e of list) { + if (used[e.s]) continue; + used[e.s] = true; + return { + s: e.s, + x: segments[e.s * 4 + (e.end === 0 ? 2 : 0)]!, + y: segments[e.s * 4 + (e.end === 0 ? 3 : 1)]!, + }; + } + return null; + }; + for (let s0 = 0; s0 < n; s0++) { + if (used[s0]) continue; + used[s0] = true; + const line = [segments[s0 * 4]!, segments[s0 * 4 + 1]!, segments[s0 * 4 + 2]!, segments[s0 * 4 + 3]!]; + // Extend at the tail, then at the head. + for (let guard = 0; guard < 2 * n + 4; guard++) { + const tx = line[line.length - 2]!; + const ty = line[line.length - 1]!; + const nb = takeNeighbor(tx, ty, -1); + if (!nb) break; + line.push(nb.x, nb.y); + } + for (let guard = 0; guard < 2 * n + 4; guard++) { + const hx = line[0]!; + const hy = line[1]!; + const nb = takeNeighbor(hx, hy, -1); + if (!nb) break; + line.unshift(nb.x, nb.y); + } + polylines.push(line); + } + return polylines; +} diff --git a/tests/region-contour.test.ts b/tests/region-contour.test.ts new file mode 100644 index 0000000..13ee2de --- /dev/null +++ b/tests/region-contour.test.ts @@ -0,0 +1,168 @@ +/** + * Region boundary primitives (components/region-contour.ts): marching + * squares, segment joining, and the color projection that recovers each + * cluster's own field from the shared blurred bitmap. Pure typed-array + * math — exact, deterministic tests. + */ +import { strict as assert } from "node:assert"; +import { describe, test } from "node:test"; +import { hslToRgb, joinContourSegments, marchingSquares, projectCrop } from "../components/region-contour.ts"; + +describe("hslToRgb", () => { + test("matches the canvas hsla() conversion at the region paint points", () => { + // hsla(h, 65%, 60%) — the exact paint colors buildRegions uses. + assert.deepEqual(hslToRgb(0, 0.65, 0.6), [219, 87, 87]); // red-ish + assert.deepEqual(hslToRgb(120, 0.65, 0.6), [87, 219, 87]); // green-ish + assert.deepEqual(hslToRgb(240, 0.65, 0.6), [87, 87, 219]); // blue-ish + // Gray: zero saturation. + assert.deepEqual(hslToRgb(37, 0, 0.6), [153, 153, 153]); + // Hue wraps. + assert.deepEqual(hslToRgb(360, 0.65, 0.6), hslToRgb(0, 0.65, 0.6)); + }); +}); + +describe("marchingSquares", () => { + /** A smooth radial bump on a w×h grid. */ + function bump(w: number, h: number, cx: number, cy: number, radius: number): Float32Array { + const f = new Float32Array(w * h); + for (let y = 0; y < h; y++) { + for (let x = 0; x < w; x++) { + const d = Math.hypot(x - cx, y - cy); + f[y * w + x] = Math.max(0, 1 - d / radius); + } + } + return f; + } + + test("a single blob yields one closed loop", () => { + // Irrational center offsets: no grid sample sits EXACTLY on the + // iso-line (a tie makes a crossing land on a sample point, which + // can split the loop into two stroked halves — visually identical, + // but this test wants the clean case). + const w = 21; + const f = bump(w, 21, 10.4, 9.7, 8); + const segs = marchingSquares(f, w, 21, 0.5); + assert.ok(segs.length >= 16, "a circle has plenty of segments"); + const lines = joinContourSegments(segs); + assert.equal(lines.length, 1, "exactly one loop"); + const l = lines[0]!; + // Closed: first point == last point. + assert.equal(l[0], l[l.length - 2]); + assert.equal(l[1], l[l.length - 1]); + // Roughly the right radius: the iso-line of 1 - d/r = 0.5 is d = r/2 = 4. + const meanR = + l.filter((_, i) => i % 2 === 0 && i < l.length - 2).length === 0 + ? 0 + : l.reduce((acc, _, i) => (i % 4 === 0 && i + 1 < l.length ? Math.hypot(l[i]! - 10, l[i + 1]! - 10) + acc : acc), 0) / + (l.filter((_, i) => i % 4 === 0 && i + 1 < l.length).length || 1); + assert.ok(Math.abs(meanR - 4) < 1.5, `iso-line radius ≈ 4 (got ${meanR.toFixed(2)})`); + }); + + test("two separated blobs yield two loops", () => { + const w = 40; + const f = bump(w, 21, 10, 10, 7); + const g = bump(w, 21, 30, 10, 7); + const both = f.map((v, i) => Math.max(v, g[i]!)); + const lines = joinContourSegments(marchingSquares(both, w, 21, 0.5)); + assert.equal(lines.length, 2); + }); + + test("a ramp clipped by the grid edge yields an OPEN polyline", () => { + const w = 16; + const f = new Float32Array(w * 16); + for (let y = 0; y < 16; y++) for (let x = 0; x < w; x++) f[y * w + x] = x / w; + const lines = joinContourSegments(marchingSquares(f, w, 16, 0.5)); + assert.equal(lines.length, 1); + const l = lines[0]!; + assert.notEqual(l[1], l[l.length - 1], "open: first y != last y"); + // The iso-line runs vertically at x = 8. + assert.ok(l.every((_, i) => i % 2 === 0 && Math.abs(l[i]! - 8) < 0.01 || i % 2 === 1)); + }); + + test("level above the peak: no segments; level at 0 on an empty field: none", () => { + const f = bump(10, 10, 5, 5, 4); + assert.equal(marchingSquares(f, 10, 10, 1.5).length, 0); + assert.equal(marchingSquares(new Float32Array(100), 10, 10, 0.5).length, 0); + }); + + test("saddle cases produce consistent, joinable segments", () => { + // A 2x2 field with tl+br inside (case 10) — must not crash, must + // yield segments whose endpoints all pair up. + const f = new Float32Array([1, 0, 0, 1]); + const segs = marchingSquares(f, 2, 2, 0.5); + assert.ok(segs.length >= 8, "two segments (4 numbers each)"); + const lines = joinContourSegments(segs); + for (const l of lines) assert.equal(l.length % 2, 0); + }); +}); + +describe("joinContourSegments", () => { + test("chains collinear segments in order, closed loops close", () => { + // A unit square contour: 4 segments. + const segs = [0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0]; + const lines = joinContourSegments(segs); + assert.equal(lines.length, 1); + const l = lines[0]!; + assert.equal(l[0], l[l.length - 2]); + assert.equal(l[1], l[l.length - 1]); + // All four corners present. + for (const [cx, cy] of [[0, 0], [1, 0], [1, 1], [0, 1]] as const) { + assert.ok(l.some((_, i) => i % 2 === 0 && l[i] === cx && l[i + 1] === cy), `corner ${cx},${cy}`); + } + }); + + test("disconnected segments stay separate polylines", () => { + const lines = joinContourSegments([0, 0, 1, 1, 5, 5, 6, 6]); + assert.equal(lines.length, 2); + }); +}); + +describe("projectCrop", () => { + test("pure-color pixels project to their alpha; blends split", () => { + // Paint color: pure red (255, 0, 0). Premultiplied bitmap (RGBA). + const pmW = 4; + const pm = new Float32Array(4 * 3 * 4); // 4x3 pixels + const set = (x: number, y: number, r: number, g: number, b: number, a: number): void => { + const i = (y * pmW + x) * 4; + pm[i] = r * a; + pm[i + 1] = g * a; + pm[i + 2] = b * a; + pm[i + 3] = a * 255; + }; + set(0, 0, 255, 0, 0, 0.4); // pure red at alpha 0.4 → 0.4 + set(1, 0, 255, 0, 0, 1.0); // opaque red → 1 + set(2, 0, 0, 255, 0, 0.8); // pure green → 0 (orthogonal) + const { field, peak } = projectCrop(pm, pmW, { x: 0, y: 0, w: 3, h: 1 }, [255, 0, 0]); + assert.ok(Math.abs(field[0]! - 0.4) < 1e-6); + assert.ok(Math.abs(field[1]! - 1) < 1e-6); + assert.ok(Math.abs(field[2]!) < 1e-6); + assert.ok(Math.abs(peak - 1) < 1e-6); + // A 50/50 red/green blend at alpha 1 splits into 0.5 each. + set(3, 0, 128, 128, 0, 1); + const { field: f2 } = projectCrop(pm, pmW, { x: 3, y: 0, w: 1, h: 1 }, [255, 0, 0]); + assert.ok(Math.abs(f2[0]! - 128 / 255) < 0.01, `blend splits (got ${f2[0]!.toFixed(3)})`); + }); + + test("stride 2: samples every other pixel, same values, quarter size", () => { + // 8x1 crop of pure red pixels at varying alpha. + const pmW = 8; + const pm = new Float32Array(8 * 4); + for (let x = 0; x < 8; x++) { + const i = x * 4; + pm[i] = 255 * (x / 14); // premultiplied alpha fraction + pm[i + 3] = 255; + } + const { field, peak } = projectCrop(pm, pmW, { x: 0, y: 0, w: 8, h: 1 }, [255, 0, 0], 2); + assert.equal(field.length, 4); // ceil(8/2) + assert.ok(Math.abs(field[0]! - 0) < 1e-6); + assert.ok(Math.abs(field[1]! - 2 / 14) < 1e-6); // sample gx=1 → pixel x=2 + assert.ok(Math.abs(field[2]! - 4 / 14) < 1e-6); // sample gx=2 → pixel x=4 + assert.ok(Math.abs(peak - 6 / 14) < 1e-6); // max over SAMPLED pixels (x=0,2,4,6) + }); + + test("zero paint color: empty field, peak 0", () => { + const { field, peak } = projectCrop(new Float32Array(16), 2, { x: 0, y: 0, w: 2, h: 2 }, [0, 0, 0]); + assert.equal(field.length, 4); + assert.equal(peak, 0); + }); +}); From 0af63113f59e48decfcca00e4a4bf779c375a542 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 16:23:21 +0300 Subject: [PATCH 28/34] Revert "Region boundaries: iso-line contours of the blurred region field" This reverts commit c7c1148989180b969ba9cab329c35878c14005d0. --- components/graph-canvas.ts | 113 +----------------- components/graph-view.tsx | 6 - components/region-contour.ts | 223 ----------------------------------- tests/region-contour.test.ts | 168 -------------------------- 4 files changed, 3 insertions(+), 507 deletions(-) delete mode 100644 components/region-contour.ts delete mode 100644 tests/region-contour.test.ts diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts index 7df06c1..b9bab57 100644 --- a/components/graph-canvas.ts +++ b/components/graph-canvas.ts @@ -54,7 +54,6 @@ import { select } from "d3-selection"; import { zoom, zoomIdentity, type ZoomBehavior, type ZoomTransform } from "d3-zoom"; import type { Simulation, SimulationLinkDatum } from "d3-force"; import { clusterHue, type Pt } from "./cluster-geometry.ts"; -import { hslToRgb, joinContourSegments, marchingSquares, projectCrop } from "./region-contour.ts"; import { CHARGE_STRENGTH, CLUSTER_PULL_STRENGTH, @@ -160,15 +159,6 @@ const COMMIT_DEBOUNCE_MS = 250; // settle → ledger PUT delay (new drags cancel * offset polygons re-rasterized at DPR 2 (the old GPU coil-whine). */ const REGION_EVERY = 3; // live-sim frames: rebuild every Nth, blit the rest const REGION_LABEL_SIZE = 14; -/** Region REBUILDS between boundary recomputes during the live sim: the - * boundary stage (projection + marching squares over ~26 crops) costs - * about as much as the whole fill pipeline, so it runs on every 2nd - * rebuild (~6th frame) and the stale paths ride along — at rest it - * always syncs with the fill. */ -const BOUNDARY_EVERY = 2; -/** Boundary stroke width, screen px at fit zoom (world units = this / kFit, - * so it zooms with the graph like every other stroke). */ -const REGION_BOUNDARY_WIDTH = 1.5; /** Dev-tunable region rendering (2026-09-13, user request): plain number * inputs in the settings panel — session-only, never persisted — so the @@ -188,10 +178,6 @@ export interface RegionTuning { bitmap: number; /** Per-stroke alpha; overlapping strokes stack toward the core. */ edgeAlpha: number; - /** Boundary iso-level as a FRACTION OF each cluster's peak field - * value (0.3 = the line where the blurred halo has faded to 30% of - * its densest point). 0 = no boundaries. */ - boundary: number; } export const DEFAULT_REGION_TUNING: RegionTuning = { @@ -199,7 +185,6 @@ export const DEFAULT_REGION_TUNING: RegionTuning = { blur: 10, bitmap: 384, edgeAlpha: 0.35, - boundary: 0.3, }; /** Clamp anything into a usable tuning (number inputs can be empty/NaN). */ @@ -212,7 +197,6 @@ export function sanitizeRegionTuning(input: Partial | null | undef blur: Math.min(Math.max(num(input?.blur, d.blur), 0), 40), bitmap: Math.round(Math.min(Math.max(num(input?.bitmap, d.bitmap), 128), 1024)), edgeAlpha: Math.min(Math.max(num(input?.edgeAlpha, d.edgeAlpha), 0.02), 1), - boundary: Math.min(Math.max(num(input?.boundary, d.boundary), 0), 0.95), }; } @@ -587,19 +571,11 @@ export class GraphCanvasRenderer { private regionRect: { minX: number; minY: number; w: number; h: number } | null = null; /** Region labels: cluster centroid + hue, computed at splat build. */ private regionMeta: { label: string | null; cx: number; cy: number; hue: number }[] = []; - /** Boundary paths (world-space Path2D per cluster), from the last - * region rebuild — stroked every frame under the camera transform. */ - private regionBoundaries: { path: Path2D; hue: number }[] = []; private regionTuning: RegionTuning = DEFAULT_REGION_TUNING; private physicsTuning: PhysicsTuning = DEFAULT_PHYSICS_TUNING; /** Gravity center for the live sim (graph bbox centroid at setData). */ private simCenter: Pt = { x: 0, y: 0 }; private regionBuiltVersion = -1; - /** geoVersion of the boundary paths (they lag one rebuild behind the - * region fill during the live sim — see BOUNDARY_EVERY). */ - private boundaryBuiltVersion = -1; - /** Rebuild counter for the boundary throttle (counts region rebuilds). */ - private regionRebuildCount = 0; private frameCount = 0; // draw counter for the live-sim splat throttle private labelLines = new Map(); private worldLabelFont = ""; @@ -1162,7 +1138,6 @@ export class GraphCanvasRenderer { this.regionCanvas = null; this.regionRect = null; this.regionMeta = []; - this.regionBoundaries = []; return; } const visible = this.visible; @@ -1172,32 +1147,23 @@ export class GraphCanvasRenderer { const s = 1 / this.kFit; // World rect over VISIBLE cluster members (regions cover members - // and their intra-cluster edges — nothing else), plus each cluster's - // OWN bbox for the boundary crops (the contour only needs to scan - // where a cluster's halo can reach). + // and their intra-cluster edges — nothing else). let minX = Infinity, minY = Infinity, maxX = -Infinity, maxY = -Infinity; - const boxes = data.clusters.map((c) => { - let a = Infinity, b = Infinity, e = -Infinity, f = -Infinity; + for (const c of data.clusters) { for (const id of c.members) { if (!isVisible(id)) continue; const n = this.nodeById.get(id); if (!n) continue; - if (n.x < a) a = n.x; - if (n.x > e) e = n.x; - if (n.y < b) b = n.y; - if (n.y > f) f = n.y; if (n.x < minX) minX = n.x; if (n.x > maxX) maxX = n.x; if (n.y < minY) minY = n.y; if (n.y > maxY) maxY = n.y; + } } - return a > e ? null : { minX: a, minY: b, maxX: e, maxY: f }; - }); if (minX > maxX) { this.regionCanvas = null; this.regionRect = null; this.regionMeta = []; - this.regionBoundaries = []; return; } @@ -1300,7 +1266,6 @@ export class GraphCanvasRenderer { // THE blur: 3 box-blur rounds on the premultiplied bitmap. Bilinear // upscaling alone is no blur at all (tent ramps — the JPEG look); // this makes the field genuinely smooth before the blit. - let blurred: Float32Array | null = null; // premultiplied field, for boundaries if (tuning.blur > 0) { const img = rctx.getImageData(0, 0, bw, bh); const n = bw * bh * 4; @@ -1334,65 +1299,6 @@ export class GraphCanvasRenderer { } } rctx.putImageData(img, 0, 0); - blurred = b; - } - - // ── Boundaries (DECIDED 2026-09-13) ────────────────────────────── - // Iso-lines of the blurred field at a FRACTION OF each cluster's - // peak. Per-cluster fields are recovered from the ONE shared - // bitmap by projecting crops of the blurred premultiplied array - // onto each cluster's paint color (pure-i pixels → their alpha; - // overlaps split proportionally — the natural divider between - // neighboring halos). One paint, one blur, one readback — the - // per-cluster work is cheap typed-array math over small crops. - // During the live sim this doubles the rebuild cost, so it runs on - // every BOUNDARY_EVERY-th region rebuild (at rest it always syncs; - // the stale paths just ride along for ~3 frames). - if (tuning.boundary <= 0 || !blurred) { - // Off (or no blurred field): never keep stale paths around. - this.regionBoundaries = []; - } else if (!this.simActive || this.regionRebuildCount % BOUNDARY_EVERY === 0) { - this.regionBoundaries = []; - for (let ci = 0; ci < data.clusters.length; ci++) { - const box = boxes[ci]; - if (!box) continue; - const cluster = data.clusters[ci]!; - // Crop in bitmap coords: the cluster bbox + the halo reach, - // clamped to the bitmap. - const cx0 = Math.max(0, Math.floor((box.minX - padWorld - minX) / cell)); - const cy0 = Math.max(0, Math.floor((box.minY - padWorld - minY) / cell)); - const cx1 = Math.min(bw - 1, Math.ceil((box.maxX + padWorld - minX) / cell)); - const cy1 = Math.min(bh - 1, Math.ceil((box.maxY + padWorld - minY) / cell)); - const cw = cx1 - cx0; - const ch = cy1 - cy0; - if (cw < 3 || ch < 3) continue; - const hue = clusterHue(cluster.id); - const rgb = hslToRgb(hue, 0.65, 0.6); - // Stride 2: sample every other bitmap pixel — the blurred field - // varies on a ~blur-radius scale, so the half-res grid loses - // nothing and the contour work quarters (measured ~1ms total). - const { field, peak } = projectCrop(blurred, bw, { x: cx0, y: cy0, w: cw, h: ch }, rgb, 2); - const level = tuning.boundary * peak; - if (peak <= 0 || level <= 0) continue; - const fw = Math.ceil(cw / 2); - const fh = Math.ceil(ch / 2); - const segments = marchingSquares(field, fw, fh, level); - if (segments.length === 0) continue; - // Grid coords (stride-2 samples) → world coords → one stroked - // path per cluster. Sample (gx, gy) is bitmap pixel - // (cx0 + gx*2, cy0 + gy*2). - const path = new Path2D(); - for (const line of joinContourSegments(segments)) { - if (line.length < 4) continue; - path.moveTo(minX + (cx0 + line[0]! * 2) * cell, minY + (cy0 + line[1]! * 2) * cell); - for (let i = 2; i < line.length; i += 2) { - path.lineTo(minX + (cx0 + line[i]! * 2) * cell, minY + (cy0 + line[i + 1]! * 2) * cell); - } - // Closed loop (first point == last point) → close the path. - if (line[0] === line[line.length - 2] && line[1] === line[line.length - 1]) path.closePath(); - } - this.regionBoundaries.push({ path, hue }); - } } this.regionRect = { minX, minY, w: spanX, h: spanY }; } @@ -1420,7 +1326,6 @@ export class GraphCanvasRenderer { this.regionBuiltVersion !== this.geoVersion && (!this.simActive || this.frameCount % REGION_EVERY === 0) ) { - this.regionRebuildCount++; this.buildRegions(); this.regionBuiltVersion = this.geoVersion; } @@ -1446,18 +1351,6 @@ export class GraphCanvasRenderer { ctx.drawImage(this.regionCanvas, minX, minY, w, h); } - // 2. Region boundaries: the cached iso-line paths, one stroke per - // cluster (hue-tinted, zoom-scaling width — see REGION_BOUNDARY_WIDTH). - if (this.regionBoundaries.length > 0) { - ctx.lineWidth = REGION_BOUNDARY_WIDTH * (1 / this.kFit); - ctx.lineJoin = "round"; - ctx.lineCap = "round"; - for (const rb of this.regionBoundaries) { - ctx.strokeStyle = `hsla(${rb.hue}, 65%, 68%, 0.55)`; - ctx.stroke(rb.path); - } - } - // 3. Edges: shafts + arrowheads from cached paths; width constant on // screen (÷ k), geometry in world units (scales with the graph). if (settings.edgeWidth > 0) { diff --git a/components/graph-view.tsx b/components/graph-view.tsx index 41a66cb..a139ddf 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -656,12 +656,6 @@ export function GraphView() { step={0.05} onChange={(v) => setRegionTuning((t) => ({ ...t, edgeAlpha: v }))} /> - setRegionTuning((t) => ({ ...t, boundary: v }))} - /> {/* Physics (user-facing): the live sim's forces. Every value feeds the SAME engine the server settle runs (createGraphSimulation); diff --git a/components/region-contour.ts b/components/region-contour.ts deleted file mode 100644 index 8294d3a..0000000 --- a/components/region-contour.ts +++ /dev/null @@ -1,223 +0,0 @@ -/** - * REGION BOUNDARIES (DECIDED 2026-09-13): iso-lines of the blurred - * region field — "draw a line along a constant luminance of the blurred - * background color". Each cluster's boundary is the contour of ITS OWN - * contribution to the field, recovered by projecting the blurred - * PREMULTIPLIED bitmap onto the cluster's paint color (a pixel painted - * only by cluster i projects to its alpha; a blend between two clusters - * splits proportionally — the projection is the natural divider where - * neighboring halos overlap). ONE paint pass, ONE blur, ONE readback; - * the per-cluster work is pure typed-array math over small crops. - * - * Pure module (no DOM, no Path2D) so it is unit-testable in node. - */ - -/** HSL → RGB (0-255 each), the same conversion the canvas performs for - * hsla() paint colors — the projection must divide by exactly what was - * painted. h in degrees, s and l in 0-1. */ -export function hslToRgb(h: number, s: number, l: number): [number, number, number] { - const c = (1 - Math.abs(2 * l - 1)) * s; - const hp = (((h % 360) + 360) % 360) / 60; - const x = c * (1 - Math.abs((hp % 2) - 1)); - let r = 0; - let g = 0; - let b = 0; - if (hp < 1) [r, g, b] = [c, x, 0]; - else if (hp < 2) [r, g, b] = [x, c, 0]; - else if (hp < 3) [r, g, b] = [0, c, x]; - else if (hp < 4) [r, g, b] = [0, x, c]; - else if (hp < 5) [r, g, b] = [x, 0, c]; - else [r, g, b] = [c, 0, x]; - const m = l - c / 2; - return [Math.round((r + m) * 255), Math.round((g + m) * 255), Math.round((b + m) * 255)]; -} - -/** - * Project a crop of the blurred premultiplied bitmap onto a paint color: - * field(p) = ⟨pm(p), rgb⟩ / |rgb|² — the fraction of pixel p's - * premultiplied color that lies along the cluster's paint direction. - * Pure-i pixels → their alpha fraction (0-1); blends split between the - * contributors. Returns the crop field and its peak (for the relative - * threshold). - * - * `stride` samples every Nth bitmap pixel (default 1): the blurred field - * varies on a ~blur-radius scale, so a stride-2 grid loses nothing and - * quarters the contour work (2026-09-13: stride-1 doubled the rebuild - * cost and broke the 120fps drag cadence every 6th frame; stride-2 - * contouring is measured ~1ms for the whole boundary stage). - */ -export function projectCrop( - pm: Float32Array, - pmW: number, - crop: { x: number; y: number; w: number; h: number }, - rgb: readonly [number, number, number], - stride = 1, -): { field: Float32Array; peak: number } { - const [r, g, b] = rgb; - const denom = r * r + g * g + b * b; - const fw = Math.ceil(crop.w / stride); - const fh = Math.ceil(crop.h / stride); - if (denom <= 0) return { field: new Float32Array(fw * fh), peak: 0 }; - const field = new Float32Array(fw * fh); - let peak = 0; - for (let y = 0; y < fh; y++) { - const src = ((crop.y + y * stride) * pmW + crop.x) * 4; - const dst = y * fw; - for (let x = 0; x < fw; x++) { - const i = src + x * stride * 4; - const v = (pm[i]! * r + pm[i + 1]! * g + pm[i + 2]! * b) / denom; - field[dst + x] = v; - if (v > peak) peak = v; - } - } - return { field, peak }; -} - -/** - * Marching squares over a scalar field: emits iso-line SEGMENTS (grid - * coordinates, linear interpolation between samples) where the field - * crosses `level`. Flat output [x1,y1,x2,y2, ...] — join with - * joinContourSegments. Saddle cases (5, 10) are disambiguated by the - * cell-center average. Grid coords: sample (x, y) is field[y*w + x]; - * a segment point may be fractional (between samples). - */ -export function marchingSquares( - field: Float32Array, - w: number, - h: number, - level: number, -): number[] { - // Hot loop (measured ~7ms over ~210k cells when the crossing helpers - // were per-cell closures — allocation dominated). All crossings are - // computed inline; zero allocations per empty cell. - const segments: number[] = []; - for (let y = 0; y < h - 1; y++) { - const row = y * w; - const rowBelow = row + w; - for (let x = 0; x < w - 1; x++) { - const tl = field[row + x]!; - const tr = field[row + x + 1]!; - const br = field[rowBelow + x + 1]!; - const bl = field[rowBelow + x]!; - const idx = - (tl >= level ? 8 : 0) | (tr >= level ? 4 : 0) | (br >= level ? 2 : 0) | (bl >= level ? 1 : 0); - if (idx === 0 || idx === 15) continue; - // Crossing points on the four cell edges (grid coords, - // linear interpolation between the two corner samples). - const topX = x + (level - tl) / (tr - tl); - const rightY = y + (level - tr) / (br - tr); - const bottomX = x + (level - bl) / (br - bl); - const leftY = y + (level - tl) / (bl - tl); - switch (idx) { - case 1: - case 14: - segments.push(x, leftY, bottomX, y + 1); - break; - case 2: - case 13: - segments.push(bottomX, y + 1, x + 1, rightY); - break; - case 3: - case 12: - segments.push(x, leftY, x + 1, rightY); - break; - case 4: - case 11: - segments.push(topX, y, x + 1, rightY); - break; - case 6: - case 9: - segments.push(topX, y, bottomX, y + 1); - break; - case 7: - case 8: - segments.push(topX, y, x, leftY); - break; - case 5: - // Saddle: tr+bl inside. Center decides connected vs split. - if ((tl + tr + br + bl) / 4 >= level) { - segments.push(topX, y, x, leftY, bottomX, y + 1, x + 1, rightY); - } else { - segments.push(topX, y, x + 1, rightY, x, leftY, bottomX, y + 1); - } - break; - case 10: - // Saddle: tl+br inside. - if ((tl + tr + br + bl) / 4 >= level) { - segments.push(topX, y, x + 1, rightY, x, leftY, bottomX, y + 1); - } else { - segments.push(topX, y, x, leftY, bottomX, y + 1, x + 1, rightY); - } - break; - } - } - } - return segments; -} - -/** - * Join iso-line segments into POLYLINES: chains of connected segments, - * closed loops where the contour is closed (first point == last point). - * Endpoints are quantized — marching-squares crossings on a shared cell - * edge are bit-identical from both cells, so any deterministic - * quantization matches them exactly. Output: one flat [x,y,x,y,...] array - * per polyline. - */ -export function joinContourSegments(segments: readonly number[]): number[][] { - const key = (x: number, y: number): string => `${Math.round(x * 1024)},${Math.round(y * 1024)}`; - // point key -> [{seg index, which end}] - const at = new Map>(); - const n = segments.length / 4; - for (let s = 0; s < n; s++) { - const x1 = segments[s * 4]!; - const y1 = segments[s * 4 + 1]!; - const x2 = segments[s * 4 + 2]!; - const y2 = segments[s * 4 + 3]!; - const k1 = key(x1, y1); - const k2 = key(x2, y2); - let l1 = at.get(k1); - if (!l1) at.set(k1, (l1 = [])); - l1.push({ s, end: 0 }); - let l2 = at.get(k2); - if (!l2) at.set(k2, (l2 = [])); - l2.push({ s, end: 1 }); - } - const used = new Array(n).fill(false); - const polylines: number[][] = []; - const takeNeighbor = (x: number, y: number, seg: number): { s: number; x: number; y: number } | null => { - const list = at.get(key(x, y)); - if (!list) return null; - for (const e of list) { - if (used[e.s]) continue; - used[e.s] = true; - return { - s: e.s, - x: segments[e.s * 4 + (e.end === 0 ? 2 : 0)]!, - y: segments[e.s * 4 + (e.end === 0 ? 3 : 1)]!, - }; - } - return null; - }; - for (let s0 = 0; s0 < n; s0++) { - if (used[s0]) continue; - used[s0] = true; - const line = [segments[s0 * 4]!, segments[s0 * 4 + 1]!, segments[s0 * 4 + 2]!, segments[s0 * 4 + 3]!]; - // Extend at the tail, then at the head. - for (let guard = 0; guard < 2 * n + 4; guard++) { - const tx = line[line.length - 2]!; - const ty = line[line.length - 1]!; - const nb = takeNeighbor(tx, ty, -1); - if (!nb) break; - line.push(nb.x, nb.y); - } - for (let guard = 0; guard < 2 * n + 4; guard++) { - const hx = line[0]!; - const hy = line[1]!; - const nb = takeNeighbor(hx, hy, -1); - if (!nb) break; - line.unshift(nb.x, nb.y); - } - polylines.push(line); - } - return polylines; -} diff --git a/tests/region-contour.test.ts b/tests/region-contour.test.ts deleted file mode 100644 index 13ee2de..0000000 --- a/tests/region-contour.test.ts +++ /dev/null @@ -1,168 +0,0 @@ -/** - * Region boundary primitives (components/region-contour.ts): marching - * squares, segment joining, and the color projection that recovers each - * cluster's own field from the shared blurred bitmap. Pure typed-array - * math — exact, deterministic tests. - */ -import { strict as assert } from "node:assert"; -import { describe, test } from "node:test"; -import { hslToRgb, joinContourSegments, marchingSquares, projectCrop } from "../components/region-contour.ts"; - -describe("hslToRgb", () => { - test("matches the canvas hsla() conversion at the region paint points", () => { - // hsla(h, 65%, 60%) — the exact paint colors buildRegions uses. - assert.deepEqual(hslToRgb(0, 0.65, 0.6), [219, 87, 87]); // red-ish - assert.deepEqual(hslToRgb(120, 0.65, 0.6), [87, 219, 87]); // green-ish - assert.deepEqual(hslToRgb(240, 0.65, 0.6), [87, 87, 219]); // blue-ish - // Gray: zero saturation. - assert.deepEqual(hslToRgb(37, 0, 0.6), [153, 153, 153]); - // Hue wraps. - assert.deepEqual(hslToRgb(360, 0.65, 0.6), hslToRgb(0, 0.65, 0.6)); - }); -}); - -describe("marchingSquares", () => { - /** A smooth radial bump on a w×h grid. */ - function bump(w: number, h: number, cx: number, cy: number, radius: number): Float32Array { - const f = new Float32Array(w * h); - for (let y = 0; y < h; y++) { - for (let x = 0; x < w; x++) { - const d = Math.hypot(x - cx, y - cy); - f[y * w + x] = Math.max(0, 1 - d / radius); - } - } - return f; - } - - test("a single blob yields one closed loop", () => { - // Irrational center offsets: no grid sample sits EXACTLY on the - // iso-line (a tie makes a crossing land on a sample point, which - // can split the loop into two stroked halves — visually identical, - // but this test wants the clean case). - const w = 21; - const f = bump(w, 21, 10.4, 9.7, 8); - const segs = marchingSquares(f, w, 21, 0.5); - assert.ok(segs.length >= 16, "a circle has plenty of segments"); - const lines = joinContourSegments(segs); - assert.equal(lines.length, 1, "exactly one loop"); - const l = lines[0]!; - // Closed: first point == last point. - assert.equal(l[0], l[l.length - 2]); - assert.equal(l[1], l[l.length - 1]); - // Roughly the right radius: the iso-line of 1 - d/r = 0.5 is d = r/2 = 4. - const meanR = - l.filter((_, i) => i % 2 === 0 && i < l.length - 2).length === 0 - ? 0 - : l.reduce((acc, _, i) => (i % 4 === 0 && i + 1 < l.length ? Math.hypot(l[i]! - 10, l[i + 1]! - 10) + acc : acc), 0) / - (l.filter((_, i) => i % 4 === 0 && i + 1 < l.length).length || 1); - assert.ok(Math.abs(meanR - 4) < 1.5, `iso-line radius ≈ 4 (got ${meanR.toFixed(2)})`); - }); - - test("two separated blobs yield two loops", () => { - const w = 40; - const f = bump(w, 21, 10, 10, 7); - const g = bump(w, 21, 30, 10, 7); - const both = f.map((v, i) => Math.max(v, g[i]!)); - const lines = joinContourSegments(marchingSquares(both, w, 21, 0.5)); - assert.equal(lines.length, 2); - }); - - test("a ramp clipped by the grid edge yields an OPEN polyline", () => { - const w = 16; - const f = new Float32Array(w * 16); - for (let y = 0; y < 16; y++) for (let x = 0; x < w; x++) f[y * w + x] = x / w; - const lines = joinContourSegments(marchingSquares(f, w, 16, 0.5)); - assert.equal(lines.length, 1); - const l = lines[0]!; - assert.notEqual(l[1], l[l.length - 1], "open: first y != last y"); - // The iso-line runs vertically at x = 8. - assert.ok(l.every((_, i) => i % 2 === 0 && Math.abs(l[i]! - 8) < 0.01 || i % 2 === 1)); - }); - - test("level above the peak: no segments; level at 0 on an empty field: none", () => { - const f = bump(10, 10, 5, 5, 4); - assert.equal(marchingSquares(f, 10, 10, 1.5).length, 0); - assert.equal(marchingSquares(new Float32Array(100), 10, 10, 0.5).length, 0); - }); - - test("saddle cases produce consistent, joinable segments", () => { - // A 2x2 field with tl+br inside (case 10) — must not crash, must - // yield segments whose endpoints all pair up. - const f = new Float32Array([1, 0, 0, 1]); - const segs = marchingSquares(f, 2, 2, 0.5); - assert.ok(segs.length >= 8, "two segments (4 numbers each)"); - const lines = joinContourSegments(segs); - for (const l of lines) assert.equal(l.length % 2, 0); - }); -}); - -describe("joinContourSegments", () => { - test("chains collinear segments in order, closed loops close", () => { - // A unit square contour: 4 segments. - const segs = [0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0]; - const lines = joinContourSegments(segs); - assert.equal(lines.length, 1); - const l = lines[0]!; - assert.equal(l[0], l[l.length - 2]); - assert.equal(l[1], l[l.length - 1]); - // All four corners present. - for (const [cx, cy] of [[0, 0], [1, 0], [1, 1], [0, 1]] as const) { - assert.ok(l.some((_, i) => i % 2 === 0 && l[i] === cx && l[i + 1] === cy), `corner ${cx},${cy}`); - } - }); - - test("disconnected segments stay separate polylines", () => { - const lines = joinContourSegments([0, 0, 1, 1, 5, 5, 6, 6]); - assert.equal(lines.length, 2); - }); -}); - -describe("projectCrop", () => { - test("pure-color pixels project to their alpha; blends split", () => { - // Paint color: pure red (255, 0, 0). Premultiplied bitmap (RGBA). - const pmW = 4; - const pm = new Float32Array(4 * 3 * 4); // 4x3 pixels - const set = (x: number, y: number, r: number, g: number, b: number, a: number): void => { - const i = (y * pmW + x) * 4; - pm[i] = r * a; - pm[i + 1] = g * a; - pm[i + 2] = b * a; - pm[i + 3] = a * 255; - }; - set(0, 0, 255, 0, 0, 0.4); // pure red at alpha 0.4 → 0.4 - set(1, 0, 255, 0, 0, 1.0); // opaque red → 1 - set(2, 0, 0, 255, 0, 0.8); // pure green → 0 (orthogonal) - const { field, peak } = projectCrop(pm, pmW, { x: 0, y: 0, w: 3, h: 1 }, [255, 0, 0]); - assert.ok(Math.abs(field[0]! - 0.4) < 1e-6); - assert.ok(Math.abs(field[1]! - 1) < 1e-6); - assert.ok(Math.abs(field[2]!) < 1e-6); - assert.ok(Math.abs(peak - 1) < 1e-6); - // A 50/50 red/green blend at alpha 1 splits into 0.5 each. - set(3, 0, 128, 128, 0, 1); - const { field: f2 } = projectCrop(pm, pmW, { x: 3, y: 0, w: 1, h: 1 }, [255, 0, 0]); - assert.ok(Math.abs(f2[0]! - 128 / 255) < 0.01, `blend splits (got ${f2[0]!.toFixed(3)})`); - }); - - test("stride 2: samples every other pixel, same values, quarter size", () => { - // 8x1 crop of pure red pixels at varying alpha. - const pmW = 8; - const pm = new Float32Array(8 * 4); - for (let x = 0; x < 8; x++) { - const i = x * 4; - pm[i] = 255 * (x / 14); // premultiplied alpha fraction - pm[i + 3] = 255; - } - const { field, peak } = projectCrop(pm, pmW, { x: 0, y: 0, w: 8, h: 1 }, [255, 0, 0], 2); - assert.equal(field.length, 4); // ceil(8/2) - assert.ok(Math.abs(field[0]! - 0) < 1e-6); - assert.ok(Math.abs(field[1]! - 2 / 14) < 1e-6); // sample gx=1 → pixel x=2 - assert.ok(Math.abs(field[2]! - 4 / 14) < 1e-6); // sample gx=2 → pixel x=4 - assert.ok(Math.abs(peak - 6 / 14) < 1e-6); // max over SAMPLED pixels (x=0,2,4,6) - }); - - test("zero paint color: empty field, peak 0", () => { - const { field, peak } = projectCrop(new Float32Array(16), 2, { x: 0, y: 0, w: 2, h: 2 }, [0, 0, 0]); - assert.equal(field.length, 4); - assert.equal(peak, 0); - }); -}); From 04a0236c368a168f1226e89295ffbe4109cde978 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sun, 13 Sep 2026 17:06:18 +0300 Subject: [PATCH 29/34] User-facing persisted region + physics settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All graph panel controls are now user-facing and server-persisted: the old session-only dev number inputs are gone. GraphRenderSettings gains nested regions/physics sections (backward compatible: old flat files load with defaults), and RegionTuning/PhysicsTuning moved from the renderer into core/render-settings.ts — one shape, one sanitizer, one source for the store, the panel, and the renderer. Physics/region clamps are sweep-validated safe ranges (each field at both ends + all four corners settles cleanly on the real graph: no NaN, no explosion — linkStrength caps at 4 since 8 explodes and 5 wobbles; friction floors at 0.1 since 0 never damps). Panel: log sliders (default = geometric mean = position 50, step 5), off-zone sliders for blur/cluster pull (position 0 = exactly 0), a fixed-set dropdown for bitmap resolution, depth-anchor modes promoted to the select. --- components/graph-canvas.ts | 109 +----------- components/graph-view.tsx | 323 +++++++++++++++++++++------------- src/core/render-settings.ts | 196 +++++++++++++++++++-- tests/graph-canvas.test.ts | 21 ++- tests/render-settings.test.ts | 90 +++++++++- 5 files changed, 492 insertions(+), 247 deletions(-) diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts index b9bab57..dfabc38 100644 --- a/components/graph-canvas.ts +++ b/components/graph-canvas.ts @@ -58,6 +58,12 @@ import { CHARGE_STRENGTH, CLUSTER_PULL_STRENGTH, createGraphSimulation, + DEFAULT_PHYSICS_TUNING, + DEFAULT_REGION_TUNING, + sanitizePhysicsTuning, + sanitizeRegionTuning, + type PhysicsTuning, + type RegionTuning, INCREMENTAL_GRAVITY, LINK_DISTANCE, type DepthAnchorMode, @@ -160,109 +166,6 @@ const COMMIT_DEBOUNCE_MS = 250; // settle → ledger PUT delay (new drags cancel const REGION_EVERY = 3; // live-sim frames: rebuild every Nth, blit the rest const REGION_LABEL_SIZE = 14; -/** Dev-tunable region rendering (2026-09-13, user request): plain number - * inputs in the settings panel — session-only, never persisted — so the - * look can be tuned live against the real graph. A cluster is its - * CONNECTED SKELETON: the region is the dilated line network ONLY (no - * per-node discs — they double-counted where nodes sit on edges and - * made hubs hot). Blur is REAL: a premultiplied 3-pass box blur on the - * bitmap (bilinear upscale alone was no blur at all — zoom-in showed - * tent-ramp chunkiness, the "low-quality JPEG" look). */ -export interface RegionTuning { - /** Dilation half-width of the round-cap edge strokes, screen px at - * fit zoom (the expansion of the cluster skeleton). */ - dilation: number; - /** Box-blur radius in bitmap px (3 passes ≈ Gaussian). 0 = off. */ - blur: number; - /** Max bitmap dimension (cell = worldSpan / this). */ - bitmap: number; - /** Per-stroke alpha; overlapping strokes stack toward the core. */ - edgeAlpha: number; -} - -export const DEFAULT_REGION_TUNING: RegionTuning = { - dilation: 28, - blur: 10, - bitmap: 384, - edgeAlpha: 0.35, -}; - -/** Clamp anything into a usable tuning (number inputs can be empty/NaN). */ -export function sanitizeRegionTuning(input: Partial | null | undefined): RegionTuning { - const d = DEFAULT_REGION_TUNING; - const num = (v: unknown, fallback: number): number => - typeof v === "number" && Number.isFinite(v) ? v : fallback; - return { - dilation: Math.min(Math.max(num(input?.dilation, d.dilation), 4), 250), - blur: Math.min(Math.max(num(input?.blur, d.blur), 0), 40), - bitmap: Math.round(Math.min(Math.max(num(input?.bitmap, d.bitmap), 128), 1024)), - edgeAlpha: Math.min(Math.max(num(input?.edgeAlpha, d.edgeAlpha), 0.02), 1), - }; -} - -/** User-facing live-sim physics controls (2026-09-13, user request): - * every value feeds createGraphSimulation — the SAME engine the server - * settle runs — so tuning the feel changes exactly what a drag relaxes - * toward. Session state, never persisted. The cluster pull is the - * "virtual node" realized as a centroid force (O(n)/tick, measured - * indistinguishable from off). */ -export interface PhysicsTuning { - /** Node repulsion (d3 charge, negative = push apart). */ - chargeStrength: number; - /** Edge attraction: link-spring stiffness multiplier (1 = d3 default). */ - linkStrength: number; - /** Link spring rest length (world units). */ - linkDistance: number; - /** Central pull toward the graph centroid. */ - gravity: number; - /** Per-cluster pull toward the cluster centroid. */ - clusterPull: number; - /** Per-tick velocity damping (d3 default 0.4). */ - velocityDecay: number; - /** Inference-depth anchoring mode (see DepthAnchor in core). */ - depthMode: DepthAnchorMode; - /** Depth-anchor pull strength. */ - depthStrength: number; - /** Spacing between inference levels (world units). */ - depthSpacing: number; - /** Inner ring radius, radial mode (world units). */ - depthBaseRadius: number; -} - -export const DEFAULT_PHYSICS_TUNING: PhysicsTuning = { - chargeStrength: CHARGE_STRENGTH, - linkStrength: 1, - linkDistance: LINK_DISTANCE, - gravity: INCREMENTAL_GRAVITY, - clusterPull: CLUSTER_PULL_STRENGTH, - velocityDecay: 0.4, - depthMode: "off", - // 0.8 by sweep on the real graph: 215/216 arrows read upward with - // organic sag (mean |dev| from level ~34u at spacing 60); 1.5 gives - // crisp rows (dev ~18) if the promoted default wants them. - depthStrength: 0.8, - depthSpacing: 60, - depthBaseRadius: 220, -}; - -/** Clamp anything into a usable tuning (number inputs can be empty/NaN). */ -export function sanitizePhysicsTuning(input: Partial | null | undefined): PhysicsTuning { - const d = DEFAULT_PHYSICS_TUNING; - const num = (v: unknown, fallback: number): number => - typeof v === "number" && Number.isFinite(v) ? v : fallback; - return { - chargeStrength: Math.min(Math.max(num(input?.chargeStrength, d.chargeStrength), -1000), -10), - linkStrength: Math.min(Math.max(num(input?.linkStrength, d.linkStrength), 0), 5), - linkDistance: Math.min(Math.max(num(input?.linkDistance, d.linkDistance), 1), 300), - gravity: Math.min(Math.max(num(input?.gravity, d.gravity), 0), 1), - clusterPull: Math.min(Math.max(num(input?.clusterPull, d.clusterPull), 0), 2), - velocityDecay: Math.min(Math.max(num(input?.velocityDecay, d.velocityDecay), 0), 0.99), - depthMode: input?.depthMode === "vertical" || input?.depthMode === "radial" ? input.depthMode : "off", - depthStrength: Math.min(Math.max(num(input?.depthStrength, d.depthStrength), 0), 2), - depthSpacing: Math.min(Math.max(num(input?.depthSpacing, d.depthSpacing), 1), 500), - depthBaseRadius: Math.min(Math.max(num(input?.depthBaseRadius, d.depthBaseRadius), 0), 2000), - }; -} /** One horizontal+vertical box-blur round over PREMULTIPLIED RGBA * (Float32, stride 4, length w*h*4). Premultiplying first is what keeps diff --git a/components/graph-view.tsx b/components/graph-view.tsx index a139ddf..60c3ff3 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -65,25 +65,33 @@ import { useDeferredValue, useEffect, useLayoutEffect, useMemo, useRef, useState import Link from "next/link"; import { FileTextIcon, Loader2Icon, SearchIcon, SettingsIcon, UploadIcon, WaypointsIcon } from "lucide-react"; import { incrementalLayout, type GraphNode, type GraphSnapshot } from "@/src/core"; -import { - DEFAULT_PHYSICS_TUNING, - DEFAULT_REGION_TUNING, - GraphCanvasRenderer, - type GraphRenderSettings, - type PhysicsTuning, - type RegionTuning, -} from "./graph-canvas.ts"; +import { GraphCanvasRenderer, type GraphRenderSettings } from "./graph-canvas.ts"; import { ARROW_LENGTH_RANGE, + CHARGE_RANGE, + CLUSTER_PULL_RANGE, + DEPTH_BASE_RADIUS_RANGE, + DEPTH_SPACING_RANGE, + DEPTH_STRENGTH_RANGE, DEFAULT_RENDER_SETTINGS, EDGE_WIDTH_RANGE, + GRAVITY_RANGE, LABEL_SIZE_RANGE, LABEL_ZOOM_MAX, LABEL_ZOOM_MIN, + LINK_DISTANCE_RANGE, + LINK_STRENGTH_RANGE, NODE_SIZE_RANGE, + REGION_BITMAP_OPTIONS, + REGION_BLUR_RANGE, + REGION_DILATION_RANGE, + REGION_EDGE_ALPHA_RANGE, + VELOCITY_DECAY_RANGE, + fromOffSliderPosition, fromSliderPosition, labelZoomDisplay, sanitizeRenderSettings, + toOffSliderPosition, toSliderPosition, } from "@/src/core"; @@ -149,33 +157,6 @@ function sizeDisplay(v: number): string { return `${v >= 1 ? Math.round(v * 10) / 10 : Math.round(v * 100) / 100}`; } -/** Dev number input: raw value in, raw value out (NaN while empty → the - * renderer's sanitizer holds the last sane tuning). Development tool. */ -function NumberField({ - label, - value, - step, - onChange, -}: { - label: string; - value: number; - step: number; - onChange: (v: number) => void; -}) { - return ( - - ); -} - function SettingSlider({ label, min, @@ -227,19 +208,16 @@ export function GraphView() { // Search panel state: the query text + the settings section toggle. const [query, setQuery] = useState(""); const [settingsOpen, setSettingsOpen] = useState(false); - // Region tuning: DEV-ONLY, session state, never persisted (unlike the - // render settings, which live on the server). Number inputs so any - // value can be tried quickly. - const [regionTuning, setRegionTuning] = useState(DEFAULT_REGION_TUNING); - // Live-sim physics controls: session state, never persisted. Changes - // rebuild the renderer's sim from the current node positions — the - // same engine the server settle runs, so the feel IS the physics. - const [physicsTuning, setPhysicsTuning] = useState(DEFAULT_PHYSICS_TUNING); // Render settings: SERVER-persisted (data/render-settings.json via - // /api/graph/render-settings). Loaded once after mount; every user - // change saves back, debounced. Defaults render until the load - // resolves — a fraction of a second at worst. - const [settings, setSettings] = useState(DEFAULT_RENDER_SETTINGS); + // /api/graph/render-settings) — sizes AND the region/physics sections + // (user-facing since 2026-09-13; the dev-session era is over). Loaded + // once after mount; every user change saves back, debounced. Defaults + // render until the load resolves — a fraction of a second at worst. + const [settings, setSettings] = useState(DEFAULT_RENDER_SETTINGS); + // The two nested sections, unpacked for the renderer feed effects + // below (same names the old session state had). + const regionTuning = settings.regions; + const physicsTuning = settings.physics; const settingsRef = useRef(settings); settingsRef.current = settings; // True once the user has moved a control this mount: the server copy @@ -625,42 +603,82 @@ export function GraphView() { className="accent-primary" /> - {/* Regions (dev): quick-tuning number inputs. Session state - only - never persisted. Bad/empty values fall back to - the last sane tuning inside the renderer. */} + {/* Regions: the blurred cluster tints. Sliders + a fixed-set + dropdown for the bitmap resolution (it's a discrete + choice, not a free number). Server-persisted like the + sizes; every range end was sweep-validated on the real + graph. */}
- Regions (dev) + Regions - setRegionTuning((t) => ({ ...t, dilation: v }))} + value={toSliderPosition(regionTuning.dilation, REGION_DILATION_RANGE.min, REGION_DILATION_RANGE.max)} + display={sizeDisplay(regionTuning.dilation)} + onChange={(p) => + changeSettings((cur) => ({ + ...cur, + regions: { ...cur.regions, dilation: fromSliderPosition(p, REGION_DILATION_RANGE.min, REGION_DILATION_RANGE.max) }, + }))} /> - setRegionTuning((t) => ({ ...t, blur: v }))} + + changeSettings((cur) => ({ + ...cur, + regions: { ...cur.regions, blur: fromOffSliderPosition(p, REGION_BLUR_RANGE.min, REGION_BLUR_RANGE.max) }, + }))} /> - setRegionTuning((t) => ({ ...t, bitmap: v }))} - /> - setRegionTuning((t) => ({ ...t, edgeAlpha: v }))} + min={0} + max={100} + step={5} + value={toSliderPosition(regionTuning.edgeAlpha, REGION_EDGE_ALPHA_RANGE.min, REGION_EDGE_ALPHA_RANGE.max)} + display={sizeDisplay(regionTuning.edgeAlpha)} + onChange={(p) => + changeSettings((cur) => ({ + ...cur, + regions: { ...cur.regions, edgeAlpha: fromSliderPosition(p, REGION_EDGE_ALPHA_RANGE.min, REGION_EDGE_ALPHA_RANGE.max) }, + }))} /> +
- {/* Physics (user-facing): the live sim's forces. Every value feeds - the SAME engine the server settle runs (createGraphSimulation); + {/* Physics: the live sim's forces. Every value feeds the SAME + engine the server settle runs (createGraphSimulation); changing one rebuilds the sim from the current positions — - drag mid-settle re-anneals, pins survive. Session-only. */} + drag mid-settle re-anneals, pins survive. Sliders over + sweep-validated ranges (src/core/render-settings.ts): + range ends and all four corners settle cleanly — no NaN, + no explosion, no non-settle. Repulsion is displayed as + the negative it is; cluster pull's far left = off. */}
Physics @@ -671,9 +689,9 @@ export function GraphView() { className="rounded border bg-background px-1 py-0.5 text-xs" value={physicsTuning.depthMode} onChange={(e) => - setPhysicsTuning((t) => ({ - ...t, - depthMode: e.target.value as PhysicsTuning["depthMode"], + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, depthMode: e.target.value as GraphRenderSettings["physics"]["depthMode"] }, })) } > @@ -682,59 +700,122 @@ export function GraphView() { - setPhysicsTuning((t) => ({ ...t, depthStrength: v }))} - /> - setPhysicsTuning((t) => ({ ...t, depthSpacing: v }))} - /> - setPhysicsTuning((t) => ({ ...t, depthBaseRadius: v }))} - /> - setPhysicsTuning((t) => ({ ...t, chargeStrength: v }))} - /> - setPhysicsTuning((t) => ({ ...t, linkStrength: v }))} - /> - setPhysicsTuning((t) => ({ ...t, linkDistance: v }))} + value={toSliderPosition(physicsTuning.depthStrength, DEPTH_STRENGTH_RANGE.min, DEPTH_STRENGTH_RANGE.max)} + display={sizeDisplay(physicsTuning.depthStrength)} + onChange={(p) => + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, depthStrength: fromSliderPosition(p, DEPTH_STRENGTH_RANGE.min, DEPTH_STRENGTH_RANGE.max) }, + }))} /> - + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, depthSpacing: fromSliderPosition(p, DEPTH_SPACING_RANGE.min, DEPTH_SPACING_RANGE.max) }, + }))} + /> + + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, depthBaseRadius: fromSliderPosition(p, DEPTH_BASE_RADIUS_RANGE.min, DEPTH_BASE_RADIUS_RANGE.max) }, + }))} + /> + + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, chargeStrength: -fromSliderPosition(p, -CHARGE_RANGE.max, -CHARGE_RANGE.min) }, + }))} + /> + + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, linkStrength: fromSliderPosition(p, LINK_STRENGTH_RANGE.min, LINK_STRENGTH_RANGE.max) }, + }))} + /> + + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, linkDistance: fromSliderPosition(p, LINK_DISTANCE_RANGE.min, LINK_DISTANCE_RANGE.max) }, + }))} + /> + setPhysicsTuning((t) => ({ ...t, gravity: v }))} + min={0} + max={100} + step={5} + value={toSliderPosition(physicsTuning.gravity, GRAVITY_RANGE.min, GRAVITY_RANGE.max)} + display={sizeDisplay(physicsTuning.gravity)} + onChange={(p) => + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, gravity: fromSliderPosition(p, GRAVITY_RANGE.min, GRAVITY_RANGE.max) }, + }))} /> - setPhysicsTuning((t) => ({ ...t, clusterPull: v }))} + min={0} + max={100} + step={5} + value={toOffSliderPosition(physicsTuning.clusterPull, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max)} + display={physicsTuning.clusterPull <= 0 ? "off" : sizeDisplay(physicsTuning.clusterPull)} + onChange={(p) => + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, clusterPull: fromOffSliderPosition(p, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max) }, + }))} /> - setPhysicsTuning((t) => ({ ...t, velocityDecay: v }))} + min={0} + max={100} + step={5} + value={toSliderPosition(physicsTuning.velocityDecay, VELOCITY_DECAY_RANGE.min, VELOCITY_DECAY_RANGE.max)} + display={sizeDisplay(physicsTuning.velocityDecay)} + onChange={(p) => + changeSettings((cur) => ({ + ...cur, + physics: { ...cur.physics, velocityDecay: fromSliderPosition(p, VELOCITY_DECAY_RANGE.min, VELOCITY_DECAY_RANGE.max) }, + }))} />
diff --git a/src/core/render-settings.ts b/src/core/render-settings.ts index edfb1ca..4f653a9 100644 --- a/src/core/render-settings.ts +++ b/src/core/render-settings.ts @@ -16,6 +16,14 @@ * Persisted values keep these units across the renderer swap. */ +import { + CHARGE_STRENGTH, + CLUSTER_PULL_STRENGTH, + INCREMENTAL_GRAVITY, + LINK_DISTANCE, + type DepthAnchorMode, +} from "./layout.ts"; + export interface GraphRenderSettings { nodeSize: number; edgeWidth: number; @@ -26,21 +34,14 @@ export interface GraphRenderSettings { labelSize: number; /** Toggle for the cluster region labels (hulls always draw). */ clusterLabels: boolean; + /** Region rendering (the blurred cluster tints). Server-persisted + * since 2026-09-13 (user request) — was session-only dev state. */ + regions: RegionTuning; + /** Live-sim physics + inference-depth anchoring. Server-persisted + * since 2026-09-13 (user request) — was session-only dev state. */ + physics: PhysicsTuning; } -// Sizes are screen px AT FIT ZOOM (the world-space renderer scales them -// with the camera, so a label reads at labelSize x k/kFit on screen). -// Defaults were re-tuned for world-scaling on 2026-09-13: the old values -// were calibrated in the screen-fixed era, so everything rendered ~2x -// too big once sizes started scaling with the graph. -export const DEFAULT_RENDER_SETTINGS: GraphRenderSettings = { - nodeSize: 2, - edgeWidth: 1, - arrowLength: 2, - labelZoom: 0.5, - labelSize: 2, - clusterLabels: true, -}; const clamp = (v: number, min: number, max: number) => Math.min(Math.max(v, min), max); @@ -64,9 +65,142 @@ export function sanitizeRenderSettings(raw: unknown): GraphRenderSettings { labelSize: num(r.labelSize, DEFAULT_RENDER_SETTINGS.labelSize, LABEL_SIZE_RANGE.min, LABEL_SIZE_RANGE.max), clusterLabels: typeof r.clusterLabels === "boolean" ? r.clusterLabels : DEFAULT_RENDER_SETTINGS.clusterLabels, + // Nested sections (2026-09-13): old files lack them → defaults. + regions: sanitizeRegionTuning(r.regions), + physics: sanitizePhysicsTuning(r.physics), }; } +// ─── Region + physics tuning (moved from the renderer, 2026-09-13) ───────────── +// +// The renderer's region/physics controls became USER-FACING persisted +// settings: one shape + one sanitizer in core, consumed by the settings +// store (this file's schema), the panel, and the renderer's +// setRegionTuning/setPhysicsTuning. Every clamp below is a SWEEP-VALIDATED +// safe range (2026-09-13, real graph, renderer's exact sim): each field at +// both range ends AND all four range corners settles cleanly — no NaN, no +// explosion (linkStrength ≥ 8 explodes the layout, 5 wobbles; cap 4), +// no non-settle (velocityDecay 0 never damps; floor 0.1), no 3x sprawl +// (gravity 0 / charge -1000 excluded). + +/** Region rendering knobs (the blurred cluster tints). */ +export interface RegionTuning { + /** Dilation half-width of the round-cap edge strokes, screen px at + * fit zoom (the expansion of the cluster skeleton). */ + dilation: number; + /** Box-blur radius in bitmap px (3 passes ≈ Gaussian). 0 = off. */ + blur: number; + /** Max bitmap dimension (cell = worldSpan / this). Fixed set: + * see REGION_BITMAP_OPTIONS (dropdown in the panel). */ + bitmap: number; + /** Per-stroke alpha; overlapping strokes stack toward the core. */ + edgeAlpha: number; +} + +export const REGION_BITMAP_OPTIONS = [128, 192, 256, 384, 512] as const; + +export const DEFAULT_REGION_TUNING: RegionTuning = { + dilation: 28, + blur: 10, + bitmap: 384, + edgeAlpha: 0.35, +}; + +/** Clamp/validate a region tuning. `bitmap` snaps to the nearest allowed + * option (it's a fixed set — a dropdown, not a free number). */ +export function sanitizeRegionTuning(input: Partial | null | undefined): RegionTuning { + const d = DEFAULT_REGION_TUNING; + const num = (v: unknown, fallback: number): number => + typeof v === "number" && Number.isFinite(v) ? v : fallback; + const bitmap = num(input?.bitmap, d.bitmap); + return { + dilation: Math.min(Math.max(num(input?.dilation, d.dilation), REGION_DILATION_RANGE.min), REGION_DILATION_RANGE.max), + blur: Math.min(Math.max(num(input?.blur, d.blur), 0), REGION_BLUR_RANGE.max), + bitmap: REGION_BITMAP_OPTIONS.reduce( + (best, o) => (Math.abs(o - bitmap) < Math.abs(best - bitmap) ? o : best), + REGION_BITMAP_OPTIONS[0] as number, + ), + edgeAlpha: Math.min(Math.max(num(input?.edgeAlpha, d.edgeAlpha), REGION_EDGE_ALPHA_RANGE.min), 1), + }; +} + +/** Live-sim physics controls (user-facing since 2026-09-13): every value + * feeds createGraphSimulation — the SAME engine the server settle runs — + * so tuning the feel changes exactly what a drag relaxes toward. */ +export interface PhysicsTuning { + /** Node repulsion (d3 charge, negative = push apart). */ + chargeStrength: number; + /** Edge attraction: link-spring stiffness multiplier (1 = d3 default). */ + linkStrength: number; + /** Link spring rest length (world units). */ + linkDistance: number; + /** Central pull toward the graph centroid. */ + gravity: number; + /** Per-cluster pull toward the cluster centroid. 0 = off. */ + clusterPull: number; + /** Per-tick velocity damping (d3 default 0.4). */ + velocityDecay: number; + /** Inference-depth anchoring mode (see DepthAnchor in core). */ + depthMode: DepthAnchorMode; + /** Depth-anchor pull strength. */ + depthStrength: number; + /** Spacing between inference levels (world units). */ + depthSpacing: number; + /** Inner ring radius, radial mode (world units). */ + depthBaseRadius: number; +} + +export const DEFAULT_PHYSICS_TUNING: PhysicsTuning = { + chargeStrength: CHARGE_STRENGTH, + linkStrength: 1, + linkDistance: LINK_DISTANCE, + gravity: INCREMENTAL_GRAVITY, + clusterPull: CLUSTER_PULL_STRENGTH, + velocityDecay: 0.4, + depthMode: "off", + // 0.8 by sweep on the real graph: 215/216 arrows read upward with + // organic sag (mean |dev| from level ~34u at spacing 60); 1.5 gives + // crisp rows (dev ~18) if the promoted default wants them. + depthStrength: 0.8, + depthSpacing: 60, + depthBaseRadius: 220, +}; + +/** Clamp anything into a usable tuning (sliders/JSON can be empty/NaN). */ +export function sanitizePhysicsTuning(input: Partial | null | undefined): PhysicsTuning { + const d = DEFAULT_PHYSICS_TUNING; + const num = (v: unknown, fallback: number): number => + typeof v === "number" && Number.isFinite(v) ? v : fallback; + return { + chargeStrength: Math.min(Math.max(num(input?.chargeStrength, d.chargeStrength), CHARGE_RANGE.min), CHARGE_RANGE.max), + linkStrength: Math.min(Math.max(num(input?.linkStrength, d.linkStrength), LINK_STRENGTH_RANGE.min), LINK_STRENGTH_RANGE.max), + linkDistance: Math.min(Math.max(num(input?.linkDistance, d.linkDistance), LINK_DISTANCE_RANGE.min), LINK_DISTANCE_RANGE.max), + gravity: Math.min(Math.max(num(input?.gravity, d.gravity), GRAVITY_RANGE.min), GRAVITY_RANGE.max), + clusterPull: Math.min(Math.max(num(input?.clusterPull, d.clusterPull), 0), CLUSTER_PULL_RANGE.max), + velocityDecay: Math.min(Math.max(num(input?.velocityDecay, d.velocityDecay), VELOCITY_DECAY_RANGE.min), VELOCITY_DECAY_RANGE.max), + depthMode: input?.depthMode === "vertical" || input?.depthMode === "radial" ? input.depthMode : "off", + depthStrength: Math.min(Math.max(num(input?.depthStrength, d.depthStrength), DEPTH_STRENGTH_RANGE.min), DEPTH_STRENGTH_RANGE.max), + depthSpacing: Math.min(Math.max(num(input?.depthSpacing, d.depthSpacing), DEPTH_SPACING_RANGE.min), DEPTH_SPACING_RANGE.max), + depthBaseRadius: Math.min(Math.max(num(input?.depthBaseRadius, d.depthBaseRadius), DEPTH_BASE_RADIUS_RANGE.min), DEPTH_BASE_RADIUS_RANGE.max), + }; +} + +// Sizes are screen px AT FIT ZOOM (the world-space renderer scales them +// with the camera, so a label reads at labelSize x k/kFit on screen). +// Defaults were re-tuned for world-scaling on 2026-09-13: the old values +// were calibrated in the screen-fixed era, so everything rendered ~2x +// too big once sizes started scaling with the graph. +export const DEFAULT_RENDER_SETTINGS: GraphRenderSettings = { + nodeSize: 2, + edgeWidth: 1, + arrowLength: 2, + labelZoom: 0.5, + labelSize: 2, + clusterLabels: true, + regions: DEFAULT_REGION_TUNING, + physics: DEFAULT_PHYSICS_TUNING, +}; + // ─── The slider mapping ──────────────────────────────────────────────────────── // // EVERY slider is log-scaled over its setting's range with the DEFAULT at @@ -81,6 +215,42 @@ export const LABEL_SIZE_RANGE = { min: 0.5, max: 8 } as const; export const LABEL_ZOOM_MIN = 0.0625; // slider far left: labels only very deep in export const LABEL_ZOOM_MAX = 4; // slider far right: labels visible at fit zoom +// ─── Region + physics slider ranges ────────────────────────────────────────── +// +// Same laws as the size sliders: log-scaled, DEFAULT AT 50 (each default is +// the geometric mean of its ends), coarse 5-step tracks. Signed/zeroed +// fields get special treatment: +// chargeStrength is NEGATIVE — the slider maps |charge| and displays "-x"; +// clusterPull and region blur have an OFF zone — position 0 is exactly 0 +// (see toOffSliderPosition), the log range starts one step in. + +export const REGION_DILATION_RANGE = { min: 7, max: 112 } as const; // gm = 28 +export const REGION_BLUR_RANGE = { min: 2.5, max: 40 } as const; // gm = 10; 0 = off +export const REGION_EDGE_ALPHA_RANGE = { min: 0.12, max: 1 } as const; // gm ≈ 0.35 + +export const CHARGE_RANGE = { min: -480, max: -30 } as const; // |gm| = 120 +export const LINK_STRENGTH_RANGE = { min: 0.25, max: 4 } as const; // gm = 1 +export const LINK_DISTANCE_RANGE = { min: 10, max: 160 } as const; // gm = 40 +export const GRAVITY_RANGE = { min: 0.03, max: 0.48 } as const; // gm = 0.12 +export const CLUSTER_PULL_RANGE = { min: 0.0225, max: 1 } as const; // gm = 0.15; 0 = off +export const VELOCITY_DECAY_RANGE = { min: 0.18, max: 0.9 } as const; // gm ≈ 0.40 +export const DEPTH_STRENGTH_RANGE = { min: 0.2, max: 3.2 } as const; // gm = 0.8 +export const DEPTH_SPACING_RANGE = { min: 15, max: 240 } as const; // gm = 60 +export const DEPTH_BASE_RADIUS_RANGE = { min: 55, max: 880 } as const; // gm = 220 + +/** Off-aware slider mapping (position 0 → value 0 = OFF; the log range + * starts one step in). Inverse of fromOffSliderPosition. */ +export function toOffSliderPosition(value: number, min: number, max: number): number { + if (value <= 0) return 0; + return toSliderPosition(value, min, max); +} + +/** Off-aware slider mapping: position 0 → 0 (off), else log-scaled. */ +export function fromOffSliderPosition(position: number, min: number, max: number): number { + if (position <= 0) return 0; + return fromSliderPosition(position, min, max); +} + /** Setting value → slider position [0, 100] (log scale, default at 50). */ export function toSliderPosition(value: number, min: number, max: number): number { const v = clamp(value, min, max); diff --git a/tests/graph-canvas.test.ts b/tests/graph-canvas.test.ts index 87aaa71..d472e2b 100644 --- a/tests/graph-canvas.test.ts +++ b/tests/graph-canvas.test.ts @@ -17,13 +17,16 @@ import { fitTransform, labelAlpha, pickNearest, - sanitizePhysicsTuning, - sanitizeRegionTuning, zoomRatioOf, - DEFAULT_PHYSICS_TUNING, - DEFAULT_REGION_TUNING, LABELS_FADE_SPAN, } from "../components/graph-canvas.ts"; +import { + sanitizePhysicsTuning, + sanitizeRegionTuning, + DEFAULT_PHYSICS_TUNING, + DEFAULT_REGION_TUNING, + REGION_BITMAP_OPTIONS, +} from "../src/core/render-settings.ts"; describe("zoomRatioOf (the sigma-compatible camera ratio)", () => { test("1 at fit zoom, smaller when zoomed in, larger when zoomed out", () => { @@ -156,7 +159,7 @@ describe("sanitizeRegionTuning (dev number inputs)", () => { const t = sanitizeRegionTuning({ dilation: Number.NaN, blur: -5, bitmap: 99999, edgeAlpha: 42 }); assert.equal(t.dilation, DEFAULT_REGION_TUNING.dilation); // NaN -> default assert.equal(t.blur, 0); - assert.equal(t.bitmap, 1024); + assert.equal(t.bitmap, REGION_BITMAP_OPTIONS[REGION_BITMAP_OPTIONS.length - 1]); // 99999 snaps to the largest option assert.equal(t.edgeAlpha, 1); }); @@ -215,10 +218,10 @@ describe("sanitizePhysicsTuning (physics controls)", () => { velocityDecay: 3, }); assert.equal(t.chargeStrength, DEFAULT_PHYSICS_TUNING.chargeStrength); - assert.equal(t.linkStrength, 5); - assert.equal(t.linkDistance, 1); - assert.equal(t.gravity, 1); + assert.equal(t.linkStrength, 4); // sweep-validated cap (8 explodes, 5 wobbles) + assert.equal(t.linkDistance, 10); + assert.equal(t.gravity, 0.48); assert.equal(t.clusterPull, 0); - assert.equal(t.velocityDecay, 0.99); + assert.equal(t.velocityDecay, 0.9); }); }); diff --git a/tests/render-settings.test.ts b/tests/render-settings.test.ts index 3fef0fe..38e45f1 100644 --- a/tests/render-settings.test.ts +++ b/tests/render-settings.test.ts @@ -9,15 +9,31 @@ import assert from "node:assert/strict"; import { describe, it as test } from "node:test"; import { ARROW_LENGTH_RANGE, + CHARGE_RANGE, + CLUSTER_PULL_RANGE, DEFAULT_RENDER_SETTINGS, + DEPTH_BASE_RADIUS_RANGE, + DEPTH_SPACING_RANGE, + DEPTH_STRENGTH_RANGE, EDGE_WIDTH_RANGE, + GRAVITY_RANGE, LABEL_SIZE_RANGE, LABEL_ZOOM_MAX, LABEL_ZOOM_MIN, + LINK_DISTANCE_RANGE, + LINK_STRENGTH_RANGE, NODE_SIZE_RANGE, + REGION_BLUR_RANGE, + REGION_DILATION_RANGE, + REGION_EDGE_ALPHA_RANGE, + VELOCITY_DECAY_RANGE, + DEFAULT_PHYSICS_TUNING, + DEFAULT_REGION_TUNING, + fromOffSliderPosition, fromSliderPosition, labelZoomDisplay, sanitizeRenderSettings, + toOffSliderPosition, toSliderPosition, } from "../src/core/render-settings.ts"; @@ -64,10 +80,45 @@ describe("sanitizeRenderSettings", () => { }); test("a fully valid object passes through unchanged", () => { - const valid = { nodeSize: 4, edgeWidth: 2, arrowLength: 3, labelZoom: 0.25, labelSize: 4, clusterLabels: false }; + const valid = { + nodeSize: 4, + edgeWidth: 2, + arrowLength: 3, + labelZoom: 0.25, + labelSize: 4, + clusterLabels: false, + regions: DEFAULT_REGION_TUNING, + physics: DEFAULT_PHYSICS_TUNING, + }; assert.deepEqual(sanitizeRenderSettings(valid), valid); }); + test("the pre-2026-09-13 stored shape (flat, no regions/physics) still loads", () => { + // data/render-settings.json on disk predates the nested sections — + // those loads must get defaults, not crash, not shift anything. + const s = sanitizeRenderSettings({ nodeSize: 4, edgeWidth: 1, arrowLength: 5, labelZoom: 0.5, clusterLabels: true }); + assert.deepEqual(s.regions, DEFAULT_REGION_TUNING); + assert.deepEqual(s.physics, DEFAULT_PHYSICS_TUNING); + assert.equal(s.nodeSize, 4); + }); + + test("nested sections sanitize: garbage clamps inside them without touching the rest", () => { + const s = sanitizeRenderSettings({ + nodeSize: 3, + regions: { blur: -5, bitmap: 999, edgeAlpha: 9 }, + physics: { linkStrength: 99, gravity: 7, depthMode: "sideways" }, + }); + assert.equal(s.nodeSize, 3); + assert.equal(s.regions.blur, 0); + assert.equal(s.regions.bitmap, 512); // snaps to the nearest fixed option + assert.equal(s.regions.edgeAlpha, 1); + assert.equal(s.regions.dilation, DEFAULT_REGION_TUNING.dilation); + assert.equal(s.physics.linkStrength, 4); // sweep-validated cap + assert.equal(s.physics.gravity, 0.48); + assert.equal(s.physics.depthMode, "off"); // unknown mode → off + assert.equal(s.physics.chargeStrength, DEFAULT_PHYSICS_TUNING.chargeStrength); + }); + test("the pre-wave stored shape (no labelSize) still loads", () => { // localStorage-era records lacked labelSize — must not shift anything. const s = sanitizeRenderSettings({ nodeSize: 4, edgeWidth: 1, arrowLength: 5, labelZoom: 0.5, clusterLabels: true }); @@ -114,4 +165,41 @@ describe("slider mapping (log scale, default-centered)", () => { assert.equal(labelZoomDisplay(4), "8×"); assert.equal(labelZoomDisplay(0.0625), "0.13×"); // rounded to 2 decimals }); + + test("every region/physics default sits dead-center at position 50", () => { + // The same law for the 2026-09-13 sliders: default = geometric mean + // of the range ends (repulsion maps |charge|; friction tolerates + // the sub-0.5° drift). + const pairs: [number, number, number][] = [ + [REGION_DILATION_RANGE.min, REGION_DILATION_RANGE.max, DEFAULT_REGION_TUNING.dilation], + [REGION_EDGE_ALPHA_RANGE.min, REGION_EDGE_ALPHA_RANGE.max, DEFAULT_REGION_TUNING.edgeAlpha], + // |charge| in [30, 480] (the range stores negative ends). + [-CHARGE_RANGE.max, -CHARGE_RANGE.min, -DEFAULT_PHYSICS_TUNING.chargeStrength], + [LINK_STRENGTH_RANGE.min, LINK_STRENGTH_RANGE.max, DEFAULT_PHYSICS_TUNING.linkStrength], + [LINK_DISTANCE_RANGE.min, LINK_DISTANCE_RANGE.max, DEFAULT_PHYSICS_TUNING.linkDistance], + [GRAVITY_RANGE.min, GRAVITY_RANGE.max, DEFAULT_PHYSICS_TUNING.gravity], + [VELOCITY_DECAY_RANGE.min, VELOCITY_DECAY_RANGE.max, DEFAULT_PHYSICS_TUNING.velocityDecay], + [DEPTH_STRENGTH_RANGE.min, DEPTH_STRENGTH_RANGE.max, DEFAULT_PHYSICS_TUNING.depthStrength], + [DEPTH_SPACING_RANGE.min, DEPTH_SPACING_RANGE.max, DEFAULT_PHYSICS_TUNING.depthSpacing], + [DEPTH_BASE_RADIUS_RANGE.min, DEPTH_BASE_RADIUS_RANGE.max, DEFAULT_PHYSICS_TUNING.depthBaseRadius], + ]; + for (const [min, max, d] of pairs) { + assert.ok(Math.abs(toSliderPosition(d, min, max) - 50) < 0.6, `default ${d} in [${min}, ${max}]`); + } + // Off-zone fields: the default sits at 50 INSIDE the on-part. + assert.ok(Math.abs(toOffSliderPosition(DEFAULT_PHYSICS_TUNING.clusterPull, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max) - 50) < 0.6); + assert.ok(Math.abs(toOffSliderPosition(DEFAULT_REGION_TUNING.blur, REGION_BLUR_RANGE.min, REGION_BLUR_RANGE.max) - 50) < 0.6); + }); + + test("off-slider mapping: position 0 ↔ value 0; the log part round-trips", () => { + assert.equal(toOffSliderPosition(0, REGION_BLUR_RANGE.min, REGION_BLUR_RANGE.max), 0); + assert.equal(fromOffSliderPosition(0, REGION_BLUR_RANGE.min, REGION_BLUR_RANGE.max), 0); + const v = fromOffSliderPosition(toOffSliderPosition(10, REGION_BLUR_RANGE.min, REGION_BLUR_RANGE.max), REGION_BLUR_RANGE.min, REGION_BLUR_RANGE.max); + assert.ok(Math.abs(v - 10) < 1e-9); + // One step in (position 5) sits just above the log floor: + // min * (max/min)^(5/100), NOT the floor itself (that's position 0, + // the off zone). + const stepIn = REGION_BLUR_RANGE.min * (REGION_BLUR_RANGE.max / REGION_BLUR_RANGE.min) ** (5 / 100); + assert.ok(Math.abs(fromOffSliderPosition(5, REGION_BLUR_RANGE.min, REGION_BLUR_RANGE.max) - stepIn) < 1e-9); + }); }); From b42fdbec1e597dac37501305586aa75b0c879eb4 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Mon, 14 Sep 2026 03:03:04 +0300 Subject: [PATCH 30/34] Region blur becomes a fixed-set dropdown (2/4/6/8/10, default 6) Non-integer blur radii broke the rendering (the log slider mapped to fractional values), so the control is now integer-only by design: REGION_BLUR_OPTIONS with snap-to-nearest sanitization for any input, same law as the bitmap resolution. The old off zone (blur 0) is gone; stored 0s snap back into the set. --- components/graph-view.tsx | 34 ++++++++++++++++++++-------------- src/core/render-settings.ts | 29 +++++++++++++++++------------ tests/graph-canvas.test.ts | 6 +++--- tests/render-settings.test.ts | 30 +++++++++++++++++++++--------- 4 files changed, 61 insertions(+), 38 deletions(-) diff --git a/components/graph-view.tsx b/components/graph-view.tsx index 60c3ff3..ecc3f38 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -83,7 +83,7 @@ import { LINK_STRENGTH_RANGE, NODE_SIZE_RANGE, REGION_BITMAP_OPTIONS, - REGION_BLUR_RANGE, + REGION_BLUR_OPTIONS, REGION_DILATION_RANGE, REGION_EDGE_ALPHA_RANGE, VELOCITY_DECAY_RANGE, @@ -625,19 +625,25 @@ export function GraphView() { regions: { ...cur.regions, dilation: fromSliderPosition(p, REGION_DILATION_RANGE.min, REGION_DILATION_RANGE.max) }, }))} /> - - changeSettings((cur) => ({ - ...cur, - regions: { ...cur.regions, blur: fromOffSliderPosition(p, REGION_BLUR_RANGE.min, REGION_BLUR_RANGE.max) }, - }))} - /> + (Math.abs(o - value) < Math.abs(best - value) ? o : best), options[0] as number); +} + +/** Clamp/validate a region tuning. `bitmap` and `blur` snap to the nearest + * allowed option (they're fixed sets — dropdowns, not free numbers). */ export function sanitizeRegionTuning(input: Partial | null | undefined): RegionTuning { const d = DEFAULT_REGION_TUNING; const num = (v: unknown, fallback: number): number => @@ -115,11 +123,8 @@ export function sanitizeRegionTuning(input: Partial | null | undef const bitmap = num(input?.bitmap, d.bitmap); return { dilation: Math.min(Math.max(num(input?.dilation, d.dilation), REGION_DILATION_RANGE.min), REGION_DILATION_RANGE.max), - blur: Math.min(Math.max(num(input?.blur, d.blur), 0), REGION_BLUR_RANGE.max), - bitmap: REGION_BITMAP_OPTIONS.reduce( - (best, o) => (Math.abs(o - bitmap) < Math.abs(best - bitmap) ? o : best), - REGION_BITMAP_OPTIONS[0] as number, - ), + blur: snapToOption(num(input?.blur, d.blur), REGION_BLUR_OPTIONS), + bitmap: snapToOption(bitmap, REGION_BITMAP_OPTIONS), edgeAlpha: Math.min(Math.max(num(input?.edgeAlpha, d.edgeAlpha), REGION_EDGE_ALPHA_RANGE.min), 1), }; } @@ -221,11 +226,11 @@ export const LABEL_ZOOM_MAX = 4; // slider far right: labels visible at fit zoom // the geometric mean of its ends), coarse 5-step tracks. Signed/zeroed // fields get special treatment: // chargeStrength is NEGATIVE — the slider maps |charge| and displays "-x"; -// clusterPull and region blur have an OFF zone — position 0 is exactly 0 -// (see toOffSliderPosition), the log range starts one step in. +// clusterPull has an OFF zone — position 0 is exactly 0 (see +// toOffSliderPosition), the log range starts one step in. Blur used to +// have one too; it's a dropdown since 2026-09-13 (non-integers broke). export const REGION_DILATION_RANGE = { min: 7, max: 112 } as const; // gm = 28 -export const REGION_BLUR_RANGE = { min: 2.5, max: 40 } as const; // gm = 10; 0 = off export const REGION_EDGE_ALPHA_RANGE = { min: 0.12, max: 1 } as const; // gm ≈ 0.35 export const CHARGE_RANGE = { min: -480, max: -30 } as const; // |gm| = 120 diff --git a/tests/graph-canvas.test.ts b/tests/graph-canvas.test.ts index d472e2b..b51ea74 100644 --- a/tests/graph-canvas.test.ts +++ b/tests/graph-canvas.test.ts @@ -150,15 +150,15 @@ describe("sanitizeRegionTuning (dev number inputs)", () => { test("defaults survive untouched; partial input keeps the rest", () => { const t = sanitizeRegionTuning(undefined); assert.deepEqual(t, DEFAULT_REGION_TUNING); - const half = sanitizeRegionTuning({ blur: 0 }); - assert.equal(half.blur, 0); + const half = sanitizeRegionTuning({ blur: 7.2 }); + assert.equal(half.blur, 8); // snaps into the fixed set assert.equal(half.dilation, DEFAULT_REGION_TUNING.dilation); }); test("garbage clamps: NaN falls back, out-of-range clamps", () => { const t = sanitizeRegionTuning({ dilation: Number.NaN, blur: -5, bitmap: 99999, edgeAlpha: 42 }); assert.equal(t.dilation, DEFAULT_REGION_TUNING.dilation); // NaN -> default - assert.equal(t.blur, 0); + assert.equal(t.blur, 2); // snaps into the fixed set assert.equal(t.bitmap, REGION_BITMAP_OPTIONS[REGION_BITMAP_OPTIONS.length - 1]); // 99999 snaps to the largest option assert.equal(t.edgeAlpha, 1); }); diff --git a/tests/render-settings.test.ts b/tests/render-settings.test.ts index 38e45f1..4d613bc 100644 --- a/tests/render-settings.test.ts +++ b/tests/render-settings.test.ts @@ -23,7 +23,8 @@ import { LINK_DISTANCE_RANGE, LINK_STRENGTH_RANGE, NODE_SIZE_RANGE, - REGION_BLUR_RANGE, + REGION_BLUR_OPTIONS, + sanitizeRegionTuning, REGION_DILATION_RANGE, REGION_EDGE_ALPHA_RANGE, VELOCITY_DECAY_RANGE, @@ -109,7 +110,7 @@ describe("sanitizeRenderSettings", () => { physics: { linkStrength: 99, gravity: 7, depthMode: "sideways" }, }); assert.equal(s.nodeSize, 3); - assert.equal(s.regions.blur, 0); + assert.equal(s.regions.blur, 2); // snaps into the fixed set {2,4,6,8,10} assert.equal(s.regions.bitmap, 512); // snaps to the nearest fixed option assert.equal(s.regions.edgeAlpha, 1); assert.equal(s.regions.dilation, DEFAULT_REGION_TUNING.dilation); @@ -188,18 +189,29 @@ describe("slider mapping (log scale, default-centered)", () => { } // Off-zone fields: the default sits at 50 INSIDE the on-part. assert.ok(Math.abs(toOffSliderPosition(DEFAULT_PHYSICS_TUNING.clusterPull, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max) - 50) < 0.6); - assert.ok(Math.abs(toOffSliderPosition(DEFAULT_REGION_TUNING.blur, REGION_BLUR_RANGE.min, REGION_BLUR_RANGE.max) - 50) < 0.6); }); test("off-slider mapping: position 0 ↔ value 0; the log part round-trips", () => { - assert.equal(toOffSliderPosition(0, REGION_BLUR_RANGE.min, REGION_BLUR_RANGE.max), 0); - assert.equal(fromOffSliderPosition(0, REGION_BLUR_RANGE.min, REGION_BLUR_RANGE.max), 0); - const v = fromOffSliderPosition(toOffSliderPosition(10, REGION_BLUR_RANGE.min, REGION_BLUR_RANGE.max), REGION_BLUR_RANGE.min, REGION_BLUR_RANGE.max); - assert.ok(Math.abs(v - 10) < 1e-9); + assert.equal(toOffSliderPosition(0, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max), 0); + assert.equal(fromOffSliderPosition(0, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max), 0); + const v = fromOffSliderPosition(toOffSliderPosition(0.15, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max), CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max); + assert.ok(Math.abs(v - 0.15) < 1e-9); // One step in (position 5) sits just above the log floor: // min * (max/min)^(5/100), NOT the floor itself (that's position 0, // the off zone). - const stepIn = REGION_BLUR_RANGE.min * (REGION_BLUR_RANGE.max / REGION_BLUR_RANGE.min) ** (5 / 100); - assert.ok(Math.abs(fromOffSliderPosition(5, REGION_BLUR_RANGE.min, REGION_BLUR_RANGE.max) - stepIn) < 1e-9); + const stepIn = CLUSTER_PULL_RANGE.min * (CLUSTER_PULL_RANGE.max / CLUSTER_PULL_RANGE.min) ** (5 / 100); + assert.ok(Math.abs(fromOffSliderPosition(5, CLUSTER_PULL_RANGE.min, CLUSTER_PULL_RANGE.max) - stepIn) < 1e-9); + }); + + test("blur is a fixed set: the default is 6, non-members snap to the nearest", () => { + // Non-integer radii broke the rendering, so the control became a + // dropdown — the sanitizer enforces the set for ANY input. + assert.equal(DEFAULT_REGION_TUNING.blur, 6); + assert.equal(sanitizeRegionTuning({ blur: 3.7 }).blur, 4); + assert.equal(sanitizeRegionTuning({ blur: 7.2 }).blur, 8); + assert.equal(sanitizeRegionTuning({ blur: -5 }).blur, 2); + assert.equal(sanitizeRegionTuning({ blur: 99 }).blur, 10); + // Stored blur 0 (the old off zone) snaps back into the set. + assert.equal(sanitizeRegionTuning({ blur: 0 }).blur, 2); }); }); From f3a3d694012cea308449a7eeda7d445576c4e523 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Mon, 14 Sep 2026 03:23:20 +0300 Subject: [PATCH 31/34] Region tints follow the search filter: visible-only edge strokes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildRegions filtered everything by visibility except the intra-cluster edge strokes: with a search active, edges between filtered-out nodes were still stroked into the bitmap — which had already shrunk to the visible members — and segments crossing that rect cut diagonal hue streaks from other clusters across the filtered regions. Strokes now require both endpoints visible, same law as the world rect, member discs, and label anchors. The no-double-counting rule survives: a member touched only by a partially-invisible edge gets its disc. --- components/graph-canvas.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts index dfabc38..77cf146 100644 --- a/components/graph-canvas.ts +++ b/components/graph-canvas.ts @@ -1124,6 +1124,11 @@ export class GraphCanvasRenderer { for (const e of data.edges) { if (clusterOf.get(e.from) !== ci) continue; if (clusterOf.get(e.to) !== ci) continue; + // Search filter: stroke only fully-visible edges. Invisible-node + // segments still stroked into the (visible-only) bitmap cut + // diagonal hue streaks across other clusters' regions (user- + // reported 2026-09-14) — rect, discs, and anchors already filter. + if (!isVisible(e.from) || !isVisible(e.to)) continue; const a = this.nodeById.get(e.from); const b = this.nodeById.get(e.to); if (!a || !b) continue; From ab42ec81e6838fb2205eba389f00b99b0e21e1d4 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Mon, 14 Sep 2026 03:53:23 +0300 Subject: [PATCH 32/34] Fix settings-load race: region/physics tuning never reached the renderer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The settings fetch resolves before the graph snapshot creates the renderer, so the load-time run of the region/physics feed effects hit a null rendererRef and never re-fired — every reload silently kept the DEFAULT region/physics tuning (a stored blur 10 rendered as 6). Feeds #4/#5 now carry isEmpty in their deps, the same guard Feed #2 (setSettings) already had. Sizes were unaffected only because their stored values happened to equal the defaults. --- components/graph-view.tsx | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/components/graph-view.tsx b/components/graph-view.tsx index ecc3f38..9ce650e 100644 --- a/components/graph-view.tsx +++ b/components/graph-view.tsx @@ -460,17 +460,20 @@ export function GraphView() { setSelected((sel) => (sel && visibleIds.has(sel.node.id) ? sel : null)); }, [visibleIds, query]); - // Feed #4 — region tuning (dev number inputs): its own update path, - // independent of the server-persisted render settings. + // Feed #4 — region tuning. `isEmpty` in the deps (same as Feed #2): + // the settings fetch resolves BEFORE the snapshot creates the renderer, + // so the load-time run hits a null renderer and would never re-fire — + // the renderer would silently keep the default tuning forever. useEffect(() => { rendererRef.current?.setRegionTuning(regionTuning); - }, [regionTuning]); + }, [regionTuning, isEmpty]); // Feed #5 — physics tuning: rebuilds the live sim in place (drag pins - // survive; a mid-settle change re-anneals, the honest response). + // survive; a mid-settle change re-anneals, the honest response). Same + // isEmpty dep as Feed #4 — see there. useEffect(() => { rendererRef.current?.setPhysicsTuning(physicsTuning); - }, [physicsTuning]); + }, [physicsTuning, isEmpty]); // Position the detail card: below the clicked node, flipped above near // the bottom edge, clamped horizontally. Imperative (ref + layout effect, From 2a80707b850f674967fbeda3f11eccf450a34064 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Mon, 14 Sep 2026 03:53:34 +0300 Subject: [PATCH 33/34] Screen-anchored region blur + cell floor: fixes tight-search blur collapse and the Infinity throw MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stroke width is anchored to kFit (screen px at fit zoom — the camera persists across searches), but the blur radius was anchored to cell, which scales with the FILTERED world span: a tight search shrank cell, and the blur's screen reach (blur*cell*kFit) collapsed with it while the stroke stayed put — the blur visibly went away for close-together matches and the knob did nothing (verified: blur 6 and blur 10 produced identical output under a tight search). The blur now uses the stroke's anchoring: its world reach is blur*cellRef, cellRef being the full-graph cell from worldBBox (the same bbox kFit comes from) — span-independent, and with nothing filtered cell == cellRef so the unfiltered look is unchanged. The cell also gains a floor: a single matching node (span 0) zeroed the cell and blew the bitmap dims to Infinity, throwing the reported getImageData TypeError. The floor is the padded rect's own reach, so dims stay finite however tight the search (and the old bitmap ballooning — ~1040px bitmap for a 3-node search — is gone too). --- components/graph-canvas.ts | 41 ++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts index 77cf146..5a35676 100644 --- a/components/graph-canvas.ts +++ b/components/graph-canvas.ts @@ -1073,8 +1073,30 @@ export class GraphCanvasRenderer { // Bitmap sizing: longest world span maps to `bitmap` px. The rect // is padded by the stroke half-width (dilation) plus the blur // reach so nothing clips at the bitmap edge. - const cell = Math.max(maxX - minX, maxY - minY) / tuning.bitmap; - const padWorld = tuning.dilation * s + (tuning.blur * cell) * 2; + // + // SCREEN-ANCHORED RADII (user report 2026-09-14: a tight search + // collapsed the blur). The stroke width is anchored to kFit (screen + // px at fit zoom — the camera persists, searches don't refit), but + // the blur radius was anchored to `cell`, which shrinks with the + // FILTERED span: close-together matches → tiny cell → blur reach + // (blur·cell·kFit) collapsed on screen while the stroke stayed put. + // The blur now uses the stroke's anchoring: its world reach is + // blur · cellRef, cellRef being the FULL-graph cell (worldBBox — + // the same bbox kFit comes from). Span-independent, and with + // nothing filtered cell == cellRef, so the look is unchanged. + const dilationW = tuning.dilation * s; + const bbox = this.worldBBox; + const cellRef = bbox + ? Math.max(bbox.maxX - bbox.minX, bbox.maxY - bbox.minY) / tuning.bitmap + : 0; + const blurW = tuning.blur * cellRef; + const padWorld = dilationW + blurW * 2; + // Cell floor: a single visible node (span 0) zeroed the cell and + // blew the bitmap dims to Infinity (getImageData TypeError). The + // floor is the padded rect's own reach, so dims stay finite (long + // side ≤ bitmap · (1 + span/2·pad)) however tight the search. + const spanNow = Math.max(maxX - minX, maxY - minY); + const cell = Math.max(spanNow, padWorld * 2) / tuning.bitmap; minX -= padWorld; minY -= padWorld; maxX += padWorld; @@ -1099,7 +1121,7 @@ export class GraphCanvasRenderer { if (!rctx) return; rctx.clearRect(0, 0, bw, bh); rctx.lineCap = "round"; - rctx.lineWidth = (tuning.dilation * s * 2) / cell; + rctx.lineWidth = (dilationW * 2) / cell; // Which cluster is a node in (for intra-cluster edges)? const clusterOf = new Map(); @@ -1147,7 +1169,7 @@ export class GraphCanvasRenderer { // stroke reached: every member is painted by its edges' caps OR // one disc, never both — the no-double-counting rule survives. rctx.fillStyle = `hsla(${hue}, 65%, 60%, ${tuning.edgeAlpha})`; - const discR = (tuning.dilation * s) / cell; + const discR = dilationW / cell; for (const id of cluster.members) { if (touched.has(id) || !isVisible(id)) continue; const n = this.nodeById.get(id); @@ -1173,8 +1195,11 @@ export class GraphCanvasRenderer { // THE blur: 3 box-blur rounds on the premultiplied bitmap. Bilinear // upscaling alone is no blur at all (tent ramps — the JPEG look); - // this makes the field genuinely smooth before the blit. + // this makes the field genuinely smooth before the blit. The + // radius is blurW in bitmap px (screen-anchored — see sizing above), + // an integer because boxBlurRound indexes with the loop counter. if (tuning.blur > 0) { + const blurPx = Math.max(1, Math.round(blurW / cell)); const img = rctx.getImageData(0, 0, bw, bh); const n = bw * bh * 4; const a = new Float32Array(n); @@ -1187,9 +1212,9 @@ export class GraphCanvasRenderer { } const b = new Float32Array(n); const tmp = new Float32Array(n); // scratch inside each round only - boxBlurRound(a, tmp, b, bw, bh, tuning.blur); // a → b - boxBlurRound(b, tmp, a, bw, bh, tuning.blur); // b → a - boxBlurRound(a, tmp, b, bw, bh, tuning.blur); // a → b (result) + boxBlurRound(a, tmp, b, bw, bh, blurPx); // a → b + boxBlurRound(b, tmp, a, bw, bh, blurPx); // b → a + boxBlurRound(a, tmp, b, bw, bh, blurPx); // a → b (result) // Unpremultiply back into pixels. for (let i = 0; i < n; i += 4) { const alpha = b[i + 3]!; From 0d67fd20738779253d07efe0c3e0cad57f39e64d Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Mon, 14 Sep 2026 04:56:15 +0300 Subject: [PATCH 34/34] Region alpha applies once after the blur: no more overlap stacking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-stroke alpha baked into every stroke/disc accumulated through source-over compositing: one shape 0.35, an orphan disc over anything 1-0.65^2 ~ 0.58, hubs toward 1.0 — a lone node's glow read as a darker double-counted spot on top of other regions. The pipeline is now dilate at full opacity -> blur -> scale the whole field by edgeAlpha once at the end. Painted at 1, the alpha channel is a pure coverage indicator (overlaps of any count contribute exactly 1.0), the blur smooths it into a coverage falloff, and the final scaling gives a constant interior alpha everywhere covered, varying only across the blur edges. The blur-off path also runs the readback now (radius 0 = copy) so the end-alpha applies uniformly without blur. Interaction change (user-flagged, accepted for now): different clusters' overlapping halos no longer blend hues proportionally — the later-painted cluster covers the earlier, ownership boundaries softened only by the blur reach. Verified: old build spread 36k of 55k tinted pixels above alpha 100 (stacking); new build concentrates at 70-90 (0.35*255=89 interior with falloff), and an orphan disc now reads no brighter than the skeleton it sits on. --- components/graph-canvas.ts | 32 +++++++++++++++++++++++--------- src/core/render-settings.ts | 5 ++++- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/components/graph-canvas.ts b/components/graph-canvas.ts index 5a35676..91950d8 100644 --- a/components/graph-canvas.ts +++ b/components/graph-canvas.ts @@ -1136,7 +1136,14 @@ export class GraphCanvasRenderer { this.regionMeta = []; data.clusters.forEach((cluster, ci) => { const hue = clusterHue(cluster.id); - rctx.strokeStyle = `hsla(${hue}, 65%, 60%, ${tuning.edgeAlpha})`; + // Full-opacity strokes: alpha is applied ONCE at the very end + // (dilate at 1 → blur → scale by edgeAlpha). Baking per-stroke + // alpha in here made source-over stacking ACCUMULATE it — an + // orphan disc overlapping anything read as a darker double- + // counted spot (user report 2026-09-14). At alpha 1 the alpha + // channel is a pure coverage indicator: overlaps of any count + // contribute exactly 1.0. + rctx.strokeStyle = `hsla(${hue}, 65%, 60%, 1)`; rctx.beginPath(); let members = 0; let sumX = 0; @@ -1168,7 +1175,7 @@ export class GraphCanvasRenderer { // radius as a round cap = dilation) covers exactly the members no // stroke reached: every member is painted by its edges' caps OR // one disc, never both — the no-double-counting rule survives. - rctx.fillStyle = `hsla(${hue}, 65%, 60%, ${tuning.edgeAlpha})`; + rctx.fillStyle = `hsla(${hue}, 65%, 60%, 1)`; const discR = dilationW / cell; for (const id of cluster.members) { if (touched.has(id) || !isVisible(id)) continue; @@ -1193,13 +1200,18 @@ export class GraphCanvasRenderer { } }); - // THE blur: 3 box-blur rounds on the premultiplied bitmap. Bilinear + // THE blur + alpha (in that order, by design — user-decided + // 2026-09-14): 3 box-blur rounds on the premultiplied bitmap, + // THEN scale the whole field by edgeAlpha once. Bilinear // upscaling alone is no blur at all (tent ramps — the JPEG look); // this makes the field genuinely smooth before the blit. The - // radius is blurW in bitmap px (screen-anchored — see sizing above), - // an integer because boxBlurRound indexes with the loop counter. - if (tuning.blur > 0) { - const blurPx = Math.max(1, Math.round(blurW / cell)); + // radius is blurW in bitmap px (screen-anchored — see sizing + // above), an integer because boxBlurRound indexes with the loop + // counter. Radius 0 (blur off) still runs this path — boxBlurRound + // copies — so the end-alpha applies uniformly with or without + // blur. + { + const blurPx = tuning.blur > 0 ? Math.max(1, Math.round(blurW / cell)) : 0; const img = rctx.getImageData(0, 0, bw, bh); const n = bw * bh * 4; const a = new Float32Array(n); @@ -1215,7 +1227,9 @@ export class GraphCanvasRenderer { boxBlurRound(a, tmp, b, bw, bh, blurPx); // a → b boxBlurRound(b, tmp, a, bw, bh, blurPx); // b → a boxBlurRound(a, tmp, b, bw, bh, blurPx); // a → b (result) - // Unpremultiply back into pixels. + // Unpremultiply back into pixels; scale the alpha by edgeAlpha + // HERE — the one, final application. Result: constant interior + // alpha everywhere covered, tapering only across the blur edges. for (let i = 0; i < n; i += 4) { const alpha = b[i + 3]!; if (alpha <= 0) { @@ -1228,7 +1242,7 @@ export class GraphCanvasRenderer { img.data[i] = Math.round((b[i]! / alpha) * 255); img.data[i + 1] = Math.round((b[i + 1]! / alpha) * 255); img.data[i + 2] = Math.round((b[i + 2]! / alpha) * 255); - img.data[i + 3] = Math.round(alpha); + img.data[i + 3] = Math.round(alpha * tuning.edgeAlpha); } } rctx.putImageData(img, 0, 0); diff --git a/src/core/render-settings.ts b/src/core/render-settings.ts index 6a1cfb2..1002985 100644 --- a/src/core/render-settings.ts +++ b/src/core/render-settings.ts @@ -95,7 +95,10 @@ export interface RegionTuning { /** Max bitmap dimension (cell = worldSpan / this). Fixed set: * see REGION_BITMAP_OPTIONS (dropdown in the panel). */ bitmap: number; - /** Per-stroke alpha; overlapping strokes stack toward the core. */ + /** Region alpha, applied ONCE after the blur (dilate at full + * opacity → blur → scale). Constant interior, soft edges; overlaps + * of any count never stack (DECIDED 2026-09-14 — per-stroke alpha + * accumulated through source-over and double-counted orphans). */ edgeAlpha: number; }