learnharness/src/core/contracts/graph.ts

/**
 * Knowledge-graph contracts — nodes, ledgers, extraction findings
 *
 * The minimal implementation of the graph vision (docs/vision.md §1, the
 * 2026-09-02 redesign): every node is a TRUE proposition; edges are follows
 * (derivable) and supports (grounds without entailing); provenance is
 * message-granular; extraction is free-form and post-hoc, fully decoupled
 * from the session's plan tree; the event ledger records quiz outcomes plus
 * the extractor's per-proposition understanding observations. Anything not
 * yet decided (the state model, strength derivation, understanding
 * treatment, selection strategies) is deliberately NOT here — transitional
 * stubs carry the renderer until those land.
 *
 * Part of the domain-split contracts (src/core/contracts/): each domain file
 * is owned outright by its stream — cross-domain schema changes go through
 * master. Pure types + protocol docs + tiny pure helpers. See docs/vision.md.
 */
// ─── Knowledge graph (vision §1 — global, cross-session) ────────────────────

/** Two relations only (DECIDED 2026-09-02): follows = b is derivable from a;
 *  supports = a grounds b without entailing it. No scalars, no encoded
 *  strength — strength is derived at read time from topology. */
import type { CallUsage, ExtractionRunUsage } from "./chat.ts";

export type EdgeType = "follows" | "supports";

/** What one extraction run merged into the graph ledgers — the merger's
 *  output, also persisted in extraction-run usage records (contracts/chat). */
export interface EmissionStats {
	/** New graph nodes appended. */
	nodes: number;
	/** New edge assertions appended. */
	edges: number;
	/** Ledger events rebuilt (quiz outcomes + observations). */
	events: number;
}

/** One provenance point — message-granular (DECIDED 2026-09-02): nodes and
 *  edges record every (chat, message) where they came up, deep-linkable to
 *  the exact transcript point. */
export interface ProvenancePoint {
	/** The chat this point anchors to — or a notes source id ("src_…"):
	 *  imports carry no message anchors (provenance is the file itself,
	 *  recorded through the mapping line + the source record). */
	chatId: string;
	messageId: string;
}

/** One imported notes file — the format-agnostic front door's bookkeeping
 *  (DECIDED 2026-09-13): personal notes in ARBITRARY formats run through the
 *  same extractor → auditor → merger pipeline, with the file itself as the
 *  provenance anchor. There is no chat: the detail card shows the file path
 *  as a STATIC entry (not a link), and understanding observations still
 *  apply (judged from the author's own voice in the notes). The run's usage
 *  log lives here, mirroring ChatRecord.extractions. Ids are "src_" + hash
 *  of the file path — deterministic, so re-importing the same file re-runs
 *  extraction against the same source id (mappings/events dedupe by it). */
export interface NotesSource {
	id: string;
	filePath: string;
	/** ISO — when this file was imported (the checkpoint's foldedAt). */
	importedAt: string;
	/** The run's usage log — same append-only discipline as
	 *  ChatRecord.extractions, idempotent per startedAt. */
	extractions?: ExtractionRunUsage[];
}

/** What one cluster-naming refresh did — the namer's ClusterRefreshStats,
 *  contract-shaped so the agent layer needs no contracts import.
 *  Structural twins stay interchangeable. */
export interface NamingRunStats {
	/** Communities the current graph divides into (all sizes). */
	detected: number;
	/** Communities that kept a label via core-survival. */
	claimed: number;
	/** Communities sent to the namer (unnamed, size >= MIN_CLUSTER_SIZE). */
	named: number;
	/** Labeling records actually appended (idempotent dedupe may shrink). */
	appended: number;
}

/** One cluster-naming run's usage record — the money channel's THIRD stage
 *  (graph cost metrics, 2026-09). The namer is graph-global (tied to no
 *  chat or source), so runs live in their own append-only ledger
 *  (config.LEDGER_FILES.namingRuns). Appended when a run actually called
 *  the model; skipped refreshes (nothing to name) never bill and never
 *  append. Idempotent per startedAt — same discipline as every usage
 *  channel. Cost is INFERRED at read time from steps + model
 *  (core/prices.ts) — dollars are never stored. */
export interface NamingRunUsage {
	/** ms epoch — the idempotency key. */
	startedAt: number;
	/** ISO — wall-clock time of the run. */
	at: string;
	durationMs: number;
	/** The labeler role's model id. */
	model: string;
	/** Every billed call (usually ONE — the namer is single-submit). */
	steps: CallUsage[];
	stats: NamingRunStats;
}

export interface GraphNode {
	/** Opaque stable id, store-assigned: "n_" + nanoid-like. */
	id: string;
	/** A very short label for the node — a compact handle like a glossary
	 *  headword ("Will to power", "Divine command theory"): what the graph
	 *  shows at a glance. Presentational only — not unique, not an identity
	 *  key, never precise: the statement carries the precision. Set at
	 *  creation (first line defines); a link-append never changes it. */
	name: string;
	/** The minimal proposition — one true, self-contained statement, at
	 *  full precision. The IDENTITY ANCHOR: the auditor recognizes a
	 *  re-encounter by judgment against this (link-or-create), not by
	 *  string matching. Shown in the detail card; the canvas label is the
	 *  name. */
	statement: string;
	/** Verbatim phrasings sessions used for this proposition (the identity
	 *  audit trail — how the world said what the node says). */
	aliases: string[];
	createdAt: string;
	/** Every (chat, message) point where this proposition came up. */
	provenance: ProvenancePoint[];
}

export interface GraphEdge {
	from: string;
	to: string;
	type: EdgeType;
	/** Every (chat, message) point where this relation was asserted. */
	provenance: ProvenancePoint[];
}

/** mappings.jsonl line — the session→graph identity map: the extraction
 *  pipeline's proposition (identified by its run-local source id) → the
 *  graph node it resolved to (created or linked). The audit trail of the
 *  auditor's identity verdicts; also the chat-scoped resolution table the
 *  merger uses to point events and edges at node ids. */
export interface NodeMapping {
	chatId: string;
	/** The extractor's run-local proposition id ("p1", …). */
	sourceId: string;
	graphNodeId: string;
}

/** sessions.jsonl line — the emission checkpoint for one chat (which chats
 *  have been folded, and through which updatedAt — never fold twice). */
export interface SessionCheckpoint {
	chatId: string;
	/** The chat record's `updatedAt` at the time of the fold. */
	foldedAt: string;
}

/** coverage.jsonl line (vision-extraction-coverage.md, settled 2026-09-15):
 *  ONE MESSAGE of one chat has been extracted — by content fingerprint over
 *  exactly what the extraction view sees. The MESSAGE is the unit of
 *  coverage; "uncovered" is derived (fingerprint the current transcript,
 *  diff) — never dual-tracked. A quiz answer merging into the asking
 *  message changes its fingerprint: the message re-opens itself, only
 *  itself. Chat extractions write these instead of whole-chat checkpoints;
 *  notes imports keep the checkpoint (a file is one shot). */
export interface CoverageEntry {
	chatId: string;
	/** sha256 of the message's view lines — "covered" means exactly this. */
	fp: string;
	/** chars/4 estimate of the message's view — display + budgeting. */
	tokens: number;
	/** ISO — when this message's last extraction run merged. */
	foldedAt: string;
}

/**
 * Event kinds (DECIDED 2026-09-02): quiz outcomes only — correct / miss /
 * dont-know — plus `observation`, the extractor's per-proposition
 * understanding note joining the ledger as a session-scoped observation
 * (named-word scale; treatment pending the state-model ideation). No
 * `taught` events at all; no misconception kind — a misconception is a
 * TRUE correction in the graph, and the wrongness lives here, in the
 * ledger, until the state model exists.
 */
export type LedgerEventKind = "correct" | "miss" | "dont-know" | "observation";

export interface LedgerEvent {
	/** null = not yet attributed. */
	nodeId: string | null;
	kind: LedgerEventKind;
	/** The chat that produced this evidence — or a notes source id
	 *  ("src_…") for imported-notes observations. */
	chatId: string;
	at: string;
	/** Idempotency key: the quiz part's toolCallId, or `obs:<sourceId>` for
	 *  understanding observations — re-emission of the same chat must never
	 *  double-record an event. */
	ref?: string;
	/** The learner's quiz note (diagnostic gold), or the understanding
	 *  LEVEL word for observations — a session-relative judgment. */
	note?: string;
	/** The quiz question text, when nodeId is null (later attribution input). */
	question?: string;
}

/**
 * Store format (src/store/graph.ts): JSONL ledgers at config.LEDGER_FILES
 * (nodes/edges/events/mappings/sessions), all append-only.
 *
 *   nodes.jsonl    one line per node — the FIRST line for an id defines
 *                  name/statement/createdAt; later lines for the
 *                  same id are link-appends (a session's re-encounter):
 *                  their statements/aliases join `aliases`, their
 *                  provenance unions; the name stays the first line's.
 *                  getGraph merges by id.
 *   edges.jsonl    one line per edge ASSERTION; readers merge by
 *                  (from,to,type), unioning provenance.
 *
 * The derived graph is rebuilt by reading the ledgers — store events,
 * derive state. sessions.jsonl is the extraction checkpoint: which chats
 * the extraction pipeline has processed, and through which updatedAt
 * (never extract the same turns twice).
 *
 * The EXTRACTION PIPELINE is the sole graph writer (extractor → auditor →
 * merger), tree-blind and post-hoc: the extractor reads only the transcript
 * and proposes propositions/edges/quiz-attribution with verbatim evidence;
 * the auditor sees the extractor's output PLUS the current graph, owns
 * faithfulness (evidence check) and identity (link-or-create: point each
 * proposition at the existing node it is, or let it be created — recording
 * the session's phrasings as aliases); the merger (core/graph.ts,
 * deterministic) applies the verdicts and appends.
 */

// ─── Extraction findings (extractor output → audit → merger input) ───────────

/**
 * The extractor's standardized per-session understanding scale (DECIDED
 * 2026-09-08): a CLOSED named-word list — no numbers, no prose, no
 * justifications (a justification, if the agent wants one, is dropped;
 * it is not stored). The level is session- and extraction-run-relative
 * judgment about the learner, shown next to its session — never a node
 * property.
 */
export type UnderstandingLevel = "solid" | "shaky" | "accepted" | "resisted" | "not yet engaged";

/** One proposed proposition — a node candidate (or a re-encounter of one). */
export interface ExtractedProposition {
	/** Extractor-local id ("p1", …) referenced by edges + attribution. */
	id: string;
	/** A very short label — a few words, a compact handle like a glossary
	 *  headword. Not a sentence, needn't be unique or self-contained: the
	 *  statement carries the precision. Becomes the node's name on create;
	 *  on link, the existing node keeps its name. */
	name: string;
	/** The minimal proposition — a true, self-contained, world-facing
	 *  statement, at full precision. No learner / teacher / session
	 *  references: misconceptions are stated as the true correction, and
	 *  wrongness lives in the event ledger, never in the node. */
	statement: string;
	/** Verbatim phrasings the session used. */
	aliases: string[];
	/** The message ids where this proposition came up — provenance. */
	messages: string[];
	/** Verbatim transcript quotes backing the proposition. */
	evidence: string[];
	/** The extractor's understanding level for this proposition IN THIS
	 *  session — one word from the UnderstandingLevel scale. Joins the
	 *  ledger as an observation event (note = the level word); treatment
	 *  pending the state-model ideation. */
	understanding?: UnderstandingLevel;
}

/** One proposed relation between two propositions. */
export interface ExtractedEdge {
	/** Proposition id that is followed / supported. */
	from: string;
	to: string;
	type: EdgeType;
	/** The message ids where this relation was asserted — provenance. */
	messages: string[];
	/** Verbatim transcript quotes backing the relation. */
	evidence: string[];
}

/**
 * The extractor's quiz attribution — its ONLY say about quiz events. Event
 * kinds, timestamps, idempotency refs, notes and question texts are all
 * rebuilt deterministically by the merger from the transcript's quiz tool
 * parts; the LLM supplies exactly the judgment call: WHICH proposition each
 * quiz tested. Tree-blind, so EVERY quiz attributes here (there is no tree
 * mapping to lean on) — retroactively against the whole transcript.
 */
export interface ExtractionAttribution {
	/** quiz toolCallId → proposition id; null / missing = unattributed (the
	 *  event lands with nodeId null + the question text, a later-
	 *  attribution input). */
	quiz: Record<string, string | null>;
}

/** The extractor's output shape. */
export interface SessionFindings {
	propositions: ExtractedProposition[];
	edges: ExtractedEdge[];
	attribution: ExtractionAttribution;
}

/** The auditor's verdict on one proposition: the findings, plus `linkTo` —
 *  the identity judgment. Absent = create a new node. */
export interface AuditedProposition extends ExtractedProposition {
	/** The existing GraphNode id this proposition IS — the auditor's
	 *  identity verdict. On link, the session's phrasings (statement,
	 *  aliases) join the node's aliases and the provenance unions.
	 *  Conservative: when unsure, leave it out — duplicates over wrong
	 *  merges. */
	linkTo?: string;
}

/** The auditor's output: the verified findings (with identity verdicts),
 * plus the drops (the audit trail of what was rejected and why). */
export interface AuditResult {
	propositions: AuditedProposition[];
	edges: ExtractedEdge[];
	attribution: ExtractionAttribution;
	/** Dropped candidates: proposition id, edge "from→to", or attribution
	 *  toolCallId — with the reason. */
	dropped: { ref: string; reason: string }[];
}

/** The GET /api/graph payload — everything the home view's PURE renderer needs. */
/** cluster-labels.jsonl line — one LLM-named community (vision §1,
 * DECIDED 2026-09-13). Append-only history: records are immutable, never
 * edited or deleted; dead labelings remain as history. IDENTITY is the
 * sorted member set ("core"), hashed into the id, so re-naming the same
 * community dedupes and a labeling survives graph edits as long as at
 * least half its core survives in some single community (core-survival;
 * under half → the community is renamed, splits can keep only one
 * successor). */
/** The embedding ledger shape (config.EMBEDDINGS_FILE): vectors for
 *  EVERY embedded node, all produced by one model at one dimensionality. */
export interface EmbeddingsLedger {
	/** The embedding model id (e.g. the HF repo) — mismatch = stale
	 *  vectors, the writer re-embeds everything. */
	model: string;
	/** Vector dimensionality (MRL-truncated on write). */
	dim: number;
	/** nodeId → L2-normalized vector (cosine similarity = dot product). */
	vectors: Record<string, number[]>;
	/** The topic partition over those vectors, computed by the SAME
	 *  embedder run (the user-decided trigger: recluster at the end of
	 *  every embedding batch, never per read — the agglomerative pass is
	 *  ~1.5s at current scale). Consumers (snapshot, settle, namer) read
	 *  this instead of recomputing. Empty when nothing is embedded. */
	clusters: Array<{ members: string[] }>;
	/** ISO timestamp of the last write. */
	updatedAt: string;
}

export interface ClusterLabeling {
	/** "clu_" + content hash of the sorted core. */
	id: string;
	/** The short domain label (1–4 words), e.g. "Differential forms". */
	label: string;
	/** The member node ids the label was minted for, sorted — the survival core. */
	core: string[];
	createdAt: string;
}

/** One rendered cluster in the snapshot payload (vision §1). Members are
 * node ids; the renderer draws the hull from member positions at draw
 * time and gates the label inversely on the node-label zoom control. */
export interface Cluster {
	/** The labeling record id when claimed; unlabeled clusters still
	 * appear (hull, no name) with a member-hash id. */
	id: string;
	/** null until the naming call has minted/claimed a labeling. */
	label: string | null;
	/** Node ids, sorted. Hull + centroid are renderer concerns. */
	members: string[];
}

export interface GraphSnapshot {
	nodes: GraphNode[];
	edges: GraphEdge[];
	/** chatId → title, for every chat that touched the graph. */
	sessions: Record<string, string>;
	/** srcId → file path, for every IMPORTED NOTES file that touched the
	 *  graph (the format-agnostic front door): a static provenance entry —
	 *  the detail card shows the path as plain text, not a link. */
	fileSources: Record<string, string>;
	/** nodeId → chatIds that touched the node (click-through provenance). */
	nodeSessions: Record<string, string[]>;
	/** nodeId → chatId → that session's understanding observation (the
	 *  extractor's per-session named-word note; session-relative data,
	 *  shown next to the session list — the confidence history). Multiple
	 *  notes for one (node, chat) collapse to the latest. */
	nodeObservations: Record<string, Record<string, string>>;
	/** Structural communities (Louvain over follows+supports), size ≥ 3,
	 * joined with persisted LLM labelings. Purely structural — never
	 * depends on layout. */
	clusters: Cluster[];
	/** Resting layout positions per node id, when a positions ledger
	 *  exists (core/layout.ts settle writes it). Absent on a fresh graph —
	 *  the view then falls back to its own one-shot forceAtlas2 pass. */
	positions?: Record<string, Position>;
}

/** A node's resting position in layout space (the positions ledger). */
export interface Position {
	x: number;
	y: number;
}

/** Derive a short chat title from the learner's first message (pure).
 *  Used on chat creation; deterministic so tests can pin it. */
export function titleFromPrompt(prompt: string): string {
	const oneLine = prompt.replace(/\s+/g, " ").trim();
	return oneLine.length > 72 ? `${oneLine.slice(0, 72)}…` : oneLine;
}