2026-08-22 10:30:00 +03:00
|
|
|
/**
|
2026-08-31 22:36:41 +03:00
|
|
|
* The knowledge graph: merger (the deterministic final stage of the
|
|
|
|
|
* extraction pipeline — vision §1 construction) + derivation (reads).
|
2026-08-22 10:30:00 +03:00
|
|
|
*
|
2026-09-08 22:19:37 +03:00
|
|
|
* The extraction pipeline is the SOLE graph writer, tree-blind and post-hoc:
|
2026-08-22 10:30:00 +03:00
|
|
|
*
|
2026-09-08 22:19:37 +03:00
|
|
|
* extractor (LLM) → SessionFindings: propositions, edges, quiz
|
|
|
|
|
* attribution, per-proposition understanding notes —
|
|
|
|
|
* each with verbatim evidence, from the transcript ALONE
|
|
|
|
|
* auditor (LLM) → AuditResult: the verified subset, with the IDENTITY
|
|
|
|
|
* VERDICT per proposition (linkTo an existing node, or
|
|
|
|
|
* create), and the drops for the log
|
2026-08-31 22:36:41 +03:00
|
|
|
* merger (HERE) → mergeFindings(): deterministic ledger appends
|
|
|
|
|
*
|
|
|
|
|
* The merger is deliberately the ONLY stage that touches the ledgers, and it
|
2026-09-08 22:19:37 +03:00
|
|
|
* is pure code: the LLM supplies exactly the judgment calls (what the
|
|
|
|
|
* propositions are, what follows/supports what, which proposition a quiz
|
|
|
|
|
* tested, whether a proposition IS an existing node), while event
|
|
|
|
|
* kinds/timestamps/refs/provenance — anything the model could silently
|
|
|
|
|
* mis-copy — are rebuilt deterministically from the transcript's tool parts
|
|
|
|
|
* and the message ids in the findings. Idempotent + incremental: refs are
|
|
|
|
|
* chat-scoped idempotency keys, edges dedupe by (from,to,type)-already-
|
|
|
|
|
* asserted-by-this-chat, mapping lines dedupe by (chat, sourceId), and
|
|
|
|
|
* sessions.jsonl checkpoints which chats have been extracted (never the
|
|
|
|
|
* same turns twice).
|
|
|
|
|
*
|
|
|
|
|
* Identity across re-encounters is the AUDITOR's judgment (link-or-create),
|
|
|
|
|
* not string matching: the conservative v1 exact-statement merge was
|
|
|
|
|
* replaced by the 2026-09-02 redesign. A re-extracted session's
|
|
|
|
|
* propositions link to the existing nodes; the link appends a node line
|
|
|
|
|
* whose phrasings join `aliases` and whose provenance unions (getGraph
|
|
|
|
|
* merges lines by id).
|
2026-08-31 22:36:41 +03:00
|
|
|
*
|
|
|
|
|
* Derivation below (getGraph / deriveNodeStates / graphSnapshot) is reads
|
|
|
|
|
* only — everything is recomputed from the ledgers. Every function takes
|
|
|
|
|
* the storage ports (core/ports.ts) as explicit parameters — pure core, no
|
|
|
|
|
* adapter imports; the composition root (src/server/stores.ts) wires the fs
|
|
|
|
|
* adapters in.
|
2026-08-22 10:30:00 +03:00
|
|
|
*/
|
|
|
|
|
|
2026-09-23 15:37:34 +03:00
|
|
|
import { randomBytes } from "node:crypto";
|
2026-09-26 01:52:47 +03:00
|
|
|
import type { LhMessage } from "./native-messages.ts";
|
|
|
|
|
import type { ToolResultMessage } from "@earendil-works/pi-ai";
|
2026-09-15 17:41:56 +03:00
|
|
|
import type { CoverageEntry } from "./contracts/graph.ts";
|
2026-08-31 19:51:17 +03:00
|
|
|
import type {
|
2026-09-08 22:19:37 +03:00
|
|
|
AuditResult,
|
2026-09-13 03:59:48 +03:00
|
|
|
Cluster,
|
2026-09-11 03:25:25 +03:00
|
|
|
EmissionStats,
|
2026-08-31 19:51:17 +03:00
|
|
|
GraphEdge,
|
|
|
|
|
GraphNode,
|
2026-09-08 22:19:37 +03:00
|
|
|
ProvenancePoint,
|
2026-08-31 19:51:17 +03:00
|
|
|
GraphSnapshot,
|
|
|
|
|
LedgerEvent,
|
|
|
|
|
LedgerEventKind,
|
2026-09-23 14:51:37 +03:00
|
|
|
UnderstandingLevel,
|
2026-08-31 19:51:17 +03:00
|
|
|
} from "./contracts.ts";
|
2026-09-23 14:51:37 +03:00
|
|
|
import { MIN_CLUSTER_SIZE } from "./clusters.ts";
|
2026-09-23 15:37:34 +03:00
|
|
|
import { birthMappingIndex, birthSessionOf, birthSessionPartition } from "./session-clusters.ts";
|
2026-08-22 10:30:00 +03:00
|
|
|
import type { QuestionAnswerData } from "./question-core.ts";
|
2026-08-31 20:00:12 +03:00
|
|
|
import type { ChatStore, GraphStore } from "./ports.ts";
|
2026-08-22 10:30:00 +03:00
|
|
|
|
2026-08-31 20:00:12 +03:00
|
|
|
/** The storage ports every graph operation needs, passed explicitly. */
|
|
|
|
|
export interface GraphDeps {
|
|
|
|
|
chatStore: ChatStore;
|
|
|
|
|
graphStore: GraphStore;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** "n_" + 12 hex chars (contracts.ts GraphNode.id — store-assigned). */
|
|
|
|
|
function newGraphNodeId(): string {
|
|
|
|
|
return `n_${randomBytes(6).toString("hex")}`;
|
|
|
|
|
}
|
2026-08-22 11:41:50 +03:00
|
|
|
|
2026-09-13 02:08:55 +03:00
|
|
|
/** What one merge run merges FROM (the caller supplies it — the merger never
|
2026-09-15 17:41:56 +03:00
|
|
|
* reads the chat store): a session's transcript + its coverage checkpoint,
|
|
|
|
|
* or — for the format-agnostic notes front door (2026-09-13) — `messages:
|
|
|
|
|
* null`: raw notes carry no tool parts, so no quiz events are rebuilt (quiz
|
|
|
|
|
* outcomes inside notes surface only through the extractor's understanding
|
|
|
|
|
* observations), and the whole-file checkpoint stamps the import.
|
|
|
|
|
*
|
|
|
|
|
* Coverage (vision-extraction-coverage.md, settled 2026-09-15): a chat merge
|
|
|
|
|
* passes the BATCH it extracted — the caller computed the fingerprints
|
|
|
|
|
* before the run, from the same snapshot the merger sees — and step 4
|
|
|
|
|
* appends one CoverageEntry per message on success ("covered" means
|
|
|
|
|
* exactly this merge happened). `batchTag` prefixes the observation refs so
|
|
|
|
|
* re-runs of the SAME batch dedupe while a re-opened (mutated) message's
|
|
|
|
|
* fresh understanding observations land as new events; the notes path
|
|
|
|
|
* passes neither and keeps the whole-file checkpoint. */
|
2026-09-13 02:08:55 +03:00
|
|
|
export interface MergeSource {
|
2026-09-26 01:52:47 +03:00
|
|
|
messages: LhMessage[] | null;
|
2026-09-13 02:08:55 +03:00
|
|
|
foldedAt: string;
|
2026-09-15 17:41:56 +03:00
|
|
|
/** The batch being merged (chat path): one entry per message in
|
|
|
|
|
* `messages`, same order — undefined (notes path / coverage-less
|
|
|
|
|
* callers) → the legacy whole-chat checkpoint is written instead. */
|
|
|
|
|
coverage?: CoverageEntry[];
|
|
|
|
|
/** The batch's deterministic tag (core/coverage.ts batchTag) — obs refs. */
|
|
|
|
|
batchTag?: string;
|
2026-09-13 02:08:55 +03:00
|
|
|
}
|
|
|
|
|
|
2026-08-31 22:36:41 +03:00
|
|
|
// ─── transcript scanning (shared by the merger + trigger) ─────────────────────
|
2026-08-22 10:30:00 +03:00
|
|
|
|
2026-09-26 01:52:47 +03:00
|
|
|
/** A flattened, order-preserving view of every tool call (+ its result
|
|
|
|
|
* state) in a native transcript. Wave D: the pairing is native — calls
|
|
|
|
|
* come from assistant content blocks; a result map built from the
|
|
|
|
|
* toolResult messages carries the outcome (a dangling call is
|
|
|
|
|
* input-available, an error result is output-error). */
|
2026-08-22 10:30:00 +03:00
|
|
|
interface ToolPartView {
|
|
|
|
|
name: string;
|
|
|
|
|
toolCallId: string;
|
|
|
|
|
state: string;
|
|
|
|
|
input: unknown;
|
|
|
|
|
output: unknown;
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-26 01:52:47 +03:00
|
|
|
function toolParts(messages: LhMessage[]): ToolPartView[] {
|
|
|
|
|
const results = new Map<string, ToolResultMessage>();
|
|
|
|
|
for (const m of messages) {
|
|
|
|
|
if ((m as { role?: string }).role === "toolResult") {
|
|
|
|
|
results.set((m as ToolResultMessage).toolCallId, m as ToolResultMessage);
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-08-22 10:30:00 +03:00
|
|
|
const out: ToolPartView[] = [];
|
2026-09-26 01:52:47 +03:00
|
|
|
for (const m of messages) {
|
|
|
|
|
if ((m as { role?: string }).role !== "assistant") continue;
|
|
|
|
|
const content = (m as { content?: unknown }).content;
|
|
|
|
|
if (!Array.isArray(content)) continue;
|
|
|
|
|
for (const block of content as Array<{ type?: string; id?: string; name?: string; arguments?: unknown }>) {
|
|
|
|
|
if (block.type !== "toolCall" || typeof block.id !== "string") continue;
|
|
|
|
|
const result = results.get(block.id);
|
2026-08-22 10:30:00 +03:00
|
|
|
out.push({
|
2026-09-26 01:52:47 +03:00
|
|
|
name: block.name ?? "",
|
|
|
|
|
toolCallId: block.id,
|
|
|
|
|
state: result ? (result.isError ? "output-error" : "output-available") : "input-available",
|
|
|
|
|
input: block.arguments,
|
|
|
|
|
output: result && !result.isError ? result.details : undefined,
|
2026-08-22 10:30:00 +03:00
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
return out;
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-08 22:19:37 +03:00
|
|
|
/** (chatId, messageIds) → provenance points; order-preserving, deduped. */
|
|
|
|
|
function pointsOf(chatId: string, messages: string[]): ProvenancePoint[] {
|
|
|
|
|
const seen = new Set<string>();
|
|
|
|
|
const out: ProvenancePoint[] = [];
|
|
|
|
|
for (const messageId of messages) {
|
|
|
|
|
if (messageId.length === 0 || seen.has(messageId)) continue;
|
|
|
|
|
seen.add(messageId);
|
|
|
|
|
out.push({ chatId, messageId });
|
|
|
|
|
}
|
|
|
|
|
return out;
|
2026-08-22 10:30:00 +03:00
|
|
|
}
|
|
|
|
|
|
2026-09-08 22:19:37 +03:00
|
|
|
const samePoint = (a: ProvenancePoint, b: ProvenancePoint): boolean =>
|
|
|
|
|
a.chatId === b.chatId && a.messageId === b.messageId;
|
|
|
|
|
|
2026-08-31 22:36:41 +03:00
|
|
|
// ─── the merger (extraction pipeline stage 3 — deterministic) ────────────────
|
2026-08-22 10:30:00 +03:00
|
|
|
|
|
|
|
|
/**
|
2026-08-31 22:36:41 +03:00
|
|
|
* Merge one chat's audited findings into the graph ledgers — the pipeline's
|
2026-09-08 22:19:37 +03:00
|
|
|
* apply stage. Deterministic + idempotent: re-merging the same verdicts (or
|
2026-08-31 22:36:41 +03:00
|
|
|
* a later re-extraction of the same chat) appends nothing but the new.
|
2026-08-22 11:41:50 +03:00
|
|
|
*
|
2026-09-08 22:19:37 +03:00
|
|
|
* Identity is the auditor's link-or-create verdict: `linkTo` an existing
|
|
|
|
|
* node (validated against the ledger — an unknown id falls back to create)
|
|
|
|
|
* appends a LINK line (the session's phrasings join the node's aliases and
|
|
|
|
|
* the provenance unions; getGraph merges lines by id); absent → a new node.
|
|
|
|
|
* Mapping lines record each verdict: one per (chat, sourceId). Idempotent:
|
|
|
|
|
* a proposition id already mapped for this chat reuses its resolution.
|
2026-08-31 22:36:41 +03:00
|
|
|
*
|
2026-09-08 22:19:37 +03:00
|
|
|
* Events are rebuilt deterministically from the transcript: quiz outcomes
|
|
|
|
|
* (correct / miss / dont-know; kinds + timestamps + refs from the tool
|
|
|
|
|
* parts, nodeId from the extractor's attribution through this run's
|
|
|
|
|
* mapping) plus one observation event per proposition carrying an
|
|
|
|
|
* understanding note (ref `obs:<sourceId>`). No taught events, ever.
|
2026-09-13 02:08:55 +03:00
|
|
|
*
|
|
|
|
|
* `chatId` is the run's source key: a real chat, or a notes source id
|
|
|
|
|
* ("src_…") for the format-agnostic import path — same ledgers, same
|
|
|
|
|
* idempotency, one graph. The `source` parameter carries the transcript
|
|
|
|
|
* being merged (null messages = raw notes: no quiz events) and the
|
|
|
|
|
* checkpoint stamp; the merger itself is chat-store-free.
|
2026-08-22 10:30:00 +03:00
|
|
|
*/
|
2026-09-13 02:08:55 +03:00
|
|
|
export function mergeFindings(chatId: string, audit: AuditResult, deps: GraphDeps, source: MergeSource): EmissionStats {
|
|
|
|
|
const { graphStore: ledgers } = deps;
|
2026-08-22 10:30:00 +03:00
|
|
|
const stats: EmissionStats = { nodes: 0, edges: 0, events: 0 };
|
|
|
|
|
|
2026-09-08 22:19:37 +03:00
|
|
|
// 1. Resolution: this run's propositions → graph node ids. Existing
|
|
|
|
|
// mapping lines for THIS chat are loaded first — the audit trail doubles
|
|
|
|
|
// as the resolution table, which makes re-merging the same chat
|
|
|
|
|
// idempotent (a re-extraction re-merges through the auditor's linkTo
|
|
|
|
|
// verdicts instead). Fresh sourceIds resolve as the verdicts say.
|
|
|
|
|
const existingIds = new Set(ledgers.readNodes().map((n) => n.id));
|
2026-08-22 10:30:00 +03:00
|
|
|
const mapping = new Map<string, string>();
|
2026-09-08 22:19:37 +03:00
|
|
|
const seenMappingKeys = new Set<string>();
|
2026-08-22 11:41:50 +03:00
|
|
|
for (const m of ledgers.readMappings()) {
|
2026-09-08 22:19:37 +03:00
|
|
|
if (m.chatId !== chatId) continue;
|
2026-08-31 22:36:41 +03:00
|
|
|
mapping.set(m.sourceId, m.graphNodeId);
|
2026-09-08 22:19:37 +03:00
|
|
|
seenMappingKeys.add(m.sourceId);
|
2026-08-22 11:41:50 +03:00
|
|
|
}
|
2026-08-22 10:30:00 +03:00
|
|
|
const now = new Date().toISOString();
|
2026-09-08 22:19:37 +03:00
|
|
|
|
|
|
|
|
for (const p of audit.propositions) {
|
|
|
|
|
if (mapping.has(p.id)) continue; // already resolved for this chat — idempotent
|
|
|
|
|
const points = pointsOf(chatId, p.messages);
|
2026-08-31 22:36:41 +03:00
|
|
|
let graphNodeId: string;
|
2026-09-08 22:19:37 +03:00
|
|
|
if (p.linkTo !== undefined && existingIds.has(p.linkTo)) {
|
|
|
|
|
// Identity verdict: this proposition IS that node. Append a link
|
|
|
|
|
// line — the session's phrasings (statement + aliases) join the
|
|
|
|
|
// node's aliases, the provenance unions. Reads merge by id.
|
|
|
|
|
graphNodeId = p.linkTo;
|
2026-09-09 00:40:02 +03:00
|
|
|
// Link-append: phrasings + provenance join at read time (the
|
|
|
|
|
// first line's name/statement/createdAt stand).
|
2026-09-08 22:19:37 +03:00
|
|
|
ledgers.appendNode({
|
|
|
|
|
id: graphNodeId,
|
2026-09-09 00:40:02 +03:00
|
|
|
name: p.name,
|
2026-08-31 22:36:41 +03:00
|
|
|
statement: p.statement,
|
|
|
|
|
aliases: [...p.aliases],
|
|
|
|
|
createdAt: now,
|
2026-09-08 22:19:37 +03:00
|
|
|
provenance: points,
|
|
|
|
|
});
|
|
|
|
|
} else {
|
|
|
|
|
const node: GraphNode = {
|
|
|
|
|
id: newGraphNodeId(),
|
2026-09-09 00:40:02 +03:00
|
|
|
name: p.name,
|
2026-09-08 22:19:37 +03:00
|
|
|
statement: p.statement,
|
|
|
|
|
aliases: [...p.aliases],
|
|
|
|
|
createdAt: now,
|
|
|
|
|
provenance: points,
|
2026-08-31 22:36:41 +03:00
|
|
|
};
|
2026-09-08 22:19:37 +03:00
|
|
|
ledgers.appendNode(node);
|
2026-08-31 22:36:41 +03:00
|
|
|
stats.nodes++;
|
2026-09-08 22:19:37 +03:00
|
|
|
graphNodeId = node.id;
|
|
|
|
|
existingIds.add(graphNodeId);
|
2026-08-31 22:36:41 +03:00
|
|
|
}
|
|
|
|
|
mapping.set(p.id, graphNodeId);
|
2026-09-08 22:19:37 +03:00
|
|
|
if (!seenMappingKeys.has(p.id)) {
|
|
|
|
|
ledgers.appendMapping({ chatId, sourceId: p.id, graphNodeId });
|
|
|
|
|
seenMappingKeys.add(p.id);
|
|
|
|
|
}
|
2026-08-22 10:30:00 +03:00
|
|
|
}
|
|
|
|
|
|
2026-09-08 22:19:37 +03:00
|
|
|
// 2. Edges. Resolve endpoints through the mapping; dedupe by
|
|
|
|
|
// (from,to,type) already asserted by THIS chat (an existing assertion
|
|
|
|
|
// by another chat stays untouched — provenance unions at read time).
|
|
|
|
|
const assertedByChat = new Set<string>();
|
2026-08-22 11:41:50 +03:00
|
|
|
for (const line of ledgers.readEdges()) {
|
2026-08-22 10:30:00 +03:00
|
|
|
const key = `${line.from}\u0000${line.to}\u0000${line.type}`;
|
2026-09-08 22:19:37 +03:00
|
|
|
if (line.provenance.some((pt) => pt.chatId === chatId)) assertedByChat.add(key);
|
2026-08-22 10:30:00 +03:00
|
|
|
}
|
2026-09-08 22:19:37 +03:00
|
|
|
for (const edge of audit.edges) {
|
2026-08-22 10:30:00 +03:00
|
|
|
const from = mapping.get(edge.from);
|
|
|
|
|
const to = mapping.get(edge.to);
|
|
|
|
|
if (!from || !to || from === to) continue; // unmapped endpoint — skip
|
2026-08-31 22:36:41 +03:00
|
|
|
const key = `${from}\u0000${to}\u0000${edge.type}`;
|
2026-09-08 22:19:37 +03:00
|
|
|
if (assertedByChat.has(key)) continue;
|
|
|
|
|
ledgers.appendEdge({ from, to, type: edge.type, provenance: pointsOf(chatId, edge.messages) });
|
|
|
|
|
assertedByChat.add(key);
|
2026-08-22 10:30:00 +03:00
|
|
|
stats.edges++;
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-08 22:19:37 +03:00
|
|
|
// 3. Events, rebuilt deterministically in transcript order. Refs are
|
|
|
|
|
// chat-scoped idempotency keys (quiz toolCallId, obs:<sourceId>) —
|
|
|
|
|
// a ref already in the ledger means this evidence was extracted before.
|
2026-08-22 10:30:00 +03:00
|
|
|
const seenRefs = new Set<string>();
|
2026-08-22 11:41:50 +03:00
|
|
|
for (const event of ledgers.readEvents()) {
|
2026-08-22 10:30:00 +03:00
|
|
|
if (event.chatId === chatId && event.ref) seenRefs.add(event.ref);
|
|
|
|
|
}
|
2026-09-13 02:08:55 +03:00
|
|
|
for (const part of source.messages ? toolParts(source.messages) : []) {
|
2026-09-08 22:19:37 +03:00
|
|
|
if (part.name !== "quiz" || part.state !== "output-available") continue;
|
|
|
|
|
const event = quizEvent(part, chatId, audit, mapping, now);
|
|
|
|
|
if (event?.ref !== undefined && !seenRefs.has(event.ref)) {
|
|
|
|
|
ledgers.appendEvent(event);
|
|
|
|
|
seenRefs.add(event.ref);
|
|
|
|
|
stats.events++;
|
2026-08-22 10:30:00 +03:00
|
|
|
}
|
|
|
|
|
}
|
2026-09-08 22:19:37 +03:00
|
|
|
// Understanding observations: one per proposition carrying a note,
|
2026-09-15 17:41:56 +03:00
|
|
|
// resolved to the node the verdict mapped it to. Refs carry the batch
|
|
|
|
|
// tag when there is one: run-local source ids ("p1"…) shift position
|
|
|
|
|
// between runs, so a bare obs:p1 could never both dedupe a re-run of
|
|
|
|
|
// the same batch AND let a re-opened message's fresh level through.
|
2026-09-08 22:19:37 +03:00
|
|
|
for (const p of audit.propositions) {
|
|
|
|
|
if (p.understanding === undefined || p.understanding.length === 0) continue;
|
2026-09-15 17:41:56 +03:00
|
|
|
const ref = source.batchTag !== undefined ? `obs:${source.batchTag}:${p.id}` : `obs:${p.id}`;
|
2026-09-08 22:19:37 +03:00
|
|
|
if (seenRefs.has(ref)) continue;
|
|
|
|
|
ledgers.appendEvent({
|
|
|
|
|
nodeId: mapping.get(p.id) ?? null,
|
|
|
|
|
kind: "observation",
|
|
|
|
|
chatId,
|
|
|
|
|
at: now,
|
|
|
|
|
ref,
|
|
|
|
|
note: p.understanding,
|
|
|
|
|
});
|
|
|
|
|
seenRefs.add(ref);
|
|
|
|
|
stats.events++;
|
|
|
|
|
}
|
2026-08-22 10:30:00 +03:00
|
|
|
|
2026-09-15 17:41:56 +03:00
|
|
|
// 4. Checkpoint — THE COVERAGE DISCIPLINE (vision-extraction-coverage.md):
|
|
|
|
|
// only now, after every append succeeded, is the batch covered. The
|
|
|
|
|
// chat path appends one CoverageEntry per message (per-batch atomic —
|
|
|
|
|
// an abort or failure mid-run leaves earlier batches covered, this
|
|
|
|
|
// batch's messages uncovered); the notes path keeps the whole-file
|
|
|
|
|
// checkpoint (a file is one shot).
|
|
|
|
|
if (source.coverage !== undefined) {
|
|
|
|
|
for (const entry of source.coverage) ledgers.appendCoverage(entry);
|
|
|
|
|
} else {
|
|
|
|
|
ledgers.appendCheckpoint({ chatId, foldedAt: source.foldedAt });
|
|
|
|
|
}
|
2026-08-22 10:30:00 +03:00
|
|
|
return stats;
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-31 22:36:41 +03:00
|
|
|
/** Build the ledger event for one answered quiz tool part (or null to skip).
|
|
|
|
|
* Deterministic in everything except the attribution-supplied nodeId. */
|
2026-08-22 10:30:00 +03:00
|
|
|
function quizEvent(
|
|
|
|
|
part: ToolPartView,
|
|
|
|
|
chatId: string,
|
2026-09-08 22:19:37 +03:00
|
|
|
audit: AuditResult,
|
2026-08-22 10:30:00 +03:00
|
|
|
mapping: Map<string, string>,
|
|
|
|
|
now: string,
|
|
|
|
|
): LedgerEvent | null {
|
|
|
|
|
const out = part.output;
|
|
|
|
|
if (typeof out !== "object" || out === null) return null;
|
|
|
|
|
const answer = out as Partial<QuestionAnswerData>;
|
|
|
|
|
if (answer.kind !== "quiz") return null; // ask questions are not graded
|
|
|
|
|
if (answer.dismissed === true) return null; // no answer — no evidence
|
|
|
|
|
|
|
|
|
|
let kind: LedgerEventKind;
|
|
|
|
|
if (answer.dontKnow === true) kind = "dont-know";
|
2026-09-08 22:19:37 +03:00
|
|
|
else if (answer.correct === true) kind = "correct";
|
|
|
|
|
else kind = "miss";
|
2026-08-22 10:30:00 +03:00
|
|
|
|
2026-09-08 22:19:37 +03:00
|
|
|
// Tree-blind: EVERY quiz attributes through the extractor's judgment —
|
|
|
|
|
// the vision's retroactive attribution, with no tree mapping to lean on.
|
|
|
|
|
const attributedId = audit.attribution.quiz[part.toolCallId];
|
|
|
|
|
const nodeId = attributedId ? (mapping.get(attributedId) ?? null) : null;
|
2026-08-22 10:30:00 +03:00
|
|
|
|
|
|
|
|
const event: LedgerEvent = {
|
|
|
|
|
nodeId,
|
|
|
|
|
kind,
|
|
|
|
|
chatId,
|
|
|
|
|
at: typeof answer.answeredAt === "string" ? answer.answeredAt : now,
|
|
|
|
|
ref: part.toolCallId,
|
|
|
|
|
};
|
|
|
|
|
if (typeof answer.note === "string" && answer.note.length > 0) event.note = answer.note;
|
2026-09-08 22:19:37 +03:00
|
|
|
// The question text is the attribution input for unattributed quizzes.
|
2026-08-22 10:30:00 +03:00
|
|
|
if (nodeId === null && typeof answer.questionText === "string") event.question = answer.questionText;
|
|
|
|
|
return event;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ─── derivation (reads only — everything below is recomputable) ─────────────
|
|
|
|
|
|
|
|
|
|
export interface DerivedGraph {
|
|
|
|
|
nodes: GraphNode[];
|
|
|
|
|
edges: GraphEdge[];
|
|
|
|
|
events: LedgerEvent[];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
2026-09-08 22:19:37 +03:00
|
|
|
* The whole graph from the ledgers. Node LINES may repeat an id (link-
|
2026-09-09 00:40:02 +03:00
|
|
|
* appends): the FIRST line defines name/statement/createdAt; later
|
|
|
|
|
* lines' statements and aliases join `aliases`, provenance unions — the
|
|
|
|
|
* name never changes after creation. Edge
|
2026-09-08 22:19:37 +03:00
|
|
|
* LINES are assertions, merged by (from,to,type) with unioned provenance,
|
|
|
|
|
* first-seen order. `store events, derive state` — this is the only reader
|
|
|
|
|
* that matters.
|
2026-08-22 10:30:00 +03:00
|
|
|
*/
|
2026-08-31 20:00:12 +03:00
|
|
|
export function getGraph(graphStore: GraphStore): DerivedGraph {
|
2026-09-08 22:19:37 +03:00
|
|
|
// Merge node lines by id (first line defines; later lines add phrasings).
|
|
|
|
|
const byId = new Map<string, GraphNode>();
|
|
|
|
|
for (const line of graphStore.readNodes()) {
|
|
|
|
|
const existing = byId.get(line.id);
|
|
|
|
|
if (!existing) {
|
|
|
|
|
byId.set(line.id, { ...line, aliases: [...line.aliases], provenance: [...line.provenance] });
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
// A later line's statement is a phrasing, not a replacement.
|
|
|
|
|
for (const phrasing of [line.statement, ...line.aliases]) {
|
|
|
|
|
if (phrasing.length > 0 && !existing.aliases.includes(phrasing)) existing.aliases.push(phrasing);
|
|
|
|
|
}
|
|
|
|
|
for (const pt of line.provenance) {
|
|
|
|
|
if (!existing.provenance.some((q) => samePoint(q, pt))) existing.provenance.push(pt);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2026-08-22 10:30:00 +03:00
|
|
|
const merged: GraphEdge[] = [];
|
|
|
|
|
const byKey = new Map<string, GraphEdge>();
|
2026-08-31 20:00:12 +03:00
|
|
|
for (const line of graphStore.readEdges()) {
|
2026-08-22 10:30:00 +03:00
|
|
|
const key = `${line.from}\u0000${line.to}\u0000${line.type}`;
|
|
|
|
|
const existing = byKey.get(key);
|
|
|
|
|
if (existing) {
|
2026-09-08 22:19:37 +03:00
|
|
|
for (const pt of line.provenance) {
|
|
|
|
|
if (!existing.provenance.some((q) => samePoint(q, pt))) existing.provenance.push(pt);
|
2026-08-22 10:30:00 +03:00
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
const edge: GraphEdge = { ...line, provenance: [...line.provenance] };
|
|
|
|
|
byKey.set(key, edge);
|
|
|
|
|
merged.push(edge);
|
|
|
|
|
}
|
|
|
|
|
}
|
2026-09-08 22:19:37 +03:00
|
|
|
return { nodes: [...byId.values()], edges: merged, events: graphStore.readEvents() };
|
2026-08-22 10:30:00 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// ─── the /api/graph payload ──────────────────────────────────────────────────
|
|
|
|
|
|
2026-08-31 19:51:17 +03:00
|
|
|
/** Everything the home graph view needs, in one fetch (shape: contracts). */
|
2026-08-31 20:00:12 +03:00
|
|
|
export function graphSnapshot(deps: GraphDeps): GraphSnapshot {
|
|
|
|
|
const { nodes, edges, events } = getGraph(deps.graphStore);
|
2026-08-22 10:30:00 +03:00
|
|
|
|
|
|
|
|
const nodeChats = new Map<string, Set<string>>();
|
|
|
|
|
const touch = (nodeId: string, chatId: string): void => {
|
|
|
|
|
let chats = nodeChats.get(nodeId);
|
|
|
|
|
if (!chats) {
|
|
|
|
|
chats = new Set();
|
|
|
|
|
nodeChats.set(nodeId, chats);
|
|
|
|
|
}
|
|
|
|
|
chats.add(chatId);
|
|
|
|
|
};
|
2026-08-31 20:00:12 +03:00
|
|
|
for (const m of deps.graphStore.readMappings()) touch(m.graphNodeId, m.chatId);
|
2026-09-08 22:19:37 +03:00
|
|
|
for (const node of nodes) {
|
|
|
|
|
for (const pt of node.provenance) touch(node.id, pt.chatId);
|
|
|
|
|
}
|
2026-08-22 10:30:00 +03:00
|
|
|
for (const edge of edges) {
|
2026-09-08 22:19:37 +03:00
|
|
|
for (const pt of edge.provenance) {
|
|
|
|
|
touch(edge.from, pt.chatId);
|
|
|
|
|
touch(edge.to, pt.chatId);
|
2026-08-22 10:30:00 +03:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
for (const event of events) {
|
|
|
|
|
if (event.nodeId !== null) touch(event.nodeId, event.chatId);
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-13 02:08:55 +03:00
|
|
|
// Labels for every source the graph can point back at: real chats
|
|
|
|
|
// (title, deep-linkable) and imported notes files (path, static — the
|
|
|
|
|
// format-agnostic front door's provenance).
|
|
|
|
|
const sourceIds = new Set<string>();
|
2026-08-22 10:30:00 +03:00
|
|
|
for (const chats of nodeChats.values()) {
|
2026-09-13 02:08:55 +03:00
|
|
|
for (const id of chats) sourceIds.add(id);
|
2026-08-22 10:30:00 +03:00
|
|
|
}
|
|
|
|
|
const sessions: Record<string, string> = {};
|
2026-09-13 02:08:55 +03:00
|
|
|
const fileSources: Record<string, string> = {};
|
|
|
|
|
for (const id of sourceIds) {
|
2026-08-31 20:00:12 +03:00
|
|
|
const chat = deps.chatStore.getChat(id);
|
2026-09-13 02:08:55 +03:00
|
|
|
if (chat) {
|
|
|
|
|
sessions[id] = chat.title;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
const source = deps.graphStore.readSources().find((s) => s.id === id);
|
|
|
|
|
if (source) fileSources[id] = source.filePath;
|
2026-08-22 10:30:00 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const nodeSessions: Record<string, string[]> = {};
|
|
|
|
|
for (const [nodeId, chats] of nodeChats) {
|
2026-09-13 02:08:55 +03:00
|
|
|
const ids = [...chats].filter((id) => sessions[id] !== undefined || fileSources[id] !== undefined);
|
2026-08-22 10:30:00 +03:00
|
|
|
if (ids.length > 0) nodeSessions[nodeId] = ids;
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-09 01:25:14 +03:00
|
|
|
// Understanding observations per (node, chat) — the extractor's
|
|
|
|
|
// session-relative note (latest wins within a chat). Read-time join for
|
|
|
|
|
// the detail card: the confidence history next to the session list.
|
|
|
|
|
const nodeObservations: Record<string, Record<string, string>> = {};
|
|
|
|
|
const latestObs = new Map<string, { at: string; note: string; index: number }>();
|
|
|
|
|
events.forEach((event, index) => {
|
|
|
|
|
if (event.kind !== "observation" || event.nodeId === null) return;
|
|
|
|
|
if (!event.note) return;
|
|
|
|
|
const key = `${event.nodeId}\u0000${event.chatId}`;
|
|
|
|
|
const current = latestObs.get(key);
|
|
|
|
|
if (!current || event.at > current.at || (event.at === current.at && index > current.index)) {
|
|
|
|
|
latestObs.set(key, { at: event.at, note: event.note, index });
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
for (const [key, { note }] of latestObs) {
|
|
|
|
|
const sep = key.indexOf("\u0000");
|
|
|
|
|
const nodeId = key.slice(0, sep);
|
|
|
|
|
const chatId = key.slice(sep + 1);
|
|
|
|
|
(nodeObservations[nodeId] ??= {})[chatId] = note;
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-23 14:51:37 +03:00
|
|
|
// BIRTH-SESSION CLUSTERS (wave 7, owner-locked 2026-09-23): embeddings
|
|
|
|
|
// are dead — a cluster is the SESSION THAT CREATED ITS PROPOSITIONS,
|
|
|
|
|
// derived on read from provenance (core/session-clusters.ts). The label
|
|
|
|
|
// is the session's NAME (chat title / file path): no LLM naming pass, no
|
|
|
|
|
// persisted partition to invalidate. MIN_CLUSTER_SIZE stays the hull
|
|
|
|
|
// presentation rule (a region needs >= 3 propositions for a hull). A
|
|
|
|
|
// birth session whose record is gone (deleted chat) keeps its members
|
|
|
|
|
// but renders unlabeled — honest, never guessed.
|
2026-09-23 15:37:34 +03:00
|
|
|
const communities = birthSessionPartition(nodes, deps.graphStore.readMappings());
|
2026-09-13 14:29:16 +03:00
|
|
|
const clusters: Cluster[] = communities
|
2026-09-23 14:51:37 +03:00
|
|
|
.filter((community) => community.members.length >= MIN_CLUSTER_SIZE)
|
|
|
|
|
.map((community) => ({
|
|
|
|
|
id: community.id,
|
|
|
|
|
label: sessions[community.id] ?? fileSources[community.id] ?? null,
|
2026-09-13 03:59:48 +03:00
|
|
|
members: community.members,
|
|
|
|
|
}));
|
|
|
|
|
|
2026-09-13 08:35:42 +03:00
|
|
|
// Resting layout (Option A): the positions ledger, when the settle has
|
|
|
|
|
// ever run. Absent on a fresh graph — the view falls back to its own
|
|
|
|
|
// one-shot pass until the first merge writes the ledger.
|
|
|
|
|
const positions = deps.graphStore.readPositions();
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
nodes,
|
|
|
|
|
edges,
|
|
|
|
|
sessions,
|
|
|
|
|
fileSources,
|
|
|
|
|
nodeSessions,
|
|
|
|
|
nodeObservations,
|
|
|
|
|
clusters,
|
|
|
|
|
...(Object.keys(positions).length > 0 ? { positions } : {}),
|
|
|
|
|
};
|
2026-08-22 10:30:00 +03:00
|
|
|
}
|
2026-09-11 03:25:25 +03:00
|
|
|
|
2026-09-23 14:51:37 +03:00
|
|
|
// ─── Prober graph tools — the data seam (wave 7, owner-locked 2026-09-23) ───
|
|
|
|
|
|
|
|
|
|
/** One session in the prober's graph view: every provenance anchor the
|
|
|
|
|
* graph can name — a real chat (deep-linkable) or an imported notes file
|
|
|
|
|
* (owner ruling 2026-09-23: files are birth-sessions too, kind "file").
|
|
|
|
|
* The prober's session-list tool renders these; the per-session view is
|
|
|
|
|
* graphSessionPropositions below. Pure derivation over the same ports
|
|
|
|
|
* the snapshot reads — Teacher's tool handlers call these in-process. */
|
|
|
|
|
export interface GraphSessionEntry {
|
|
|
|
|
/** chatId or notes source id ("src_…") — the birth-session key. */
|
|
|
|
|
id: string;
|
|
|
|
|
/** The chat's title, or the file's path — the cluster label's source. */
|
|
|
|
|
name: string;
|
|
|
|
|
/** A real chat (its record still exists) or an imported notes file. */
|
|
|
|
|
kind: "chat" | "file";
|
|
|
|
|
/** When the session began: the chat's createdAt, or the file's
|
|
|
|
|
* importedAt. null when no record can date it. */
|
|
|
|
|
createdAt: string | null;
|
|
|
|
|
/** Most recent activity: the chat's updatedAt, or the file's
|
|
|
|
|
* importedAt. null when no record can date it. */
|
|
|
|
|
updatedAt: string | null;
|
|
|
|
|
/** How many propositions were BORN in this session (birth-session
|
|
|
|
|
* clustering's count — shared/re-encountered nodes count for their
|
|
|
|
|
* birth session only). */
|
|
|
|
|
propositions: number;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** The prober's session list: every birth session, most-recently-active
|
|
|
|
|
* first (undatable records sink, in stable id order). */
|
|
|
|
|
export function graphSessionIndex(deps: GraphDeps): GraphSessionEntry[] {
|
|
|
|
|
const counts = new Map<string, number>();
|
2026-09-23 15:37:34 +03:00
|
|
|
const born = getGraph(deps.graphStore);
|
|
|
|
|
const mappingFallback = birthMappingIndex(deps.graphStore.readMappings());
|
|
|
|
|
for (const cluster of birthSessionPartition(born.nodes, deps.graphStore.readMappings())) {
|
2026-09-23 14:51:37 +03:00
|
|
|
counts.set(cluster.id, cluster.members.length);
|
|
|
|
|
}
|
|
|
|
|
const chats = new Map(deps.chatStore.listChats().map((chat) => [chat.id, chat]));
|
|
|
|
|
const sources = new Map(deps.graphStore.readSources().map((s) => [s.id, s]));
|
|
|
|
|
const entries: GraphSessionEntry[] = [];
|
|
|
|
|
for (const [id, propositions] of counts) {
|
|
|
|
|
const chat = chats.get(id); if (chat) {
|
|
|
|
|
entries.push({
|
|
|
|
|
id,
|
|
|
|
|
name: chat.title,
|
|
|
|
|
kind: "chat",
|
|
|
|
|
createdAt: chat.createdAt,
|
|
|
|
|
updatedAt: chat.updatedAt,
|
|
|
|
|
propositions,
|
|
|
|
|
});
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
const source = sources.get(id);
|
|
|
|
|
entries.push({
|
|
|
|
|
id,
|
|
|
|
|
name: source?.filePath ?? id,
|
|
|
|
|
kind: "file",
|
|
|
|
|
createdAt: source?.importedAt ?? null,
|
|
|
|
|
updatedAt: source?.importedAt ?? null,
|
|
|
|
|
propositions,
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
return entries.sort(
|
|
|
|
|
(a, b) =>
|
|
|
|
|
(b.updatedAt ?? "").localeCompare(a.updatedAt ?? "") ||
|
|
|
|
|
(b.createdAt ?? "").localeCompare(a.createdAt ?? "") ||
|
|
|
|
|
(a.id < b.id ? -1 : 1),
|
|
|
|
|
);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** One birth proposition of one session, for the prober's per-session
|
|
|
|
|
* view. `understanding` is the extractor's latest observation for
|
|
|
|
|
* (node, session) — the named-word scale (solid/shaky/accepted/resisted/
|
|
|
|
|
* not yet engaged) — and `evaluatedAt` is THAT event's own timestamp
|
|
|
|
|
* (owner ruling 2026-09-23: quiz outcomes are evidence in the ledger,
|
|
|
|
|
* but they never move the evaluation date — the level's event does). */
|
|
|
|
|
export interface GraphSessionProposition {
|
|
|
|
|
nodeId: string;
|
|
|
|
|
/** The compact handle — the same name the canvas shows. */
|
|
|
|
|
name: string;
|
|
|
|
|
understanding: UnderstandingLevel | null;
|
|
|
|
|
/** The `at` of the event that minted `understanding`; null when the
|
|
|
|
|
* session never evaluated this proposition. */
|
|
|
|
|
evaluatedAt: string | null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/** The propositions BORN in one session, with the session's latest
|
|
|
|
|
* understanding of each. Latest observation wins by `at`, then by
|
|
|
|
|
* ledger order (the same tie rule the snapshot's nodeObservations
|
|
|
|
|
* uses). Members sorted by node id — deterministic. */
|
|
|
|
|
export function graphSessionPropositions(deps: GraphDeps, sessionId: string): GraphSessionProposition[] {
|
2026-09-23 15:37:34 +03:00
|
|
|
const mappingFallback = birthMappingIndex(deps.graphStore.readMappings());
|
|
|
|
|
const born = getGraph(deps.graphStore).nodes.filter((node) => birthSessionOf(node, mappingFallback) === sessionId);
|
2026-09-23 14:51:37 +03:00
|
|
|
const latest = new Map<string, { at: string; note: string; index: number }>();
|
|
|
|
|
deps.graphStore.readEvents().forEach((event, index) => {
|
|
|
|
|
if (event.kind !== "observation" || event.nodeId === null || event.chatId !== sessionId) return;
|
|
|
|
|
if (!event.note) return;
|
|
|
|
|
const current = latest.get(event.nodeId);
|
|
|
|
|
if (!current || event.at > current.at || (event.at === current.at && index > current.index)) {
|
|
|
|
|
latest.set(event.nodeId, { at: event.at, note: event.note, index });
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
return born
|
|
|
|
|
.map((node) => {
|
|
|
|
|
const observation = latest.get(node.id);
|
|
|
|
|
return {
|
|
|
|
|
nodeId: node.id,
|
|
|
|
|
name: node.name,
|
|
|
|
|
understanding: (observation?.note as UnderstandingLevel | undefined) ?? null,
|
|
|
|
|
evaluatedAt: observation?.at ?? null,
|
|
|
|
|
};
|
|
|
|
|
})
|
|
|
|
|
.sort((a, b) => (a.nodeId < b.nodeId ? -1 : 1));
|
|
|
|
|
}
|
|
|
|
|
|
2026-09-11 03:25:25 +03:00
|
|
|
// Re-export: the contract owns the shape; legacy importers still reach it.
|
|
|
|
|
export type { EmissionStats };
|