Clusters phases B+C: labeling records, the naming call, snapshot payload
Core-survival labelings (vision section 1, DECIDED 2026-09-13): ClusterLabeling records in a new append-only ledger (data/graph/cluster-labels.jsonl), immutable, id = clu_ + sha256 of the sorted core so re-naming races and re-runs dedupe at the store seam. matchClusterLabelings (core/clusters.ts, pure): a community keeps a record's label while at least half its core survives in it; both sides single-successor (a split keeps one name; greedy by overlap, ties to the newer record). The namer (src/agent/subagents/cluster-namer.ts): ONE singular cheap LLM call (new 'labeler' role, glm-flash/low) names every new/changed community at once, given member names + statements (capped at 12 members per cluster). refreshClusterLabels runs POST-EXTRACTION (user-decided) from both front doors, fire-and-forget — naming failure never fails the run; refreshes serialize behind a chain so parallel imports never mint duplicate labels. import-notes.ts awaits clusterNamingSettled before exit. Payload: GraphSnapshot.clusters (>= MIN_CLUSTER_SIZE 3) joins detection with labelings at read time — claimed clusters carry the record's label and id; unlabeled ones carry a member-hash id and null label. Detection is deterministic so no memo in v1 (ms at current scale; revisit at vault scale). Tests (372/372): matcher half-rule/split/merge/greedy/tie cases; naming end-to-end through importNotes (one call, one record, claim on re-refresh), naming failure non-fatal; snapshot derivation with labeling join; 11 test pipelines now pin an explicit labeler seam.
This commit is contained in:
@@ -47,6 +47,9 @@ export const MODELS = {
|
|||||||
extractor: { model: "z-ai/glm-5.3-flash", effort: "low" },
|
extractor: { model: "z-ai/glm-5.3-flash", effort: "low" },
|
||||||
/** The auditor verifies evidence exists; keep it cheap. */
|
/** The auditor verifies evidence exists; keep it cheap. */
|
||||||
auditor: { model: "z-ai/glm-5.3-flash", effort: "low" },
|
auditor: { model: "z-ai/glm-5.3-flash", effort: "low" },
|
||||||
|
/** Names detected communities — one singular call, given member names
|
||||||
|
* + statements. Purely a labeling flourish; keep it cheap. */
|
||||||
|
labeler: { model: "z-ai/glm-5.3-flash", effort: "low" },
|
||||||
} satisfies Record<string, ModelSpec>;
|
} satisfies Record<string, ModelSpec>;
|
||||||
|
|
||||||
export type ModelRole = keyof typeof MODELS;
|
export type ModelRole = keyof typeof MODELS;
|
||||||
@@ -71,6 +74,8 @@ export const LEDGER_FILES = {
|
|||||||
events: path.join(PATHS.graph, "events.jsonl"),
|
events: path.join(PATHS.graph, "events.jsonl"),
|
||||||
mappings: path.join(PATHS.graph, "mappings.jsonl"),
|
mappings: path.join(PATHS.graph, "mappings.jsonl"),
|
||||||
sessions: path.join(PATHS.graph, "sessions.jsonl"),
|
sessions: path.join(PATHS.graph, "sessions.jsonl"),
|
||||||
|
/** LLM-named community records (contracts/graph.ts ClusterLabeling). */
|
||||||
|
clusterLabels: path.join(PATHS.graph, "cluster-labels.jsonl"),
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -15,6 +15,7 @@
|
|||||||
import * as fs from "node:fs";
|
import * as fs from "node:fs";
|
||||||
import * as path from "node:path";
|
import * as path from "node:path";
|
||||||
import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts";
|
import { createExtractionPipeline } from "../src/agent/subagents/extraction.ts";
|
||||||
|
import { clusterNamingSettled } from "../src/agent/subagents/cluster-namer.ts";
|
||||||
import { getGraph } from "../src/core/graph.ts";
|
import { getGraph } from "../src/core/graph.ts";
|
||||||
import { stores } from "../src/server/stores.ts";
|
import { stores } from "../src/server/stores.ts";
|
||||||
|
|
||||||
@@ -52,10 +53,16 @@ async function main() {
|
|||||||
});
|
});
|
||||||
await Promise.all(workers);
|
await Promise.all(workers);
|
||||||
|
|
||||||
|
// The post-merge cluster naming is fire-and-forget inside the pipeline;
|
||||||
|
// settle it before the process exits so labels actually land.
|
||||||
|
await clusterNamingSettled();
|
||||||
|
|
||||||
const graph = getGraph(stores.graphStore);
|
const graph = getGraph(stores.graphStore);
|
||||||
|
const labels = stores.graphStore.readClusterLabels().length;
|
||||||
console.log(
|
console.log(
|
||||||
`[import-notes] done: +${totals.nodes} nodes, +${totals.edges} edges, +${totals.events} events, ${totals.failed} failed — ` +
|
`[import-notes] done: +${totals.nodes} nodes, +${totals.edges} edges, +${totals.events} events, ${totals.failed} failed — ` +
|
||||||
`graph now ${graph.nodes.length} nodes, ${graph.edges.length} edges, ${graph.events.length} events`,
|
`graph now ${graph.nodes.length} nodes, ${graph.edges.length} edges, ${graph.events.length} events, ` +
|
||||||
|
`${labels} named clusters`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,200 @@
|
|||||||
|
/**
|
||||||
|
* CLUSTER NAMING (vision §1, DECIDED 2026-09-13): the garnish layer over
|
||||||
|
* community detection. Detection is purely structural and deterministic
|
||||||
|
* (core/clusters.ts); naming is ONE singular cheap LLM call that labels
|
||||||
|
* every NEW/CHANGED community at once, given member names + statements.
|
||||||
|
*
|
||||||
|
* Core-survival: a community keeps an existing labeling while at least
|
||||||
|
* half its core survives in it (matchClusterLabelings); only communities
|
||||||
|
* with no surviving label AND at least MIN_CLUSTER_SIZE members are sent
|
||||||
|
* to the namer. Records are append-only and idempotent (content-hashed
|
||||||
|
* ids dedupe at the store seam), so a naming race or a re-run appends
|
||||||
|
* nothing new.
|
||||||
|
*
|
||||||
|
* Runs POST-EXTRACTION (user-decided 2026-09-13): both front doors call
|
||||||
|
* refreshClusterLabels after a successful merge. Failure is NON-FATAL —
|
||||||
|
* the graph and hulls render fine unlabeled; the next run retries.
|
||||||
|
*/
|
||||||
|
import { z } from "zod";
|
||||||
|
import { lhTool, toPiTools } from "../lh-tool.ts";
|
||||||
|
import type { PiEngine } from "../models.ts";
|
||||||
|
import { runSubagent } from "./pi-run.ts";
|
||||||
|
import type { GraphNode } from "../../core/contracts.ts";
|
||||||
|
import { getGraph, clusterContentId } from "../../core/graph.ts";
|
||||||
|
import {
|
||||||
|
detectCommunities,
|
||||||
|
matchClusterLabelings,
|
||||||
|
MIN_CLUSTER_SIZE,
|
||||||
|
type Community,
|
||||||
|
} from "../../core/clusters.ts";
|
||||||
|
import type { GraphStore } from "../../core/ports.ts";
|
||||||
|
|
||||||
|
/** How many member propositions the namer sees per cluster — enough to
|
||||||
|
* recognize the domain without paying for the whole region. */
|
||||||
|
const MAX_MEMBERS_PER_CLUSTER = 12;
|
||||||
|
|
||||||
|
const CLUSTER_NAMER_INSTRUCTIONS = `You name communities of propositions in a personal knowledge graph.
|
||||||
|
|
||||||
|
You are given clusters. Each cluster is a set of related propositions
|
||||||
|
(theory fragments the author is learning) plus, for context, a sketch of
|
||||||
|
every OTHER cluster already named. For each UNNAMED cluster, mint ONE
|
||||||
|
short DOMAIN LABEL: 1-4 words naming the region of the world model those
|
||||||
|
propositions belong to together — like a section heading in a textbook.
|
||||||
|
Use the members' own vocabulary ("Differential forms", "Sales
|
||||||
|
psychology", "Server architecture"), not generic words ("Topics",
|
||||||
|
"Various concepts"). If the propositions are unrelated, that is fine:
|
||||||
|
name the best common denominator anyway — a reader should recognize the
|
||||||
|
region at a glance.
|
||||||
|
|
||||||
|
Your entire deliverable is ONE submit_labels call. Include EVERY
|
||||||
|
top-level field even when empty (labels: [] when nothing to name).`;
|
||||||
|
|
||||||
|
const labelsSchema = z.object({
|
||||||
|
labels: z
|
||||||
|
.array(
|
||||||
|
z.object({
|
||||||
|
/** The cluster key from the prompt ("c1", "c2", ...). */
|
||||||
|
key: z.string(),
|
||||||
|
label: z.string(),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.default([]),
|
||||||
|
});
|
||||||
|
|
||||||
|
export interface UnnamedCluster {
|
||||||
|
key: string;
|
||||||
|
members: string[];
|
||||||
|
nodes: Pick<GraphNode, "name" | "statement">[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One single-submit namer run (the same shape as every extraction stage):
|
||||||
|
* the whole deliverable is ONE zod-validated submit call. */
|
||||||
|
async function nameClusters(
|
||||||
|
engine: PiEngine | undefined,
|
||||||
|
signal: AbortSignal | undefined,
|
||||||
|
unnamed: UnnamedCluster[],
|
||||||
|
): Promise<{ key: string; label: string }[]> {
|
||||||
|
let submitted: { key: string; label: string }[] | undefined;
|
||||||
|
await runSubagent(
|
||||||
|
toPiTools({
|
||||||
|
submit_labels: lhTool({
|
||||||
|
name: "submit_labels",
|
||||||
|
description: "Submit the cluster labels.",
|
||||||
|
schema: labelsSchema,
|
||||||
|
execute: async (labels) => {
|
||||||
|
submitted = labels.labels;
|
||||||
|
return labels;
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
{
|
||||||
|
role: "labeler",
|
||||||
|
engine,
|
||||||
|
systemPrompt: CLUSTER_NAMER_INSTRUCTIONS,
|
||||||
|
prompt: JSON.stringify(
|
||||||
|
{
|
||||||
|
unnamedClusters: unnamed.map((c) => ({
|
||||||
|
key: c.key,
|
||||||
|
members: c.nodes.map((n) => ({ name: n.name, statement: n.statement })),
|
||||||
|
})),
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2,
|
||||||
|
),
|
||||||
|
maxTurns: 4,
|
||||||
|
abortSignal: signal,
|
||||||
|
shouldStop: (_message, toolResults) =>
|
||||||
|
toolResults.some((r) => r.toolName === "submit_labels" && !r.isError),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
if (!submitted) throw new Error("the cluster namer never submitted labels");
|
||||||
|
return submitted;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ClusterRefreshStats {
|
||||||
|
/** Communities the current graph divides into (all sizes). */
|
||||||
|
detected: number;
|
||||||
|
/** Communities that kept a label via core-survival. */
|
||||||
|
claimed: number;
|
||||||
|
/** Communities sent to the namer (unnamed, size >= MIN_CLUSTER_SIZE). */
|
||||||
|
named: number;
|
||||||
|
/** Labeling records actually appended (idempotent dedupe may shrink this). */
|
||||||
|
appended: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The post-extraction garnish: detect communities on the CURRENT graph,
|
||||||
|
* join persisted labelings (core-survival), and name what is new — one
|
||||||
|
* singular LLM call. Appends ClusterLabeling records; the NEXT snapshot
|
||||||
|
* read joins them (the payload is derived, never stored).
|
||||||
|
*
|
||||||
|
* Serialized: concurrent merges (parallel notes imports) chain here, so
|
||||||
|
* two runs never mint duplicate labels for the same community.
|
||||||
|
*/
|
||||||
|
async function doRefresh(deps: {
|
||||||
|
graphStore: GraphStore;
|
||||||
|
/** Test seam: a scripted engine (production uses the labeler role). */
|
||||||
|
engine?: PiEngine;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}): Promise<ClusterRefreshStats> {
|
||||||
|
const { graphStore, engine, signal } = deps;
|
||||||
|
const { nodes, edges } = getGraph(graphStore);
|
||||||
|
const detection = detectCommunities(nodes, edges);
|
||||||
|
const labelings = graphStore.readClusterLabels();
|
||||||
|
const claims = matchClusterLabelings(detection.communities, labelings);
|
||||||
|
const byId = new Map(nodes.map((n) => [n.id, n]));
|
||||||
|
|
||||||
|
const claimed = claims.filter((c) => c !== null).length;
|
||||||
|
const unnamed = detection.communities
|
||||||
|
.filter((c, i) => !claims[i] && c.members.length >= MIN_CLUSTER_SIZE)
|
||||||
|
.map((c: Community, i) => {
|
||||||
|
const nodesFor = c.members
|
||||||
|
.map((id) => byId.get(id))
|
||||||
|
.filter((n): n is GraphNode => n !== undefined)
|
||||||
|
.slice(0, MAX_MEMBERS_PER_CLUSTER)
|
||||||
|
.map((n) => ({ name: n.name, statement: n.statement }));
|
||||||
|
return { key: `c${i}`, members: c.members, nodes: nodesFor };
|
||||||
|
});
|
||||||
|
|
||||||
|
if (unnamed.length === 0) return { detected: detection.communities.length, claimed, named: 0, appended: 0 };
|
||||||
|
|
||||||
|
const labels = await nameClusters(engine, signal, unnamed);
|
||||||
|
let appended = 0;
|
||||||
|
const seen = new Set<string>();
|
||||||
|
for (const { key, members } of unnamed) {
|
||||||
|
if (seen.has(key)) continue;
|
||||||
|
const label = labels.find((l) => l.key === key)?.label?.trim();
|
||||||
|
if (!label) continue;
|
||||||
|
seen.add(key);
|
||||||
|
graphStore.appendClusterLabel({
|
||||||
|
id: clusterContentId(members),
|
||||||
|
label,
|
||||||
|
core: members,
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
appended++;
|
||||||
|
}
|
||||||
|
return { detected: detection.communities.length, claimed, named: unnamed.length, appended };
|
||||||
|
}
|
||||||
|
|
||||||
|
let namingTail: Promise<unknown> = Promise.resolve();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Fire the refresh, serialized behind any refresh already running (a
|
||||||
|
* parallel import's merge waits its turn; each run reads fresh ledgers).
|
||||||
|
* Waivable: call sites may fire-and-forget; scripts await the return.
|
||||||
|
*/
|
||||||
|
export function refreshClusterLabels(deps: {
|
||||||
|
graphStore: GraphStore;
|
||||||
|
engine?: PiEngine;
|
||||||
|
signal?: AbortSignal;
|
||||||
|
}): Promise<ClusterRefreshStats> {
|
||||||
|
const run = namingTail.then(() => doRefresh(deps));
|
||||||
|
namingTail = run.catch(() => {});
|
||||||
|
return run;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Await every refresh fired so far (scripts: don't exit mid-naming). */
|
||||||
|
export async function clusterNamingSettled(): Promise<void> {
|
||||||
|
await namingTail;
|
||||||
|
}
|
||||||
@@ -16,6 +16,7 @@ import type {
|
|||||||
SessionFindings,
|
SessionFindings,
|
||||||
} from "../../core/contracts.ts";
|
} from "../../core/contracts.ts";
|
||||||
import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts";
|
import { getGraph, mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts";
|
||||||
|
import { refreshClusterLabels } from "./cluster-namer.ts";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The extraction pipeline (vision §1 construction, 2026-09-02 redesign) — the
|
* The extraction pipeline (vision §1 construction, 2026-09-02 redesign) — the
|
||||||
@@ -548,6 +549,8 @@ export interface ExtractionModels {
|
|||||||
/** Test seam: a scripted engine per stage (production uses the roles). */
|
/** Test seam: a scripted engine per stage (production uses the roles). */
|
||||||
extractor?: PiEngine;
|
extractor?: PiEngine;
|
||||||
auditor?: PiEngine;
|
auditor?: PiEngine;
|
||||||
|
/** Test seam for the cluster namer (post-extraction garnish). */
|
||||||
|
labeler?: PiEngine;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {}
|
export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {}
|
||||||
@@ -716,6 +719,13 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
|||||||
`[extraction ${chatId}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` +
|
`[extraction ${chatId}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` +
|
||||||
`(${Math.round(performance.now() - t0)}ms total)`,
|
`(${Math.round(performance.now() - t0)}ms total)`,
|
||||||
);
|
);
|
||||||
|
// Post-extraction garnish (DECIDED 2026-09-13): name new/changed
|
||||||
|
// clusters. Fire-and-forget — naming never fails the run.
|
||||||
|
refreshClusterLabels({
|
||||||
|
graphStore,
|
||||||
|
engine: deps.labeler,
|
||||||
|
signal: runningExtractions.get(chatId)?.controller.signal,
|
||||||
|
}).catch(() => {});
|
||||||
return stats;
|
return stats;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
persistRunUsage(chatId, Math.round(performance.now() - t0), undefined);
|
persistRunUsage(chatId, Math.round(performance.now() - t0), undefined);
|
||||||
@@ -819,6 +829,11 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
|||||||
`[extraction ${id}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` +
|
`[extraction ${id}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` +
|
||||||
`(${Math.round(performance.now() - t0)}ms total)`,
|
`(${Math.round(performance.now() - t0)}ms total)`,
|
||||||
);
|
);
|
||||||
|
refreshClusterLabels({
|
||||||
|
graphStore,
|
||||||
|
engine: deps.labeler,
|
||||||
|
signal: runningExtractions.get(id)?.controller.signal,
|
||||||
|
}).catch(() => {});
|
||||||
return stats;
|
return stats;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
persistRunUsage(id, Math.round(performance.now() - t0), undefined);
|
persistRunUsage(id, Math.round(performance.now() - t0), undefined);
|
||||||
|
|||||||
@@ -26,6 +26,7 @@
|
|||||||
*/
|
*/
|
||||||
import Graph from "graphology";
|
import Graph from "graphology";
|
||||||
import louvain from "graphology-communities-louvain";
|
import louvain from "graphology-communities-louvain";
|
||||||
|
import type { ClusterLabeling } from "./contracts.ts";
|
||||||
|
|
||||||
/** A detected community. Members are SORTED node ids — the member set IS
|
/** A detected community. Members are SORTED node ids — the member set IS
|
||||||
* the identity a labeling record can outlive a reshuffle with. */
|
* the identity a labeling record can outlive a reshuffle with. */
|
||||||
@@ -33,6 +34,11 @@ export interface Community {
|
|||||||
members: string[];
|
members: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Communities below this size are not rendered, not named — presentation
|
||||||
|
* rule (user-decided 2026-09-13): a region of the world model needs at
|
||||||
|
* least three propositions to be worth a hull and a label. */
|
||||||
|
export const MIN_CLUSTER_SIZE = 3;
|
||||||
|
|
||||||
export interface CommunityDetection {
|
export interface CommunityDetection {
|
||||||
communities: Community[];
|
communities: Community[];
|
||||||
/** Modularity of the found partition — the one quality number. */
|
/** Modularity of the found partition — the one quality number. */
|
||||||
@@ -116,3 +122,47 @@ export function detectCommunities(
|
|||||||
);
|
);
|
||||||
return { communities, modularity: details.modularity };
|
return { communities, modularity: details.modularity };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Claim labeling records for detected communities — CORE-SURVIVAL (DECIDED
|
||||||
|
* 2026-09-13): a community keeps a record's label while AT LEAST HALF of
|
||||||
|
* the record's core survives in it (overlap * 2 >= core.length; exactly
|
||||||
|
* half survives). Both sides are single-successor: one community holds at
|
||||||
|
* most one label, one record is claimed by at most one community — on a
|
||||||
|
* split, only the branch carrying the majority of the core keeps the name;
|
||||||
|
* on a merge, the best-overlapping record wins.
|
||||||
|
*
|
||||||
|
* Greedy assignment by overlap size (largest first; ties broken by newer
|
||||||
|
* record). Returns a claim (or null) per community, in input order —
|
||||||
|
* unclaimed communities are the namer's input. Pure: same inputs, same
|
||||||
|
* claims.
|
||||||
|
*/
|
||||||
|
export function matchClusterLabelings(
|
||||||
|
communities: readonly Community[],
|
||||||
|
labelings: readonly ClusterLabeling[],
|
||||||
|
): (ClusterLabeling | null)[] {
|
||||||
|
interface Pair {
|
||||||
|
community: number;
|
||||||
|
labeling: ClusterLabeling;
|
||||||
|
overlap: number;
|
||||||
|
}
|
||||||
|
const pairs: Pair[] = [];
|
||||||
|
communities.forEach((community, ci) => {
|
||||||
|
const members = new Set(community.members);
|
||||||
|
for (const labeling of labelings) {
|
||||||
|
const overlap = labeling.core.reduce((n, id) => n + (members.has(id) ? 1 : 0), 0);
|
||||||
|
if (overlap * 2 >= labeling.core.length && overlap > 0) {
|
||||||
|
pairs.push({ community: ci, labeling, overlap });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
pairs.sort((a, b) => b.overlap - a.overlap || (a.labeling.createdAt < b.labeling.createdAt ? 1 : -1));
|
||||||
|
const claims: (ClusterLabeling | null)[] = communities.map(() => null);
|
||||||
|
const takenRecords = new Set<string>();
|
||||||
|
for (const pair of pairs) {
|
||||||
|
if (claims[pair.community] || takenRecords.has(pair.labeling.id)) continue;
|
||||||
|
claims[pair.community] = pair.labeling;
|
||||||
|
takenRecords.add(pair.labeling.id);
|
||||||
|
}
|
||||||
|
return claims;
|
||||||
|
}
|
||||||
|
|||||||
@@ -270,6 +270,37 @@ export interface AuditResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** The GET /api/graph payload — everything the home view's PURE renderer needs. */
|
/** The GET /api/graph payload — everything the home view's PURE renderer needs. */
|
||||||
|
/** cluster-labels.jsonl line — one LLM-named community (vision §1,
|
||||||
|
* DECIDED 2026-09-13). Append-only history: records are immutable, never
|
||||||
|
* edited or deleted; dead labelings remain as history. IDENTITY is the
|
||||||
|
* sorted member set ("core"), hashed into the id, so re-naming the same
|
||||||
|
* community dedupes and a labeling survives graph edits as long as at
|
||||||
|
* least half its core survives in some single community (core-survival;
|
||||||
|
* under half → the community is renamed, splits can keep only one
|
||||||
|
* successor). */
|
||||||
|
export interface ClusterLabeling {
|
||||||
|
/** "clu_" + content hash of the sorted core. */
|
||||||
|
id: string;
|
||||||
|
/** The short domain label (1–4 words), e.g. "Differential forms". */
|
||||||
|
label: string;
|
||||||
|
/** The member node ids the label was minted for, sorted — the survival core. */
|
||||||
|
core: string[];
|
||||||
|
createdAt: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One rendered cluster in the snapshot payload (vision §1). Members are
|
||||||
|
* node ids; the renderer draws the hull from member positions at draw
|
||||||
|
* time and gates the label inversely on the node-label zoom control. */
|
||||||
|
export interface Cluster {
|
||||||
|
/** The labeling record id when claimed; unlabeled clusters still
|
||||||
|
* appear (hull, no name) with a member-hash id. */
|
||||||
|
id: string;
|
||||||
|
/** null until the naming call has minted/claimed a labeling. */
|
||||||
|
label: string | null;
|
||||||
|
/** Node ids, sorted. Hull + centroid are renderer concerns. */
|
||||||
|
members: string[];
|
||||||
|
}
|
||||||
|
|
||||||
export interface GraphSnapshot {
|
export interface GraphSnapshot {
|
||||||
nodes: GraphNode[];
|
nodes: GraphNode[];
|
||||||
edges: GraphEdge[];
|
edges: GraphEdge[];
|
||||||
@@ -286,6 +317,10 @@ export interface GraphSnapshot {
|
|||||||
* shown next to the session list — the confidence history). Multiple
|
* shown next to the session list — the confidence history). Multiple
|
||||||
* notes for one (node, chat) collapse to the latest. */
|
* notes for one (node, chat) collapse to the latest. */
|
||||||
nodeObservations: Record<string, Record<string, string>>;
|
nodeObservations: Record<string, Record<string, string>>;
|
||||||
|
/** Structural communities (Louvain over follows+supports), size ≥ 3,
|
||||||
|
* joined with persisted LLM labelings. Purely structural — never
|
||||||
|
* depends on layout. */
|
||||||
|
clusters: Cluster[];
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Derive a short chat title from the learner's first message (pure).
|
/** Derive a short chat title from the learner's first message (pure).
|
||||||
|
|||||||
+30
-2
@@ -38,10 +38,11 @@
|
|||||||
* adapters in.
|
* adapters in.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { randomBytes } from "node:crypto";
|
import { createHash, randomBytes } from "node:crypto";
|
||||||
import { isToolUIPart, type UIMessage } from "ai";
|
import { isToolUIPart, type UIMessage } from "ai";
|
||||||
import type {
|
import type {
|
||||||
AuditResult,
|
AuditResult,
|
||||||
|
Cluster,
|
||||||
EmissionStats,
|
EmissionStats,
|
||||||
GraphEdge,
|
GraphEdge,
|
||||||
GraphNode,
|
GraphNode,
|
||||||
@@ -50,6 +51,7 @@ import type {
|
|||||||
LedgerEvent,
|
LedgerEvent,
|
||||||
LedgerEventKind,
|
LedgerEventKind,
|
||||||
} from "./contracts.ts";
|
} from "./contracts.ts";
|
||||||
|
import { detectCommunities, matchClusterLabelings, MIN_CLUSTER_SIZE } from "./clusters.ts";
|
||||||
import type { QuestionAnswerData } from "./question-core.ts";
|
import type { QuestionAnswerData } from "./question-core.ts";
|
||||||
import type { ChatStore, GraphStore } from "./ports.ts";
|
import type { ChatStore, GraphStore } from "./ports.ts";
|
||||||
|
|
||||||
@@ -330,6 +332,16 @@ export function pendingQuizAnswers(chatId: string, deps: GraphDeps): number {
|
|||||||
|
|
||||||
// ─── derivation (reads only — everything below is recomputable) ─────────────
|
// ─── derivation (reads only — everything below is recomputable) ─────────────
|
||||||
|
|
||||||
|
/** The content identity of a community: "clu_" + sha256 of the sorted
|
||||||
|
* member ids (12 hex). Deterministic — the same member set always mints
|
||||||
|
* the same id — so re-naming races and idempotent re-runs dedupe at the
|
||||||
|
* store seam (appendClusterLabel). Lives here (server-only core), not in
|
||||||
|
* clusters.ts, because clusters.ts is barrel-exported and must stay
|
||||||
|
* browser-safe (no node:crypto). */
|
||||||
|
export function clusterContentId(members: readonly string[]): string {
|
||||||
|
return "clu_" + createHash("sha256").update([...members].sort().join("\n")).digest("hex").slice(0, 12);
|
||||||
|
}
|
||||||
|
|
||||||
export interface DerivedGraph {
|
export interface DerivedGraph {
|
||||||
nodes: GraphNode[];
|
nodes: GraphNode[];
|
||||||
edges: GraphEdge[];
|
edges: GraphEdge[];
|
||||||
@@ -485,7 +497,23 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot {
|
|||||||
(nodeObservations[nodeId] ??= {})[chatId] = note;
|
(nodeObservations[nodeId] ??= {})[chatId] = note;
|
||||||
}
|
}
|
||||||
|
|
||||||
return { nodes, edges, sessions, fileSources, nodeSessions, nodeObservations };
|
// Structural communities (vision §1): Louvain over the undirected
|
||||||
|
// follows+supports structure, joined with persisted labelings —
|
||||||
|
// core-survival matching, size-filtered at the presentation threshold.
|
||||||
|
// Detection is DETERMINISTIC (seeded), so no memo in v1: a snapshot
|
||||||
|
// read recomputes it (ms at current scale; revisit at vault scale).
|
||||||
|
const detection = detectCommunities(nodes, edges);
|
||||||
|
const claims = matchClusterLabelings(detection.communities, deps.graphStore.readClusterLabels());
|
||||||
|
const clusters: Cluster[] = detection.communities
|
||||||
|
.map((community, index) => ({ community, claim: claims[index] }))
|
||||||
|
.filter(({ community }) => community.members.length >= MIN_CLUSTER_SIZE)
|
||||||
|
.map(({ community, claim }) => ({
|
||||||
|
id: claim?.id ?? clusterContentId(community.members),
|
||||||
|
label: claim?.label ?? null,
|
||||||
|
members: community.members,
|
||||||
|
}));
|
||||||
|
|
||||||
|
return { nodes, edges, sessions, fileSources, nodeSessions, nodeObservations, clusters };
|
||||||
}
|
}
|
||||||
|
|
||||||
// Re-export: the contract owns the shape; legacy importers still reach it.
|
// Re-export: the contract owns the shape; legacy importers still reach it.
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ import type {
|
|||||||
SessionCheckpoint,
|
SessionCheckpoint,
|
||||||
TurnUsage,
|
TurnUsage,
|
||||||
NotesSource,
|
NotesSource,
|
||||||
|
ClusterLabeling,
|
||||||
} from "./contracts.ts";
|
} from "./contracts.ts";
|
||||||
|
|
||||||
// ─── ChatStore — the session record's home (data/chats) ─────────────────────
|
// ─── ChatStore — the session record's home (data/chats) ─────────────────────
|
||||||
@@ -82,6 +83,11 @@ export interface GraphStore {
|
|||||||
appendCheckpoint(checkpoint: SessionCheckpoint): void;
|
appendCheckpoint(checkpoint: SessionCheckpoint): void;
|
||||||
readCheckpoints(): SessionCheckpoint[];
|
readCheckpoints(): SessionCheckpoint[];
|
||||||
|
|
||||||
|
/** Append one cluster labeling — idempotent by id (the id hashes the
|
||||||
|
* sorted core, so identical member sets dedupe). */
|
||||||
|
appendClusterLabel(labeling: ClusterLabeling): void;
|
||||||
|
readClusterLabels(): ClusterLabeling[];
|
||||||
|
|
||||||
/** Upsert one imported-notes source record (config.SOURCES_FILE — a
|
/** Upsert one imported-notes source record (config.SOURCES_FILE — a
|
||||||
* JSON map, not a ledger; an upsert keeps the existing usage log). */
|
* JSON map, not a ledger; an upsert keeps the existing usage log). */
|
||||||
upsertSource(source: NotesSource): void;
|
upsertSource(source: NotesSource): void;
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ import * as fs from "node:fs";
|
|||||||
import * as path from "node:path";
|
import * as path from "node:path";
|
||||||
import { LEDGER_FILES, SOURCES_FILE } from "../../config.ts";
|
import { LEDGER_FILES, SOURCES_FILE } from "../../config.ts";
|
||||||
import type {
|
import type {
|
||||||
|
ClusterLabeling,
|
||||||
GraphEdge,
|
GraphEdge,
|
||||||
GraphNode,
|
GraphNode,
|
||||||
ExtractionRunUsage,
|
ExtractionRunUsage,
|
||||||
@@ -86,6 +87,14 @@ function readValid<T>(file: string, isValid: (v: unknown) => v is T): T[] {
|
|||||||
const isProvenancePoint = (v: unknown): v is ProvenancePoint =>
|
const isProvenancePoint = (v: unknown): v is ProvenancePoint =>
|
||||||
isRecord(v) && isString(v.chatId) && isString(v.messageId);
|
isRecord(v) && isString(v.chatId) && isString(v.messageId);
|
||||||
|
|
||||||
|
const isClusterLabeling = (v: unknown): v is ClusterLabeling =>
|
||||||
|
isRecord(v) &&
|
||||||
|
isString(v.id) &&
|
||||||
|
isString(v.label) &&
|
||||||
|
Array.isArray(v.core) &&
|
||||||
|
v.core.every(isString) &&
|
||||||
|
isString(v.createdAt);
|
||||||
|
|
||||||
const isGraphNode = (v: unknown): v is GraphNode =>
|
const isGraphNode = (v: unknown): v is GraphNode =>
|
||||||
isRecord(v) &&
|
isRecord(v) &&
|
||||||
isString(v.id) &&
|
isString(v.id) &&
|
||||||
@@ -164,6 +173,18 @@ export const graphStore: GraphStore = {
|
|||||||
return readValid(LEDGER_FILES.sessions, isCheckpoint);
|
return readValid(LEDGER_FILES.sessions, isCheckpoint);
|
||||||
},
|
},
|
||||||
|
|
||||||
|
// ── cluster labelings (config.LEDGER_FILES.clusterLabels) ──
|
||||||
|
|
||||||
|
/** Idempotent by id: the id is a hash of the sorted core, so re-appending
|
||||||
|
* the same member set (re-naming races, idempotent re-runs) is a no-op. */
|
||||||
|
appendClusterLabel(labeling: ClusterLabeling): void {
|
||||||
|
if (this.readClusterLabels().some((l) => l.id === labeling.id)) return;
|
||||||
|
appendLine(LEDGER_FILES.clusterLabels, labeling);
|
||||||
|
},
|
||||||
|
readClusterLabels(): ClusterLabeling[] {
|
||||||
|
return readValid(LEDGER_FILES.clusterLabels, isClusterLabeling);
|
||||||
|
},
|
||||||
|
|
||||||
// ── imported-notes bookkeeping (config.SOURCES_FILE — a JSON map) ──
|
// ── imported-notes bookkeeping (config.SOURCES_FILE — a JSON map) ──
|
||||||
|
|
||||||
/** Upsert: keeps the existing record's usage log when re-importing. */
|
/** Upsert: keeps the existing record's usage log when re-importing. */
|
||||||
|
|||||||
+61
-4
@@ -6,8 +6,9 @@
|
|||||||
*/
|
*/
|
||||||
import { strict as assert } from "node:assert";
|
import { strict as assert } from "node:assert";
|
||||||
import { describe, test } from "node:test";
|
import { describe, test } from "node:test";
|
||||||
import { detectCommunities } from "../src/core/clusters.ts";
|
import { detectCommunities, matchClusterLabelings, MIN_CLUSTER_SIZE } from "../src/core/clusters.ts";
|
||||||
import type { Community } from "../src/core/clusters.ts";
|
import type { Community } from "../src/core/clusters.ts";
|
||||||
|
import type { ClusterLabeling } from "../src/core/contracts.ts";
|
||||||
|
|
||||||
const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${prefix}${i}`);
|
const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${prefix}${i}`);
|
||||||
|
|
||||||
@@ -15,6 +16,62 @@ const ids = (n: number, prefix = "n") => [...Array(n).keys()].map((i) => `${pref
|
|||||||
const clique = (nodeIds: string[]) =>
|
const clique = (nodeIds: string[]) =>
|
||||||
nodeIds.flatMap((a, i) => nodeIds.slice(i + 1).map((b) => ({ from: a, to: b })));
|
nodeIds.flatMap((a, i) => nodeIds.slice(i + 1).map((b) => ({ from: a, to: b })));
|
||||||
|
|
||||||
|
const labeling = (id: string, core: string[], createdAt = "2026-09-01T00:00:00.000Z"): ClusterLabeling => ({
|
||||||
|
id,
|
||||||
|
label: `label-${id}`,
|
||||||
|
core,
|
||||||
|
createdAt,
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("matchClusterLabelings", () => {
|
||||||
|
test("claims a labeling while at least half its core survives in the community", () => {
|
||||||
|
const community: Community = { members: ["a", "b", "c", "d", "e"] };
|
||||||
|
const records = [labeling("clu_1", ["a", "b", "x", "y"])]; // 2 of 4 survive — exactly half
|
||||||
|
const [claim] = matchClusterLabelings([community], records);
|
||||||
|
assert.equal(claim?.id, "clu_1");
|
||||||
|
// One fewer survivor: under half → relabel.
|
||||||
|
const [dead] = matchClusterLabelings([{ members: ["a", "c", "d", "e", "f"] }], records);
|
||||||
|
assert.equal(dead, null);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("single successor: a split core keeps its label in only one community", () => {
|
||||||
|
const records = [labeling("clu_1", ["a", "b", "c", "d"])];
|
||||||
|
const left: Community = { members: ["a", "b", "z"] }; // 2 of 4 — half
|
||||||
|
const right: Community = { members: ["c", "d", "w"] }; // 2 of 4 — half
|
||||||
|
// Tie: larger overlap wins; equal → both eligible, greedy takes first come.
|
||||||
|
const claims = matchClusterLabelings([left, right], records);
|
||||||
|
const claimed = claims.filter((c) => c !== null);
|
||||||
|
assert.equal(claimed.length, 1);
|
||||||
|
assert.equal(claimed[0]?.id, "clu_1");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("greedy: the best-overlapping community wins the record", () => {
|
||||||
|
const records = [labeling("clu_1", ["a", "b", "c", "d"])];
|
||||||
|
const weak: Community = { members: ["a", "b", "q", "r", "s"] }; // 2/4
|
||||||
|
const strong: Community = { members: ["a", "b", "c", "x", "y"] }; // 3/4
|
||||||
|
const claims = matchClusterLabelings([weak, strong], records);
|
||||||
|
assert.equal(claims[0], null);
|
||||||
|
assert.equal(claims[1]?.id, "clu_1");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("one record per community; newest record breaks ties", () => {
|
||||||
|
const oldR = labeling("clu_old", ["a", "b"], "2026-09-01T00:00:00.000Z");
|
||||||
|
const newR = labeling("clu_new", ["a", "b"], "2026-09-02T00:00:00.000Z");
|
||||||
|
const community: Community = { members: ["a", "b", "c"] };
|
||||||
|
const [claim] = matchClusterLabelings([community], [oldR, newR]);
|
||||||
|
assert.equal(claim?.id, "clu_new");
|
||||||
|
});
|
||||||
|
|
||||||
|
test("no labelings: every community unclaimed; empty communities array ok", () => {
|
||||||
|
assert.deepEqual(matchClusterLabelings([{ members: ["a", "b"] }], []), [null]);
|
||||||
|
assert.deepEqual(matchClusterLabelings([], [labeling("clu_1", ["a"])]), []);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("MIN_CLUSTER_SIZE is 3 (the render/naming threshold)", () => {
|
||||||
|
assert.equal(MIN_CLUSTER_SIZE, 3);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("detectCommunities", () => {
|
describe("detectCommunities", () => {
|
||||||
test("two dense cliques joined by a single bridge split into two communities", () => {
|
test("two dense cliques joined by a single bridge split into two communities", () => {
|
||||||
const left = ids(4, "L");
|
const left = ids(4, "L");
|
||||||
@@ -57,7 +114,7 @@ describe("detectCommunities", () => {
|
|||||||
{ from: "b", to: "c" },
|
{ from: "b", to: "c" },
|
||||||
]);
|
]);
|
||||||
assert.equal(result.communities.length, 1);
|
assert.equal(result.communities.length, 1);
|
||||||
assert.deepEqual(result.communities[0].members.sort(), ["a", "b", "c"]);
|
assert.deepEqual(result.communities[0]!.members.sort(), ["a", "b", "c"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("self-loops and unknown-endpoint edges are skipped defensively", () => {
|
test("self-loops and unknown-endpoint edges are skipped defensively", () => {
|
||||||
@@ -69,7 +126,7 @@ describe("detectCommunities", () => {
|
|||||||
{ from: "a", to: "b" }, // the one real edge
|
{ from: "a", to: "b" }, // the one real edge
|
||||||
]);
|
]);
|
||||||
assert.equal(result.communities.length, 1);
|
assert.equal(result.communities.length, 1);
|
||||||
assert.deepEqual(result.communities[0].members.sort(), ["a", "b"]);
|
assert.deepEqual(result.communities[0]!.members.sort(), ["a", "b"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
test("members are sorted; communities ordered by size then id", () => {
|
test("members are sorted; communities ordered by size then id", () => {
|
||||||
@@ -84,7 +141,7 @@ describe("detectCommunities", () => {
|
|||||||
assert.deepEqual(c.members, sorted);
|
assert.deepEqual(c.members, sorted);
|
||||||
}
|
}
|
||||||
// iso (isolated) is last: size 1 < 4.
|
// iso (isolated) is last: size 1 < 4.
|
||||||
const last = result.communities[result.communities.length - 1];
|
const last = result.communities[result.communities.length - 1]!;
|
||||||
assert.deepEqual(last.members, ["iso"]);
|
assert.deepEqual(last.members, ["iso"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ import type { UIMessage } from "ai";
|
|||||||
import type { PiEngine } from "../src/agent/models.ts";
|
import type { PiEngine } from "../src/agent/models.ts";
|
||||||
import type {
|
import type {
|
||||||
AuditResult,
|
AuditResult,
|
||||||
|
ClusterLabeling,
|
||||||
ExtractionRunUsage,
|
ExtractionRunUsage,
|
||||||
GraphEdge,
|
GraphEdge,
|
||||||
GraphNode,
|
GraphNode,
|
||||||
@@ -38,6 +39,7 @@ import type {
|
|||||||
} from "../src/core/contracts.ts";
|
} from "../src/core/contracts.ts";
|
||||||
import type { GraphStore } from "../src/core/ports.ts";
|
import type { GraphStore } from "../src/core/ports.ts";
|
||||||
import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts";
|
import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts";
|
||||||
|
import { clusterNamingSettled, refreshClusterLabels } from "../src/agent/subagents/cluster-namer.ts";
|
||||||
import { chatStore } from "../src/store/chats.ts";
|
import { chatStore } from "../src/store/chats.ts";
|
||||||
import { PATHS } from "../config.ts";
|
import { PATHS } from "../config.ts";
|
||||||
|
|
||||||
@@ -63,6 +65,7 @@ function memoryLedgers() {
|
|||||||
const mappings: NodeMapping[] = [];
|
const mappings: NodeMapping[] = [];
|
||||||
const checkpoints: SessionCheckpoint[] = [];
|
const checkpoints: SessionCheckpoint[] = [];
|
||||||
const sources = new Map<string, NotesSource>();
|
const sources = new Map<string, NotesSource>();
|
||||||
|
const clusterLabels: ClusterLabeling[] = [];
|
||||||
return {
|
return {
|
||||||
appendNode: (n: GraphNode) => void nodes.push(n),
|
appendNode: (n: GraphNode) => void nodes.push(n),
|
||||||
readNodes: () => nodes.map((n) => ({ ...n })),
|
readNodes: () => nodes.map((n) => ({ ...n })),
|
||||||
@@ -87,6 +90,11 @@ function memoryLedgers() {
|
|||||||
source.extractions = [...(source.extractions ?? []), entry];
|
source.extractions = [...(source.extractions ?? []), entry];
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
// Cluster labelings — mirrors the real store: append dedupes by id.
|
||||||
|
appendClusterLabel: (l: ClusterLabeling) => {
|
||||||
|
if (!clusterLabels.some((x) => x.id === l.id)) clusterLabels.push(l);
|
||||||
|
},
|
||||||
|
readClusterLabels: () => clusterLabels.map((l) => ({ ...l, core: [...l.core] })),
|
||||||
} satisfies GraphStore;
|
} satisfies GraphStore;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -306,6 +314,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
|||||||
graphStore: ledgers,
|
graphStore: ledgers,
|
||||||
extractor: submitModel("submit_findings", FINDINGS),
|
extractor: submitModel("submit_findings", FINDINGS),
|
||||||
auditor: submitModel("submit_review", verdict),
|
auditor: submitModel("submit_review", verdict),
|
||||||
|
labeler: submitModel("submit_labels", { labels: [] }),
|
||||||
});
|
});
|
||||||
const stats = await pipeline.runExtraction(chatId);
|
const stats = await pipeline.runExtraction(chatId);
|
||||||
|
|
||||||
@@ -346,6 +355,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
|||||||
graphStore: ledgers,
|
graphStore: ledgers,
|
||||||
extractor: submitModel("submit_findings", FINDINGS),
|
extractor: submitModel("submit_findings", FINDINGS),
|
||||||
auditor: silentModel(), // never submits the review
|
auditor: silentModel(), // never submits the review
|
||||||
|
labeler: submitModel("submit_labels", { labels: [] }),
|
||||||
});
|
});
|
||||||
await assert.rejects(pipeline.runExtraction(chatId), /the auditor never submitted a review/);
|
await assert.rejects(pipeline.runExtraction(chatId), /the auditor never submitted a review/);
|
||||||
assert.equal(ledgers.readNodes().length, 0);
|
assert.equal(ledgers.readNodes().length, 0);
|
||||||
@@ -356,6 +366,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
|||||||
graphStore: ledgers,
|
graphStore: ledgers,
|
||||||
extractor: silentModel(),
|
extractor: silentModel(),
|
||||||
auditor: submitModel("submit_review", { ...FINDINGS, dropped: [] }),
|
auditor: submitModel("submit_review", { ...FINDINGS, dropped: [] }),
|
||||||
|
labeler: submitModel("submit_labels", { labels: [] }),
|
||||||
});
|
});
|
||||||
await assert.rejects(failingExtractor.runExtraction(chatId), /the extractor never submitted findings/);
|
await assert.rejects(failingExtractor.runExtraction(chatId), /the extractor never submitted findings/);
|
||||||
assert.equal(ledgers.readCheckpoints().length, 0);
|
assert.equal(ledgers.readCheckpoints().length, 0);
|
||||||
@@ -388,6 +399,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
|||||||
// registry must already carry this chat at the right phase.
|
// registry must already carry this chat at the right phase.
|
||||||
extractor: submitModel("submit_findings", FINDINGS, () => seen.push(phaseOf())),
|
extractor: submitModel("submit_findings", FINDINGS, () => seen.push(phaseOf())),
|
||||||
auditor: submitModel("submit_review", verdict, () => seen.push(phaseOf())),
|
auditor: submitModel("submit_review", verdict, () => seen.push(phaseOf())),
|
||||||
|
labeler: submitModel("submit_labels", { labels: [] }),
|
||||||
});
|
});
|
||||||
await pipeline.runExtraction(chatId);
|
await pipeline.runExtraction(chatId);
|
||||||
assert.deepEqual(seen, ["extracting", "auditing"]);
|
assert.deepEqual(seen, ["extracting", "auditing"]);
|
||||||
@@ -422,6 +434,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
|||||||
auditor: submitModel("submit_review", verdict, () =>
|
auditor: submitModel("submit_review", verdict, () =>
|
||||||
usageSeen.push(extractionStatuses().find((r) => r.chatId === chatId)?.usage.extractor?.steps.at(-1)?.inputTokens),
|
usageSeen.push(extractionStatuses().find((r) => r.chatId === chatId)?.usage.extractor?.steps.at(-1)?.inputTokens),
|
||||||
),
|
),
|
||||||
|
labeler: submitModel("submit_labels", { labels: [] }),
|
||||||
});
|
});
|
||||||
await pipeline.runExtraction(chatId);
|
await pipeline.runExtraction(chatId);
|
||||||
|
|
||||||
@@ -458,6 +471,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
|||||||
graphStore: ledgers,
|
graphStore: ledgers,
|
||||||
extractor: hangingModel(), // hangs until aborted — no step finishes
|
extractor: hangingModel(), // hangs until aborted — no step finishes
|
||||||
auditor: submitModel("submit_review", { propositions: [], edges: [], attribution: { quiz: {} }, dropped: [] }),
|
auditor: submitModel("submit_review", { propositions: [], edges: [], attribution: { quiz: {} }, dropped: [] }),
|
||||||
|
labeler: submitModel("submit_labels", { labels: [] }),
|
||||||
});
|
});
|
||||||
const runPromise = pipeline.runExtraction(chatId);
|
const runPromise = pipeline.runExtraction(chatId);
|
||||||
|
|
||||||
@@ -508,6 +522,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
|||||||
graphStore: ledgers,
|
graphStore: ledgers,
|
||||||
extractor: gatedExtractor,
|
extractor: gatedExtractor,
|
||||||
auditor: silentModel(),
|
auditor: silentModel(),
|
||||||
|
labeler: submitModel("submit_labels", { labels: [] }),
|
||||||
});
|
});
|
||||||
|
|
||||||
const first = pipeline.runExtraction(chatId);
|
const first = pipeline.runExtraction(chatId);
|
||||||
@@ -525,6 +540,7 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
|||||||
graphStore: ledgers,
|
graphStore: ledgers,
|
||||||
extractor: silentModel(),
|
extractor: silentModel(),
|
||||||
auditor: silentModel(),
|
auditor: silentModel(),
|
||||||
|
labeler: submitModel("submit_labels", { labels: [] }),
|
||||||
});
|
});
|
||||||
await assert.rejects(pipeline.runExtraction("chat_does_not_exist"), /Chat not found/);
|
await assert.rejects(pipeline.runExtraction("chat_does_not_exist"), /Chat not found/);
|
||||||
});
|
});
|
||||||
@@ -562,6 +578,70 @@ const NOTES_VERDICT = {
|
|||||||
dropped: [],
|
dropped: [],
|
||||||
} as const;
|
} as const;
|
||||||
|
|
||||||
|
const CLUSTER_FINDINGS = {
|
||||||
|
propositions: [
|
||||||
|
{ id: "p1", name: "Modal editing", statement: "Vim edits text through modal command modes.", aliases: [], messages: [], evidence: ["Normal mode keys are commands."], understanding: "solid" },
|
||||||
|
{ id: "p2", name: "Verb-object grammar", statement: "Vim commands compose as verb and object.", aliases: [], messages: [], evidence: ["d then motion deletes."], understanding: "solid" },
|
||||||
|
{ id: "p3", name: "Operator-pending", statement: "Vim enters operator-pending mode between verb and object.", aliases: [], messages: [], evidence: ["After d, a motion is expected."], understanding: "solid" },
|
||||||
|
],
|
||||||
|
edges: [
|
||||||
|
{ from: "p1", to: "p2", type: "follows", messages: [], evidence: ["e"] },
|
||||||
|
{ from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] },
|
||||||
|
],
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
const CLUSTER_VERDICT = {
|
||||||
|
propositions: CLUSTER_FINDINGS.propositions,
|
||||||
|
edges: CLUSTER_FINDINGS.edges,
|
||||||
|
dropped: [],
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
describe("cluster naming (the post-extraction garnish)", () => {
|
||||||
|
it("names a new community after a notes merge — one record, claimed on the next refresh", async () => {
|
||||||
|
const ledgers = memoryLedgers();
|
||||||
|
const namerCalls: number[] = [];
|
||||||
|
const labeler = submitModel("submit_labels", { labels: [{ key: "c0", label: "Vim modal grammar" }] }, () => namerCalls.push(1));
|
||||||
|
const pipeline = createExtractionPipeline({
|
||||||
|
chatStore,
|
||||||
|
graphStore: ledgers,
|
||||||
|
extractor: submitModel("submit_findings", CLUSTER_FINDINGS),
|
||||||
|
auditor: submitModel("submit_review", CLUSTER_VERDICT),
|
||||||
|
labeler,
|
||||||
|
});
|
||||||
|
const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes, any format" });
|
||||||
|
assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 });
|
||||||
|
// The refresh is fire-and-forget inside the pipeline — settle it.
|
||||||
|
await clusterNamingSettled();
|
||||||
|
assert.equal(namerCalls.length, 1, "ONE singular naming call");
|
||||||
|
const records = ledgers.readClusterLabels();
|
||||||
|
assert.equal(records.length, 1);
|
||||||
|
assert.equal(records[0]!.label, "Vim modal grammar");
|
||||||
|
assert.equal(records[0]!.core.length, 3);
|
||||||
|
assert.match(records[0]!.id, /^clu_/);
|
||||||
|
// Second refresh: core-survival claims it — nothing new to name.
|
||||||
|
const again = await refreshClusterLabels({ graphStore: ledgers, engine: labeler });
|
||||||
|
assert.equal(again.named, 0);
|
||||||
|
assert.equal(again.appended, 0);
|
||||||
|
assert.ok(again.claimed >= 1);
|
||||||
|
assert.equal(ledgers.readClusterLabels().length, 1, "no duplicate records");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("naming failure never fails the run", async () => {
|
||||||
|
const ledgers = memoryLedgers();
|
||||||
|
const pipeline = createExtractionPipeline({
|
||||||
|
chatStore,
|
||||||
|
graphStore: ledgers,
|
||||||
|
extractor: submitModel("submit_findings", CLUSTER_FINDINGS),
|
||||||
|
auditor: submitModel("submit_review", CLUSTER_VERDICT),
|
||||||
|
labeler: silentModel(), // never submits — the namer throws
|
||||||
|
});
|
||||||
|
const stats = await pipeline.importNotes({ filePath: "/vault/modal.md", text: "raw notes" });
|
||||||
|
assert.deepEqual(stats, { nodes: 3, edges: 2, events: 3 }, "the merge itself succeeded");
|
||||||
|
await clusterNamingSettled();
|
||||||
|
assert.equal(ledgers.readClusterLabels().length, 0, "nothing named, nothing appended");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe("the notes front door (importNotes — format-agnostic)", () => {
|
describe("the notes front door (importNotes — format-agnostic)", () => {
|
||||||
it("imports raw notes end to end: source record, file-scoped provenance, no quiz events", async () => {
|
it("imports raw notes end to end: source record, file-scoped provenance, no quiz events", async () => {
|
||||||
const ledgers = memoryLedgers();
|
const ledgers = memoryLedgers();
|
||||||
@@ -575,6 +655,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
|
|||||||
const run = extractionStatuses().find((r) => r.chatId.startsWith("src_"));
|
const run = extractionStatuses().find((r) => r.chatId.startsWith("src_"));
|
||||||
seen.push({ kind: run?.kind, label: run?.label });
|
seen.push({ kind: run?.kind, label: run?.label });
|
||||||
}),
|
}),
|
||||||
|
labeler: submitModel("submit_labels", { labels: [] }),
|
||||||
});
|
});
|
||||||
const stats = await pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
|
const stats = await pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
|
||||||
|
|
||||||
@@ -618,6 +699,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
|
|||||||
graphStore: ledgers,
|
graphStore: ledgers,
|
||||||
extractor: submitModel("submit_findings", NOTES_FINDINGS),
|
extractor: submitModel("submit_findings", NOTES_FINDINGS),
|
||||||
auditor: submitModel("submit_review", NOTES_VERDICT),
|
auditor: submitModel("submit_review", NOTES_VERDICT),
|
||||||
|
labeler: submitModel("submit_labels", { labels: [] }),
|
||||||
});
|
});
|
||||||
const first = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
|
const first = await mk().importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
|
||||||
assert.deepEqual(first, { nodes: 1, edges: 0, events: 1 });
|
assert.deepEqual(first, { nodes: 1, edges: 0, events: 1 });
|
||||||
@@ -637,6 +719,7 @@ describe("the notes front door (importNotes — format-agnostic)", () => {
|
|||||||
graphStore: ledgers,
|
graphStore: ledgers,
|
||||||
extractor: hangingModel(),
|
extractor: hangingModel(),
|
||||||
auditor: submitModel("submit_review", NOTES_VERDICT),
|
auditor: submitModel("submit_review", NOTES_VERDICT),
|
||||||
|
labeler: submitModel("submit_labels", { labels: [] }),
|
||||||
});
|
});
|
||||||
const runPromise = pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
|
const runPromise = pipeline.importNotes({ filePath: "/vault/learn vim.md", text: NOTES_TEXT });
|
||||||
|
|
||||||
|
|||||||
+48
-1
@@ -20,7 +20,7 @@ import { after, describe, it } from "node:test";
|
|||||||
import type { UIMessage } from "ai";
|
import type { UIMessage } from "ai";
|
||||||
import type { AuditResult } from "../src/core/contracts.ts";
|
import type { AuditResult } from "../src/core/contracts.ts";
|
||||||
import type { QuestionAnswerData } from "../src/core/question-core.ts";
|
import type { QuestionAnswerData } from "../src/core/question-core.ts";
|
||||||
import { getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts";
|
import { clusterContentId, getGraph, graphSnapshot, mergeFindings, pendingQuizAnswers, type GraphDeps } from "../src/core/graph.ts";
|
||||||
import { graphStore } from "../src/store/graph.ts";
|
import { graphStore } from "../src/store/graph.ts";
|
||||||
import { chatStore } from "../src/store/chats.ts";
|
import { chatStore } from "../src/store/chats.ts";
|
||||||
import { LEDGER_FILES, PATHS, SOURCES_FILE } from "../config.ts";
|
import { LEDGER_FILES, PATHS, SOURCES_FILE } from "../config.ts";
|
||||||
@@ -407,6 +407,53 @@ describe("the merger (mergeFindings)", () => {
|
|||||||
assert.equal(snap.nodeObservations[node.id]!["src_snapfixture"], "solid");
|
assert.equal(snap.nodeObservations[node.id]!["src_snapfixture"], "solid");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("derives structural clusters in graphSnapshot, joined with persisted labelings", () => {
|
||||||
|
// A self-contained 3-clique: three propositions, all mutually linked.
|
||||||
|
const tri = ["Alpha.", "Beta.", "Gamma cluster."] as const;
|
||||||
|
const merge = mergeFindings(
|
||||||
|
"src_clusterfixture",
|
||||||
|
{
|
||||||
|
propositions: tri.map((statement, i) => ({
|
||||||
|
id: `p${i + 1}`,
|
||||||
|
name: `Cluster node ${i + 1}`,
|
||||||
|
statement,
|
||||||
|
aliases: [],
|
||||||
|
messages: [],
|
||||||
|
evidence: ["fixture evidence"],
|
||||||
|
})),
|
||||||
|
edges: [
|
||||||
|
{ from: "p1", to: "p2", type: "follows", messages: [], evidence: ["e"] },
|
||||||
|
{ from: "p2", to: "p3", type: "follows", messages: [], evidence: ["e"] },
|
||||||
|
{ from: "p1", to: "p3", type: "follows", messages: [], evidence: ["e"] },
|
||||||
|
],
|
||||||
|
attribution: { quiz: {} },
|
||||||
|
dropped: [],
|
||||||
|
},
|
||||||
|
graphDeps,
|
||||||
|
{ messages: null, foldedAt: "2026-09-13T00:00:00.000Z" },
|
||||||
|
);
|
||||||
|
assert.equal(merge.nodes, 3);
|
||||||
|
const snap = graphSnapshot(graphDeps);
|
||||||
|
// Presentation threshold: every cluster is >= 3 members.
|
||||||
|
assert.ok(snap.clusters.every((c) => c.members.length >= 3));
|
||||||
|
// Our trio is exactly one cluster, unlabeled until the namer runs.
|
||||||
|
const mine = snap.clusters.find((c) => c.members.length === 3 && tri.every((s) => c.members.includes(snap.nodes.find((n) => n.statement === s)!.id)));
|
||||||
|
assert.ok(mine, "the 3-clique is a cluster");
|
||||||
|
assert.equal(mine!.label, null);
|
||||||
|
assert.match(mine!.id, /^clu_/);
|
||||||
|
// Mint the labeling the namer would append — the next read joins it.
|
||||||
|
graphStore.appendClusterLabel({
|
||||||
|
id: clusterContentId(mine!.members),
|
||||||
|
label: "Test region",
|
||||||
|
core: mine!.members,
|
||||||
|
createdAt: "2026-09-13T00:00:00.000Z",
|
||||||
|
});
|
||||||
|
const snap2 = graphSnapshot(graphDeps);
|
||||||
|
const claimed = snap2.clusters.find((c) => c.members.length === 3 && tri.every((s) => c.members.includes(snap2.nodes.find((n) => n.statement === s)!.id)))!;
|
||||||
|
assert.equal(claimed.label, "Test region");
|
||||||
|
assert.equal(claimed.id, clusterContentId(mine!.members), "the claimed cluster carries the record's id");
|
||||||
|
});
|
||||||
|
|
||||||
it("derives sessions and per-session understanding observations in graphSnapshot", () => {
|
it("derives sessions and per-session understanding observations in graphSnapshot", () => {
|
||||||
const snap = graphSnapshot(graphDeps);
|
const snap = graphSnapshot(graphDeps);
|
||||||
const idByStatement = new Map(snap.nodes.map((n) => [n.statement, n.id]));
|
const idByStatement = new Map(snap.nodes.map((n) => [n.statement, n.id]));
|
||||||
|
|||||||
Reference in New Issue
Block a user