Wave 7 close-out: the embedding ledger seam dies; notes birth via mapping lines
Part 1 — the ledger + remnants interlude (master-dispatched, owner ruling 2026-09-23: nothing configures the graph beyond chat models): - The embedding ledger surface is gone: readEmbeddings/writeEmbeddings ports, the EmbeddingsLedger contract, EMBEDDINGS_FILE (config, store impls, graphRevision source list), and graph-deletion's vector-pruning step. On-disk data files stay untouched (inert history). - The extraction embedder seam dies wholesale: ExtractionPipelineDeps drops the embedder dep (chat-handler + backfill/import-notes stubs go with it), the 'embedding' phase leaves ExtractionPhase, the usage capture/persist helpers die, and stages.embedding leaves ExtractionRunUsage (historical records keep their shape — reads are lenient). Extraction is two stages again, like the local-embedder era. - Consumer-less remnants: latestQuizOutcomes + QUIZ_EVENT_KINDS, clusterContentId, matchClusterLabelings + Community + ClusterLabeling, the readClusterLabels/appendClusterLabel port seams + store impls + LEDGER_FILES.clusterLabels. NamingRunUsage/readNamingRuns STAY (standing ruling — the cost pill renders historical naming spend); clusterHue + MIN_CLUSTER_SIZE stay (live). - The 'labeler' role dies: BACKGROUND_MODEL_ROLES, ROLE_SLOT, ModelRole, the catalog pin projections, and the UI roster strings. RolePins keys shrink with the union. The notes birth fallback (owner-approved 2026-09-23, surfaced by the interlude's live smoke: production notes imports create provenance-EMPTY nodes (importNotes passes messages: null — the file's anchor is the mapping line), so the provenance-only birth rule left every imported file unclustered and invisible to the prober's session list, against the locked file-birth ruling. A provenance-empty node now births under its FIRST mapping line's chatId (ledger order); only notes-born nodes ever hit the fallback (chat nodes always carry provenance; a re-encountered node's link-append re-opens the provenance rule). Deletion-safe: mapping lines of deleted chats are stripped alongside their provenance. Live-verified on port 3004: a fresh notes import clusters under its src_ id with the file path as the label, end to end with no embedder. 845/845 tests, typecheck/lint/build green. EOF )
This commit is contained in:
@@ -482,13 +482,11 @@ function runNotesAuditor(
|
||||
// ─── the pipeline status registry ─────────────────────────────────────────────
|
||||
|
||||
/** The pipeline phases, in order — what the home status endpoint shows.
|
||||
* `embedding` was its own phase (accuracy wave, 2026-09-19): the post-batch
|
||||
* embed-re-cluster tail used to hide inside `merging`. Wave 7 stubbed the
|
||||
* tail to a no-op — the phase survives only until Platform's
|
||||
* ledger-port interlude deletes the seam wholesale; it transitions
|
||||
* instantly today. The phases must tell the truth about what the loop is
|
||||
* actually waiting on. */
|
||||
export type ExtractionPhase = "extracting" | "auditing" | "merging" | "embedding";
|
||||
* The post-batch embedding tail was its own phase (accuracy wave,
|
||||
* 2026-09-19) until wave 7 killed the embedder seam wholesale (this
|
||||
* interlude) — birth-session clusters need no post-merge pass. The
|
||||
* phases tell the truth about what the loop is actually waiting on. */
|
||||
export type ExtractionPhase = "extracting" | "auditing" | "merging";
|
||||
|
||||
/** One in-flight pipeline run. Module-level: one process, one registry.
|
||||
* Exported for the status payload's consumer typing (the dock). */
|
||||
@@ -504,13 +502,10 @@ export interface RunningExtraction {
|
||||
phase: ExtractionPhase;
|
||||
startedAt: number;
|
||||
/** Live per-stage usage, accumulated as steps bill (the dock's cost /
|
||||
* context-window blobs read this through /api/graph/status). Wave 6:
|
||||
* the provider-embedding tail became a billable stage — recorded from
|
||||
* the embedder's reported usage when present. */
|
||||
* context-window blobs read this through /api/graph/status). */
|
||||
usage: {
|
||||
extractor?: ExtractionStageUsage;
|
||||
auditor?: ExtractionStageUsage;
|
||||
embedding?: ExtractionStageUsage;
|
||||
};
|
||||
/** True-cancellation handle (the dock's abort button): one controller per
|
||||
* run, shared across phases; abort() cancels the in-flight model call. */
|
||||
@@ -574,15 +569,7 @@ export interface ExtractionModels {
|
||||
engineFor?: (role: ModelRole) => PiEngine | undefined;
|
||||
}
|
||||
|
||||
export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {
|
||||
/** The post-merge embedding batch — WAVE 7 STUB: the provider
|
||||
* embedder and semantic clustering are dead (birth-session clusters
|
||||
* + session-name labels), so production injects a no-op that
|
||||
* reports skipped; tests fake writes to the (still-live) ledger
|
||||
* port. The whole seam dies with Platform's ledger-port interlude
|
||||
* — kept one round so that removal is a pure deletion. */
|
||||
embedder: (graphStore: GraphStore) => Promise<unknown>;
|
||||
}
|
||||
export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {}
|
||||
|
||||
export interface NotesImportInput {
|
||||
/** The source's file path — its identity (deterministic src id) AND its
|
||||
@@ -620,7 +607,7 @@ export function callUsage(usage: Usage): CallUsage {
|
||||
}
|
||||
|
||||
export function createExtractionPipeline(deps: ExtractionPipelineDeps): ExtractionPipeline {
|
||||
const { chatStore, graphStore, embedder: embed } = deps;
|
||||
const { chatStore, graphStore } = deps;
|
||||
// Stage-engine resolution: a direct test value wins; otherwise the
|
||||
// engineFor chain (role override → slot → nothing, resolved FRESH per
|
||||
// stage call server-side) — the job runner's exact treatment. undefined
|
||||
@@ -668,35 +655,6 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
run.usage[stage]!.steps.push(usage);
|
||||
}
|
||||
|
||||
/** The embedding tail's failure log (non-fatal by contract). Wave-6
|
||||
* follow-up (02bd1b0): the production embedder no longer THROWS on
|
||||
* embed failures (usage-on-error — landed vectors + partial usage ride
|
||||
* the result), so this catch is belt-and-suspenders for injected
|
||||
* embedders that still throw. */
|
||||
function recordEmbedFailure(chatId: string): (err: unknown) => undefined {
|
||||
return (err) => {
|
||||
console.error(`[extraction ${chatId}] embedding failed (non-fatal):`, err);
|
||||
return undefined;
|
||||
};
|
||||
}
|
||||
|
||||
/** Wave 6: fold the provider-embedding tail's reported usage into the
|
||||
* run's capture — one ExtractionStageUsage per run (the embedder
|
||||
* already aggregated its API calls into steps). Skipped embeds
|
||||
* (unconfigured) carry no usage; FAILED embeds still carry the landed
|
||||
* batches' steps (usage-on-error, 02bd1b0) — a mid-run failure leaves
|
||||
* a three-stage run's partial embedding spend on the ledger. A run
|
||||
* that embedded nothing stays two-stage, like the whole
|
||||
* local-embedder era. */
|
||||
function captureEmbedUsage(chatId: string, result: unknown): void {
|
||||
const stage = (result as { usage?: ExtractionStageUsage } | undefined)?.usage;
|
||||
if (!stage) return;
|
||||
const run = runningExtractions.get(chatId);
|
||||
if (!run) return;
|
||||
run.usage.embedding = stage;
|
||||
}
|
||||
|
||||
|
||||
async function extract(chatId: string, view: string): Promise<SessionFindings> {
|
||||
const signal = runningExtractions.get(chatId)?.controller.signal;
|
||||
signal?.throwIfAborted(); // an abort during a sync gap ends the run here
|
||||
@@ -771,7 +729,6 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
stages: {
|
||||
extractor: run.usage.extractor ?? { model: "unknown", steps: [] },
|
||||
auditor: run.usage.auditor ?? { model: "unknown", steps: [] },
|
||||
...(run.usage.embedding ? { embedding: run.usage.embedding } : {}),
|
||||
},
|
||||
};
|
||||
if (run.kind === "notes") graphStore.appendSourceUsage(chatId, entry);
|
||||
@@ -856,13 +813,6 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
);
|
||||
}
|
||||
const stats = total;
|
||||
// The embedding tail gets its own phase — the honest label for what
|
||||
// the run is waiting on here (see ExtractionPhase above).
|
||||
setPhase(chatId, "embedding", "chat");
|
||||
// The embedding tail (wave 7 stub — see ExtractionPipelineDeps):
|
||||
// the seam stays until Platform's ledger-port interlude; the
|
||||
// failure contract is unchanged (non-fatal, by-result).
|
||||
await captureEmbedUsage(chatId, await embed(graphStore).catch(recordEmbedFailure(chatId)));
|
||||
// Option A: the layout ledger evolves with the merge (sync — the
|
||||
// next snapshot fetch must already reflect the new nodes).
|
||||
settleGraphLayout(graphStore);
|
||||
@@ -977,12 +927,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
// (messages: null) — nothing to attribute, nothing to build.
|
||||
const auditResult: AuditResult = { ...notesResult, attribution: { quiz: {} } };
|
||||
const stats = mergeFindings(id, auditResult, { chatStore, graphStore }, { messages: null, foldedAt: input.importedAt });
|
||||
// Same post-merge chain as the chat path: embeddings first
|
||||
// (topics from meaning), then the settle, then the namer —
|
||||
// under their own phase, exactly like the chat path.
|
||||
setPhase(id, "embedding", "notes", input.filePath);
|
||||
await captureEmbedUsage(id, await embed(graphStore).catch(recordEmbedFailure(id)));
|
||||
// Option A: same post-merge settle as the chat path.
|
||||
// Same post-merge settle as the chat path.
|
||||
settleGraphLayout(graphStore);
|
||||
persistRunUsage(id, Math.round(performance.now() - t0), stats);
|
||||
console.info(
|
||||
|
||||
+6
-63
@@ -1,71 +1,14 @@
|
||||
/**
|
||||
* CLUSTER LABEL MATCHING (vision §1). DETECTION is birth-session
|
||||
* CLUSTER PRESENTATION RULES (vision §1). DETECTION is birth-session
|
||||
* clustering since wave 7 (core/session-clusters.ts; the semantic
|
||||
* embedder and its Louvain predecessor are both dead). What REMAINS here
|
||||
* is the
|
||||
* cluster contract + core-survival label matching, which is
|
||||
* detection-agnostic: it consumes the same { members } shape semantic
|
||||
* clustering emits.
|
||||
* embedder, its Louvain predecessor, and the LLM label matcher are all
|
||||
* dead — labels are session names now). What remains here is the
|
||||
* one presentation rule the snapshot and the renderer share.
|
||||
*
|
||||
* Labelings key to sorted MEMBER SETS, never to detection indices —
|
||||
* a labeling record outlives any reshuffle of the partition.
|
||||
*
|
||||
* Browser-safe pure logic (no node builtins, no fs): the server runs
|
||||
* matching for the snapshot payload, tests run it directly.
|
||||
* Browser-safe pure logic (no node builtins, no fs).
|
||||
*/
|
||||
import type { ClusterLabeling } from "./contracts.ts";
|
||||
|
||||
/** A detected community. Members are SORTED node ids — the member set IS
|
||||
* the identity a labeling record can outlive a reshuffle with. */
|
||||
export interface Community {
|
||||
members: string[];
|
||||
}
|
||||
|
||||
/** Communities below this size are not rendered, not named — presentation
|
||||
* rule (user-decided 2026-09-13): a region of the world model needs at
|
||||
* least three propositions to be worth a hull and a label. */
|
||||
export const MIN_CLUSTER_SIZE = 3;
|
||||
|
||||
/**
|
||||
* Claim labeling records for detected communities — CORE-SURVIVAL (DECIDED
|
||||
* 2026-09-13): a community keeps a record's label while AT LEAST HALF of
|
||||
* the record's core survives in it (overlap * 2 >= core.length; exactly
|
||||
* half survives). Both sides are single-successor: one community holds at
|
||||
* most one label, one record is claimed by at most one community — on a
|
||||
* split, only the branch carrying the majority of the core keeps the name;
|
||||
* on a merge, the best-overlapping record wins.
|
||||
*
|
||||
* Greedy assignment by overlap size (largest first; ties broken by newer
|
||||
* record). Returns a claim (or null) per community, in input order —
|
||||
* unclaimed communities are the namer's input. Pure: same inputs, same
|
||||
* claims.
|
||||
*/
|
||||
export function matchClusterLabelings(
|
||||
communities: readonly Community[],
|
||||
labelings: readonly ClusterLabeling[],
|
||||
): (ClusterLabeling | null)[] {
|
||||
interface Pair {
|
||||
community: number;
|
||||
labeling: ClusterLabeling;
|
||||
overlap: number;
|
||||
}
|
||||
const pairs: Pair[] = [];
|
||||
communities.forEach((community, ci) => {
|
||||
const members = new Set(community.members);
|
||||
for (const labeling of labelings) {
|
||||
const overlap = labeling.core.reduce((n, id) => n + (members.has(id) ? 1 : 0), 0);
|
||||
if (overlap * 2 >= labeling.core.length && overlap > 0) {
|
||||
pairs.push({ community: ci, labeling, overlap });
|
||||
}
|
||||
}
|
||||
});
|
||||
pairs.sort((a, b) => b.overlap - a.overlap || (a.labeling.createdAt < b.labeling.createdAt ? 1 : -1));
|
||||
const claims: (ClusterLabeling | null)[] = communities.map(() => null);
|
||||
const takenRecords = new Set<string>();
|
||||
for (const pair of pairs) {
|
||||
if (claims[pair.community] || takenRecords.has(pair.labeling.id)) continue;
|
||||
claims[pair.community] = pair.labeling;
|
||||
takenRecords.add(pair.labeling.id);
|
||||
}
|
||||
return claims;
|
||||
}
|
||||
export const MIN_CLUSTER_SIZE = 3;
|
||||
@@ -89,10 +89,6 @@ export interface ExtractionRunUsage {
|
||||
stages: {
|
||||
extractor: ExtractionStageUsage;
|
||||
auditor: ExtractionStageUsage;
|
||||
/** Wave 6 (2026-09-23): the provider-embedding tail — present only
|
||||
* when an embedding model is configured and the run embedded
|
||||
* (local embeddings were free, so this post-dates them). */
|
||||
embedding?: ExtractionStageUsage;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -93,7 +93,8 @@ export interface NamingRunUsage {
|
||||
/** ISO — wall-clock time of the run. */
|
||||
at: string;
|
||||
durationMs: number;
|
||||
/** The labeler role's model id. */
|
||||
/** The naming run's labeler role model id (the role died in wave 7;
|
||||
* the field stays as the historical record's shape). */
|
||||
model: string;
|
||||
/** Every billed call (usually ONE — the namer is single-submit). */
|
||||
steps: CallUsage[];
|
||||
@@ -323,43 +324,6 @@ export interface AuditResult {
|
||||
}
|
||||
|
||||
/** The GET /api/graph payload — everything the home view's PURE renderer needs. */
|
||||
/** cluster-labels.jsonl line — one LLM-named community (vision §1,
|
||||
* DECIDED 2026-09-13). Append-only history: records are immutable, never
|
||||
* edited or deleted; dead labelings remain as history. IDENTITY is the
|
||||
* sorted member set ("core"), hashed into the id, so re-naming the same
|
||||
* community dedupes and a labeling survives graph edits as long as at
|
||||
* least half its core survives in some single community (core-survival;
|
||||
* under half → the community is renamed, splits can keep only one
|
||||
* successor). */
|
||||
/** The embedding ledger shape (config.EMBEDDINGS_FILE): vectors for
|
||||
* EVERY embedded node, all produced by one model at one dimensionality. */
|
||||
export interface EmbeddingsLedger {
|
||||
/** The embedding model id (e.g. the HF repo) — mismatch = stale
|
||||
* vectors, the writer re-embeds everything. */
|
||||
model: string;
|
||||
/** Vector dimensionality (MRL-truncated on write). */
|
||||
dim: number;
|
||||
/** nodeId → L2-normalized vector (cosine similarity = dot product). */
|
||||
vectors: Record<string, number[]>;
|
||||
/** The topic partition over those vectors, computed by the SAME
|
||||
* embedder run (the user-decided trigger: recluster at the end of
|
||||
* every embedding batch, never per read — the agglomerative pass is
|
||||
* ~1.5s at current scale). Consumers (snapshot, settle, namer) read
|
||||
* this instead of recomputing. Empty when nothing is embedded. */
|
||||
clusters: Array<{ members: string[] }>;
|
||||
/** ISO timestamp of the last write. */
|
||||
updatedAt: string;
|
||||
}
|
||||
|
||||
export interface ClusterLabeling {
|
||||
/** "clu_" + content hash of the sorted core. */
|
||||
id: string;
|
||||
/** The short domain label (1–4 words), e.g. "Differential forms". */
|
||||
label: string;
|
||||
/** The member node ids the label was minted for, sorted — the survival core. */
|
||||
core: string[];
|
||||
createdAt: string;
|
||||
}
|
||||
|
||||
/** One rendered cluster in the snapshot payload (vision §1). Members are
|
||||
* node ids; the renderer draws the hull from member positions at draw
|
||||
|
||||
@@ -249,29 +249,5 @@ export function deleteChatTraces(graphStore: GraphStore, chatId: string, mode: C
|
||||
graphStore.writePositions(prunedPositions);
|
||||
}
|
||||
|
||||
// 6. Embeddings: prune dead vectors and dead cluster members — a dead
|
||||
// member in a hull list would break rendering (its position is
|
||||
// gone). Clusters emptied by pruning drop out; sub-minimum ones
|
||||
// vanish at the read-side filter. The model/dim headers stay.
|
||||
const embeddings = graphStore.readEmbeddings();
|
||||
const prunedVectors = Object.fromEntries(
|
||||
Object.entries(embeddings.vectors).filter(([id]) => live.has(id) || !knownNodeIds.has(id)),
|
||||
);
|
||||
const prunedClusters = embeddings.clusters
|
||||
.map((cluster) => ({ members: cluster.members.filter((id) => live.has(id) || !knownNodeIds.has(id)) }))
|
||||
.filter((cluster) => cluster.members.length > 0);
|
||||
if (
|
||||
Object.keys(prunedVectors).length !== Object.keys(embeddings.vectors).length ||
|
||||
prunedClusters.length !== embeddings.clusters.length ||
|
||||
prunedClusters.some((c, i) => c.members.length !== embeddings.clusters[i]!.members.length)
|
||||
) {
|
||||
graphStore.writeEmbeddings({
|
||||
...embeddings,
|
||||
vectors: prunedVectors,
|
||||
clusters: prunedClusters,
|
||||
updatedAt: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
+8
-45
@@ -38,7 +38,7 @@
|
||||
* adapters in.
|
||||
*/
|
||||
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { isToolUIPart, type UIMessage } from "ai";
|
||||
import type { CoverageEntry } from "./contracts/graph.ts";
|
||||
import type {
|
||||
@@ -54,7 +54,7 @@ import type {
|
||||
UnderstandingLevel,
|
||||
} from "./contracts.ts";
|
||||
import { MIN_CLUSTER_SIZE } from "./clusters.ts";
|
||||
import { birthSessionPartition } from "./session-clusters.ts";
|
||||
import { birthMappingIndex, birthSessionOf, birthSessionPartition } from "./session-clusters.ts";
|
||||
import type { QuestionAnswerData } from "./question-core.ts";
|
||||
import type { ChatStore, GraphStore } from "./ports.ts";
|
||||
|
||||
@@ -333,16 +333,6 @@ function quizEvent(
|
||||
|
||||
// ─── derivation (reads only — everything below is recomputable) ─────────────
|
||||
|
||||
/** The content identity of a community: "clu_" + sha256 of the sorted
|
||||
* member ids (12 hex). Deterministic — the same member set always mints
|
||||
* the same id — so re-naming races and idempotent re-runs dedupe at the
|
||||
* store seam (appendClusterLabel). Lives here (server-only core), not in
|
||||
* clusters.ts, because clusters.ts is barrel-exported and must stay
|
||||
* browser-safe (no node:crypto). */
|
||||
export function clusterContentId(members: readonly string[]): string {
|
||||
return "clu_" + createHash("sha256").update([...members].sort().join("\n")).digest("hex").slice(0, 12);
|
||||
}
|
||||
|
||||
export interface DerivedGraph {
|
||||
nodes: GraphNode[];
|
||||
edges: GraphEdge[];
|
||||
@@ -394,34 +384,6 @@ export function getGraph(graphStore: GraphStore): DerivedGraph {
|
||||
return { nodes: [...byId.values()], edges: merged, events: graphStore.readEvents() };
|
||||
}
|
||||
|
||||
const QUIZ_EVENT_KINDS: ReadonlySet<string> = new Set(["correct", "miss", "dont-know"]);
|
||||
|
||||
/**
|
||||
* The latest quiz outcome per node — a raw EVENT FACT, not a node state
|
||||
* (the transitional node-state classes were removed 2026-09-08: nothing
|
||||
* colors nodes by quiz results anymore). Its last consumer, the
|
||||
* graph-as-context selection, died in wave 7 (the injection is gone —
|
||||
* the prober reads its map through tools now); the event fact stays as
|
||||
* raw ledger truth. Nodes with no quiz events have no entry.
|
||||
*/
|
||||
export function latestQuizOutcomes(nodes: GraphNode[], events: LedgerEvent[]): Map<string, LedgerEventKind> {
|
||||
const latest = new Map<string, { at: string; kind: LedgerEventKind; index: number }>();
|
||||
events.forEach((event, index) => {
|
||||
if (event.nodeId === null) return;
|
||||
if (!QUIZ_EVENT_KINDS.has(event.kind)) return;
|
||||
const current = latest.get(event.nodeId);
|
||||
if (!current || event.at > current.at || (event.at === current.at && index > current.index)) {
|
||||
latest.set(event.nodeId, { at: event.at, kind: event.kind, index });
|
||||
}
|
||||
});
|
||||
const out = new Map<string, LedgerEventKind>();
|
||||
for (const node of nodes) {
|
||||
const last = latest.get(node.id);
|
||||
if (last) out.set(node.id, last.kind);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ─── the /api/graph payload ──────────────────────────────────────────────────
|
||||
|
||||
/** Everything the home graph view needs, in one fetch (shape: contracts). */
|
||||
@@ -505,7 +467,7 @@ export function graphSnapshot(deps: GraphDeps): GraphSnapshot {
|
||||
// presentation rule (a region needs >= 3 propositions for a hull). A
|
||||
// birth session whose record is gone (deleted chat) keeps its members
|
||||
// but renders unlabeled — honest, never guessed.
|
||||
const communities = birthSessionPartition(nodes);
|
||||
const communities = birthSessionPartition(nodes, deps.graphStore.readMappings());
|
||||
const clusters: Cluster[] = communities
|
||||
.filter((community) => community.members.length >= MIN_CLUSTER_SIZE)
|
||||
.map((community) => ({
|
||||
@@ -562,7 +524,9 @@ export interface GraphSessionEntry {
|
||||
* first (undatable records sink, in stable id order). */
|
||||
export function graphSessionIndex(deps: GraphDeps): GraphSessionEntry[] {
|
||||
const counts = new Map<string, number>();
|
||||
for (const cluster of birthSessionPartition(getGraph(deps.graphStore).nodes)) {
|
||||
const born = getGraph(deps.graphStore);
|
||||
const mappingFallback = birthMappingIndex(deps.graphStore.readMappings());
|
||||
for (const cluster of birthSessionPartition(born.nodes, deps.graphStore.readMappings())) {
|
||||
counts.set(cluster.id, cluster.members.length);
|
||||
}
|
||||
const chats = new Map(deps.chatStore.listChats().map((chat) => [chat.id, chat]));
|
||||
@@ -619,9 +583,8 @@ export interface GraphSessionProposition {
|
||||
* ledger order (the same tie rule the snapshot's nodeObservations
|
||||
* uses). Members sorted by node id — deterministic. */
|
||||
export function graphSessionPropositions(deps: GraphDeps, sessionId: string): GraphSessionProposition[] {
|
||||
const born = getGraph(deps.graphStore).nodes.filter(
|
||||
(node) => node.provenance[0] !== undefined && node.provenance[0]!.chatId === sessionId,
|
||||
);
|
||||
const mappingFallback = birthMappingIndex(deps.graphStore.readMappings());
|
||||
const born = getGraph(deps.graphStore).nodes.filter((node) => birthSessionOf(node, mappingFallback) === sessionId);
|
||||
const latest = new Map<string, { at: string; note: string; index: number }>();
|
||||
deps.graphStore.readEvents().forEach((event, index) => {
|
||||
if (event.kind !== "observation" || event.nodeId === null || event.chatId !== sessionId) return;
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import type { GraphEdge, GraphNode } from "./contracts.ts";
|
||||
import type { GraphEdge, GraphNode, NodeMapping, Position } from "./contracts.ts";
|
||||
import { incrementalLayout } from "./layout.ts";
|
||||
import { birthSessionPartition } from "./session-clusters.ts";
|
||||
import type { Position } from "./contracts.ts";
|
||||
|
||||
/**
|
||||
* The merge-time settle: recompute the resting layout from the CURRENT
|
||||
@@ -20,6 +19,7 @@ import type { Position } from "./contracts.ts";
|
||||
export function settleGraphLayout(graphStore: {
|
||||
readNodes(): GraphNode[];
|
||||
readEdges(): GraphEdge[];
|
||||
readMappings(): NodeMapping[];
|
||||
readPositions(): Record<string, Position>;
|
||||
writePositions(positions: Record<string, Position>): void;
|
||||
}): void {
|
||||
@@ -39,7 +39,7 @@ export function settleGraphLayout(graphStore: {
|
||||
// core/session-clusters.ts, wave 7) — one cluster concept everywhere.
|
||||
// The unique list is the deduped set the partition needs (lines may
|
||||
// repeat an id — first defines).
|
||||
const communities = birthSessionPartition(unique);
|
||||
const communities = birthSessionPartition(unique, graphStore.readMappings());
|
||||
const next = incrementalLayout(unique, edges, previous, { clusters: communities });
|
||||
graphStore.writePositions(Object.fromEntries(next));
|
||||
}
|
||||
@@ -27,9 +27,7 @@ import type {
|
||||
SessionCheckpoint,
|
||||
TurnUsage,
|
||||
NotesSource,
|
||||
ClusterLabeling,
|
||||
NamingRunUsage,
|
||||
EmbeddingsLedger,
|
||||
} from "./contracts.ts";
|
||||
import type { UserSettings } from "./settings.ts";
|
||||
|
||||
@@ -102,11 +100,6 @@ export interface GraphStore {
|
||||
appendCoverage(entry: CoverageEntry): void;
|
||||
readCoverage(): CoverageEntry[];
|
||||
|
||||
/** Append one cluster labeling — idempotent by id (the id hashes the
|
||||
* sorted core, so identical member sets dedupe). */
|
||||
appendClusterLabel(labeling: ClusterLabeling): void;
|
||||
readClusterLabels(): ClusterLabeling[];
|
||||
|
||||
/** Append one cluster-naming run's usage (graph-global money channel) —
|
||||
* idempotent per startedAt. */
|
||||
appendNamingUsage(entry: NamingRunUsage): void;
|
||||
@@ -128,16 +121,6 @@ export interface GraphStore {
|
||||
/** Replace the whole ledger (the settle is a full-graph computation). */
|
||||
writePositions(positions: Record<string, Position>): void;
|
||||
|
||||
/** The embeddings ledger (config.EMBEDDINGS_FILE) — INERT DATA since
|
||||
* wave 7 (owner ruling 2026-09-23: the provider embedder is dead;
|
||||
* clustering is birth-session, labels are session names; the
|
||||
* graph-context injection is gone with it). The whole seam dies in
|
||||
* the follow-up interlude; graph-deletion's vector pruning is
|
||||
* harmless housekeeping meanwhile. */
|
||||
readEmbeddings(): EmbeddingsLedger;
|
||||
/** Replace the whole ledger (graph-deletion's vector pruning only). */
|
||||
writeEmbeddings(ledger: EmbeddingsLedger): void;
|
||||
|
||||
/** THE DESTRUCTIVE SEAM (session-deletion wave 6, owner 2026-09-22 —
|
||||
* the first non-append operation on these ledgers): atomically
|
||||
* replace the full contents of any subset of the six rewritable
|
||||
|
||||
@@ -21,14 +21,24 @@
|
||||
*
|
||||
* Notes imports are birth-sessions too (owner ruling 2026-09-23): their
|
||||
* nodes carry a "src_…" source id instead of a chat id; the caller labels
|
||||
* them with the file's path (kind: file in the session views).
|
||||
* them with the file's path (kind: file in the session views). The
|
||||
* mapping-line fallback (owner-approved 2026-09-23, surfaced by the
|
||||
* interlude's live smoke): production notes imports create provenance-
|
||||
* EMPTY nodes (the notes front door passes messages: null — the file's
|
||||
* anchor is the mapping line), so a provenance-empty node's birth session
|
||||
* is its FIRST mapping line's chatId in ledger order. Only notes-born
|
||||
* nodes ever hit the fallback — chat extraction always writes provenance
|
||||
* points, and a re-encountered node's link-append adds one, which
|
||||
* re-opens the provenance rule above. Deletion-safe: a deleted chat's
|
||||
* mapping lines are stripped alongside its provenance; notes mappings
|
||||
* survive because files are never deleted.
|
||||
*
|
||||
* Browser-safe pure logic (no node builtins, no fs): same contract style
|
||||
* as the rest of the core graph derivation. Not re-exported through the
|
||||
* core barrel — the shell reads clusters from the snapshot; only the
|
||||
* server-side derivations (snapshot, settle) and tests import this.
|
||||
*/
|
||||
import type { GraphNode } from "./contracts.ts";
|
||||
import type { GraphNode, NodeMapping } from "./contracts.ts";
|
||||
|
||||
/** One birth-session cluster: the session id + the SORTED member node
|
||||
* ids born in it. The shape is deliberately the { members } contract
|
||||
@@ -41,11 +51,23 @@ export interface BirthSessionCluster {
|
||||
members: string[];
|
||||
}
|
||||
|
||||
/** A node's birth session: its first surviving provenance point's chatId,
|
||||
* or null when the node carries no provenance at all (unclustered). */
|
||||
export function birthSessionOf(node: GraphNode): string | null {
|
||||
/** A node's birth session: its first surviving provenance point's chatId;
|
||||
* for a provenance-empty node, its first mapping line's chatId (the notes
|
||||
* fallback — see the module doc). null when neither exists. */
|
||||
export function birthSessionOf(node: GraphNode, mappingFallback?: ReadonlyMap<string, string>): string | null {
|
||||
const first = node.provenance[0];
|
||||
return first === undefined ? null : first.chatId;
|
||||
if (first !== undefined) return first.chatId;
|
||||
return mappingFallback?.get(node.id) ?? null;
|
||||
}
|
||||
|
||||
/** The notes fallback index: nodeId → the chatId of the node's FIRST
|
||||
* mapping line in ledger order. */
|
||||
export function birthMappingIndex(mappings: readonly NodeMapping[]): Map<string, string> {
|
||||
const index = new Map<string, string>();
|
||||
for (const mapping of mappings) {
|
||||
if (!index.has(mapping.graphNodeId)) index.set(mapping.graphNodeId, mapping.chatId);
|
||||
}
|
||||
return index;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -53,15 +75,20 @@ export function birthSessionOf(node: GraphNode): string | null {
|
||||
* (first line defines — the getGraph rule), groups by birth session, and
|
||||
* returns the groups size-desc with members sorted (deterministic output:
|
||||
* ties break by session id ascending). Nodes with no surviving provenance
|
||||
* are left unclustered — they belong to no session the graph can name.
|
||||
* fall back to their first mapping line (notes-born); a node with neither
|
||||
* is left unclustered — it belongs to no session the graph can name.
|
||||
*/
|
||||
export function birthSessionPartition(nodes: readonly GraphNode[]): BirthSessionCluster[] {
|
||||
export function birthSessionPartition(
|
||||
nodes: readonly GraphNode[],
|
||||
mappings: readonly NodeMapping[] = [],
|
||||
): BirthSessionCluster[] {
|
||||
const mappingFallback = birthMappingIndex(mappings);
|
||||
const members = new Map<string, string[]>();
|
||||
const seen = new Set<string>();
|
||||
for (const node of nodes) {
|
||||
if (seen.has(node.id)) continue;
|
||||
seen.add(node.id);
|
||||
const birth = birthSessionOf(node);
|
||||
const birth = birthSessionOf(node, mappingFallback);
|
||||
if (birth === null) continue;
|
||||
const bucket = members.get(birth);
|
||||
if (bucket) bucket.push(node.id);
|
||||
|
||||
@@ -45,7 +45,6 @@ export const BACKGROUND_MODEL_ROLES = [
|
||||
"visualMaker",
|
||||
"extractor",
|
||||
"auditor",
|
||||
"labeler",
|
||||
] as const;
|
||||
|
||||
export type BackgroundModelRole = (typeof BACKGROUND_MODEL_ROLES)[number];
|
||||
@@ -64,7 +63,7 @@ export interface RoleModelOverride {
|
||||
/** The capability tiers a user can pin models into (owner-settled mapping):
|
||||
* strong = prober + teacher (also the per-session default-teacher
|
||||
* starting point); cheap = researcher, demo-builder, extractor, auditor,
|
||||
* labeler; vision = visual-maker + pdf-reader. Resolution per role:
|
||||
; vision = visual-maker + pdf-reader. Resolution per role:
|
||||
* ROLE OVERRIDE → slot → factory pin. */
|
||||
export const MODEL_SLOTS = ["strong", "cheap", "vision"] as const;
|
||||
export type ModelSlotName = (typeof MODEL_SLOTS)[number];
|
||||
@@ -174,7 +173,6 @@ export const ROLE_SLOT: Readonly<Record<ModelRole, ModelSlotName>> = {
|
||||
demoBuilder: "cheap",
|
||||
extractor: "cheap",
|
||||
auditor: "cheap",
|
||||
labeler: "cheap",
|
||||
visualMaker: "vision",
|
||||
pdfReader: "vision",
|
||||
};
|
||||
|
||||
@@ -61,11 +61,6 @@ const jobRunner = createJobRunner({
|
||||
* (/api/graph/import) hit the same single-flight registry. */
|
||||
export const extraction = createExtractionPipeline({
|
||||
...stores,
|
||||
// Wave 7 (owner ruling 2026-09-23): embeddings are DEAD — the pipeline's
|
||||
// embed seam becomes a no-op until Teacher's pass removes the phase
|
||||
// outright. Nothing embedded, nothing spent, the ledger untouched
|
||||
// (its data is inert anyway: nothing reads it but the dying injection).
|
||||
embedder: async () => ({ embedded: 0, total: 0, skipped: "embeddings removed (wave 7)" }),
|
||||
// BYOK (owner, 2026-09-22): the extraction stages resolve the chain —
|
||||
// role override → slot → NOTHING — read FRESH per stage call; an empty
|
||||
// chain flows through as undefined and the pipeline refuses the run
|
||||
|
||||
+1
-3
@@ -1,5 +1,5 @@
|
||||
import * as fs from "node:fs";
|
||||
import { EMBEDDINGS_FILE, LEDGER_FILES, POSITIONS_FILE } from "../../config.ts";
|
||||
import { LEDGER_FILES, POSITIONS_FILE } from "../../config.ts";
|
||||
import type { GraphSnapshot } from "../core/contracts.ts";
|
||||
import type { GraphCostSummary } from "../core/cost-summary.ts";
|
||||
import { graphCostSummary } from "../core/cost-summary.ts";
|
||||
@@ -54,10 +54,8 @@ export function graphRevision(): number {
|
||||
LEDGER_FILES.mappings,
|
||||
LEDGER_FILES.sessions,
|
||||
LEDGER_FILES.coverage,
|
||||
LEDGER_FILES.clusterLabels,
|
||||
LEDGER_FILES.namingRuns,
|
||||
POSITIONS_FILE,
|
||||
EMBEDDINGS_FILE,
|
||||
]) {
|
||||
try {
|
||||
revision = Math.max(revision, fs.statSync(file).mtimeMs);
|
||||
|
||||
@@ -87,7 +87,7 @@ export function rolePinsFor(
|
||||
return pins;
|
||||
}
|
||||
|
||||
/** The six background roles' RESOLVED pins: the user's agent-models
|
||||
/** The background roles' RESOLVED pins: the user's agent-models
|
||||
* override → tier slot → nothing — the same resolution the agent wiring
|
||||
* uses (src/server/stores.ts effectiveSettings), so the caption can
|
||||
* never drift from what actually runs. */
|
||||
|
||||
+1
-80
@@ -29,10 +29,8 @@
|
||||
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { EMBEDDINGS_FILE, LEDGER_FILES, POSITIONS_FILE, SOURCES_FILE } from "../../config.ts";
|
||||
import { LEDGER_FILES, POSITIONS_FILE, SOURCES_FILE } from "../../config.ts";
|
||||
import type {
|
||||
ClusterLabeling,
|
||||
EmbeddingsLedger,
|
||||
GraphEdge,
|
||||
GraphNode,
|
||||
ExtractionRunUsage,
|
||||
@@ -104,14 +102,6 @@ const isNamingRun = (v: unknown): v is NamingRunUsage =>
|
||||
Array.isArray(v.steps) &&
|
||||
isNamingStats(v.stats);
|
||||
|
||||
const isClusterLabeling = (v: unknown): v is ClusterLabeling =>
|
||||
isRecord(v) &&
|
||||
isString(v.id) &&
|
||||
isString(v.label) &&
|
||||
Array.isArray(v.core) &&
|
||||
v.core.every(isString) &&
|
||||
isString(v.createdAt);
|
||||
|
||||
const isGraphNode = (v: unknown): v is GraphNode =>
|
||||
isRecord(v) &&
|
||||
isString(v.id) &&
|
||||
@@ -200,18 +190,6 @@ export const graphStore: GraphStore = {
|
||||
return readValid(LEDGER_FILES.coverage, isCoverageEntry);
|
||||
},
|
||||
|
||||
// ── cluster labelings (config.LEDGER_FILES.clusterLabels) ──
|
||||
|
||||
/** Idempotent by id: the id is a hash of the sorted core, so re-appending
|
||||
* the same member set (re-naming races, idempotent re-runs) is a no-op. */
|
||||
appendClusterLabel(labeling: ClusterLabeling): void {
|
||||
if (this.readClusterLabels().some((l) => l.id === labeling.id)) return;
|
||||
appendLine(LEDGER_FILES.clusterLabels, labeling);
|
||||
},
|
||||
readClusterLabels(): ClusterLabeling[] {
|
||||
return readValid(LEDGER_FILES.clusterLabels, isClusterLabeling);
|
||||
},
|
||||
|
||||
// ── cluster-naming run usage (config.LEDGER_FILES.namingRuns) ──
|
||||
|
||||
/** Idempotent per startedAt — the same discipline as every usage
|
||||
@@ -254,12 +232,6 @@ export const graphStore: GraphStore = {
|
||||
writePositions(positions: Record<string, Position>): void {
|
||||
writePositionMap(positions);
|
||||
},
|
||||
readEmbeddings(): EmbeddingsLedger {
|
||||
return readEmbeddingsLedger();
|
||||
},
|
||||
writeEmbeddings(ledger: EmbeddingsLedger): void {
|
||||
writeEmbeddingsLedger(ledger);
|
||||
},
|
||||
|
||||
/** THE DESTRUCTIVE SEAM (wave 6 session deletion): atomic full-content
|
||||
* replacement per provided ledger — write a uniquely-named temp file
|
||||
@@ -339,57 +311,6 @@ function writePositionMap(positions: Record<string, Position>): void {
|
||||
);
|
||||
}
|
||||
|
||||
/** embeddings.json — the whole EmbeddingsLedger at once. Missing/corrupt
|
||||
* file reads as an EMPTY ledger (model "") so consumers see "nothing
|
||||
* embedded yet" rather than an error; the embedder backfills. */
|
||||
function readEmbeddingsLedger(): EmbeddingsLedger {
|
||||
let raw: string;
|
||||
try {
|
||||
raw = fs.readFileSync(EMBEDDINGS_FILE, "utf-8");
|
||||
} catch {
|
||||
return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" };
|
||||
}
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (!isRecord(parsed)) return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" };
|
||||
const model = typeof parsed.model === "string" ? parsed.model : "";
|
||||
const dim = typeof parsed.dim === "number" && Number.isFinite(parsed.dim) ? parsed.dim : 0;
|
||||
const vectors: Record<string, number[]> = {};
|
||||
if (isRecord(parsed.vectors)) {
|
||||
for (const [id, value] of Object.entries(parsed.vectors)) {
|
||||
if (
|
||||
Array.isArray(value) &&
|
||||
value.length > 0 &&
|
||||
value.every((x) => typeof x === "number" && Number.isFinite(x))
|
||||
) {
|
||||
vectors[id] = value as number[];
|
||||
}
|
||||
}
|
||||
}
|
||||
const clusters: Array<{ members: string[] }> = [];
|
||||
if (Array.isArray(parsed.clusters)) {
|
||||
for (const c of parsed.clusters) {
|
||||
if (
|
||||
isRecord(c) &&
|
||||
Array.isArray(c.members) &&
|
||||
c.members.length > 0 &&
|
||||
c.members.every((m) => typeof m === "string")
|
||||
) {
|
||||
clusters.push({ members: [...(c.members as string[])] });
|
||||
}
|
||||
}
|
||||
}
|
||||
return { model, dim, vectors, clusters, updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : "" };
|
||||
} catch {
|
||||
return { model: "", dim: 0, vectors: {}, clusters: [], updatedAt: "" };
|
||||
}
|
||||
}
|
||||
|
||||
function writeEmbeddingsLedger(ledger: EmbeddingsLedger): void {
|
||||
fs.mkdirSync(path.dirname(EMBEDDINGS_FILE), { recursive: true });
|
||||
fs.writeFileSync(EMBEDDINGS_FILE, JSON.stringify(ledger, null, "\t"), "utf-8");
|
||||
}
|
||||
|
||||
const isPosition = (v: unknown): v is Position =>
|
||||
isRecord(v) && typeof v.x === "number" && Number.isFinite(v.x) && typeof v.y === "number" && Number.isFinite(v.y);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user