diff --git a/.gitignore b/.gitignore index 57ef722..975197b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ data/ .DS_Store *.tsbuildinfo .env.local +.memory/ diff --git a/README.md b/README.md index 8e3fd36..e82ec94 100644 --- a/README.md +++ b/README.md @@ -26,13 +26,11 @@ npx tsx scripts/bench-graph.ts # force-layout benchmark (synthetic graphs) ## Environment (`.env.local`) +Secrets and machine-specific paths only. Model choice and thinking budgets are product tuning, not environment — they live in **`config.ts` → `MODELS`** (teacher, researcher, demoBuilder, visualMaker, extractor, auditor; each with an OpenRouter model id + reasoning `effort`). The extraction trigger threshold is there too (`EXTRACTION.everyNQuizAnswers`). + | Variable | Purpose | |---|---| | `OPENROUTER_API_KEY` | required — all LLM calls go through OpenRouter | -| `LH_MODEL` | teacher model (default `anthropic/claude-sonnet-4.5`) | -| `LH_RESEARCH_MODEL`, `LH_DEMO_MODEL`, `LH_EXTRACT_MODEL` | per-sub-agent overrides (fall back to `LH_MODEL`) | -| `LH_EXTRACT_N` | answered quizzes per chat before the extraction pipeline fires (default `3`) | -| `LH_VISUAL_MODEL` | visual-maker model — must be **vision-capable** (it looks at rendered SVGs) | | `GOOGLE_SEARCH_API_KEY` + `GOOGLE_CSE_ID` | researcher's `web_search` (Google Custom Search) | | `CHROME_PATH` | optional explicit Chrome path for headless verification (puppeteer-core uses system Chrome; nothing is downloaded) | @@ -161,9 +159,9 @@ data/ chats/ · graph/ · jobs/ — runtime state, git **The formal plan tree.** The teacher authors the session's dependency tree with schema-validated tools (`plan_init`, `plan_add_node`, … `plan_set_current`, `plan_mark_done`). The tree persists in the chat record; the tree panel renders from the latest plan tool output in the transcript. Quiz calls can tag a `nodeId`, so outcomes feed the graph per-node. -**The async sub-agent protocol.** `research` / `demo` / `visual` are also client-side tools: the server turn ends at the call, `onFinish` spawns a background runner (fire-and-forget), the job store tracks a live activity line, the client polls `/api/jobs` while parts are pending, and on completion the runner rewrites the tool part in the chat record to `output-available` — the client re-syncs, the continuation fires, the teacher receives the result as a normal tool output. Sub-agents **verify, never one-shot**: the demo-builder headlessly executes its HTML in system Chrome before delivery; the visual-maker renders each SVG and looks at it (needs a vision-capable `LH_VISUAL_MODEL`). +**The async sub-agent protocol.** `research` / `demo` / `visual` are also client-side tools: the server turn ends at the call, `onFinish` spawns a background runner (fire-and-forget), the job store tracks a live activity line, the client polls `/api/jobs` while parts are pending, and on completion the runner rewrites the tool part in the chat record to `output-available` — the client re-syncs, the continuation fires, the teacher receives the result as a normal tool output. Sub-agents **verify, never one-shot**: the demo-builder headlessly executes its HTML in system Chrome before delivery; the visual-maker renders each SVG and looks at it (needs a vision-capable model — `MODELS.visualMaker` in config.ts). -**The knowledge graph (the two channels).** The write channel: the extraction pipeline — extractor (LLM: propositions, edges, probe attribution, each with verbatim evidence) → auditor (LLM: drops anything unbacked) → merger (`mergeFindings`, deterministic: exact/alias identity resolution, event rebuild from the transcript's tool parts, dedupe, checkpoint). Triggered once a chat accumulates `EXTRACTION.everyNQuizAnswers` (default 3, `LH_EXTRACT_N`) new answered quizzes — a v1 placeholder trigger isolated in one seam (`turn-service.ts`), swappable without touching the pipeline. Failure checkpoints nothing; the next trigger re-runs. The read channel: the home view derives node states from the ledgers (`GET /api/graph` is a pure read — forceAtlas2 run-then-freeze, no idle simulation), and at session start the derived graph is rendered into a compact markdown block appended to the teacher's instructions (`buildGraphContext`) — whole graph while small, hotspots + 1-hop neighborhood past 150 nodes. +**The knowledge graph (the two channels).** The write channel: the extraction pipeline — extractor (LLM: propositions, edges, probe attribution, each with verbatim evidence) → auditor (LLM: drops anything unbacked) → merger (`mergeFindings`, deterministic: exact/alias identity resolution, event rebuild from the transcript's tool parts, dedupe, checkpoint). Triggered once a chat accumulates `EXTRACTION.everyNQuizAnswers` (default 3, config.ts) new answered quizzes — a v1 placeholder trigger isolated in one seam (`turn-service.ts`), swappable without touching the pipeline. Failure checkpoints nothing; the next trigger re-runs. The read channel: the home view derives node states from the ledgers (`GET /api/graph` is a pure read — forceAtlas2 run-then-freeze, no idle simulation), and at session start the derived graph is rendered into a compact markdown block appended to the teacher's instructions (`buildGraphContext`) — whole graph while small, hotspots + 1-hop neighborhood past 150 nodes. ## Testing diff --git a/config.ts b/config.ts index b0ceb62..f3877a2 100644 --- a/config.ts +++ b/config.ts @@ -14,15 +14,48 @@ export const PATHS = { graph: path.join(ROOT, "data", "graph"), } as const; +// ─── Models ──────────────────────────────────────────────────────────────── +// THE single place every LLM is chosen and tuned. .env stays for secrets +// (API keys) only — model choice and thinking budget are product tuning, +// not deployment environment, and this section is the one place they grow. +// `effort` is OpenRouter's reasoning scale (docs: reasoning-tokens): +// "none" | "minimal" | "low" | "medium" | "high" | "xhigh". +// "none" disables reasoning tokens entirely. + +export type ReasoningEffort = "none" | "minimal" | "low" | "medium" | "high" | "xhigh"; + +export interface ModelSpec { + /** OpenRouter model id (e.g. "z-ai/glm-5.3-flash"). */ + model: string; + /** Thinking budget — what the model may spend on reasoning tokens. */ + effort: ReasoningEffort; +} + +export const MODELS = { + /** The teacher — streaming, lesson-facing: smart, but keep it lively. */ + teacher: { model: "z-ai/glm-5.3-flash", effort: "medium" }, + /** Background sub-agents: fast and cheap beats deep thought. */ + researcher: { model: "z-ai/glm-5.3-flash", effort: "low" }, + demoBuilder: { model: "z-ai/glm-5.3-flash", effort: "low" }, + /** Must be vision-capable: it looks at rendered SVGs. */ + visualMaker: { model: "anthropic/claude-sonnet-4.5", effort: "low" }, + /** Extraction is judgment over given evidence — light thinking suffices. */ + extractor: { model: "z-ai/glm-5.3-flash", effort: "low" }, + /** The auditor verifies evidence exists; keep it cheap. */ + auditor: { model: "z-ai/glm-5.3-flash", effort: "low" }, +} satisfies Record; + +export type ModelRole = keyof typeof MODELS; + /** * The extraction pipeline (vision §1 construction): the sole graph writer. * Triggered per chat once this many NEW quiz answers have landed since the - * chat's last extraction checkpoint (LH_EXTRACT_N). The trigger mechanism is - * a v1 placeholder — deliberately isolated behind one seam + * chat's last extraction checkpoint. The trigger mechanism is a v1 + * placeholder — deliberately isolated behind one seam * (server/turn-service.ts) so it can be swapped without touching the * pipeline. */ -export const EXTRACTION = { everyNQuizAnswers: Number(process.env.LH_EXTRACT_N ?? 3) } as const; +export const EXTRACTION = { everyNQuizAnswers: 3 } as const; /** * The LIVE graph store (src/store/graph.ts) — owned by the app's extraction diff --git a/docs/architecture.md b/docs/architecture.md index e62f1cc..91fcff6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -81,7 +81,7 @@ chat record ──▶ EXTRACTOR (LLM) ──▶ AUDITOR (LLM) ──▶ MERGER ( **Probe attribution** (vision open question, resolved structurally): tagged events (quiz `nodeId`, `plan_mark_done`'s tree node) resolve deterministically through the proposition mapping. The extractor's judgment is consulted only for **untagged probe quizzes** — retroactively, against the whole transcript, so a pre-tree probe maps to a proposition the tree only articulates later. Attribution can also rescue a tagged-but-unknown id; genuinely unattributable probes land `nodeId: null` with the question text, exactly as they always have. -**The trigger is a v1 placeholder, isolated in ONE seam.** `src/server/turn-service.ts` (`triggerExtraction`) counts answered quizzes not yet in the ledger (`pendingQuizAnswers`, core) and fires the pipeline async once a chat crosses `EXTRACTION.everyNQuizAnswers` (`LH_EXTRACT_N`, default 3) — single-flight per chat. The pipeline is trigger-agnostic: when a better mechanism lands (session end, idle, explicit flush), only this seam changes. +**The trigger is a v1 placeholder, isolated in ONE seam.** `src/server/turn-service.ts` (`triggerExtraction`) counts answered quizzes not yet in the ledger (`pendingQuizAnswers`, core) and fires the pipeline async once a chat crosses `EXTRACTION.everyNQuizAnswers` (default 3, config.ts) — single-flight per chat. The pipeline is trigger-agnostic: when a better mechanism lands (session end, idle, explicit flush), only this seam changes. **The read channel is pure derivation.** `GET /api/graph` reads the ledgers (`graphSnapshot`: merged edges, derived node states, session provenance) — the home view never triggers a write. At session start, `buildGraphContext` renders the derived graph into a compact markdown block appended to the teacher's instructions (whole graph while small; diagnostic hotspots + 1-hop neighborhood past 150 nodes) — this is what makes probing precise and personalization compound. `store events, derive state`: node states are always recomputed from the ledger, never stored. diff --git a/next-env.d.ts b/next-env.d.ts index ce4e94a..a419cbe 100644 --- a/next-env.d.ts +++ b/next-env.d.ts @@ -1,7 +1,7 @@ /// /// -import "./.next/types/routes.d.ts"; -import "./.next/types/root-params.d.ts"; +import "./.next/dev/types/routes.d.ts"; +import "./.next/dev/types/root-params.d.ts"; // NOTE: This file should not be edited // see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/src/agent/models.ts b/src/agent/models.ts new file mode 100644 index 0000000..aa44f0c --- /dev/null +++ b/src/agent/models.ts @@ -0,0 +1,17 @@ +import { openrouter } from "@openrouter/ai-sdk-provider"; +import type { LanguageModel } from "ai"; +import { MODELS, type ModelRole } from "../../config.ts"; + +/** + * The one factory every agent's model comes from. Model choice and thinking + * budget live in config.ts (MODELS) — the single tuning table; this module + * only wires them to OpenRouter. `effort: "none"` is OpenRouter's off-switch + * for reasoning tokens; higher levels let the model think longer (and bill + * accordingly). + */ +export function modelFor(role: ModelRole): LanguageModel { + const spec = MODELS[role]; + return openrouter(spec.model, { + reasoning: { enabled: true, effort: spec.effort }, + }); +} diff --git a/src/agent/subagents/demo-builder.ts b/src/agent/subagents/demo-builder.ts index 374a4cd..37ed194 100644 --- a/src/agent/subagents/demo-builder.ts +++ b/src/agent/subagents/demo-builder.ts @@ -1,6 +1,6 @@ import { ToolLoopAgent, stepCountIs, tool, type LanguageModel, type ToolSet } from "ai"; -import { openrouter } from "@openrouter/ai-sdk-provider"; import { z } from "zod"; +import { modelFor } from "../models.ts"; import type { DemoResult } from "../../core/contracts.ts"; import { getHeadless, type DemoRunResult, type HeadlessApi } from "./headless.ts"; @@ -20,9 +20,6 @@ import { getHeadless, type DemoRunResult, type HeadlessApi } from "./headless.ts * The job wrapper treats html === "" as failure and marks the job error. */ -const DEFAULT_DEMO_MODEL = - process.env.LH_DEMO_MODEL ?? process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5"; - export interface DemoBuilderInput { title: string; brief: string; @@ -174,7 +171,7 @@ export async function runDemoBuilder(input: DemoBuilderInput, opts: DemoBuilderO const state: BuilderState = { html: null, testedHtml: null, lastResult: null }; const agent = new ToolLoopAgent({ - model: opts.model ?? openrouter(DEFAULT_DEMO_MODEL), + model: opts.model ?? modelFor("demoBuilder"), instructions: INSTRUCTIONS, tools: createDemoBuilderTools(state, headless, opts.onActivity), stopWhen: stepCountIs(opts.maxSteps ?? 24), diff --git a/src/agent/subagents/extraction.ts b/src/agent/subagents/extraction.ts index fbe1695..6b9b30b 100644 --- a/src/agent/subagents/extraction.ts +++ b/src/agent/subagents/extraction.ts @@ -1,6 +1,6 @@ import { ToolLoopAgent, stepCountIs, tool, type LanguageModel, type UIMessage } from "ai"; -import { openrouter } from "@openrouter/ai-sdk-provider"; import { z } from "zod"; +import { modelFor } from "../models.ts"; import type { AuditResult, ChatRecord, SessionFindings } from "../../core/contracts.ts"; import { mergeFindings, type EmissionStats, type GraphDeps } from "../../core/graph.ts"; @@ -27,9 +27,6 @@ import { mergeFindings, type EmissionStats, type GraphDeps } from "../../core/gr * the next trigger re-runs the whole thing. */ -const DEFAULT_EXTRACTION_MODEL = - process.env.LH_EXTRACT_MODEL ?? process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5"; - // ─── the transcript view (what the LLM sees) ──────────────────────────────── /** @@ -175,6 +172,12 @@ function createExtractor(model: LanguageModel) { }), }, stopWhen: stepCountIs(4), + // One line per step: a background worker's progress must be legible + // from the server log (§5 observability baseline). + onStepFinish: (step) => + console.info( + `[extraction] extractor step ${step.stepNumber}: ${step.finishReason}, tools: ${step.toolCalls.map((c) => c.toolName).join(", ") || "(none)"}`, + ), }); } @@ -224,6 +227,10 @@ function createAuditor(model: LanguageModel) { }), }, stopWhen: stepCountIs(4), + onStepFinish: (step) => + console.info( + `[extraction] auditor step ${step.stepNumber}: ${step.finishReason}, tools: ${step.toolCalls.map((c) => c.toolName).join(", ") || "(none)"}`, + ), }); } @@ -246,14 +253,15 @@ export interface ExtractionPipeline { export function createExtractionPipeline(deps: ExtractionPipelineDeps): ExtractionPipeline { const { chatStore, graphStore } = deps; - const extractorAgent = createExtractor(deps.extractor ?? openrouter(DEFAULT_EXTRACTION_MODEL)); - const auditorAgent = createAuditor(deps.auditor ?? openrouter(DEFAULT_EXTRACTION_MODEL)); + const extractorAgent = createExtractor(deps.extractor ?? modelFor("extractor")); + const auditorAgent = createAuditor(deps.auditor ?? modelFor("auditor")); const inFlight = new Set(); async function extract(chatId: string): Promise { const record = chatStore.getChat(chatId); if (!record) throw new Error(`Chat not found: ${chatId}`); const view = transcriptView(record); + console.info(`[extraction ${chatId}] extractor starting (${view.split("\n").length} transcript lines)`); const extraction = await extractorAgent.generate({ prompt: `Transcript:\n\n${view}` }); for (const part of extraction.content) { @@ -268,6 +276,10 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti const record = chatStore.getChat(chatId); if (!record) throw new Error(`Chat not found: ${chatId}`); const view = transcriptView(record); + console.info( + `[extraction ${chatId}] auditor starting (${findings.propositions.length} propositions, ${findings.edges.length} edges, ` + + `${Object.keys(findings.attribution.quiz).length} attributions)`, + ); const review = await auditorAgent.generate({ prompt: `Transcript:\n\n${view}\n\nProposed findings:\n\n${JSON.stringify(findings, null, 2)}`, @@ -286,6 +298,10 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti ); } const { dropped: _dropped, ...verified } = auditResult; + console.info( + `[extraction ${chatId}] audit kept ${verified.propositions.length}/${findings.propositions.length} propositions, ` + + `${verified.edges.length}/${findings.edges.length} edges`, + ); return verified; } @@ -293,6 +309,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti if (inFlight.has(chatId)) return { nodes: 0, edges: 0, events: 0 }; inFlight.add(chatId); try { + console.info(`[extraction ${chatId}] pipeline start`); const findings = await extract(chatId); const verified = await audit(chatId, findings); // The merger re-reads the chat record itself — events are rebuilt diff --git a/src/agent/subagents/researcher.ts b/src/agent/subagents/researcher.ts index 6741b9f..9eeb1d2 100644 --- a/src/agent/subagents/researcher.ts +++ b/src/agent/subagents/researcher.ts @@ -1,6 +1,6 @@ import { ToolLoopAgent, stepCountIs, type LanguageModel } from "ai"; -import { openrouter } from "@openrouter/ai-sdk-provider"; import { webTools } from "../tools/web.ts"; +import { modelFor } from "../models.ts"; /** * The researcher sub-agent (vision §2.5): a SEPARATE ToolLoopAgent instance @@ -12,9 +12,6 @@ import { webTools } from "../tools/web.ts"; * The model is injectable so tests run the loop with a MockLanguageModelV4. */ -const DEFAULT_RESEARCHER_MODEL = - process.env.LH_RESEARCH_MODEL ?? process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5"; - const INSTRUCTIONS = `You are a research specialist. You work for a teacher agent inside a learning application: given a research question, conduct thorough web research and produce a focused, well-sourced brief that the teacher can teach from. @@ -61,7 +58,7 @@ What couldn't be answered. Suggested next steps.`; export function createResearcherAgent(model?: LanguageModel) { return new ToolLoopAgent({ - model: model ?? openrouter(DEFAULT_RESEARCHER_MODEL), + model: model ?? modelFor("researcher"), instructions: INSTRUCTIONS, tools: webTools, stopWhen: stepCountIs(12), diff --git a/src/agent/subagents/visual-maker.ts b/src/agent/subagents/visual-maker.ts index 7a590b3..d5058c8 100644 --- a/src/agent/subagents/visual-maker.ts +++ b/src/agent/subagents/visual-maker.ts @@ -1,6 +1,6 @@ import { generateText, tool, type LanguageModel, type ModelMessage, type ToolResultPart } from "ai"; -import { openrouter } from "@openrouter/ai-sdk-provider"; import { z } from "zod"; +import { modelFor } from "../models.ts"; import type { VisualResult } from "../../core/contracts.ts"; import { getHeadless, type HeadlessApi } from "./headless.ts"; @@ -26,9 +26,6 @@ import { getHeadless, type HeadlessApi } from "./headless.ts"; * Failure semantics mirror the demo-builder: { svg: "", notes: "FAILED: …" }. */ -const DEFAULT_VISUAL_MODEL = - process.env.LH_VISUAL_MODEL ?? process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5"; - export interface VisualMakerInput { brief: string; } @@ -127,7 +124,7 @@ export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerO try { for (let iteration = 0; iteration < maxIterations; iteration++) { - const result = await generateText({ model: opts.model ?? openrouter(DEFAULT_VISUAL_MODEL), system: INSTRUCTIONS, messages, tools }); + const result = await generateText({ model: opts.model ?? modelFor("visualMaker"), system: INSTRUCTIONS, messages, tools }); messages = [...messages, ...result.response.messages]; const toolCalls = result.toolCalls; diff --git a/src/agent/teacher-agent.ts b/src/agent/teacher-agent.ts index 3d3f4eb..f251005 100644 --- a/src/agent/teacher-agent.ts +++ b/src/agent/teacher-agent.ts @@ -1,9 +1,9 @@ import * as fs from "node:fs"; import * as path from "node:path"; import { ToolLoopAgent, stepCountIs, type LanguageModel, type ToolSet } from "ai"; -import { openrouter } from "@openrouter/ai-sdk-provider"; import { PATHS } from "../../config.ts"; import { createTeacherTools, jobTools, questionTools } from "./tools.ts"; +import { modelFor } from "./models.ts"; /** * The teacher agent. Everything pi used to provide (agent loop, tool calling, @@ -22,8 +22,6 @@ import { createTeacherTools, jobTools, questionTools } from "./tools.ts"; * fragments with a numeric order key: 10-plan-tree.md, 20-jobs.md, … */ -const DEFAULT_MODEL = process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5"; - const PROMPTS_DIR = path.join(PATHS.root, "src", "agent", "prompts"); function composeInstructions(): string { @@ -54,13 +52,9 @@ function withGraphContext(graphContext: string | null | undefined): string { export function createTeacherAgent(model?: LanguageModel, tools?: ToolSet, graphContext?: string | null) { return new ToolLoopAgent({ - model: - model ?? - openrouter(DEFAULT_MODEL, { - // Ask OpenRouter for reasoning tokens when the model supports them; - // they stream as reasoning parts into the UI's collapsible block. - reasoning: { enabled: true, effort: "medium" }, - }), + // Model choice + thinking budget live in config.ts (MODELS.teacher). + // Reasoning tokens stream as reasoning parts into the UI's collapsible block. + model: model ?? modelFor("teacher"), instructions: withGraphContext(graphContext), // Tools are per-request (plan tools close over the injected ctx); // chat-handler passes createTeacherTools({ chatId, chatStore }). The