From 87b04cc86e60089ca37ff492ec4b0d4dbe9a1101 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Mon, 26 Oct 2026 01:55:57 +0200 Subject: [PATCH] fix: deliver Claude teaching prose through an output tool --- src/agent/subagents/extraction.ts | 21 +- src/agent/teacher-agent.ts | 38 +++- src/agent/tools/send-to-user.ts | 36 +++ src/core/coverage.ts | 52 ++--- src/core/native-messages.ts | 49 ++++ src/core/session-markdown.ts | 22 +- src/core/view.ts | 28 ++- tests/send-to-user.test.ts | 366 ++++++++++++++++++++++++++++++ 8 files changed, 550 insertions(+), 62 deletions(-) create mode 100644 src/agent/tools/send-to-user.ts create mode 100644 tests/send-to-user.test.ts diff --git a/src/agent/subagents/extraction.ts b/src/agent/subagents/extraction.ts index aca547f..24cc975 100644 --- a/src/agent/subagents/extraction.ts +++ b/src/agent/subagents/extraction.ts @@ -28,7 +28,7 @@ import type { GraphStore } from "../../core/ports.ts"; import { emit } from "../../core/events.ts"; import { settleGraphLayout } from "../../core/layout-settle.ts"; import { batchRuns, coverageOf, messageViewLines } from "../../core/coverage.ts"; -import type { LhMessage } from "../../core/native-messages.ts"; +import { failedOutputCallIds, type LhMessage } from "../../core/native-messages.ts"; import { auditorGraphTools } from "../tools/graph.ts"; import { runSubmitAgent, healStringifiedArrays } from "./submit-agent.ts"; @@ -124,16 +124,20 @@ export function transcriptView(record: ChatRecord): string { } /** The extraction view under the teacher-prose ruling (owner, 2026-09-28): - * ONLY the teacher agent's text blocks — no learner messages, no - * thinking, no tool calls or results, no quiz blocks — each labeled with + * ONLY learner-facing teacher prose (text or send_to_user) — no learner + * messages, thinking, other tool calls/results, or quiz blocks — labeled with * its ACTUAL message id (owner, 2026-10-13): the view lines ARE * core/coverage.ts messageViewLines ("TEACHER [id] text"), the exact * lines the coverage fingerprints hash, so the extractor can anchor * every proposition to the passage it came from (the session submit's * required messageId). The shared markdown engine stays reserved for the * future user-facing export. */ -function proseView(messages: ReadonlyArray, title?: string): string { - const lines = messages.flatMap((message) => messageViewLines(message, message.id)); +function proseView( + messages: ReadonlyArray, + title?: string, + failedOutputCalls: ReadonlySet = failedOutputCallIds(messages), +): string { + const lines = messages.flatMap((message) => messageViewLines(message, message.id, failedOutputCalls)); const head = title && title.length > 0 ? `# ${title}\n\n` : ""; return head + lines.join("\n\n"); } @@ -1113,6 +1117,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti // stage runs (typed — the trigger skips, routes surface 503). refuseIfStagesUnready(); const covered = coverageOf(record.messages, graphStore.readCoverage(), chatId); + const failedOutputCalls = failedOutputCallIds(record.messages); if (covered.uncovered.length === 0) { console.info(`[extraction ${chatId}] nothing uncovered — no-op`); return { nodes: 0, chains: 0 }; @@ -1148,14 +1153,14 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti // fingerprints' messageViewLines ("TEACHER [id] text"), one // shared seam. const messages = batch.map((slot) => slot.message); - const view = proseView(messages, record.title); + const view = proseView(messages, record.title, failedOutputCalls); // The batch's teacher-prose ids — the ONLY anchors this // job's submit may name (validateSessionAnchors at the // submit seam refuses anything else: ids from other // batches — even real ones of this chat — other chats, // and non-prose messages). const batchIds = new Set( - messages.filter((m) => messageViewLines(m, m.id).length > 0).map((m) => m.id), + messages.filter((m) => messageViewLines(m, m.id, failedOutputCalls).length > 0).map((m) => m.id), ); setPhase(jobKey, "queued", "chat", undefined, part); const findings = await withExtractSlot(jobKey, async () => { @@ -1188,7 +1193,7 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti const batchStats = mergeFindings(chatId, findings, auditResult, { chatStore, graphStore }, { foldedAt, coverage: batch - .filter((slot) => messageViewLines(slot.message, slot.message.id).length > 0) + .filter((slot) => slot.tokens > 0) .map((slot) => ({ chatId, fp: slot.fp, tokens: slot.tokens, foldedAt })), }); // The layout ledger evolves with the merge (sync — the diff --git a/src/agent/teacher-agent.ts b/src/agent/teacher-agent.ts index 08caa42..6fe6ac1 100644 --- a/src/agent/teacher-agent.ts +++ b/src/agent/teacher-agent.ts @@ -15,10 +15,11 @@ import { type TeacherToolContext, } from "./tools.ts"; import { practiceTools, type PracticeToolContext } from "./tools/practice.ts"; -import { healToolRecord } from "./tools/heal.ts"; +import { healToolRecord, withArgumentHeal } from "./tools/heal.ts"; +import { sendToUserTool } from "./tools/send-to-user.ts"; import { quizInputObject, quizInputProblems } from "../core/question-core.ts"; import { QUESTION_TOOLS, type EnabledTools } from "../core/pending.ts"; -import { toLlmMessages, nativeInputCause } from "../core/native-messages.ts"; +import { toLlmMessages, nativeInputCause, PRACTICE_PROBLEM_TOOL } from "../core/native-messages.ts"; import type { ChatGPTRequestCycle } from "./chatgpt-request-cycle.ts"; import { usageWasAdmitted } from "./subscription-request-cycle.ts"; @@ -145,22 +146,27 @@ export interface AgentBuildOptions { * belt-and-suspenders HALF for the mixed batch (a question + other calls * in one message) that terminate-only semantics would let run on — the * learner cannot answer while a turn is in flight (per-chat 409), so a - * cleanly-posed question MUST always end the turn. maxTurns is gone + * cleanly-posed question MUST always end the turn. Practice's problem pose + * has the same wait-for-the-learner contract, including mixed output/problem + * batches; failed poses must still retry. maxTurns is gone * (owner ruling 2026-09-25: no maximum-agent-turns net on the teaching * turn). */ -function makeQuestionStopper() { +function makeQuestionStopper(kind: AgentKind) { return (context: { message: { content: Array<{ type: string; id?: string; name?: string }> }; toolResults: Array<{ toolCallId: string; isError?: boolean }>; }): boolean => { const called = new Set( context.message.content - .filter((c) => c.type === "toolCall" && c.name !== undefined && (QUESTION_TOOLS as readonly string[]).includes(c.name!)) + .filter((c) => c.type === "toolCall" && c.name !== undefined && ( + (QUESTION_TOOLS as readonly string[]).includes(c.name) || + (kind === "practice" && c.name === PRACTICE_PROBLEM_TOOL) + )) .map((c) => c.id!), ); if (called.size === 0) return false; - // Stop only when every question call of this turn resolved cleanly. + // Stop only when every question/problem pose resolved cleanly. for (const result of context.toolResults) { if (called.has(result.toolCallId) && result.isError) return false; } @@ -174,7 +180,7 @@ function baseAgentOptions(kind: AgentKind, opts: AgentBuildOptions, tools: Agent // conversation-spec resolution as teaching); the cast keeps AgentKind // ("practice" is not a ModelRole) honest about that. const engine = opts.engine ?? piEngineFor(kind as "prober" | "teacher"); - const stopper = makeQuestionStopper(); + const stopper = makeQuestionStopper(kind); const systemPrompt = opts.promptSwitches ? composeInstructions(kind, opts.promptSwitches) : composedInstructions[kind]!; @@ -190,7 +196,16 @@ function baseAgentOptions(kind: AgentKind, opts: AgentBuildOptions, tools: Agent systemPrompt, model: engine.model, thinkingLevel: engine.level, - tools, + // Conditional registry inclusion, like the graph-tool gates: when + // closed, neither the tool nor its usage instructions reach the model. + // Match Claude across providers (including OpenRouter), not the API + // transport. Explicit test registries remain caller-owned. + tools: [ + ...tools, + ...(opts.tools === undefined && engine.model.id.toLowerCase().includes("claude") + ? [withArgumentHeal(sendToUserTool)] + : []), + ], messages: opts.messages ?? [], }, streamFn: engine.streamFn, @@ -312,10 +327,9 @@ export function bareQuestionTools(): Record> { * from turn 1 of a kind:"practice" session — no prober→teacher split, * no plan tools, no question tools. Its ENTIRE registry is * problem + conclude (tools/practice.ts); the doctrine prompt - * (prompts/practice/) carries the method. The question-stop rule does - * not apply (problem rides terminate: true natively — - * QUESTION_TOOLS does not include it), so baseAgentOptions's stopper is - * a no-op here; it stays wired for uniformity. + * (prompts/practice/) carries the method. A successful problem pose ends + * the run, natively for a problem-only batch and via the shared stop hook + * for a mixed batch (for example, send_to_user + problem). */ export function createPracticeAgent(ctx: PracticeToolContext, opts: AgentBuildOptions = {}) { const tools = opts.tools ?? practiceTools(ctx); diff --git a/src/agent/tools/send-to-user.ts b/src/agent/tools/send-to-user.ts new file mode 100644 index 0000000..fb0e409 --- /dev/null +++ b/src/agent/tools/send-to-user.ts @@ -0,0 +1,36 @@ +import { Type } from "typebox"; +import type { AgentTool } from "@earendil-works/pi-agent-core"; +import { SEND_TO_USER_TOOL } from "../../core/native-messages.ts"; +import { serverExecute } from "../tool-execute.ts"; + +const sendToUserInput = Type.Object({ + message: Type.String({ + minLength: 1, + pattern: "\\S", + description: "The complete content to display to the learner. Markdown and LaTeX are supported.", + }), +}); + +/** Claude's documented verbatim mid-turn output channel. Availability is + * model-gated at the shared conversation-agent construction boundary; + * the native call remains in history, and the view renders its message. */ +export const sendToUserTool: AgentTool = { + name: SEND_TO_USER_TOOL, + label: SEND_TO_USER_TOOL, + description: + "## REQUIRED OUTPUT CHANNEL — ALWAYS USE `send_to_user`\n\n" + + "**ALL learner-facing messages MUST be sent using `send_to_user`.** This includes explanations, teaching, " + + "feedback, corrections, hints, introductions, progress messages, and final responses. " + + "There is no exception for short messages or the end of a turn.\n\n" + + "Ordinary response text may be converted into summarized thinking instead of reaching the learner. " + + "**Never rely on ordinary response text to communicate with the learner. Thinking is not delivery.**\n\n" + + "Put the complete message in `message`. Markdown and LaTeX are supported. " + + "Do not duplicate it in ordinary response text.\n\n" + + "Question content still belongs in the appropriate question tool; send any accompanying teaching or " + + "commentary through `send_to_user` before calling that tool.\n\n" + + "This tool does not pause or end your turn. Continue with other tools when needed. " + + "After delivering your final message, end the turn without additional ordinary response text.", + parameters: sendToUserInput, + executionMode: "sequential", + execute: serverExecute(SEND_TO_USER_TOOL, sendToUserInput, () => "Message received for display."), +}; diff --git a/src/core/coverage.ts b/src/core/coverage.ts index fe5b523..50c3ce0 100644 --- a/src/core/coverage.ts +++ b/src/core/coverage.ts @@ -1,9 +1,9 @@ /** * Extraction coverage — the pure core. THE TEACHER'S PROSE IS THE UNIT * (owner ruling, 2026-09-28): the extractor sees ONLY the teacher agent's - * text blocks — no learner messages, no thinking, no tool calls or - * results, no quiz blocks. Everything else is INVISIBLE to coverage: it - * neither opens nor joins a run, so answering a quiz or editing a learner + * prose — ordinary text blocks and explicit send_to_user messages. No + * learner messages, thinking, other tool calls/results, or quiz blocks. + * Everything else is INVISIBLE to coverage: it neither opens nor joins a run, so answering a quiz or editing a learner * message triggers nothing. "Covered" is by construction exactly "what * the extractor saw": the fingerprint (sha256 of a message's view lines) * and the extractor's prompt share ONE selection — the teacher prose. @@ -22,33 +22,27 @@ import { createHash } from "node:crypto"; import type { CoverageEntry } from "./contracts/graph.ts"; -import type { LhMessage } from "./native-messages.ts"; +import { failedOutputCallIds, userFacingTextOf, type LhMessage } from "./native-messages.ts"; export type { CoverageEntry }; -/** The user-facing text of a native message (string or text blocks). */ -function textOf(content: unknown): string[] { - if (typeof content === "string") return content.trim() ? [content] : []; - if (!Array.isArray(content)) return []; - return content - .filter((c): c is { type: "text"; text: string } => (c as { type?: string }).type === "text") - .map((c) => c.text) - .filter((t) => t.trim().length > 0); -} - /** * The extraction-relevant lines of ONE native message — the shared seam * between the extractor's prompt and messageFingerprint (coverage). * TEACHER PROSE ONLY (owner, 2026-09-28): an assistant message yields one - * line per text block; everything else (user messages, tool results, - * markers) yields NOTHING and is invisible to coverage. `mid` is the - * message id — part of the hash input, so the same prose in two messages + * line per learner-facing prose block; everything else (user messages, + * tool results, markers) yields NOTHING and is invisible to coverage. + * `mid` is the message id — part of the hash input, so the same prose in two messages * is two slots (per-message coverage). */ -export function messageViewLines(message: LhMessage, mid: string): string[] { +export function messageViewLines( + message: LhMessage, + mid: string, + failedOutputCalls: ReadonlySet = failedOutputCallIds([message]), +): string[] { const lines: string[] = []; if ((message as { role?: string }).role !== "assistant") return lines; - for (const text of textOf((message as { content: unknown }).content)) { + for (const text of userFacingTextOf((message as { content: unknown }).content, failedOutputCalls)) { lines.push(`TEACHER [${mid}] ${text.trim()}`); } return lines; @@ -62,9 +56,10 @@ export function messageViewLines(message: LhMessage, mid: string): string[] { */ export function teacherProseBlocks(messages: ReadonlyArray): string[] { const out: string[] = []; + const failedOutputCalls = failedOutputCallIds(messages); for (const message of messages) { if ((message as { role?: string }).role !== "assistant") continue; - for (const text of textOf((message as { content: unknown }).content)) out.push(text.trim()); + for (const text of userFacingTextOf((message as { content: unknown }).content, failedOutputCalls)) out.push(text.trim()); } return out; } @@ -79,9 +74,9 @@ export function viewTokens(lines: string[]): number { * teacher prose, with the message id as the tag). Changes exactly when * the teacher prose of the message changes — nothing else can re-open it. */ -export function messageFingerprint(message: LhMessage): string { +export function messageFingerprint(message: LhMessage, failedOutputCalls?: ReadonlySet): string { const h = createHash("sha256"); - h.update(JSON.stringify([(message as { role?: string }).role, messageViewLines(message, message.id)])); + h.update(JSON.stringify([(message as { role?: string }).role, messageViewLines(message, message.id, failedOutputCalls)])); return h.digest("hex"); } @@ -95,7 +90,12 @@ export interface CoverageSlot { /** Fingerprint a whole transcript, in order (used by the trigger, the run, * and the coverage view — one derivation everywhere). */ export function coverageSlots(messages: ReadonlyArray): CoverageSlot[] { - return messages.map((message) => ({ message, fp: messageFingerprint(message), tokens: viewTokens(messageViewLines(message, message.id)) })); + const failedOutputCalls = failedOutputCallIds(messages); + return messages.map((message) => ({ + message, + fp: messageFingerprint(message, failedOutputCalls), + tokens: viewTokens(messageViewLines(message, message.id, failedOutputCalls)), + })); } /** The covered-fingerprint set for one chat, from the ledger. */ @@ -131,16 +131,14 @@ export function coverageOf( let uncoveredTokens = 0; for (let i = 0; i < slots.length; i++) { const slot = slots[i]!; - if (messageViewLines(slot.message, slot.message.id).length === 0) continue; // invisible + if (slot.tokens === 0) continue; // invisible (including failed output calls) if (covered.has(slot.fp)) coveredTokens += slot.tokens; else { uncovered.push(i); uncoveredTokens += slot.tokens; } } - const visibleCount = slots.length - slots.filter( - (s) => messageViewLines(s.message, s.message.id).length === 0, - ).length; + const visibleCount = slots.filter((s) => s.tokens > 0).length; return { slots, uncovered, diff --git a/src/core/native-messages.ts b/src/core/native-messages.ts index 2983e4f..63f2720 100644 --- a/src/core/native-messages.ts +++ b/src/core/native-messages.ts @@ -98,6 +98,55 @@ export function asAgentMessages(messages: readonly LhMessage[]): AgentMessage[] return messages as unknown as AgentMessage[]; } +// ─── explicit learner-facing output ───────────────────────────────────────── + +export const SEND_TO_USER_TOOL = "send_to_user"; + +/** Read the output tool's message without changing the native call. Also + * accepts partial arguments while streaming; malformed/empty inputs carry + * no prose. This projection is independent of the currently selected model. */ +export function sendToUserText(block: unknown): string | undefined { + if (!block || typeof block !== "object") return undefined; + const call = block as { type?: unknown; name?: unknown; arguments?: unknown }; + if (call.type !== "toolCall" || call.name !== SEND_TO_USER_TOOL) return undefined; + if (!call.arguments || typeof call.arguments !== "object") return undefined; + const message = (call.arguments as { message?: unknown }).message; + return typeof message === "string" && message.trim() ? message : undefined; +} + +/** Learner-facing prose, in content order. Only the explicit output tool + * joins ordinary text here — thinking and all other tool inputs stay out. + * Shared by coverage/extraction and Markdown; native history stays intact. */ +export function userFacingTextOf(content: unknown, failedOutputCalls?: ReadonlySet): string[] { + if (typeof content === "string") return content.trim() ? [content] : []; + if (!Array.isArray(content)) return []; + const out: string[] = []; + for (const block of content) { + if (!block || typeof block !== "object") continue; + if (block.type === "toolCall" && failedOutputCalls?.has(block.id)) continue; + const text = block.type === "text" ? block.text : sendToUserText(block); + if (typeof text === "string" && text.trim()) out.push(text); + } + return out; +} + +/** Failed output calls remain retry context, not delivered teaching. + * Truncated calls can have valid-looking partial arguments. Failed/aborted + * generations do not execute their calls at all, so may have no result. */ +export function failedOutputCallIds(messages: readonly LhMessage[]): ReadonlySet { + const failed = new Set(); + for (const message of messages) { + if (message.role === "toolResult" && message.toolName === SEND_TO_USER_TOOL && message.isError) { + failed.add(message.toolCallId); + } else if (message.role === "assistant" && (message.stopReason === "error" || message.stopReason === "aborted")) { + for (const block of message.content) { + if (block.type === "toolCall" && block.name === SEND_TO_USER_TOOL) failed.add(block.id); + } + } + } + return failed; +} + // ─── id minting ────────────────────────────────────────────────────────────── /** The app-wide message id shape (same as the UIMessage-era mint). */ diff --git a/src/core/session-markdown.ts b/src/core/session-markdown.ts index 64a37b6..107e276 100644 --- a/src/core/session-markdown.ts +++ b/src/core/session-markdown.ts @@ -22,22 +22,15 @@ import type { LhMessage } from "./native-messages.ts"; import { + failedOutputCallIds, PRACTICE_CONCLUDE_TOOL, PRACTICE_PROBLEM_TOOL, problemInputForConclude, + SEND_TO_USER_TOOL, + userFacingTextOf, } from "./native-messages.ts"; import type { ToolResultMessage } from "@earendil-works/pi-ai"; -/** The user-facing text blocks of a native message (string or text parts). */ -function textOf(content: unknown): string[] { - if (typeof content === "string") return content.trim() ? [content] : []; - if (!Array.isArray(content)) return []; - return content - .filter((c): c is { type: "text"; text: string } => (c as { type?: string }).type === "text") - .map((c) => c.text) - .filter((t) => t.trim().length > 0); -} - /** The practice join context (owner-approved, 2026-10-06 export round): * practice tool RESULTS are acks — the statement, outcome, and verdict * all live in the assistant's call arguments, and the model solution @@ -48,6 +41,7 @@ export interface MarkdownContext { messages: ReadonlyArray; /** toolCallId → the call part's name + arguments. */ calls: Map | undefined }>; + failedOutputCalls?: ReadonlySet; } function markdownContext(messages: ReadonlyArray): MarkdownContext { @@ -66,7 +60,7 @@ function markdownContext(messages: ReadonlyArray): MarkdownContext { } } } - return { messages, calls }; + return { messages, calls, failedOutputCalls: failedOutputCallIds(messages) }; } /** solvedWithHints → "Solved with hints" — the outcome enum reads as prose. */ @@ -172,16 +166,16 @@ export function messageMarkdown( const parts: string[] = []; if (role === "user") { - for (const text of textOf((message as { content: unknown }).content)) { + for (const text of userFacingTextOf((message as { content: unknown }).content)) { parts.push(`### Learner${anchor}\n\n${text.trim()}`); } } else if (role === "assistant") { - for (const text of textOf((message as { content: unknown }).content)) { + for (const text of userFacingTextOf((message as { content: unknown }).content, ctx?.failedOutputCalls ?? failedOutputCallIds([message]))) { parts.push(`### Teacher${anchor}\n\n${text.trim()}`); } } else if (role === "toolResult") { const result = message as ToolResultMessage; - if (result.isError) return null; // retry context, not learning evidence + if (result.isError || result.toolName === SEND_TO_USER_TOOL) return null; // retry context or output ack, not learning evidence const { heading, body } = toolResultBody(result, ctx); parts.push(`### ${heading}${anchor}\n\n${body.join("\n\n")}`); } diff --git a/src/core/view.ts b/src/core/view.ts index 609de06..166d8d5 100644 --- a/src/core/view.ts +++ b/src/core/view.ts @@ -34,7 +34,7 @@ import type { AssistantMessage, ToolResultMessage } from "@earendil-works/pi-ai"; import type { LhMarkerMessage, LhMessage } from "./native-messages.ts"; -import { modelSolutionForConclude, PRACTICE_CONCLUDE_TOOL, problemInputForConclude } from "./native-messages.ts"; +import { modelSolutionForConclude, PRACTICE_CONCLUDE_TOOL, problemInputForConclude, SEND_TO_USER_TOOL, sendToUserText } from "./native-messages.ts"; import { activeCompactionOf, type CompactionDetails } from "./compaction.ts"; import { QUESTION_TOOLS } from "./pending.ts"; import type { SubagentArtifactData, SubagentResultMeta } from "./contracts/jobs.ts"; @@ -124,6 +124,14 @@ function lastSuccessfulPlanInit(parts: LhViewPart[]): number { * view messages. Pure: the same input always projects to the same list. */ export function projectView(messages: readonly LhMessage[], live?: LiveTurnState): LhViewMessage[] { const out: LhViewMessage[] = []; + // Output calls become text rather than tool parts, so read settled + // failures up front: a failed call must retain the normal error path. + const outputErrors = new Map(); + for (const message of messages) { + if (message.role === "toolResult" && message.toolName === SEND_TO_USER_TOOL && message.isError) { + outputErrors.set(message.toolCallId, message); + } + } const lastAssistant = (): LhViewMessage | undefined => { const last = out[out.length - 1]; @@ -218,6 +226,24 @@ export function projectView(messages: readonly LhMessage[], live?: LiveTurnState reasoning.push({ type: "reasoning", id: `${m.id}-r${reasoningN++}`, text: block.thinking, state: "done" }); } else if (block.type === "toolCall") { const part = toolPartOf(block, live, streamingMessage); + if (block.name === SEND_TO_USER_TOOL) { + // Failed/aborted generations never execute their calls; + // the provider error/abort UI already owns that outcome. + if (assistant.stopReason === "error" || assistant.stopReason === "aborted") continue; + const error = outputErrors.get(block.id); + if (error) applyResult(part, error); + if (part.state === "output-error") { + rest.push(part); + } else { + const text = sendToUserText(block); + if (text !== undefined) { + rest.push({ type: "text", id: `${m.id}-t${textN++}`, text, state: "done" }); + } + } + // No live tool card or acknowledgement: this channel is + // ordinary prose, including while its arguments stream. + continue; + } // The conclusion's reveal join (owner round, 2026-10-05): the // hero of a conclude card is the modelSolution committed at POSE // time — a record fold, enriched here so the card renders pure. diff --git a/tests/send-to-user.test.ts b/tests/send-to-user.test.ts new file mode 100644 index 0000000..beaeae3 --- /dev/null +++ b/tests/send-to-user.test.ts @@ -0,0 +1,366 @@ +/** Explicit Claude output: closed-network synthetic fixtures only. Native + * history stays native; UI, extraction, and Markdown read the same prose. */ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { after, describe, it } from "node:test"; +import { Value } from "typebox/value"; +import { + createFauxCore, fauxAssistantMessage, fauxText, fauxThinking, fauxToolCall, + type FauxResponseStep, +} from "@earendil-works/pi-ai"; +import type { PiEngine } from "../src/agent/models.ts"; +import type { AgentBuildOptions } from "../src/agent/teacher-agent.ts"; +import type { LhMessage } from "../src/core/native-messages.ts"; + +// Set the data/auth seams BEFORE importing app modules. Do not read local +// keys, recordings, or learner sessions, even if a dev server is running. +const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "lh-send-to-user-")); +const originalEnv = process.env; +const originalFetch = globalThis.fetch; +process.env = { + ...(originalEnv.NODE_TEST_CONTEXT ? { NODE_TEST_CONTEXT: originalEnv.NODE_TEST_CONTEXT } : {}), + ...(originalEnv.NODE_OPTIONS ? { NODE_OPTIONS: originalEnv.NODE_OPTIONS } : {}), + NODE_ENV: "test", + LH_DATA_DIR: path.join(scratch, "data"), + LH_AUTH_FILE: path.join(scratch, "auth.json"), + HOME: path.join(scratch, "home"), +}; +globalThis.fetch = async () => { throw new Error("CLOSED: unexpected network request"); }; +after(() => { + process.env = originalEnv; + globalThis.fetch = originalFetch; + fs.rmSync(scratch, { recursive: true, force: true }); +}); + +const { sendToUserTool } = await import("../src/agent/tools/send-to-user.ts"); +const { createProberAgent, createTeacherAgent, createPracticeAgent, composeInstructions } = await import("../src/agent/teacher-agent.ts"); +const { SEND_TO_USER_TOOL, activeProblemOf, asAgentMessages, newUserMessage, sendToUserText, toLlmMessages, userFacingTextOf } = await import("../src/core/native-messages.ts"); +const { coverageOf, messageFingerprint, messageViewLines, teacherProseBlocks } = await import("../src/core/coverage.ts"); +const { projectView } = await import("../src/core/view.ts"); +const { sessionMarkdown } = await import("../src/core/session-markdown.ts"); +const { transcriptView } = await import("../src/agent/subagents/extraction.ts"); +const { runSubagent } = await import("../src/agent/subagents/pi-run.ts"); +const { persistTurn } = await import("../src/server/turn-service.ts"); +const { runPiTurn } = await import("../src/server/pi/ui-stream.ts"); +const { SessionModel } = await import("../components/session-core.ts"); +const { chatStore } = await import("../src/store/chats.ts"); +const { jobStore } = await import("../src/store/jobs.ts"); + +function scripted(provider: string, id: string, steps: FauxResponseStep[] = []) { + const faux = createFauxCore({ provider, models: [{ id }] }); + faux.setResponses(steps); + const engine: PiEngine = { model: faux.getModel(), streamFn: faux.streamSimple }; + return { engine, faux }; +} + +const chatId = chatStore.createChat("Synthetic output gate").id; +const factories = { + prober: (opts: AgentBuildOptions) => createProberAgent({ chatId, chatStore, jobStore, deliver: () => {}, spawn: () => {} }, opts), + teacher: (opts: AgentBuildOptions) => createTeacherAgent({ chatId, chatStore, jobStore, spawn: () => {} }, opts), + practice: (opts: AgentBuildOptions) => createPracticeAgent({ chatId, chatStore }, opts), +}; + +function outputCall(message: string | number | null = "An explanation.", id = "send-1") { + return { type: "toolCall" as const, id, name: SEND_TO_USER_TOOL, arguments: { message } }; +} +function assistant(content: unknown[], id = "a1"): LhMessage { + return { ...fauxAssistantMessage(""), content, id } as LhMessage; +} +function ack(callId = "send-1", isError = false): LhMessage { + return { + id: `result-${callId}`, role: "toolResult", toolCallId: callId, toolName: SEND_TO_USER_TOOL, + content: [{ type: "text", text: isError ? "Invalid tool input" : "Message received for display." }], + details: null, isError, timestamp: 1, + }; +} +function prose(messages: ReturnType): string[] { + return messages.filter((m) => m.role === "assistant").flatMap((m) => m.parts.filter((p) => p.type === "text").map((p) => p.text!)); +} + +describe("send_to_user model gate", () => { + for (const [provider, id, enabled] of [ + ["anthropic", "claude-opus-5-5", true], + ["anthropic", "claude-haiku-4-5-20251001", true], + ["openrouter", "anthropic/claude-opus-5.5", true], + ["openrouter", "anthropic/claude-sonnet-5.5", true], + ["amazon-bedrock", "us.anthropic.claude-opus-5-5-v1:0", true], + ["fixture", "CLAUDE-fixture", true], + ["openrouter", "z-ai/glm-5.3-flash", false], + ["openai", "gpt-4o-mini", false], + ["anthropic", "opaque-alias", false], + ] as const) { + it(`${provider}/${id}: ${enabled ? "includes" : "omits"} tool and description in all conversation agents`, () => { + const { engine } = scripted(provider, id); + for (const [kind, factory] of Object.entries(factories)) { + const agent = factory({ engine }); + const output = agent.state.tools.find((tool) => tool.name === SEND_TO_USER_TOOL); + assert.equal(Boolean(output), enabled, kind); + if (enabled) { + assert.equal(output!.description, sendToUserTool.description); + assert.equal(typeof output!.prepareArguments, "function", "standard argument healing is wired"); + } + assert.equal(agent.state.systemPrompt, composeInstructions(kind as keyof typeof factories)); + assert.ok(!agent.state.systemPrompt.includes(SEND_TO_USER_TOOL), "no added prompt instructions"); + } + }); + } + + it("explicitly injected teacher/practice test registries remain unchanged", () => { + const { engine } = scripted("anthropic", "claude-opus-5-5"); + for (const factory of [factories.teacher, factories.practice]) { + assert.deepEqual(factory({ engine, tools: {} }).state.tools, []); + } + }); + + it("background agents do not acquire a learner-output tool", async () => { + let calls = 0; + const { engine } = scripted("anthropic", "claude-opus-5-5", [(context) => { + calls++; + assert.ok(!context.messages.some((m) => m.role === "system" && m.toolsAdded?.some((t) => t.name === SEND_TO_USER_TOOL))); + return fauxAssistantMessage("Synthetic background result."); + }]); + await runSubagent([], { role: "researcher", engine, systemPrompt: "Synthetic fixture.", prompt: "Finish.", maxTurns: 2 }); + assert.equal(calls, 1); + }); +}); + +describe("send_to_user mechanics", () => { + it("requires nonblank text and acknowledges without echoing or terminating", async () => { + assert.equal(sendToUserTool.name, SEND_TO_USER_TOOL); + for (const input of [{}, { message: "" }, { message: " \n\t" }, { message: 3 }, { message: null }]) { + assert.equal(Value.Check(sendToUserTool.parameters, input), false); + } + const message = "**A full explanation**\n\n$$a = b$$"; + assert.equal(Value.Check(sendToUserTool.parameters, { message }), true); + const result = await sendToUserTool.execute("send-1", { message }); + assert.ok(!result.isError); + assert.ok(!result.terminate); + assert.ok(!JSON.stringify(result).includes(message)); + assert.deepEqual(result.content, [{ type: "text", text: "Message received for display." }]); + }); + + it("only extracts the output tool's nonblank message, without modifying it", () => { + const message = " **Keep spacing and math** $x$\n"; + assert.equal(sendToUserText(outputCall(message)), message); + for (const block of [null, {}, { type: "text", text: "ordinary" }, outputCall(2), outputCall(" "), { ...outputCall(), name: "other" }]) { + assert.equal(sendToUserText(block), undefined); + } + assert.deepEqual(userFacingTextOf([null, fauxThinking("Not lesson prose."), fauxText("First"), outputCall(message), fauxToolCall("quiz", { question: "Q?" })]), ["First", message]); + }); +}); + +describe("send_to_user transcript projection", () => { + it("renders in place as prose, with thinking unchanged and no tool/ack duplication", () => { + const messages = [assistant([ + fauxText("Before."), outputCall("**The lesson.** $x = 1$"), + fauxThinking("Private reasoning."), fauxToolCall("quiz", { question: "Q?" }, { id: "quiz-1" }), + fauxText("After."), + ]), ack()]; + const before = structuredClone(messages); + const view = projectView(messages); + assert.deepEqual(view[0]!.parts.map((p) => p.type), ["step-start", "reasoning", "text", "text", "tool-quiz", "text"]); + assert.deepEqual(prose(view), ["Before.", "**The lesson.** $x = 1$", "After."]); + assert.equal(view[0]!.parts.find((p) => p.type === "reasoning")!.text, "Private reasoning."); + assert.deepEqual(view[0]!.parts.filter((p) => p.type === "text").map((p) => p.id), ["a1-t0", "a1-t1", "a1-t2"]); + assert.deepEqual(messages, before, "projection never rewrites native content"); + assert.deepEqual(projectView(JSON.parse(JSON.stringify(messages))), view, "reload is identical, without consulting a provider gate"); + }); + + it("streams available arguments as replacing prose, never as a live tool card", () => { + const live = { inFlight: true, streamingIds: new Set(["a1"]), executions: new Map() }; + for (const text of ["A", "A growing", "A growing explanation."]) { + const view = projectView([assistant([outputCall(text)])], live); + assert.deepEqual(prose(view), [text]); + assert.ok(!view[0]!.parts.some((p) => p.type.startsWith("tool-"))); + } + const empty = projectView([assistant([{ ...outputCall(), arguments: {} }])], live); + assert.deepEqual(prose(empty), []); + assert.ok(!empty[0]!.parts.some((p) => p.type.startsWith("tool-"))); + }); + + it("malformed messages contain no prose; failures keep the existing error path live and after reload", () => { + assert.deepEqual(prose(projectView([assistant([outputCall(3)])])), []); + const messages = [assistant([outputCall("Incomplete teaching.")]), ack("send-1", true)]; + const settled = projectView(messages); + assert.deepEqual(prose(settled), []); + assert.equal(settled[0]!.parts[1]!.type, "tool-send_to_user"); + assert.equal(settled[0]!.parts[1]!.state, "output-error"); + const live = projectView(messages.slice(0, 1), { + inFlight: true, streamingIds: new Set(["a1"]), + executions: new Map([["send-1", { isError: true, errorText: "Invalid tool input" }]]), + }); + assert.deepEqual(prose(live), []); + assert.equal(live[0]!.parts[1]!.state, "output-error"); + }); +}); + +describe("send_to_user extraction and Markdown", () => { + it("selects output prose once and in order, preserving old fingerprints", () => { + const message = assistant([fauxText("First."), outputCall("Second."), fauxThinking("Hidden."), outputCall("Third.", "send-2")]); + const messages = [message, ack(), ack("send-2")]; + assert.deepEqual(messageViewLines(message, "a1"), ["TEACHER [a1] First.", "TEACHER [a1] Second.", "TEACHER [a1] Third."]); + assert.deepEqual(teacherProseBlocks(messages), ["First.", "Second.", "Third."]); + assert.equal(coverageOf(messages, [], "fixture").uncoveredCount, 1); + const md = sessionMarkdown(messages, { anchors: false }); + assert.equal(md, "### Teacher\n\nFirst.\n\n### Teacher\n\nSecond.\n\n### Teacher\n\nThird."); + const record = chatStore.createChat("Synthetic extraction"); + assert.equal(transcriptView({ ...record, messages }), "# Synthetic extraction\n\nTEACHER [a1] First.\n\nTEACHER [a1] Second.\n\nTEACHER [a1] Third."); + const legacy = assistant([fauxText("Old text."), fauxThinking("Old thinking.")], "legacy"); + const oldFingerprint = createHash("sha256").update(JSON.stringify(["assistant", ["TEACHER [legacy] Old text."]])).digest("hex"); + assert.equal(messageFingerprint(legacy), oldFingerprint, "no re-extraction/migration of legacy prose"); + }); + + it("does not extract/export failed output calls, including partial calls from truncated responses", () => { + const message = assistant([fauxText("Keep this."), outputCall("Do not treat this as delivered.")]); + const messages = [message, ack("send-1", true)]; + assert.deepEqual(teacherProseBlocks(messages), ["Keep this."]); + assert.equal(sessionMarkdown(messages, { anchors: false }), "### Teacher\n\nKeep this."); + const record = chatStore.createChat("Failed output fixture"); + assert.ok(!transcriptView({ ...record, messages }).includes("Do not treat")); + const coverage = coverageOf([assistant([outputCall()]), ack("send-1", true)], [], "fixture"); + assert.equal(coverage.uncoveredCount, 0); + assert.equal(coverage.uncoveredTokens, 0); + }); +}); + +describe("send_to_user interrupted generations", () => { + for (const stopReason of ["error", "aborted"] as const) { + it(`${stopReason} without a tool result never becomes delivered teaching; ordinary text stays unchanged`, () => { + const message = { ...assistant([fauxText("Keep ordinary text."), outputCall("Unfinished output.")]), stopReason }; + const messages = [message]; + assert.deepEqual(prose(projectView(messages)), ["Keep ordinary text."]); + assert.deepEqual(prose(projectView(JSON.parse(JSON.stringify(messages)))), ["Keep ordinary text."]); + assert.deepEqual(messageViewLines(message, message.id), ["TEACHER [a1] Keep ordinary text."]); + assert.deepEqual(teacherProseBlocks(messages), ["Keep ordinary text."]); + assert.equal(sessionMarkdown(messages, { anchors: false }), "### Teacher\n\nKeep ordinary text."); + const onlyOutput = { ...assistant([outputCall("Unfinished output.")]), stopReason }; + assert.equal(coverageOf([onlyOutput], [], "fixture").uncoveredCount, 0); + }); + } +}); + +describe("send_to_user native flow (offline)", () => { + it("output alone continues the loop, so the model receives its native acknowledgement", async () => { + const { engine, faux } = scripted("anthropic", "claude-opus-5-5", [ + fauxAssistantMessage([outputCall("Read this first.")], { stopReason: "toolUse" }), + (context) => { + assert.ok(context.messages.some((m) => m.role === "toolResult" && m.toolName === SEND_TO_USER_TOOL)); + return fauxAssistantMessage([fauxText("Final plain reply.")]); + }, + ]); + const agent = createTeacherAgent(undefined, { engine }); + await agent.prompt("Synthetic request."); + assert.equal(faux.state.callCount, 2); + }); + + for (const outputFirst of [true, false]) { + it(`output + practice problem stops after the pose (${outputFirst ? "output first" : "problem first"})`, async () => { + const output = outputCall("Try this practice problem."); + const problem = fauxToolCall("problem", { + statement: "What is two plus two?", mode: "recall", modelSolution: "Four.", + }, { id: "problem-1" }); + const { engine, faux } = scripted("anthropic", "claude-opus-5-5", [ + fauxAssistantMessage(outputFirst ? [output, problem] : [problem, output], { stopReason: "toolUse" }), + fauxAssistantMessage("Premature continuation must not run."), + ]); + const agent = createPracticeAgent({ chatId, chatStore }, { engine }); + await agent.prompt("Pose a synthetic problem."); + assert.equal(faux.state.callCount, 1, "wait for the learner after a successful mixed-batch pose"); + const results = agent.state.messages.filter((m) => m.role === "toolResult"); + assert.equal(results.length, 2); + assert.ok(results.every((m) => !m.isError)); + }); + } + + for (const stopReason of ["error", "aborted"] as const) { + it(`${stopReason} mixed output/problem generation never executes or activates the pose`, async () => { + const { engine, faux } = scripted("anthropic", "claude-opus-5-5", [ + fauxAssistantMessage([ + outputCall("Undelivered introduction."), + fauxToolCall("problem", { + statement: "What is two plus two?", mode: "recall", modelSolution: "Four.", + }, { id: "unfinished-problem" }), + ], { stopReason }), + ]); + const agent = createPracticeAgent({ chatId, chatStore }, { engine }); + await agent.prompt("Pose a synthetic problem."); + assert.equal(faux.state.callCount, 1); + assert.ok(!agent.state.messages.some((m) => m.role === "toolResult")); + const messages = agent.state.messages.map((m, i) => ({ ...m, id: `interrupted-${i}` })) as LhMessage[]; + assert.equal(activeProblemOf(messages), null); + assert.deepEqual(prose(projectView(messages)), []); + }); + } + + it("a rejected mixed-batch practice pose retries, then stops after the successful pose", async () => { + const { engine, faux } = scripted("anthropic", "claude-opus-5-5", [ + fauxAssistantMessage([ + outputCall("Try this practice problem."), + fauxToolCall("problem", { statement: "Incomplete problem.", mode: "recall" }, { id: "bad-problem" }), + ], { stopReason: "toolUse" }), + fauxAssistantMessage([ + outputCall("Here is the corrected problem.", "send-2"), + fauxToolCall("problem", { + statement: "What is two plus two?", mode: "recall", modelSolution: "Four.", + }, { id: "good-problem" }), + ], { stopReason: "toolUse" }), + fauxAssistantMessage("Premature continuation must not run."), + ]); + const agent = createPracticeAgent({ chatId, chatStore }, { engine }); + await agent.prompt("Pose a synthetic problem."); + assert.equal(faux.state.callCount, 2, "failed authoring retries; successful pose waits for the learner"); + const results = agent.state.messages.filter((m) => m.role === "toolResult" && m.toolName === "problem"); + assert.equal(results.length, 2); + assert.ok(results[0]?.role === "toolResult" && results[0].isError); + assert.ok(results[1]?.role === "toolResult" && !results[1].isError); + }); + + it("output + question streams, persists natively, replays, and reloads as prose before the question", async () => { + const lesson = "**Synthetic lesson:** four is twice two. $2 + 2 = 4$"; + const quiz = { + type: "check", nodeId: "synthetic", question: "What is two plus two?", + options: [{ label: "Four", value: "4" }, { label: "Five", value: "5" }], + correctAnswer: "4", explanation: "Add two pairs.", + }; + const { engine, faux } = scripted("openrouter", "anthropic/claude-opus-5.5", [ + fauxAssistantMessage([ + fauxThinking("Synthetic private thought."), outputCall(lesson), + fauxToolCall("quiz", quiz, { id: "quiz-1" }), + ], { stopReason: "toolUse" }), + ]); + const chat = chatStore.createChat("Synthetic native output flow"); + const user = newUserMessage("Explain synthetic arithmetic."); + chatStore.saveMessages(chat.id, [user]); + const agent = createTeacherAgent(undefined, { engine, messages: asAgentMessages([user]) }); + const handle = runPiTurn({ + agent, + updateFlushMs: 0, + onFinish: (messages, aborted) => { persistTurn(chatStore, chat.id, 1, messages, aborted); }, + }); + const wire = await new Response(handle.subscribe()).text(); + assert.ok(wire.endsWith("data: [DONE]\n\n")); + assert.equal(faux.state.callCount, 1, "question stops the mixed batch as before"); + const saved = chatStore.getChat(chat.id)!.messages; + const response = saved.find((m) => m.role === "assistant")!; + assert.equal(response.role, "assistant"); + if (response.role !== "assistant") assert.fail("missing assistant"); + assert.deepEqual(response.content.map((c) => c.type), ["thinking", "toolCall", "toolCall"]); + assert.deepEqual(response.content[1], outputCall(lesson)); + assert.ok(saved.some((m) => m.role === "toolResult" && m.toolName === SEND_TO_USER_TOOL), "native output ack persists"); + assert.ok(!saved.some((m) => m.role === "toolResult" && m.toolName === "quiz"), "question placeholder remains transient"); + const providerHistory = toLlmMessages(asAgentMessages(saved)); + assert.deepEqual(providerHistory.find((m) => m.role === "assistant")!.content, response.content, "next request keeps native call/thinking structure"); + assert.deepEqual(prose(projectView(saved)), [lesson]); + const parts = projectView(saved).flatMap((m) => m.parts); + assert.ok(parts.findIndex((p) => p.type === "text") < parts.findIndex((p) => p.type === "tool-quiz")); + const client = new SessionModel({ chatId: chat.id, initialMessages: [user], fetch: globalThis.fetch }); + await client.consumeStream(new Response(wire)); + assert.deepEqual(prose(client.getState().messages), [lesson], "native event replay shows the lesson exactly once"); + assert.deepEqual(prose(projectView(JSON.parse(JSON.stringify(saved)))), [lesson], "reload preserves the lesson"); + assert.deepEqual(teacherProseBlocks(saved), [lesson]); + assert.equal(sessionMarkdown(saved).split(lesson).length - 1, 1); + }); +});