v3: AI SDK pivot — pi removed, Next.js 16 + AI SDK 7 + AI Elements teacher app
The teaching agent no longer runs on pi. Stack: - ToolLoopAgent (ai@7) + OpenRouter (OPENROUTER_API_KEY), model via LH_MODEL, default anthropic/claude-sonnet-4.5 with reasoning tokens - quiz/ask_user_question = client-side tools (no server execute): the call lands as a typed tool part in input-available state; the card grades via src/core/question-core.ts; addToolOutput sends the verdict; the teacher's reaction continues the same assistant message - resume-mid-quiz is free: the pending tool part persists in the transcript - persistence: one JSON per chat (data/chats/), full UIMessage transcripts, saved onFinish (also on abort) - UI: AI Elements (owned copy-paste components, pruned to the five in use) + button-based QuizCard/AnsweredCard (ask answers never graded) - pi fully removed: TUI, extensions, Fastify+WS server, bridge, custom protocol all deleted (recoverable from git history) - tests: question-core units carry over; chat-flow integration drives handleChatRequest with MockLanguageModelV4 (48 tests, zero tokens); render smoke tests guard the cards (renderToStaticMarkup) Carried over unchanged: teacher system prompt, question-core, schema/ ledger/derive/extract pipeline, data model.
This commit is contained in:
Amos
@@ -1,171 +0,0 @@
|
||||
/**
|
||||
* Unit tests — the question bridge (state machine, fake backends only).
|
||||
*
|
||||
* The bridge is globalThis-singleton state shared with other tests in the same
|
||||
* process, so every test creates its own bridge instance via the class-free
|
||||
* export path... (it doesn't export the class — we exercise the singleton and
|
||||
* reset it between tests with clear()).
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { beforeEach, describe, it } from "node:test";
|
||||
import {
|
||||
type AnswerRequest,
|
||||
type QuestionData,
|
||||
QUESTION_ANSWERED_ENTRY,
|
||||
QUESTION_ENTRY,
|
||||
} from "../extensions/core/question-core.ts";
|
||||
import { getBridge } from "../extensions/core/bridge.ts";
|
||||
|
||||
interface Recorded {
|
||||
answered: unknown[];
|
||||
messages: string[];
|
||||
entries: Array<{ type: string; customType?: string; data?: unknown }>;
|
||||
}
|
||||
|
||||
function fakeBackend(kind: "quiz" | "ask"): { recorded: Recorded } {
|
||||
const recorded: Recorded = { answered: [], messages: [], entries: [] };
|
||||
getBridge().registerBackend({
|
||||
kind,
|
||||
appendAnswered: (data) => recorded.answered.push(data),
|
||||
sendUserMessage: (text) => recorded.messages.push(text),
|
||||
getEntries: () => recorded.entries,
|
||||
});
|
||||
return { recorded };
|
||||
}
|
||||
|
||||
function quizData(id: string): QuestionData {
|
||||
return {
|
||||
id,
|
||||
kind: "quiz",
|
||||
question: `Question ${id}?`,
|
||||
mode: "single-select",
|
||||
options: [
|
||||
{ label: "Right", value: "right" },
|
||||
{ label: "Wrong", value: "wrong" },
|
||||
],
|
||||
correctIndices: [1],
|
||||
explanation: "because",
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
describe("question bridge", () => {
|
||||
let quiz: { recorded: Recorded };
|
||||
let ask: { recorded: Recorded };
|
||||
|
||||
beforeEach(() => {
|
||||
const bridge = getBridge();
|
||||
bridge.clear("quiz");
|
||||
bridge.clear("ask");
|
||||
quiz = fakeBackend("quiz");
|
||||
ask = fakeBackend("ask");
|
||||
});
|
||||
|
||||
it("publish makes a question answerable; answer grades and sends a user message", () => {
|
||||
const bridge = getBridge();
|
||||
const qd = quizData("q1");
|
||||
bridge.publish(qd);
|
||||
|
||||
const outcome = bridge.answer("q1", { values: ["right"] });
|
||||
assert.equal(outcome.ok, true);
|
||||
|
||||
assert.equal(quiz.recorded.answered.length, 1);
|
||||
const ad = quiz.recorded.answered[0] as { correct: boolean; questionId: string };
|
||||
assert.equal(ad.questionId, "q1");
|
||||
assert.equal(ad.correct, true);
|
||||
|
||||
assert.equal(quiz.recorded.messages.length, 1);
|
||||
assert.ok(quiz.recorded.messages[0].startsWith("[Quiz answer]"));
|
||||
// ask backend untouched
|
||||
assert.equal(ask.recorded.messages.length, 0);
|
||||
});
|
||||
|
||||
it("answering a question twice fails the second time", () => {
|
||||
const bridge = getBridge();
|
||||
bridge.publish(quizData("q2"));
|
||||
assert.equal(bridge.answer("q2", { values: ["wrong"] }).ok, true);
|
||||
const second = bridge.answer("q2", { values: ["right"] });
|
||||
assert.equal(second.ok, false);
|
||||
assert.match(second.error ?? "", /already-answered/);
|
||||
});
|
||||
|
||||
it("answering an unknown question fails", () => {
|
||||
const outcome = getBridge().answer("nope", { values: [] });
|
||||
assert.equal(outcome.ok, false);
|
||||
});
|
||||
|
||||
it("dismiss records a dismissed answer and notifies the agent", () => {
|
||||
const bridge = getBridge();
|
||||
bridge.publish(quizData("q3"));
|
||||
assert.equal(bridge.dismiss("q3").ok, true);
|
||||
|
||||
const ad = quiz.recorded.answered[0] as { dismissed: boolean };
|
||||
assert.equal(ad.dismissed, true);
|
||||
assert.match(quiz.recorded.messages[0], /dismissed/);
|
||||
});
|
||||
|
||||
it("dontKnow answers are never correct", () => {
|
||||
const bridge = getBridge();
|
||||
bridge.publish(quizData("q4"));
|
||||
bridge.answer("q4", { dontKnow: true });
|
||||
const ad = quiz.recorded.answered[0] as { dontKnow: boolean; correct: boolean };
|
||||
assert.equal(ad.dontKnow, true);
|
||||
assert.equal(ad.correct, false);
|
||||
});
|
||||
|
||||
it("rearm picks up unanswered questions from session entries", () => {
|
||||
const bridge = getBridge();
|
||||
// Simulate a resumed session whose entries contain an unanswered question.
|
||||
quiz.recorded.entries = [
|
||||
{ type: "custom", customType: QUESTION_ENTRY, data: quizData("q5") },
|
||||
{
|
||||
type: "custom",
|
||||
customType: QUESTION_ANSWERED_ENTRY,
|
||||
data: { questionId: "other", answers: [] },
|
||||
},
|
||||
];
|
||||
bridge.rearm("quiz");
|
||||
|
||||
// Now answerable through the bridge.
|
||||
assert.equal(bridge.answer("q5", { values: ["right"] }).ok, true);
|
||||
});
|
||||
|
||||
it("rearm ignores already-answered questions", () => {
|
||||
const bridge = getBridge();
|
||||
quiz.recorded.entries = [
|
||||
{ type: "custom", customType: QUESTION_ENTRY, data: quizData("q6") },
|
||||
{
|
||||
type: "custom",
|
||||
customType: QUESTION_ANSWERED_ENTRY,
|
||||
data: { questionId: "q6", answers: [], dontKnow: true },
|
||||
},
|
||||
];
|
||||
bridge.rearm("quiz");
|
||||
const outcome = bridge.answer("q6", { values: ["right"] });
|
||||
assert.equal(outcome.ok, false);
|
||||
});
|
||||
|
||||
it("clear removes only its kind's pending questions", () => {
|
||||
const bridge = getBridge();
|
||||
bridge.publish(quizData("q7"));
|
||||
const askQd: QuestionData = { ...quizData("q8"), kind: "ask", correctIndices: undefined };
|
||||
bridge.publish(askQd);
|
||||
|
||||
bridge.clear("quiz");
|
||||
assert.equal(bridge.answer("q7", { values: ["right"] }).ok, false);
|
||||
assert.equal(bridge.answer("q8", { other: "x" }).ok, true);
|
||||
});
|
||||
|
||||
it("ask answers route to the ask backend ungraded", () => {
|
||||
const bridge = getBridge();
|
||||
const askQd: QuestionData = { ...quizData("q9"), kind: "ask", correctIndices: undefined };
|
||||
bridge.publish(askQd);
|
||||
|
||||
bridge.answer("q9", { other: "custom answer" });
|
||||
const ad = ask.recorded.answered[0] as { other?: string; correct?: boolean };
|
||||
assert.equal(ad.other, "custom answer");
|
||||
assert.equal(ad.correct, undefined);
|
||||
assert.ok(ask.recorded.messages[0].startsWith("[Answer to your question]"));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,218 @@
|
||||
/**
|
||||
* Scripted integration test — the full teacher loop with zero tokens:
|
||||
*
|
||||
* learner opens chat → mock model calls `quiz` → server streams + persists
|
||||
* → client grades and sends the answer back as the tool output
|
||||
* → mock model reacts → transcript persisted.
|
||||
*
|
||||
* Drives handleChatRequest directly (plain Request/Response — no Next server
|
||||
* needed) with a MockLanguageModelV4. This is the AI SDK equivalent of the
|
||||
* old pi dispatch contract test: the quiz bridge is gone, the tool part IS
|
||||
* the pending state, and the persisted transcript is the resume record.
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { after, describe, it } from "node:test";
|
||||
import { MockLanguageModelV4, simulateReadableStream } from "ai/test";
|
||||
import type { LanguageModelV4StreamPart } from "@ai-sdk/provider";
|
||||
import { createTeacherAgent } from "../src/agent/teacher-agent.ts";
|
||||
import { handleChatRequest } from "../src/server/chat-handler.ts";
|
||||
import { createChat, getChat, saveMessages } from "../src/store/chats.ts";
|
||||
import { buildAnswer, coerceCorrectAnswer, normalizeOptions, resolveCorrect, type QuestionData } from "../src/core/question-core.ts";
|
||||
import { findPendingToolCall } from "../src/core/pending.ts";
|
||||
import { PATHS } from "../config.ts";
|
||||
|
||||
const QUIZ_INPUT = {
|
||||
question: "Which planet is closest to the sun?",
|
||||
options: [
|
||||
{ label: "Venus" },
|
||||
{ label: "Mercury" },
|
||||
{ label: "Mars" },
|
||||
],
|
||||
correctAnswer: "Mercury",
|
||||
explanation: "Mercury is the innermost planet.",
|
||||
};
|
||||
|
||||
function usage() {
|
||||
return {
|
||||
inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 },
|
||||
outputTokens: { total: 5, text: 5, reasoning: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
/** Turn 1: a bit of reasoning, then the quiz tool call. Turn 2: plain text. */
|
||||
function scriptedModel(calls: string[][]) {
|
||||
let turn = 0;
|
||||
return new MockLanguageModelV4({
|
||||
doStream: async (options) => {
|
||||
const promptTexts = options.prompt.map((m) => JSON.stringify(m.content));
|
||||
calls.push(promptTexts);
|
||||
turn++;
|
||||
const chunks: LanguageModelV4StreamPart[] =
|
||||
turn === 1
|
||||
? [
|
||||
{ type: "stream-start", warnings: [] },
|
||||
{ type: "reasoning-start", id: "r1" },
|
||||
{ type: "reasoning-delta", id: "r1", delta: "Probe the basics first." },
|
||||
{ type: "reasoning-end", id: "r1" },
|
||||
{
|
||||
type: "tool-call",
|
||||
toolCallId: "call-1",
|
||||
toolName: "quiz",
|
||||
input: JSON.stringify(QUIZ_INPUT),
|
||||
},
|
||||
{ type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() },
|
||||
]
|
||||
: [
|
||||
{ type: "stream-start", warnings: [] },
|
||||
{ type: "text-start", id: "t1" },
|
||||
{ type: "text-delta", id: "t1", delta: "Right — Mercury it is." },
|
||||
{ type: "text-end", id: "t1" },
|
||||
{ type: "finish", finishReason: { unified: "stop", raw: "stop" }, usage: usage() },
|
||||
];
|
||||
return { stream: simulateReadableStream({ chunks, initialDelayInMs: 0, chunkDelayInMs: 0 }) };
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function drain(res: Response): Promise<string> {
|
||||
if (res.status !== 200) {
|
||||
assert.fail(`expected 200, got ${res.status}: ${await res.text()}`);
|
||||
}
|
||||
return res.text(); // consumes the SSE stream to completion (fires onFinish)
|
||||
}
|
||||
|
||||
function post(chatId: string, messages: unknown[]): Request {
|
||||
return new Request("http://test/api/chat", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ chatId, messages }),
|
||||
});
|
||||
}
|
||||
|
||||
const createdChats: string[] = [];
|
||||
after(() => {
|
||||
for (const id of createdChats) {
|
||||
try {
|
||||
fs.unlinkSync(path.join(PATHS.chats, `${id}.json`));
|
||||
} catch {
|
||||
/* already gone */
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
describe("teacher chat flow (scripted model)", () => {
|
||||
it("quiz tool call persists as a pending part; the answer continues the lesson", async () => {
|
||||
const calls: string[][] = [];
|
||||
const agent = createTeacherAgent(scriptedModel(calls));
|
||||
const chat = createChat("Test: solar system");
|
||||
createdChats.push(chat.id);
|
||||
|
||||
// ── Turn 1: learner opens with a topic; teacher probes with a quiz ──
|
||||
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "teach me the solar system" }] };
|
||||
await drain(await handleChatRequest(post(chat.id, [firstUser]), agent));
|
||||
|
||||
const saved1 = getChat(chat.id)!;
|
||||
assert.equal(saved1.messages.length, 2, "user message + assistant response persisted");
|
||||
const assistantMsg = saved1.messages[1]!;
|
||||
assert.equal(assistantMsg.role, "assistant");
|
||||
|
||||
const reasoningPart = assistantMsg.parts.find((p) => p.type === "reasoning");
|
||||
assert.ok(reasoningPart && reasoningPart.type === "reasoning", "reasoning part streamed + persisted");
|
||||
assert.match(reasoningPart.text, /Probe the basics/);
|
||||
|
||||
const quizPart = assistantMsg.parts.find((p) => p.type === "tool-quiz");
|
||||
assert.ok(quizPart && quizPart.type === "tool-quiz", "quiz tool part persisted");
|
||||
assert.equal(quizPart.state, "input-available", "unanswered quiz = pending state (resume record)");
|
||||
assert.deepEqual(quizPart.input, QUIZ_INPUT);
|
||||
|
||||
// The pending-question gate sees it.
|
||||
const pending = findPendingToolCall(saved1.messages);
|
||||
assert.deepEqual(pending, { toolCallId: "call-1", tool: "quiz" });
|
||||
|
||||
// ── The learner answers: client grades, output goes back on the part ──
|
||||
const options = normalizeOptions(QUIZ_INPUT.options);
|
||||
const qd: QuestionData = {
|
||||
id: "call-1",
|
||||
kind: "quiz",
|
||||
question: QUIZ_INPUT.question,
|
||||
mode: "single-select",
|
||||
options,
|
||||
correctIndices: resolveCorrect(coerceCorrectAnswer(QUIZ_INPUT.correctAnswer), options).indices,
|
||||
explanation: QUIZ_INPUT.explanation,
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
const answer = buildAnswer(qd, { values: ["Mercury"] });
|
||||
assert.equal(answer.correct, true);
|
||||
|
||||
const messages2 = saved1.messages.map((m) =>
|
||||
m.id === assistantMsg.id
|
||||
? {
|
||||
...m,
|
||||
parts: m.parts.map((p) =>
|
||||
p.type === "tool-quiz" ? { ...p, state: "output-available" as const, output: answer } : p,
|
||||
),
|
||||
}
|
||||
: m,
|
||||
);
|
||||
await drain(await handleChatRequest(post(chat.id, messages2), agent));
|
||||
|
||||
// ── The teacher reacts — as a CONTINUATION of the same assistant message
|
||||
// (the last client message is the assistant message carrying the tool
|
||||
// output, so the new text appends to that message's parts). ──
|
||||
const saved2 = getChat(chat.id)!;
|
||||
assert.equal(saved2.messages.length, 2, "the reaction continues the same assistant message");
|
||||
const finalParts = saved2.messages[1]!.parts;
|
||||
const textPart = finalParts.find((p) => p.type === "text");
|
||||
assert.ok(textPart && textPart.type === "text", "reaction text appended after the tool part");
|
||||
assert.match(textPart.text, /Mercury it is/);
|
||||
// The answered quiz part is still there, now with its output.
|
||||
const answeredQuiz = finalParts.find((p) => p.type === "tool-quiz");
|
||||
assert.ok(answeredQuiz && answeredQuiz.type === "tool-quiz" && answeredQuiz.state === "output-available");
|
||||
|
||||
// The model actually received the graded answer as a tool result.
|
||||
assert.equal(calls.length, 2, "model saw exactly two prompts");
|
||||
const toolResultSent = calls[1]!.some((c) => c.includes("tool-result") || c.includes("Mercury"));
|
||||
assert.ok(toolResultSent, "continuation prompt carries the tool result to the model");
|
||||
|
||||
// No pending question remains.
|
||||
assert.equal(findPendingToolCall(saved2.messages), null);
|
||||
});
|
||||
|
||||
it("rejects unknown chats", async () => {
|
||||
const res = await handleChatRequest(post("chat_nonexistent", []));
|
||||
assert.equal(res.status, 404);
|
||||
});
|
||||
|
||||
it("persists even when the stream is aborted (resume keeps partial state)", async () => {
|
||||
const calls: string[][] = [];
|
||||
const agent = createTeacherAgent(scriptedModel(calls));
|
||||
const chat = createChat("Test: abort");
|
||||
createdChats.push(chat.id);
|
||||
|
||||
const req = post(chat.id, [{ id: "u1", role: "user", parts: [{ type: "text", text: "quiz me" }] }]);
|
||||
const res = await handleChatRequest(req, agent);
|
||||
assert.equal(res.status, 200);
|
||||
// Read one chunk then cancel — simulates the browser closing mid-stream.
|
||||
const reader = res.body!.getReader();
|
||||
await reader.read();
|
||||
await reader.cancel();
|
||||
// Give onFinish a tick to fire.
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
const saved = getChat(chat.id)!;
|
||||
assert.ok(saved.messages.length >= 1, "partial transcript persisted");
|
||||
});
|
||||
});
|
||||
|
||||
describe("chat store", () => {
|
||||
it("round-trips and lists chats", async () => {
|
||||
const chat = createChat("Store test");
|
||||
createdChats.push(chat.id);
|
||||
saveMessages(chat.id, [{ id: "m1", role: "user", parts: [{ type: "text", text: "hi" }] }]);
|
||||
const loaded = getChat(chat.id)!;
|
||||
assert.equal(loaded.messages.length, 1);
|
||||
assert.equal(loaded.title, "Store test");
|
||||
});
|
||||
});
|
||||
@@ -1,246 +0,0 @@
|
||||
/**
|
||||
* Integration test — the full teaching flow with a SCRIPTED model.
|
||||
*
|
||||
* Real pi agent loop, real web extensions (quiz-web/ask-web), real bridge,
|
||||
* real session persistence — the only fake is the LLM. Verifies the dispatch
|
||||
* contract end to end:
|
||||
*
|
||||
* prompt → scripted turn calls quiz → extension persists lh_question and
|
||||
* returns immediately → agent ends turn → bridge.answer() grades and sends
|
||||
* the [Quiz answer] user message → scripted final turn → settle.
|
||||
*
|
||||
* Then reloads the session file into a NEW runtime (resume) and verifies the
|
||||
* pending-question re-arm from persisted entries.
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { after, before, describe, it } from "node:test";
|
||||
import { createAgentSessionRuntime, SessionManager } from "@earendil-works/pi-coding-agent";
|
||||
import { PATHS } from "../config.ts";
|
||||
import { buildTeacherRuntimeFactory } from "../src/teacher/create-session.ts";
|
||||
import {
|
||||
QUESTION_ANSWERED_ENTRY,
|
||||
QUESTION_ENTRY,
|
||||
findPendingQuestion,
|
||||
toClientQuestion,
|
||||
type QuestionAnswerData,
|
||||
type QuestionData,
|
||||
} from "../extensions/core/question-core.ts";
|
||||
import { getBridge } from "../extensions/core/bridge.ts";
|
||||
import { createScriptedRuntime } from "../src/testing/scripted-model.ts";
|
||||
|
||||
const QUIZ_ARGS = {
|
||||
question: "Which is closest to the Sun?",
|
||||
options: [
|
||||
{ label: "Mercury", value: "mercury" },
|
||||
{ label: "Venus", value: "venus" },
|
||||
],
|
||||
correctAnswer: "mercury",
|
||||
explanation: "Mercury is the innermost planet.",
|
||||
};
|
||||
|
||||
let tmpDir: string;
|
||||
let sessionDir: string;
|
||||
|
||||
before(() => {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "lh-int-"));
|
||||
sessionDir = path.join(tmpDir, "sessions");
|
||||
fs.mkdirSync(sessionDir, { recursive: true });
|
||||
});
|
||||
|
||||
after(() => {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function entriesOf(sm: SessionManager) {
|
||||
return sm.getEntries() as Array<{ type: string; customType?: string; data?: unknown; id: string }>;
|
||||
}
|
||||
|
||||
function customEntries(sm: SessionManager, customType: string) {
|
||||
return entriesOf(sm).filter((e) => e.type === "custom" && e.customType === customType);
|
||||
}
|
||||
|
||||
function userMessages(sm: SessionManager) {
|
||||
return entriesOf(sm)
|
||||
.filter((e) => e.type === "message" && (e as { message: { role: string } }).message.role === "user")
|
||||
.map((e) => (e as unknown as { message: { content: Array<{ type: string; text?: string }> } }).message);
|
||||
}
|
||||
|
||||
function waitForIdle(session: { waitForIdle(): Promise<void> }, timeoutMs = 10_000): Promise<void> {
|
||||
return Promise.race([
|
||||
session.waitForIdle(),
|
||||
new Promise<never>((_, reject) => setTimeout(() => reject(new Error("timeout waiting for idle")), timeoutMs)),
|
||||
]);
|
||||
}
|
||||
|
||||
/** Poll until pred() is true (bridge sendUserMessage is fire-and-forget async,
|
||||
* so waitForIdle alone can win a race against the incoming prompt). */
|
||||
function waitFor(pred: () => boolean, what: string, timeoutMs = 10_000): Promise<void> {
|
||||
const start = Date.now();
|
||||
return new Promise((resolve, reject) => {
|
||||
const tick = () => {
|
||||
if (pred()) return resolve();
|
||||
if (Date.now() - start > timeoutMs) return reject(new Error(`timeout waiting for ${what}`));
|
||||
setTimeout(tick, 25);
|
||||
};
|
||||
tick();
|
||||
});
|
||||
}
|
||||
|
||||
describe("scripted teaching flow (web dispatch mode)", () => {
|
||||
it("runs the quiz dispatch cycle and persists gold-standard data", async () => {
|
||||
const scripted = createScriptedRuntime([
|
||||
// Turn 1: call quiz, end turn (toolUse stop reason).
|
||||
{ toolCalls: [{ name: "quiz", arguments: QUIZ_ARGS }] },
|
||||
// Turn 2 (after the tool result, model declines to continue): final text.
|
||||
{ text: "Take a shot." },
|
||||
// Turn 3 (after the [Quiz answer] user message): wrap up.
|
||||
{ text: "Right — Mercury it is. Lesson over." },
|
||||
]);
|
||||
|
||||
const sessionManager = SessionManager.create(PATHS.root, sessionDir);
|
||||
const runtime = await createAgentSessionRuntime(buildTeacherRuntimeFactory("web", {
|
||||
model: scripted.model,
|
||||
modelRuntime: scripted.modelRuntime,
|
||||
}), {
|
||||
cwd: PATHS.root,
|
||||
agentDir: PATHS.agentDir,
|
||||
sessionManager,
|
||||
});
|
||||
assert.ok(!runtime.diagnostics.some((d) => d.type === "error"), "no fatal diagnostics");
|
||||
|
||||
// ── Turn 1: prompt → quiz presented ────────────────────────────────
|
||||
const promptDone = runtime.session.prompt("Teach me about the solar system.");
|
||||
await waitForIdle(runtime.session);
|
||||
await promptDone;
|
||||
|
||||
const questions = customEntries(sessionManager, QUESTION_ENTRY) as Array<{ data?: QuestionData }>;
|
||||
assert.equal(questions.length, 1, "exactly one lh_question entry");
|
||||
const qd = questions[0]!.data!;
|
||||
assert.equal(qd.kind, "quiz");
|
||||
assert.equal(qd.mode, "single-select");
|
||||
assert.deepEqual(qd.correctIndices, [qd.options.findIndex((o) => o.value === "mercury") + 1], "correct index resolved post-shuffle");
|
||||
assert.equal(qd.explanation, QUIZ_ARGS.explanation);
|
||||
|
||||
// Pending question visible without leaking the answer.
|
||||
const pending = findPendingQuestion(entriesOf(sessionManager));
|
||||
assert.ok(pending, "pending question found");
|
||||
const clientPayload = JSON.stringify(toClientQuestion(qd));
|
||||
assert.ok(!clientPayload.includes("correctIndices"));
|
||||
assert.ok(!clientPayload.includes("Mercury is the innermost"));
|
||||
|
||||
// ── Answer: graded + injected as user message ──────────────────────
|
||||
const outcome = getBridge().answer(qd.id, { values: ["mercury"] });
|
||||
assert.equal(outcome.ok, true);
|
||||
await waitFor(() => userMessages(sessionManager).length === 2, "quiz answer user message");
|
||||
await waitForIdle(runtime.session);
|
||||
|
||||
const answers = customEntries(sessionManager, QUESTION_ANSWERED_ENTRY) as Array<{
|
||||
data?: QuestionAnswerData;
|
||||
}>;
|
||||
assert.equal(answers.length, 1, "exactly one lh_question_answered entry");
|
||||
const ad = answers[0]!.data!;
|
||||
assert.equal(ad.correct, true);
|
||||
assert.equal(ad.questionId, qd.id);
|
||||
assert.equal(ad.answers.length, 1);
|
||||
assert.equal(ad.answers[0]!.value, "mercury");
|
||||
|
||||
const users = userMessages(sessionManager);
|
||||
assert.equal(users.length, 2, "topic prompt + quiz answer");
|
||||
const answerMsg = users[1]!.content.find((c) => c.type === "text")!.text!;
|
||||
assert.ok(answerMsg.startsWith("[Quiz answer]"), "answer arrives as user message");
|
||||
assert.match(answerMsg, /Result: CORRECT/);
|
||||
assert.match(answerMsg, /Selected: \d\. Mercury/);
|
||||
|
||||
// Pending is cleared after answering.
|
||||
assert.equal(findPendingQuestion(entriesOf(sessionManager)), null);
|
||||
|
||||
// The scripted model saw the answer in its context (turn 3).
|
||||
assert.ok(scripted.contextsSeen.length >= 3);
|
||||
const lastContext = scripted.contextsSeen[scripted.contextsSeen.length - 1]!;
|
||||
const lastMessages = JSON.stringify(lastContext.messages);
|
||||
assert.ok(lastMessages.includes("[Quiz answer]"), "answer message reached the LLM context");
|
||||
// …but the tool RESULT must never reveal the answer (the correct answer
|
||||
// and explanation live only in the lh_question custom entry, which is
|
||||
// excluded from LLM context). The toolCall arguments legitimately contain
|
||||
// them — the model wrote those itself.
|
||||
for (const ctx of scripted.contextsSeen) {
|
||||
const messages = ctx.messages as Array<{ role?: string; content?: Array<{ type?: string; text?: string; name?: string }> }>;
|
||||
for (const m of messages) {
|
||||
if (m.role !== "toolResult") continue;
|
||||
for (const c of m.content ?? []) {
|
||||
if (c.type === "text" && c.text) {
|
||||
assert.ok(!c.text.includes("Mercury is the innermost"), "no explanation leak via tool result");
|
||||
assert.ok(!/correct/i.test(c.text), "no correct-answer leak via tool result");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
runtime.session.dispose();
|
||||
|
||||
// ── Resume: pending re-arm from the session file ───────────────────
|
||||
const sessionFile = sessionManager.getSessionFile()!;
|
||||
const sm2 = SessionManager.open(sessionFile, sessionDir, PATHS.root);
|
||||
const scripted2 = createScriptedRuntime([{ text: "resumed" }]);
|
||||
const runtime2 = await createAgentSessionRuntime(buildTeacherRuntimeFactory("web", {
|
||||
model: scripted2.model,
|
||||
modelRuntime: scripted2.modelRuntime,
|
||||
}), {
|
||||
cwd: PATHS.root,
|
||||
agentDir: PATHS.agentDir,
|
||||
sessionManager: sm2,
|
||||
});
|
||||
// session_start fired during construction → extensions re-armed.
|
||||
assert.equal(findPendingQuestion(entriesOf(sm2)), null, "answered question is NOT re-armed");
|
||||
assert.equal(customEntries(sm2, QUESTION_ENTRY).length, 1, "question entry survived on disk");
|
||||
runtime2.session.dispose();
|
||||
|
||||
// ── Resume with an UNANSWERED question pending ─────────────────────
|
||||
// New session, fresh script: quiz presented but never answered.
|
||||
const scripted3 = createScriptedRuntime([
|
||||
{ toolCalls: [{ name: "quiz", arguments: QUIZ_ARGS }] },
|
||||
{ text: "Waiting for your answer." },
|
||||
]);
|
||||
const sm3 = SessionManager.create(PATHS.root, sessionDir);
|
||||
const runtime3 = await createAgentSessionRuntime(buildTeacherRuntimeFactory("web", {
|
||||
model: scripted3.model,
|
||||
modelRuntime: scripted3.modelRuntime,
|
||||
}), {
|
||||
cwd: PATHS.root,
|
||||
agentDir: PATHS.agentDir,
|
||||
sessionManager: sm3,
|
||||
});
|
||||
const p3 = runtime3.session.prompt("Teach me something else.");
|
||||
await waitForIdle(runtime3.session);
|
||||
await p3;
|
||||
runtime3.session.dispose();
|
||||
|
||||
const file3 = sm3.getSessionFile()!;
|
||||
const sm4 = SessionManager.open(file3, sessionDir, PATHS.root);
|
||||
const scripted4 = createScriptedRuntime([{ text: "ok" }]);
|
||||
const runtime4 = await createAgentSessionRuntime(buildTeacherRuntimeFactory("web", {
|
||||
model: scripted4.model,
|
||||
modelRuntime: scripted4.modelRuntime,
|
||||
}), {
|
||||
cwd: PATHS.root,
|
||||
agentDir: PATHS.agentDir,
|
||||
sessionManager: sm4,
|
||||
});
|
||||
const reArmed = findPendingQuestion(entriesOf(sm4));
|
||||
assert.ok(reArmed, "unanswered question re-arms on resume");
|
||||
assert.equal(reArmed!.kind, "quiz");
|
||||
// And it is answerable through the bridge (the re-armed id).
|
||||
const answerOutcome = getBridge().answer(reArmed!.id, { dontKnow: true, note: "forgot" });
|
||||
assert.equal(answerOutcome.ok, true);
|
||||
const answeredOnDisk = customEntries(sm4, QUESTION_ANSWERED_ENTRY) as Array<{
|
||||
data?: QuestionAnswerData;
|
||||
}>;
|
||||
assert.equal(answeredOnDisk.length, 1);
|
||||
assert.equal(answeredOnDisk[0]!.data!.dontKnow, true);
|
||||
runtime4.session.dispose();
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
/**
|
||||
* Unit tests — question-core (pure logic, no pi, no server, no LLM).
|
||||
* Unit tests — question-core (pure logic, no server, no LLM).
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
shuffleOptions,
|
||||
type QuestionData,
|
||||
toClientQuestion,
|
||||
} from "../extensions/core/question-core.ts";
|
||||
} from "../src/core/question-core.ts";
|
||||
|
||||
// ─── normalizeOptions ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
/**
|
||||
* Render smoke tests — the lesson from the v2 bugs: protocol tests don't see
|
||||
* rendering. These render the interaction cards to static HTML and assert on
|
||||
* the output. No DOM needed (react-dom/server).
|
||||
*
|
||||
* tsx note: tsx ignores tsconfig's `jsx` field and uses the CLASSIC JSX
|
||||
* transform (React.createElement), so `React` must be a resolvable global.
|
||||
* The shim below provides it; Vite/Next use the automatic runtime and don't
|
||||
* care.
|
||||
*/
|
||||
|
||||
import React from "react";
|
||||
(globalThis as Record<string, unknown>).React = React;
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { QuizCard } from "../components/quiz-card.tsx";
|
||||
import { AnsweredCard } from "../components/answered-card.tsx";
|
||||
import { buildAnswer, coerceCorrectAnswer, normalizeOptions, resolveCorrect, type QuestionData } from "../src/core/question-core.ts";
|
||||
|
||||
const quizInput = {
|
||||
question: "Which planet is closest to the sun?",
|
||||
options: [{ label: "Venus" }, { label: "Mercury" }, { label: "Mars" }],
|
||||
correctAnswer: "Mercury",
|
||||
explanation: "Mercury is the innermost planet.",
|
||||
};
|
||||
|
||||
function quizQD(): QuestionData {
|
||||
const options = normalizeOptions(quizInput.options);
|
||||
return {
|
||||
id: "q1",
|
||||
kind: "quiz",
|
||||
question: quizInput.question,
|
||||
mode: "single-select",
|
||||
options,
|
||||
correctIndices: resolveCorrect(coerceCorrectAnswer(quizInput.correctAnswer), options).indices,
|
||||
explanation: quizInput.explanation,
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
describe("render smoke tests", () => {
|
||||
it("QuizCard renders question, options, and the automatic 'I don't know'", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<QuizCard toolName="quiz" toolCallId="q1" input={quizInput} onSubmit={() => {}} />,
|
||||
);
|
||||
assert.match(html, /Which planet is closest to the sun\?/);
|
||||
assert.match(html, /Mercury/);
|
||||
assert.match(html, /I don't know|I don't know/);
|
||||
// The v1 desync bug: options are buttons with ARIA roles, not native inputs.
|
||||
assert.ok(!html.includes('type="radio"'), "no native radio inputs");
|
||||
assert.ok(html.includes('role="radio"'), "button-based radio role present");
|
||||
});
|
||||
|
||||
it("QuizCard never leaks the explanation or the correct answer before submit", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<QuizCard toolName="quiz" toolCallId="q1" input={quizInput} onSubmit={() => {}} />,
|
||||
);
|
||||
assert.ok(!html.includes("innermost"), "explanation must not render pre-answer");
|
||||
});
|
||||
|
||||
it("ask card renders Other + note and no grading affordance", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<QuizCard
|
||||
toolName="ask_user_question"
|
||||
toolCallId="a1"
|
||||
input={{ question: "How confident are you?", options: [{ label: "Very" }, { label: "Somewhat" }] }}
|
||||
onSubmit={() => {}}
|
||||
/>,
|
||||
);
|
||||
assert.match(html, /Other/);
|
||||
assert.ok(!html.includes("I don't know"), "ask has no 'I don't know'");
|
||||
});
|
||||
|
||||
it("AnsweredCard grades quiz answers (✓/✗ + explanation)", () => {
|
||||
const answer = buildAnswer(quizQD(), { values: ["Venus"] });
|
||||
const html = renderToStaticMarkup(<AnsweredCard answer={answer} />);
|
||||
assert.match(html, /incorrect/);
|
||||
assert.match(html, /innermost/, "explanation renders post-answer");
|
||||
assert.ok(html.includes("✓") && html.includes("✗"));
|
||||
});
|
||||
|
||||
it("AnsweredCard does NOT grade ask answers", () => {
|
||||
const qd: QuestionData = {
|
||||
id: "a1",
|
||||
kind: "ask",
|
||||
question: "How confident are you?",
|
||||
mode: "single-select",
|
||||
options: normalizeOptions([{ label: "Very" }, { label: "Somewhat" }]),
|
||||
createdAt: new Date().toISOString(),
|
||||
};
|
||||
const answer = buildAnswer(qd, { values: ["Somewhat"] });
|
||||
const html = renderToStaticMarkup(<AnsweredCard answer={answer} />);
|
||||
assert.ok(!html.includes("incorrect") && !html.includes("✗"), "ask answers never get quiz grading UI");
|
||||
assert.match(html, /Somewhat/);
|
||||
});
|
||||
|
||||
it("AnsweredCard renders dismissal as skipped", () => {
|
||||
const answer = { ...buildAnswer(quizQD(), { dontKnow: true }), dismissed: true };
|
||||
const html = renderToStaticMarkup(<AnsweredCard answer={answer} />);
|
||||
assert.match(html, /skipped/);
|
||||
});
|
||||
});
|
||||
@@ -1,160 +0,0 @@
|
||||
/**
|
||||
* Render smoke tests — the layer every earlier bug slipped through.
|
||||
*
|
||||
* Components are rendered to static HTML (no browser, no DOM) and we assert
|
||||
* the expected content is actually IN the output. These would have caught:
|
||||
* - Markdown never receiving its text (assistant messages rendered empty)
|
||||
* - ask answers rendered as graded quiz answers (red ✗)
|
||||
* - quiz options built from native radio inputs (the desync bug)
|
||||
*
|
||||
* Mermaid: SSR skips effects, so diagrams don't render here — that's fine,
|
||||
* we only assert the surrounding markup doesn't crash.
|
||||
*
|
||||
* Note: components carry `import React from "react"` because tsx (the test
|
||||
* runner) uses the classic JSX transform, which needs React in scope. Vite
|
||||
* uses the automatic runtime, where the import is simply unused.
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import { createElement as h } from "react";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import { Markdown } from "../web/src/components/Markdown.tsx";
|
||||
import { QuizCard } from "../web/src/components/QuizCard.tsx";
|
||||
import { AnsweredCard } from "../web/src/components/AnsweredCard.tsx";
|
||||
import { EntryBlock } from "../web/src/views/SessionView.tsx";
|
||||
import type { ClientQuestion, QuestionAnswerData } from "../src/shared/protocol.ts";
|
||||
|
||||
describe("render smoke tests", () => {
|
||||
it("Markdown renders the actual text content", () => {
|
||||
const html = renderToStaticMarkup(h(Markdown, { text: "Hello **bold** world" }));
|
||||
assert.match(html, /Hello/);
|
||||
assert.match(html, /<strong>bold<\/strong>/);
|
||||
assert.match(html, /world/);
|
||||
});
|
||||
|
||||
it("Markdown renders LaTeX via KaTeX", () => {
|
||||
const html = renderToStaticMarkup(h(Markdown, { text: "Energy is $E = mc^2$ here." }));
|
||||
assert.match(html, /here\./);
|
||||
assert.match(html, /katex/i, "KaTeX markup present");
|
||||
});
|
||||
|
||||
it("Markdown survives a mermaid block without crashing", () => {
|
||||
const html = renderToStaticMarkup(h(Markdown, { text: "Before\n\n```mermaid\ngraph TD\nA-->B\n```\n\nAfter" }));
|
||||
assert.match(html, /Before/);
|
||||
assert.match(html, /After/);
|
||||
});
|
||||
|
||||
it("QuizCard renders options as buttons (no native radio/checkbox inputs)", () => {
|
||||
const q: ClientQuestion = {
|
||||
id: "q1",
|
||||
kind: "quiz",
|
||||
question: "Which planet?",
|
||||
mode: "single-select",
|
||||
options: [
|
||||
{ label: "Mercury", value: "mercury" },
|
||||
{ label: "Venus", value: "venus" },
|
||||
],
|
||||
multiSelect: false,
|
||||
};
|
||||
const html = renderToStaticMarkup(h(QuizCard, { question: q, send: () => {} }));
|
||||
assert.match(html, /Mercury/);
|
||||
assert.match(html, /Venus/);
|
||||
assert.match(html, /I don't know/);
|
||||
assert.ok(!/<input[^>]*type="radio"/.test(html), "no native radio inputs (v1 desync bug)");
|
||||
assert.ok(!/<input[^>]*type="checkbox"/.test(html), "no native checkbox inputs");
|
||||
assert.match(html, /<button[^>]*role="radio"/, "options are buttons with radio role");
|
||||
});
|
||||
|
||||
it("QuizCard multi-select uses checkbox role on buttons", () => {
|
||||
const q: ClientQuestion = {
|
||||
id: "q2",
|
||||
kind: "quiz",
|
||||
question: "Pick all.",
|
||||
mode: "multi-select",
|
||||
options: [
|
||||
{ label: "A", value: "a" },
|
||||
{ label: "B", value: "b" },
|
||||
],
|
||||
multiSelect: true,
|
||||
};
|
||||
const html = renderToStaticMarkup(h(QuizCard, { question: q, send: () => {} }));
|
||||
assert.match(html, /<button[^>]*role="checkbox"/);
|
||||
});
|
||||
|
||||
it("AnsweredCard: quiz answers are graded", () => {
|
||||
const data: QuestionAnswerData = {
|
||||
questionId: "q1",
|
||||
kind: "quiz",
|
||||
questionText: "Which planet?",
|
||||
mode: "single-select",
|
||||
options: [
|
||||
{ index: 1, label: "Mercury" },
|
||||
{ index: 2, label: "Venus" },
|
||||
],
|
||||
answers: [{ index: 1, label: "Mercury", value: "mercury" }],
|
||||
correctIndices: [1],
|
||||
correct: true,
|
||||
explanation: "Because.",
|
||||
};
|
||||
const html = renderToStaticMarkup(h(AnsweredCard, { data }));
|
||||
assert.match(html, /Correct/);
|
||||
assert.match(html, /✓/);
|
||||
});
|
||||
|
||||
it("AnsweredCard: ask answers are NOT graded (no ✗, no verdict)", () => {
|
||||
const data: QuestionAnswerData = {
|
||||
questionId: "q1",
|
||||
kind: "ask",
|
||||
questionText: "What do you prefer?",
|
||||
mode: "single-select",
|
||||
options: [
|
||||
{ index: 1, label: "Option A" },
|
||||
{ index: 2, label: "Option B" },
|
||||
],
|
||||
answers: [{ index: 2, label: "Option B", value: "b" }],
|
||||
};
|
||||
const html = renderToStaticMarkup(h(AnsweredCard, { data }));
|
||||
assert.match(html, /Option B/);
|
||||
assert.ok(!html.includes("✗"), "ask answers never show ✗ (v1 grading bug)");
|
||||
assert.ok(!/Incorrect/.test(html), "no grading verdict for ask");
|
||||
assert.ok(!/Correct<\/div>/.test(html));
|
||||
});
|
||||
|
||||
it("AnsweredCard: dismissed questions render as skipped", () => {
|
||||
const data: QuestionAnswerData = {
|
||||
questionId: "q1",
|
||||
kind: "quiz",
|
||||
questionText: "Q?",
|
||||
mode: "single-select",
|
||||
options: [],
|
||||
answers: [],
|
||||
dismissed: true,
|
||||
};
|
||||
const html = renderToStaticMarkup(h(AnsweredCard, { data }));
|
||||
assert.match(html, /Skipped/);
|
||||
});
|
||||
|
||||
it("EntryBlock: toolResult entries (LLM plumbing) are never rendered", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
h(EntryBlock, {
|
||||
entry: {
|
||||
type: "toolResult",
|
||||
id: "e1",
|
||||
toolName: "quiz",
|
||||
text: "Quiz displayed to the learner. End your turn now and wait…",
|
||||
},
|
||||
}),
|
||||
);
|
||||
assert.ok(!html.includes("End your turn"), "dispatch instructions must not leak into the transcript");
|
||||
});
|
||||
|
||||
it("EntryBlock: quiz-answer user messages are not duplicated in the transcript", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
h(EntryBlock, { entry: { type: "user", id: "e2", text: "[Quiz answer] Selected: Mercury" } }),
|
||||
);
|
||||
assert.ok(!html.includes("Mercury"), "already rendered as an answer card");
|
||||
const normal = renderToStaticMarkup(h(EntryBlock, { entry: { type: "user", id: "e3", text: "Teach me physics" } }));
|
||||
assert.match(normal, /Teach me physics/);
|
||||
});
|
||||
});
|
||||
@@ -1,330 +0,0 @@
|
||||
/**
|
||||
* Integration test — the web server with a SCRIPTED model runtime.
|
||||
*
|
||||
* Real Fastify server, real WebSocket protocol, real web extensions and
|
||||
* bridge, real session files — the only fake is the LLM. This is the test
|
||||
* layer that catches exactly the class of bugs reported from the browser:
|
||||
*
|
||||
* - exactly-once entry delivery (no duplicated user messages, ever —
|
||||
* including sockets that connect after the turn already happened)
|
||||
* - thinking + text deltas streamed live
|
||||
* - quiz answer flow over WS
|
||||
* - multiple sockets seeing the same history
|
||||
*
|
||||
* Deterministic: zero tokens, zero variance, runs in milliseconds.
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { after, before, describe, it } from "node:test";
|
||||
import { createAgentSessionRuntime } from "@earendil-works/pi-coding-agent";
|
||||
import { PATHS } from "../config.ts";
|
||||
import { buildTeacherRuntimeFactory } from "../src/teacher/create-session.ts";
|
||||
import { createScriptedRuntime } from "../src/testing/scripted-model.ts";
|
||||
import { startServer } from "../src/server/index.ts";
|
||||
import type { ServerMessage } from "../src/shared/protocol.ts";
|
||||
|
||||
const QUIZ_ARGS = {
|
||||
question: "Which is closest to the Sun?",
|
||||
options: [
|
||||
{ label: "Mercury", value: "mercury" },
|
||||
{ label: "Venus", value: "venus" },
|
||||
],
|
||||
correctAnswer: "mercury",
|
||||
explanation: "Mercury is the innermost planet.",
|
||||
};
|
||||
|
||||
let port: number;
|
||||
let closeServer: () => Promise<void>;
|
||||
|
||||
/** The script for the NEXT session the server activates (tests swap it). */
|
||||
let currentScript: import("../src/testing/scripted-model.ts").ScriptedTurn[] = [
|
||||
// Turn 1: think, then call quiz.
|
||||
{
|
||||
thinking: "I should probe the learner's prior knowledge first.",
|
||||
toolCalls: [{ name: "quiz", arguments: QUIZ_ARGS }],
|
||||
},
|
||||
// Turn 2 (after the tool result): end turn.
|
||||
{ text: "Take a shot." },
|
||||
// Turn 3 (after the [Quiz answer] user message): wrap up.
|
||||
{ text: "Right — Mercury it is. Lesson over." },
|
||||
];
|
||||
|
||||
before(async () => {
|
||||
const runtimeFactory = async (options: {
|
||||
cwd: string;
|
||||
agentDir: string;
|
||||
sessionManager: import("@earendil-works/pi-coding-agent").SessionManager;
|
||||
}) => {
|
||||
const scripted = createScriptedRuntime(currentScript);
|
||||
return createAgentSessionRuntime(
|
||||
buildTeacherRuntimeFactory("web", { model: scripted.model, modelRuntime: scripted.modelRuntime }),
|
||||
options,
|
||||
) as never;
|
||||
};
|
||||
const server = await startServer({ port: 0, runtimeFactory });
|
||||
port = server.port;
|
||||
closeServer = server.close;
|
||||
});
|
||||
|
||||
after(async () => {
|
||||
await closeServer();
|
||||
// The tests created real session files — remove them so the learner's
|
||||
// data/sessions stays clean.
|
||||
for (const file of createdFiles) {
|
||||
try {
|
||||
await fs.rm(path.join(PATHS.sessions, file), { force: true });
|
||||
} catch {
|
||||
// best effort
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
/** Session files this test created (cleaned up in after()). */
|
||||
const createdFiles: string[] = [];
|
||||
|
||||
async function createTestSession(topic: string): Promise<{ id: string; file: string }> {
|
||||
const res = await fetch(`http://127.0.0.1:${port}/api/sessions`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({ topic }),
|
||||
});
|
||||
assert.ok(res.ok, `create failed: ${res.status}`);
|
||||
const created = (await res.json()) as { id: string; file: string };
|
||||
assert.ok(created.file.endsWith(".jsonl"), "create returns the session file");
|
||||
createdFiles.push(created.file);
|
||||
return created;
|
||||
}
|
||||
|
||||
// ─── WS test helper ─────────────────────────────────────────────────────────
|
||||
|
||||
interface Collected {
|
||||
ws: WebSocket;
|
||||
msgs: ServerMessage[];
|
||||
}
|
||||
|
||||
async function openSocket(): Promise<Collected> {
|
||||
const ws = new WebSocket(`ws://127.0.0.1:${port}/ws`);
|
||||
const msgs: ServerMessage[] = [];
|
||||
const opened = new Promise<void>((resolve, reject) => {
|
||||
ws.onopen = () => resolve();
|
||||
ws.onerror = () => reject(new Error("ws connect failed"));
|
||||
});
|
||||
ws.onmessage = (ev) => {
|
||||
msgs.push(JSON.parse(String(ev.data)) as ServerMessage);
|
||||
};
|
||||
await opened;
|
||||
return { ws, msgs };
|
||||
}
|
||||
|
||||
function waitMsg(
|
||||
c: Collected,
|
||||
pred: (m: ServerMessage) => boolean,
|
||||
what: string,
|
||||
timeoutMs = 5000,
|
||||
): Promise<ServerMessage> {
|
||||
const found = c.msgs.find(pred);
|
||||
if (found) return Promise.resolve(found);
|
||||
return new Promise((resolve, reject) => {
|
||||
const interval = setInterval(() => {
|
||||
const m = c.msgs.find(pred);
|
||||
if (m) {
|
||||
clearInterval(interval);
|
||||
clearTimeout(timer);
|
||||
resolve(m);
|
||||
}
|
||||
}, 15);
|
||||
const timer = setTimeout(() => {
|
||||
clearInterval(interval);
|
||||
reject(new Error(`timeout waiting for ${what}`));
|
||||
}, timeoutMs);
|
||||
});
|
||||
}
|
||||
|
||||
function waitSilence(c: Collected, settleMs = 300): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
let last = c.msgs.length;
|
||||
const interval = setInterval(() => {
|
||||
if (c.msgs.length === last) {
|
||||
clearInterval(interval);
|
||||
resolve();
|
||||
}
|
||||
last = c.msgs.length;
|
||||
}, settleMs / 3);
|
||||
});
|
||||
}
|
||||
|
||||
/** Every entry id a socket has seen (backfill + live), in order. */
|
||||
function seenEntryIds(msgs: ServerMessage[]): string[] {
|
||||
const ids: string[] = [];
|
||||
for (const m of msgs) {
|
||||
if (m.type === "backfill") ids.push(...m.entries.map((e) => e.id));
|
||||
if (m.type === "entry") ids.push(m.entry.id);
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
function seenUserTexts(msgs: ServerMessage[]): string[] {
|
||||
const texts: string[] = [];
|
||||
for (const m of msgs) {
|
||||
if (m.type === "backfill") texts.push(...m.entries.filter((e) => e.type === "user").map((e) => e.text));
|
||||
if (m.type === "entry" && m.entry.type === "user") texts.push(m.entry.text);
|
||||
}
|
||||
return texts;
|
||||
}
|
||||
|
||||
// ─── the test ───────────────────────────────────────────────────────────────
|
||||
|
||||
describe("web server protocol (scripted model)", () => {
|
||||
it("delivers every entry exactly once, streams thinking, and handles the answer flow", async () => {
|
||||
const TOPIC = "Teach me the solar system.";
|
||||
|
||||
// Connect BEFORE creating the session so we see every live event.
|
||||
const c1 = await openSocket();
|
||||
|
||||
// ── create the session ────────────────────────────────────────────
|
||||
const created = await createTestSession(TOPIC);
|
||||
assert.ok(created.file.endsWith(".jsonl"));
|
||||
|
||||
// ── the first turn: thinking delta, quiz presented, agent settles ──
|
||||
const thinkingDelta = await waitMsg(
|
||||
c1,
|
||||
(m) => m.type === "delta" && m.kind === "thinking",
|
||||
"thinking delta",
|
||||
);
|
||||
assert.equal((thinkingDelta as { delta: string }).delta.length > 0, true);
|
||||
|
||||
const pendingMsg = await waitMsg(
|
||||
c1,
|
||||
(m) => m.type === "pending" && m.question !== null,
|
||||
"pending question",
|
||||
);
|
||||
const question = (pendingMsg as { question: { id: string; question: string; kind: string } }).question;
|
||||
assert.equal(question.kind, "quiz");
|
||||
// No leak in the client payload.
|
||||
const pendingJson = JSON.stringify(pendingMsg);
|
||||
assert.ok(!pendingJson.includes("Mercury is the innermost"), "no explanation leak to client");
|
||||
|
||||
await waitMsg(c1, (m) => m.type === "streaming" && m.value === false, "settle after quiz turn");
|
||||
|
||||
// The topic user message must have arrived EXACTLY once.
|
||||
await waitSilence(c1);
|
||||
const topicCount = seenUserTexts(c1.msgs).filter((t) => t === TOPIC).length;
|
||||
assert.equal(topicCount, 1, "topic message delivered exactly once (the v1 duplication bug)");
|
||||
const ids1 = seenEntryIds(c1.msgs);
|
||||
assert.equal(ids1.length, new Set(ids1).size, "no duplicate entry ids on socket 1");
|
||||
|
||||
// ── answer the quiz ───────────────────────────────────────────────
|
||||
c1.ws.send(JSON.stringify({ type: "answer", questionId: question.id, values: ["mercury"] }));
|
||||
|
||||
await waitMsg(
|
||||
c1,
|
||||
(m) => m.type === "entry" && m.entry.type === "questionAnswered",
|
||||
"questionAnswered entry",
|
||||
);
|
||||
const answerUserMsg = await waitMsg(
|
||||
c1,
|
||||
(m) => m.type === "entry" && m.entry.type === "user" && m.entry.text.startsWith("[Quiz answer]"),
|
||||
"[Quiz answer] user entry",
|
||||
);
|
||||
assert.match((answerUserMsg as { entry: { text: string } }).entry.text, /Result: CORRECT/);
|
||||
|
||||
// Pending cleared after answering.
|
||||
await waitMsg(c1, (m) => m.type === "pending" && m.question === null, "pending cleared");
|
||||
// Agent continues after the answer and settles.
|
||||
await waitMsg(c1, (m) => m.type === "streaming" && m.value === false, "settle after answer");
|
||||
await waitSilence(c1);
|
||||
|
||||
const ids1b = seenEntryIds(c1.msgs);
|
||||
assert.equal(ids1b.length, new Set(ids1b).size, "still no duplicate entry ids after answering");
|
||||
|
||||
// ── a SECOND socket connects late: same history, exactly once ─────
|
||||
const c2 = await openSocket();
|
||||
await waitMsg(c2, (m) => m.type === "backfill", "socket 2 backfill");
|
||||
await waitSilence(c2);
|
||||
|
||||
const ids2 = seenEntryIds(c2.msgs);
|
||||
assert.equal(ids2.length, new Set(ids2).size, "no duplicate entry ids on socket 2");
|
||||
assert.deepEqual([...ids2].sort(), [...ids1b].sort(), "late socket sees the same history");
|
||||
|
||||
// Thinking survives in the persisted assistant entry.
|
||||
const assistantEntry = c2.msgs
|
||||
.flatMap((m) => (m.type === "backfill" ? m.entries : []))
|
||||
.find((e) => e.type === "assistant" && (e as { thinking?: string }).thinking);
|
||||
assert.ok(assistantEntry, "persisted assistant entry carries thinking");
|
||||
assert.match((assistantEntry as { thinking: string }).thinking, /probe the learner/);
|
||||
|
||||
// ── follow-up prompt: both sockets get it exactly once ────────────
|
||||
c1.ws.send(JSON.stringify({ type: "prompt", text: "thanks, that's enough for now" }));
|
||||
await waitMsg(
|
||||
c1,
|
||||
(m) => m.type === "entry" && m.entry.type === "user" && m.entry.text.includes("enough for now"),
|
||||
"follow-up user entry on socket 1",
|
||||
);
|
||||
await waitMsg(
|
||||
c2,
|
||||
(m) => m.type === "entry" && m.entry.type === "user" && m.entry.text.includes("enough for now"),
|
||||
"follow-up user entry on socket 2",
|
||||
);
|
||||
await waitMsg(c1, (m) => m.type === "streaming" && m.value === false, "settle after follow-up");
|
||||
await waitSilence(c1);
|
||||
await waitSilence(c2);
|
||||
|
||||
for (const [name, c] of [
|
||||
["socket1", c1],
|
||||
["socket2", c2],
|
||||
] as const) {
|
||||
const ids = seenEntryIds(c.msgs);
|
||||
assert.equal(ids.length, new Set(ids).size, `no duplicate entry ids on ${name}`);
|
||||
const texts = seenUserTexts(c.msgs);
|
||||
assert.equal(texts.filter((t) => t === TOPIC).length, 1, `${name}: topic once`);
|
||||
assert.equal(texts.filter((t) => t.includes("enough for now")).length, 1, `${name}: follow-up once`);
|
||||
}
|
||||
|
||||
// ── the pending-question guard ────────────────────────────────────
|
||||
// (No pending question now, so a prompt must NOT be rejected:)
|
||||
// verified implicitly above — the follow-up prompt went through.
|
||||
|
||||
c1.ws.close();
|
||||
c2.ws.close();
|
||||
});
|
||||
|
||||
it("rejects prompts while a question is pending", async () => {
|
||||
currentScript = [
|
||||
{ toolCalls: [{ name: "quiz", arguments: QUIZ_ARGS }] },
|
||||
{ text: "Take a shot." },
|
||||
];
|
||||
await createTestSession("Teach me something with a quiz.");
|
||||
const c = await openSocket();
|
||||
const pendingMsg = await waitMsg(
|
||||
c,
|
||||
(m) => m.type === "pending" && m.question !== null,
|
||||
"pending question",
|
||||
);
|
||||
const question = (pendingMsg as { question: { id: string } }).question;
|
||||
c.ws.send(JSON.stringify({ type: "prompt", text: "hello?" }));
|
||||
const err = await waitMsg(c, (m) => m.type === "error", "guard rejection");
|
||||
assert.match((err as { message: string }).message, /Answer the pending question/);
|
||||
|
||||
// Clean up: dismiss so subsequent state is clean.
|
||||
c.ws.send(JSON.stringify({ type: "dismiss", questionId: question.id }));
|
||||
await waitMsg(
|
||||
c,
|
||||
(m) => m.type === "entry" && m.entry.type === "questionAnswered",
|
||||
"dismissal recorded",
|
||||
);
|
||||
c.ws.close();
|
||||
});
|
||||
|
||||
it("lists sessions including the created ones", async () => {
|
||||
const res = await fetch(`http://127.0.0.1:${port}/api/sessions`);
|
||||
assert.ok(res.ok);
|
||||
const data = (await res.json()) as { sessions: Array<{ file: string; name?: string }> };
|
||||
assert.ok(data.sessions.length >= 2, "at least the two sessions from this test");
|
||||
assert.ok(
|
||||
data.sessions.some((s) => s.name === "Teach me the solar system."),
|
||||
"session named after its topic",
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user