diff --git a/e2e/application-flows.spec.ts b/e2e/application-flows.spec.ts new file mode 100644 index 0000000..28290d1 --- /dev/null +++ b/e2e/application-flows.spec.ts @@ -0,0 +1,214 @@ +import type { Locator, Page } from "@playwright/test"; +import { test, expect, syntheticModel, syntheticUserMessage, type HarnessState, type ApplicationFixture } from "./fixtures"; +import type { ChatRecord } from "../src/core/contracts/chat"; +import type { GraphNode } from "../src/core/contracts/graph"; +import type { DemoResult, JobRecord } from "../src/core/contracts/jobs"; +import type { LhMessage } from "../src/core/native-messages"; + +// Inline, synthetic native records; never copy golden-data's personal content. +const id = "chat_e2e_application"; +const title = "Synthetic arithmetic lesson"; +const timestamp = 1_800_000_000_000; +function assistant(messageId: string, content: unknown[], stopReason = "stop"): LhMessage { + return { id: messageId, role: "assistant", provider: "openai", api: "openai-responses", model: syntheticModel.id, + content, stopReason, timestamp, + usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + } as LhMessage; +} +function record(messages: LhMessage[]): ChatRecord { + return { version: 1, id, title, createdAt: new Date(timestamp).toISOString(), updatedAt: new Date(timestamp).toISOString(), messages, turns: [] }; +} +function configure(harness: HarnessState, language: "en" | "ru" = "en"): ApplicationFixture { + const slot = { model: { provider: "openai", id: syntheticModel.id }, thinking: "off" as const }; + harness.keyPresent = true; + harness.settings = { version: 1, general: { language }, graph: { rendering: false }, models: { strong: slot, cheap: slot, vision: slot } }; + const node: GraphNode = { id: "n_synthetic_sum", name: "Addition", statement: "Two plus two equals four.", aliases: [], + provenance: [{ chatId: id, messageId: "lesson-opening" }], createdAt: new Date(timestamp).toISOString() }; + const app: ApplicationFixture = { chats: [], jobs: [], inputs: [], reads: [], reply: null, failures: [], + graph: { nodes: [node], propositions: [node], edges: [], logicalEdges: [], impLinks: [], chains: [], + sessions: { [id]: title }, fileSources: {}, nodeSessions: { [node.id]: [id] }, clusters: [] } }; + harness.application = app; + return app; +} +async function openSession(page: Page) { + const open = page.getByRole("button", { name: "Open sidebar", exact: true }); + if (await open.isVisible()) await open.click(); + await page.getByTestId("session-row").filter({ hasText: title }).click(); + await expect(page).toHaveURL(`/chat/${id}`); +} +async function withinViewport(control: Locator, page: Page) { + await expect(control).toBeVisible(); + // Sidebar/dock have CSS transitions: assert their settled layout, not a frame mid-animation. + await expect.poll(async () => { + const box = await control.boundingBox(); + const viewport = page.viewportSize()!; + return !!box && box.x >= 0 && box.y >= 0 && box.x + box.width <= viewport.width + 1 && box.y + box.height <= viewport.height + 1; + }, { message: "Control must fit inside the settled viewport" }).toBe(true); + const box = await control.boundingBox(); + expect(box).not.toBeNull(); + const viewport = page.viewportSize()!; + expect(box!.x).toBeGreaterThanOrEqual(0); + expect(box!.y).toBeGreaterThanOrEqual(0); + expect(box!.x + box!.width).toBeLessThanOrEqual(viewport.width + 1); + expect(box!.y + box!.height).toBeLessThanOrEqual(viewport.height + 1); + // Geometry alone misses occlusion/overflow clipping; hit-test its center too. + expect(await control.evaluate((el) => { + const r = el.getBoundingClientRect(); + const hit = document.elementFromPoint(r.x + r.width / 2, r.y + r.height / 2); + return hit === el || (hit !== null && el.contains(hit)); + })).toBe(true); +} + +function demoJob(): JobRecord { + const result: DemoResult = { title: "Synthetic addition demo", tested: true, + html: "" }; + return { id: "job_synthetic_demo", kind: "demo", chatId: id, toolCallId: "demo-call", status: "done", activity: "", createdAt: new Date(timestamp).toISOString(), updatedAt: new Date(timestamp).toISOString(), result }; +} +// buildSubagentSteer's native marker shape, with synthetic demo failure payloads. +function syntheticSteer(job: JobRecord): LhMessage { + const result: DemoResult = job.status === "error" + ? { title: "Synthetic failed demo", tested: true, html: "", notes: job.error } + : job.result as DemoResult; + const content = job.status === "error" + ? `The demo "${result.title}" could not be built: ${result.notes ?? "unknown reason"}. Fall back to explaining in prose; if the brief was the problem, re-call demo with a tighter one.` + : `The demo "${result.title}" is built, headlessly tested, and rendered in the lesson. Builder notes: ${result.notes ?? "(none)"}.\nTeach with it — reference its controls by name and point the learner at what to try.`; + return { id: `steer_${job.id}`, role: "custom", customType: "subagent-steer", display: true, timestamp, content, + details: { jobId: job.id, kind: job.kind, toolCallId: job.toolCallId, result } }; +} + +test("entrypoint, learning chains, chat, sandboxed artifact and settings compose without duplicate sends", async ({ page, harness }) => { + const app = configure(harness); + const job = demoJob(); + app.jobs.push(job); + const chat = record([ + { ...syntheticUserMessage("Teach synthetic addition"), id: "learner-opening" }, + assistant("lesson-opening", [{ type: "text", text: "Synthetic addition lesson: two plus two equals four." }, + { type: "toolCall", id: job.toolCallId, name: "demo", arguments: { description: "Show addition" } }], "toolUse"), + { id: "demo-ack", role: "toolResult", toolCallId: job.toolCallId, toolName: "demo", content: [{ type: "text", text: "Dispatched synthetic demo" }], isError: false, timestamp }, + { ...syntheticSteer(job), id: "demo-steer" }, + ]); + await harness.seedChat(chat); + await page.goto("/"); + await page.getByRole("button", { name: "Chains", exact: true }).click(); + await page.getByTestId(`cluster-toggle-${id}`).click(); + await page.getByRole("button", { name: "Expand chain Addition", exact: true }).click(); + await expect(page.getByText("Two plus two equals four.", { exact: true })).toBeVisible(); + await page.getByTestId("chain-table-search").fill("nothing-matches-this-synthetic-query"); + await expect(page.getByTestId("chain-table-nomatch")).toBeVisible(); + await page.getByTestId("chain-table-search").fill(""); + await expect(page.getByTestId(`cluster-group-${id}`)).toBeVisible(); + await page.getByTestId("chain-table-close").click(); + await openSession(page); + await expect(page.getByText("Synthetic addition lesson: two plus two equals four.", { exact: true })).toHaveCount(1); + const frame = page.frameLocator('iframe[title="Synthetic addition demo"]'); + await frame.getByRole("button", { name: "Reveal sum" }).click(); + await expect(frame.getByRole("button", { name: "Four" })).toBeVisible(); + await expect(page.locator('iframe[title="Synthetic addition demo"]')).toHaveAttribute("sandbox", "allow-scripts"); + const inverted = page.waitForResponse((response) => response.url().endsWith(`/api/chats/${id}/artifact-ui`) && response.request().method() === "POST"); + await page.getByRole("switch", { name: "Invert demo brightness", exact: true }).click(); + expect((await inverted).ok()).toBe(true); + await expect.poll(() => harness.writes.filter((w) => w.path.endsWith("/artifact-ui"))).toEqual([ + { path: `/api/chats/${id}/artifact-ui`, body: { input: { messageId: "demo-steer", inverted: true } } }, + ]); + await page.getByRole("button", { name: "Settings", exact: true }).click(); + await page.getByTestId("settings-nav-general").click(); + await expect(page.getByTestId("language-select-language")).toBeVisible(); + await page.getByTestId("settings-close").click(); + await expect(page).toHaveURL(`/chat/${id}`); + await expect(frame.getByRole("button", { name: "Four" })).toBeVisible(); + await page.getByRole("link", { name: "Home", exact: true }).click(); + await expect(page).toHaveURL("/"); + await openSession(page); + await expect(page.getByRole("switch", { name: "Invert demo brightness", exact: true })).toHaveAttribute("aria-checked", "true"); + app.reply = assistant("generated-reply", [{ type: "text", text: "Synthetic streamed explanation received." }]); + await page.getByRole("textbox", { name: "Ask, answer, or steer the lesson…", exact: true }).fill("Explain the synthetic sum"); + await page.getByRole("textbox", { name: "Ask, answer, or steer the lesson…", exact: true }).press("Enter"); + await expect(page.getByText("Synthetic streamed explanation received.", { exact: true })).toBeVisible(); + await expect.poll(() => app.inputs).toEqual([{ chatId: id, input: { type: "text", text: "Explain the synthetic sum" } }]); + expect(app.reads.filter((p) => p.endsWith("/stream"))).toEqual([`/api/chats/${id}/stream`]); + await page.reload(); + await expect(page.getByText("Synthetic streamed explanation received.", { exact: true })).toHaveCount(1); + expect(app.inputs).toHaveLength(1); +}); + +for (const width of [1280, 390]) { + test(`quiz controls remain reachable and submit exactly once at ${width}px`, async ({ page, harness }) => { + await page.setViewportSize({ width, height: 900 }); + const app = configure(harness); + const question = { type: "predict", nodeId: "synthetic-addition", question: "Which expression equals four?", options: [ + { label: "Two plus two", value: "sum" }, { label: "Two minus two", value: "difference" }, + ], correctAnswer: "sum", explanation: "Adding two pairs gives four." }; + await harness.seedChat(record([assistant("quiz-message", [{ type: "toolCall", id: "quiz-call", name: "quiz", arguments: question }], "toolUse")])); + app.reply = assistant("quiz-feedback", [{ type: "text", text: "Synthetic quiz feedback received." }]); + await page.goto(`/chat/${id}`); + // Server-rendered controls precede hydration; jobs is a client mount effect. + await expect.poll(() => app.reads.includes("/api/jobs")).toBe(true); + const close = page.getByRole("button", { name: "Close sidebar", exact: true }); + if (await close.isVisible()) await close.click(); + await expect(page.getByTestId("question-dock")).toBeVisible(); + await expect(page.getByText(question.question, { exact: true })).toBeVisible(); + const answer = page.getByRole("radio", { name: "Two plus two", exact: true }); + await withinViewport(answer, page); + await answer.click(); + const submit = page.getByRole("button", { name: "Submit answer", exact: true }); + await withinViewport(submit, page); + await expect(submit).toBeEnabled(); + const controls = page.getByTestId("question-dock").locator('button, textarea, [role="radio"], [role="slider"]'); + for (const control of await controls.all()) { + if (await control.isVisible() && await control.isEnabled()) await withinViewport(control, page); + } + await submit.click(); + await expect(page.getByText("Synthetic quiz feedback received.", { exact: true })).toBeVisible(); + await expect.poll(() => app.inputs.length).toBe(1); + expect(app.inputs[0]!.chatId).toBe(id); + expect(app.inputs[0]!.input).toMatchObject({ type: "answer", toolCallId: "quiz-call", answer: { answers: [{ value: "sum", label: "Two plus two" }], correct: true, confidence: 50 } }); + await expect(page.getByTestId("question-dock")).toHaveCount(0); + expect(app.reads.filter((p) => p.endsWith("/stream"))).toHaveLength(1); + await page.reload(); + await expect(page.getByText("Synthetic quiz feedback received.", { exact: true })).toHaveCount(1); + expect(app.inputs).toHaveLength(1); + }); +} + +test("failed response displays Retry and visibly recovers through the real acknowledged stream", async ({ page, harness }) => { + const app = configure(harness); + const failed = { ...assistant("failed-response", [], "error"), errorMessage: "Synthetic upstream response failed" } as LhMessage; + await harness.seedChat(record([syntheticUserMessage("Explain synthetic addition"), failed])); + app.reply = assistant("retry-success", [{ type: "text", text: "Synthetic retry recovered." }]); + app.failures.push({ status: 500, body: { error: "Synthetic retry still unavailable" } }); + await page.goto(`/chat/${id}`); + await expect(page.getByText("Something went wrong: Synthetic upstream response failed", { exact: true })).toBeVisible(); + await page.getByRole("button", { name: "Retry", exact: true }).click(); + // Non-2xx opening acknowledgements surface their raw response text. + await expect(page.getByText('Something went wrong: {"error":"Synthetic retry still unavailable"}', { exact: true })).toBeVisible(); + await page.getByRole("button", { name: "Retry", exact: true }).click(); + await expect(page.getByText("Synthetic retry recovered.", { exact: true })).toBeVisible(); + await expect(page.getByTestId("retry-failed-turn")).toHaveCount(0); + await expect(page.getByText(/Something went wrong:/)).toHaveCount(0); + expect(app.inputs).toEqual([{ chatId: id, input: { type: "retry" } }, { chatId: id, input: { type: "retry" } }]); + expect(app.reads.filter((p) => p.endsWith("/stream"))).toHaveLength(1); +}); + +for (const language of ["en", "ru"] as const) { + test(`settled job ${language} keeps localized completion, error and refusal registers`, async ({ page, harness }) => { + const app = configure(harness, language); + const done = demoJob(); + const error: JobRecord = { ...done, id: "job_synthetic_error", toolCallId: "error-call", status: "error", result: undefined, error: "Synthetic demo build failed" }; + const refused: JobRecord = { ...error, id: "job_synthetic_refused", toolCallId: "refused-call", error: "No model configured — set up your models first" }; + app.jobs.push(done, error, refused); + const messages: LhMessage[] = [assistant("dispatches", [done, error, refused].map((job) => ({ type: "toolCall", id: job.toolCallId, name: "demo", arguments: { description: "Synthetic job" } })), "toolUse")]; + for (const job of app.jobs) messages.push({ id: `${job.id}-ack`, role: "toolResult", toolCallId: job.toolCallId, toolName: "demo", content: [{ type: "text", text: "Dispatched synthetic job" }], isError: false, timestamp }); + for (const job of app.jobs) messages.push(syntheticSteer(job)); + await harness.seedChat(record(messages)); + await page.goto(`/chat/${id}`); + await expect(page.getByText(language === "ru" ? "завершено" : "finished", { exact: true })).toHaveCount(1); + // The error and refusal result markers each have a failed artifact card. + await expect(page.getByTestId("demo-card-failed")).toHaveCount(2); + await expect(page.getByTestId("demo-card-failed").getByText(language === "ru" ? "ошибка" : "failed", { exact: true })).toHaveCount(2); + await expect(page.getByRole("button", { name: language === "ru" ? "Настроить модели" : "Set up models", exact: true })).toBeVisible(); + await expect(page.getByText("Synthetic demo build failed", { exact: true })).toBeVisible(); + // Stored builder notes stay verbatim; the dispatch refusal CTA is localized. + await expect(page.getByText("No model configured — set up your models first", { exact: true })).toBeVisible(); + expect(app.inputs).toEqual([]); + }); +} diff --git a/e2e/fixtures.ts b/e2e/fixtures.ts index 6eaea93..885abde 100644 --- a/e2e/fixtures.ts +++ b/e2e/fixtures.ts @@ -1,4 +1,4 @@ -import { mkdir, rm, writeFile } from "node:fs/promises"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import { test as base, expect, type Route } from "@playwright/test"; import type { ModelCatalogEntry, ModelCatalogResponse } from "../src/core/contracts/models"; @@ -6,6 +6,12 @@ import type { UserSettings } from "../src/core/settings"; import type { ChatGPTStatus } from "../src/core/contracts/chatgpt"; import type { NativeSubscriptionStatus } from "../src/core/contracts/subscriptions"; import type { ProviderKeyInfo } from "../components/settings/keys-section"; +import type { ChatRecord } from "../src/core/contracts/chat"; +import type { GraphSnapshot } from "../src/core/contracts/graph"; +import type { JobRecord } from "../src/core/contracts/jobs"; +import type { LhMessage } from "../src/core/native-messages"; +import { compactAnswerForModel } from "../src/core/question-core"; +import type { ChatInput } from "../src/core/input-events"; /** Setup/teardown owns only the UUID run directory beneath this worktree. */ export default function setup() { @@ -24,6 +30,15 @@ export const syntheticModel: ModelCatalogEntry = { const provider: ProviderKeyInfo = { id: "openai", envVar: "OPENAI_API_KEY", label: "OpenAI", }; +export interface ApplicationFixture { + chats: ChatRecord[]; + jobs: JobRecord[]; + graph: GraphSnapshot; + inputs: Array<{ chatId: string; input: ChatInput }>; + reads: string[]; + reply: LhMessage | null; + failures: Array<{ status: number; body: unknown }>; +} export interface HarnessState { keyPresent: boolean; settings: UserSettings | null; @@ -35,8 +50,20 @@ export interface HarnessState { modelsWrite: { status: number; wait?: Promise }; transcription: { configured: boolean; status: number; body: { text?: string; message?: string } }; seedGeneral: (general: NonNullable) => Promise; + application: ApplicationFixture | null; + seedChat: (chat: ChatRecord) => Promise; } const json = (route: Route, body: unknown, status = 200) => route.fulfill({ status, json: body }); +// Native shapes from native-messages.ts; avoid its transitive locale JSON imports +// in Playwright's Node loader. These messages never leave the offline boundary. +export function syntheticUserMessage(text: string): LhMessage { + return { id: `msg_${crypto.randomUUID()}`, role: "user", content: text, timestamp: Date.now() }; +} +function hasPendingQuiz(chat: ChatRecord): boolean { + const newest = chat.messages.findLast((message) => message.role === "assistant"); + return newest?.role === "assistant" && newest.content.some((block) => block.type === "toolCall" && block.name === "quiz" + && !chat.messages.some((message) => message.role === "toolResult" && message.toolCallId === block.id)); +} export const test = base.extend<{ harness: HarnessState }>({ harness: async ({ context }, use) => { @@ -45,8 +72,25 @@ export const test = base.extend<{ harness: HarnessState }>({ setup(); // validate run directory before any filesystem access const settingsFile = path.join(process.env.LH_E2E_RUN_DIR!, "data", "settings.json"); await mkdir(path.dirname(settingsFile), { recursive: true }); + const ownedChats = new Set(); + const streams = new Map(); + const persistChat = async (chat: ChatRecord) => { + if (!/^chat_[a-z0-9_]+$/.test(chat.id)) throw new Error("Invalid synthetic chat id"); + const file = path.join(path.dirname(settingsFile), "chats", `${chat.id}.json`); + await mkdir(path.dirname(file), { recursive: true }); + ownedChats.add(file); + await writeFile(file, JSON.stringify(chat), "utf8"); + }; const state: HarnessState = { keyPresent: false, settings: null, writes: [], blocked: [], generation: [], catalogFailure: false, + application: null, + seedChat: async (chat) => { + if (!state.application) throw new Error("Application fixture must be explicitly enabled"); + state.application.chats = [...state.application.chats.filter((c) => c.id !== chat.id), chat]; + await persistChat(chat); + // RSC pages read these synthetic records/settings through production stores. + await writeFile(settingsFile, JSON.stringify(state.settings), "utf8"); + }, modelsWrite: { status: 200 }, transcription: { configured: true, status: 200, body: { text: "Synthetic transcript" } }, seedGeneral: async (general) => { @@ -80,6 +124,61 @@ export const test = base.extend<{ harness: HarnessState }>({ if (method === "GET" || method === "HEAD") return route.continue(); return block(); } + const app = state.application; + if (app) { + if (method === "GET") app.reads.push(p); + if (p === "/api/graph" && method === "GET") return json(route, app.graph); + if (p === "/api/chats" && method === "GET") return json(route, { chats: app.chats.map((chat) => ({ + id: chat.id, title: chat.title, createdAt: chat.createdAt, updatedAt: chat.updatedAt, + kind: chat.kind ?? "teach", messageCount: chat.messages.length, phase: "probe", + pendingQuestion: hasPendingQuiz(chat), pendingProblem: false, busy: false, compacting: false, + })) }); + if (p === "/api/jobs" && method === "GET") return json(route, { jobs: app.jobs.filter((job) => !url.searchParams.has("chatId") || job.chatId === url.searchParams.get("chatId")), turnInFlight: false, compacting: false }); + const chatPath = p.match(/^\/api\/chats\/(chat_[a-z0-9_]+)(?:\/(stream|artifact-ui))?$/); + if (chatPath) { + const chat = app.chats.find((c) => c.id === chatPath[1]); + if (!chat) return json(route, { error: "Not found" }, 404); + if (!chatPath[2] && method === "GET") return json(route, { ...chat, turnInFlight: false, compacting: false }); + if (chatPath[2] === "artifact-ui" && method === "POST") { + const body = request.postDataJSON() as { input: { messageId: string; inverted: boolean } }; + state.writes.push({ path: p, body }); + // Exercise the real silent persistence route against our owned chat. + const response = await route.fetch(); + if (response.ok()) { + const saved = JSON.parse(await readFile(path.join(path.dirname(settingsFile), "chats", `${chat.id}.json`), "utf8")) as ChatRecord; + chat.messages = saved.messages; + } + return route.fulfill({ response }); + } + if (chatPath[2] === "stream" && method === "GET") { + const reply = streams.get(chat.id); + streams.delete(chat.id); + if (reply) { chat.messages.push(reply); await persistChat(chat); } + // Pi SSE carries no app record id; SessionModel assigns local ids. + const wire = reply && Object.fromEntries(Object.entries(reply).filter(([key]) => key !== "id")); + const events = reply ? [{ type: "agent_start" }, { type: "message_start", message: wire }, { type: "message_update", message: wire }, { type: "message_end", message: wire }, { type: "agent_end", messages: [wire] }] : [{ type: "turn-settled" }]; + return route.fulfill({ contentType: "text/event-stream", body: events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join("") + "data: [DONE]\n\n" }); + } + } + if (p === "/api/chat" && method === "POST" && app.reply) { + const body = request.postDataJSON() as { chatId: string; input: ChatInput }; + app.inputs.push(body); + const failure = app.failures.shift(); + if (failure) return json(route, failure.body, failure.status); + const chat = app.chats.find((c) => c.id === body.chatId); + if (!chat) throw new Error("Generation targeted an unseeded chat"); + if (body.input.type === "text") chat.messages.push(syntheticUserMessage(body.input.text)); + else if (body.input.type === "answer") chat.messages.push({ + id: `msg_${crypto.randomUUID()}`, role: "toolResult", toolCallId: body.input.toolCallId, toolName: "quiz", + content: [{ type: "text", text: JSON.stringify(compactAnswerForModel(body.input.answer)) }], + details: body.input.answer as unknown as Extract["details"], isError: false, timestamp: Date.now(), + }); + else if (body.input.type !== "retry") throw new Error(`Unscripted input: ${body.input.type}`); + await persistChat(chat); + streams.set(chat.id, app.reply); + return json(route, { ok: true }); + } + } if ((/\/(turn|generate|transcribe)(\/|$)/.test(p) && p !== "/api/stt/transcribe") || (p.startsWith("/api/chats") && method !== "GET")) { state.generation.push(`${method} ${p}`); return block(); @@ -148,7 +247,10 @@ export const test = base.extend<{ harness: HarnessState }>({ return block(); }); try { await use(state); } - finally { await rm(settingsFile, { force: true }); } + finally { + await rm(settingsFile, { force: true }); + for (const file of ownedChats) await rm(file, { force: true }); + } expect(state.blocked, "Unexpected network request (blocked before dispatch)").toEqual([]); expect(state.generation, "Setup/settings must not generate or send").toEqual([]); },