diff --git a/.gitignore b/.gitignore index c15f3bb..c81c771 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,11 @@ xai-credentials.json* # dev-fresh launcher scratch (fresh-user sim): disposable per launch .fresh-data/ +# Playwright: synthetic runtime data and failure diagnostics (never fixtures) +.playwright-data/ +test-results/ +playwright-report/ + # ── GSD baseline (auto-generated) ── .gsd .gsd-worktrees/ diff --git a/e2e/fixtures.ts b/e2e/fixtures.ts new file mode 100644 index 0000000..ab5d937 --- /dev/null +++ b/e2e/fixtures.ts @@ -0,0 +1,129 @@ +import { rm } from "node:fs/promises"; +import path from "node:path"; +import { test as base, expect, type Route } from "@playwright/test"; +import type { ModelCatalogEntry, ModelCatalogResponse } from "../src/core/contracts/models"; +import type { UserSettings } from "../src/core/settings"; +import type { ChatGPTStatus } from "../src/core/contracts/chatgpt"; +import type { NativeSubscriptionStatus } from "../src/core/contracts/subscriptions"; +import type { ProviderKeyInfo } from "../components/settings/keys-section"; + +/** Setup/teardown owns only the UUID run directory beneath this worktree. */ +export default function setup() { + const root = process.env.LH_E2E_RUN_DIR; + if (!root || path.dirname(root) !== path.resolve(".playwright-data") || !/^run-[a-f0-9-]{36}$/.test(path.basename(root))) { + throw new Error("Unsafe/missing Playwright data directory; refusing to run or clean up."); + } + return async () => { await rm(root, { recursive: true, force: true }); }; +} + +export const syntheticModel: ModelCatalogEntry = { + id: "synthetic-smoke-model", name: "Synthetic smoke model", provider: "openai", + reasoning: false, input: ["text", "image"], contextWindow: 8192, maxTokens: 1024, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, supportedLevels: ["off"], +}; +const provider: ProviderKeyInfo = { + id: "openai", envVar: "OPENAI_API_KEY", label: "OpenAI", +}; +export interface HarnessState { + keyPresent: boolean; + settings: UserSettings | null; + writes: { path: string; body: unknown }[]; + blocked: string[]; + generation: string[]; + catalogFailure: boolean; +} +const json = (route: Route, body: unknown, status = 200) => route.fulfill({ status, json: body }); + +export const test = base.extend<{ harness: HarnessState }>({ + harness: async ({ context }, use) => { + const state: HarnessState = { + keyPresent: false, settings: null, writes: [], blocked: [], generation: [], catalogFailure: false, + }; + await context.routeWebSocket("**/*", (socket) => { + const url = new URL(socket.url()); + if (url.origin === "ws://127.0.0.1:3005" && url.pathname.startsWith("/_next/")) { + socket.connectToServer(); // Next dev HMR only; no provider sockets. + return; + } + state.blocked.push(`WS ${url.origin}${url.pathname}`); + socket.close({ code: 1008, reason: "Offline test boundary" }); + }); + // Installed on the CONTEXT before a page navigates, including popups. + // Never let a missed provider fixture fall through to a live endpoint. + await context.route("**/*", async (route) => { + const request = route.request(); + const url = new URL(request.url()); + const method = request.method(); + const p = url.pathname; + const block = async () => { + state.blocked.push(`${method} ${url.origin}${p}`); // no query strings/payloads in diagnostics + await route.abort("blockedbyclient"); + }; + if (url.origin !== "http://127.0.0.1:3005") return block(); + if (!p.startsWith("/api/")) { + if (method === "GET" || method === "HEAD") return route.continue(); + return block(); + } + if (/\/(turn|generate|transcribe)(\/|$)/.test(p) || (p.startsWith("/api/chats") && method !== "GET")) { + state.generation.push(`${method} ${p}`); + return block(); + } + if (p === "/api/keys" && method === "GET") return json(route, { + keys: state.keyPresent ? [{ id: provider.envVar, info: null, present: true, source: "file", masked: "synthetic…only (41 chars)", shadowedByEnv: false }] : [], + providers: [provider], + }); + if (p === "/api/keys/OPENAI_API_KEY" && method === "PUT") { + const body: unknown = request.postDataJSON(); + state.writes.push({ path: p, body }); + if (JSON.stringify(body) !== JSON.stringify({ value: "synthetic-smoke-key-not-a-credential" })) return json(route, { error: "Only the synthetic fixture key is accepted" }, 400); + state.keyPresent = true; + return json(route, { status: { id: provider.envVar, info: null, present: true, source: "file", masked: "synthetic…only (41 chars)", shadowedByEnv: false } }); + } + if (p === "/api/models" && method === "GET") { + if (state.catalogFailure) return json(route, { error: "Synthetic catalog unavailable" }, 503); + const slots = { + strong: state.settings?.models?.strong?.model ? { ...state.settings.models.strong, model: state.settings.models.strong.model } : null, + cheap: state.settings?.models?.cheap?.model ? { ...state.settings.models.cheap, model: state.settings.models.cheap.model } : null, + vision: state.settings?.models?.vision?.model ? { ...state.settings.models.vision, model: state.settings.models.vision.model } : null, + }; + const catalog: ModelCatalogResponse = { + models: state.keyPresent ? [syntheticModel] : [], defaults: slots.strong, slots, + rolePins: { researcher: slots.cheap, demoBuilder: slots.cheap, visualMaker: slots.vision, extractor: slots.cheap, auditor: slots.cheap }, + providers: [{ id: "openai", envVar: provider.envVar, keyPresent: state.keyPresent }], + }; + return json(route, catalog); + } + if (p === "/api/settings" && method === "GET") return json(route, { settings: state.settings }); + if (p === "/api/settings/models" && method === "PUT") { + const body = request.postDataJSON() as NonNullable; + state.writes.push({ path: p, body }); + state.settings = { ...state.settings, version: 1, models: body }; + return json(route, { settings: state.settings }); + } + if (p === "/api/subscriptions" && method === "GET") { + const chatgpt: ChatGPTStatus = { available: false, pending: false, allowBackgroundUse: false, activeId: null, accounts: [], error: null }; + const native: NativeSubscriptionStatus[] = ["github-copilot", "xai"].map((providerId) => ({ + providerId: providerId as NativeSubscriptionStatus["providerId"], available: false, connected: false, + pending: false, allowBackgroundUse: false, connectionId: null, epoch: 0, authorityRevision: 0, revision: 0, + })); + return json(route, { chatgpt, native }); + } + if (p === "/api/custom-provider" && method === "GET") return json(route, { + baseUrl: "", authMode: "keyless", models: [], overrides: {}, manualModels: [], + keyPresent: false, keySource: "none", keyReuseRequired: false, + }); + if (p === "/api/stt/token" && method === "GET") return json(route, { message: "No DEEPGRAM_API_KEY — synthetic offline fixture." }, 503); + // Real, read-only application contracts against empty fixture-owned data. + if (method === "GET" && ["/api/activity", "/api/chats", "/api/events", "/api/graph", "/api/graph/status", "/api/graph/costs", "/api/graph/render-settings"].includes(p)) return route.continue(); + return block(); + }); + await use(state); + expect(state.blocked, "Unexpected network request (blocked before dispatch)").toEqual([]); + expect(state.generation, "Setup/settings must not generate or send").toEqual([]); + }, + page: async ({ page, harness }, use) => { + void harness; // force interception before the test receives its page + await use(page); + }, +}); +export { expect }; diff --git a/e2e/smoke.spec.ts b/e2e/smoke.spec.ts new file mode 100644 index 0000000..1fc12c4 --- /dev/null +++ b/e2e/smoke.spec.ts @@ -0,0 +1,74 @@ +import { test, expect, syntheticModel } from "./fixtures"; + +test("first run configures all model slots and opens settings without sending the draft", async ({ page, harness }) => { + await page.goto("/"); + const draft = page.getByRole("textbox", { name: "What do you want to learn?", exact: true }); + await expect(page.getByTestId("setup-cta-keys")).toBeVisible(); + await draft.fill("Synthetic learning topic — do not send"); + await page.getByRole("button", { name: "Start session", exact: true }).click(); + await expect(page.getByTestId("setup-stage-keys")).toBeVisible(); + await expect(page.getByTestId("setup-stage-pickers")).toHaveCount(0); + await page.getByTestId("setup-key-provider").click(); + await page.getByTestId("provider-option-openai").click(); + await page.getByTestId("setup-key-input").fill("synthetic-smoke-key-not-a-credential"); + await page.getByTestId("setup-key-save").click(); + await expect(page.getByTestId("setup-stage-pickers")).toBeVisible(); + await expect(page.getByTestId("setup-save")).toBeDisabled(); + for (const slot of ["strong", "cheap", "vision"]) { + await page.getByTestId(`setup-trigger-${slot}`).click(); + await page.getByTestId(`model-option-${syntheticModel.id}`).click(); + } + await expect(page.getByTestId("setup-save")).toBeEnabled(); + await page.getByTestId("setup-save").click(); + await expect(page.getByTestId("setup-dialog")).toHaveCount(0); + await expect(page.getByTestId("setup-cta")).toHaveCount(0); + await expect(draft).toHaveText("Synthetic learning topic — do not send"); + const pin = { model: { provider: "openai", id: syntheticModel.id }, thinkingLevel: "off" }; + expect(harness.writes).toEqual([ + { path: "/api/keys/OPENAI_API_KEY", body: { value: "synthetic-smoke-key-not-a-credential" } }, + { path: "/api/settings/models", body: { strong: pin, cheap: pin, vision: pin } }, + ]); + await page.getByRole("button", { name: "Settings", exact: true }).click(); + await expect(page.getByTestId("settings-dialog")).toBeVisible(); + await expect(page.getByTestId("settings-nav-keys")).toBeVisible(); + await page.getByTestId("settings-close").click(); + await expect(draft).toHaveText("Synthetic learning topic — do not send"); + expect(harness.generation).toEqual([]); + await expect(page).toHaveURL("http://127.0.0.1:3005/"); +}); + +test("catalog failure is visible and cannot save incomplete slots", async ({ page, harness }) => { + harness.catalogFailure = true; + await page.goto("/"); + await page.getByTestId("setup-cta").click(); + await expect(page.getByTestId("setup-catalog-error")).toContainText("Model catalog could not be loaded. Check the connection and retry."); + await expect(page.getByTestId("setup-stage-keys")).toHaveCount(0); + await expect(page.getByTestId("setup-save")).toHaveCount(0); + harness.catalogFailure = false; + await page.getByTestId("setup-retry").click(); + await expect(page.getByTestId("setup-stage-keys")).toBeVisible(); + await expect(page.getByTestId("setup-catalog-error")).toHaveCount(0); + expect(harness.writes).toEqual([]); +}); + +test("network boundary rejects external requests and unknown API writes", async ({ page, harness }) => { + await page.goto("/"); + await page.evaluate(async () => { + await Promise.all([ + fetch("https://example.invalid/must-not-leave-host").catch(() => undefined), + fetch("/api/unexpected-write", { method: "POST", body: "synthetic" }).catch(() => undefined), + new Promise((resolve) => { + const socket = new WebSocket("wss://example.invalid/must-not-connect"); + socket.onclose = () => resolve(); + socket.onerror = () => resolve(); + }), + ]); + }); + expect(harness.blocked.sort()).toEqual([ + "GET https://example.invalid/must-not-leave-host", + "POST http://127.0.0.1:3005/api/unexpected-write", + "WS wss://example.invalid/must-not-connect", + ]); + // Only these deliberately asserted denials are consumed; every other denial fails teardown. + harness.blocked.length = 0; +}); diff --git a/package-lock.json b/package-lock.json index 0e46b7b..12c2038 100644 --- a/package-lock.json +++ b/package-lock.json @@ -63,6 +63,7 @@ }, "devDependencies": { "@next/eslint-plugin-next": "^16.4.3", + "@playwright/test": "^1.64.1", "@tailwindcss/postcss": "^4.3.3", "@types/node": "^24.0.0", "@types/react": "^19.2.18", @@ -1823,9 +1824,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1842,9 +1840,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1861,9 +1856,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1880,9 +1872,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1899,9 +1888,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1918,9 +1904,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1937,9 +1920,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1956,9 +1936,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "LGPL-3.0-or-later", "optional": true, "os": [ @@ -1975,9 +1952,6 @@ "cpu": [ "arm" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2000,9 +1974,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2025,9 +1996,6 @@ "cpu": [ "ppc64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2050,9 +2018,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2075,9 +2040,6 @@ "cpu": [ "s390x" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2100,9 +2062,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2125,9 +2084,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2150,9 +2106,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "Apache-2.0", "optional": true, "os": [ @@ -2581,9 +2534,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2604,9 +2554,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2627,9 +2574,6 @@ "cpu": [ "riscv64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2650,9 +2594,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2673,9 +2614,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2807,9 +2745,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2826,9 +2761,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -2845,9 +2777,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -2864,9 +2793,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -3014,6 +2940,21 @@ "node": ">=14.18.0" } }, + "node_modules/@playwright/test": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.64.1.tgz", + "integrity": "sha512-IaYGml/nSOiZWROfPtuIL7neuu9qDOZ3TNvNljUIrhDMxiGTgPjzj/g9p9RxpS2tSepPDRY+UwAaEeiKtUvIAg==", + "devOptional": true, + "dependencies": { + "playwright": "1.64.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -4705,9 +4646,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4724,9 +4662,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4743,9 +4678,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -4762,9 +4694,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -6198,9 +6127,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6215,9 +6141,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6232,9 +6155,6 @@ "loong64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6249,9 +6169,6 @@ "loong64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6266,9 +6183,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6283,9 +6197,6 @@ "riscv64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6300,9 +6211,6 @@ "riscv64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -6317,9 +6225,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6334,9 +6239,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -6351,9 +6253,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -15022,6 +14921,33 @@ "node": ">=16.0.0" } }, + "node_modules/playwright": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.64.1.tgz", + "integrity": "sha512-HCrVhHr5QQuwYoswEUGTb7gGp522yflbv/ST8nF4sFWGuLjQCt17Q/EubpGGSNKYn1tsTCPZxqHhHCfVURu7UA==", + "devOptional": true, + "dependencies": { + "playwright-core": "1.64.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.64.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.64.1.tgz", + "integrity": "sha512-BC4s8y6SEVA6Q1rSxZTto8hI1KUI3xXlHGZgvGqP4IqPdI/a6FX9H29IDNz7vKo9l21b6LfYosScPL6M3xxyDQ==", + "devOptional": true, + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/plist": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/plist/-/plist-3.1.0.tgz", diff --git a/package.json b/package.json index 652ccfa..a419130 100644 --- a/package.json +++ b/package.json @@ -56,8 +56,8 @@ "mdast-util-from-markdown": "2.0.3", "mdast-util-math": "3.0.0", "media-chrome": "^4.19.2", - "micromark-extension-math": "3.1.0", "mermaid": "^11.17.0", + "micromark-extension-math": "3.1.0", "motion": "^13.1.1", "nanoid": "^6.0.1", "next": "^16.4.3", @@ -82,6 +82,7 @@ }, "devDependencies": { "@next/eslint-plugin-next": "^16.4.3", + "@playwright/test": "^1.64.1", "@tailwindcss/postcss": "^4.3.3", "@types/node": "^24.0.0", "@types/react": "^19.2.18", diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000..1333fa2 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,45 @@ +import { randomUUID } from "node:crypto"; +import path from "node:path"; +import { defineConfig, devices } from "@playwright/test"; + +// The worker inherits this exact run path; never touch the default auth/data. +const runDir = process.env.LH_E2E_RUN_DIR ?? path.resolve(".playwright-data", `run-${randomUUID()}`); +if (path.dirname(runDir) !== path.resolve(".playwright-data") || !/^run-[a-f0-9-]{36}$/.test(path.basename(runDir))) { + throw new Error("Unsafe Playwright data directory; expected a fixture-owned UUID path under .playwright-data."); +} +process.env.LH_E2E_RUN_DIR = runDir; + +export default defineConfig({ + testDir: "./e2e", + fullyParallel: false, + workers: 1, + retries: 0, + timeout: 60_000, + expect: { timeout: 15_000 }, + globalSetup: "./e2e/fixtures.ts", + outputDir: "test-results", + reporter: [["list"], ["html", { open: "never" }]], + use: { + baseURL: "http://127.0.0.1:3005", + trace: "retain-on-failure", + screenshot: "only-on-failure", + serviceWorkers: "block", + locale: "en-US", + launchOptions: { chromiumSandbox: true }, + }, + projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }], + webServer: { + command: "npm run dev -- --hostname 127.0.0.1 --port 3005", + url: "http://127.0.0.1:3005", + reuseExistingServer: false, + timeout: 120_000, + stdout: "pipe", + stderr: "pipe", + env: { + LH_IGNORE_ENV_KEYS: "1", + LH_DATA_DIR: path.join(runDir, "data"), + LH_AUTH_FILE: path.join(runDir, "auth.json"), + NEXT_TELEMETRY_DISABLED: "1", + }, + }, +}); diff --git a/tests/svg-image.test.tsx b/tests/svg-image.test.tsx index e15fbf4..e4fa3df 100644 --- a/tests/svg-image.test.tsx +++ b/tests/svg-image.test.tsx @@ -179,7 +179,9 @@ it("the fullscreen button opens the same SVG image and inversion; download keeps // Intercept only the download anchor; never navigate or write a real file. dom.document.createElement = ((...args: Parameters) => { const element = originalCreateElement.apply(dom.document, args); - if (args[0] === "a") { + // Playwright imports Electron's final createElement('webview') overload; + // Parameters<> sees that overload, not the runtime string tag. + if ((args[0] as string) === "a") { element.click = () => { clicked = { name: element.getAttribute("download")!, href: element.getAttribute("href")! }; }; } return element;