sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

test: Added sandboxed, isolated Chromium first-run coverage with stat...

Browse Source 
- package.json
- package-lock.json
- .gitignore
- playwright.config.ts
- e2e/fixtures.ts
- e2e/smoke.spec.ts
- tests/svg-image.test.tsx

GSD context:
- Milestone: M001 - Merge upstream v0.4.0 into the Gitea fork
- Slice: S02 - Verify combined application behavior
- Task: T01 - Added sandboxed, isolated Chromium first-run coverage with stateful synthetic API fixtures and fail-closed network boundaries.

GSD-Task: S02/T01
This commit is contained in:
sorokin.oleg.edu
2026-10-31 16:20:49 +03:00
parent 9727516926
commit ece228e8ed
7 changed files with 301 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files
e2e/fixtures.ts
+129
View File
@@ -0,0 +1,129 @@
import { rm } from "node:fs/promises";
import path from "node:path";
import { test as base, expect, type Route } from "@playwright/test";
import type { ModelCatalogEntry, ModelCatalogResponse } from "../src/core/contracts/models";
import type { UserSettings } from "../src/core/settings";
import type { ChatGPTStatus } from "../src/core/contracts/chatgpt";
import type { NativeSubscriptionStatus } from "../src/core/contracts/subscriptions";
import type { ProviderKeyInfo } from "../components/settings/keys-section";
/** Setup/teardown owns only the UUID run directory beneath this worktree. */
export default function setup() {
const root = process.env.LH_E2E_RUN_DIR;
if (!root || path.dirname(root) !== path.resolve(".playwright-data") || !/^run-[a-f0-9-]{36}$/.test(path.basename(root))) {
throw new Error("Unsafe/missing Playwright data directory; refusing to run or clean up.");
}
return async () => { await rm(root, { recursive: true, force: true }); };
}
export const syntheticModel: ModelCatalogEntry = {
id: "synthetic-smoke-model", name: "Synthetic smoke model", provider: "openai",
reasoning: false, input: ["text", "image"], contextWindow: 8192, maxTokens: 1024,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, supportedLevels: ["off"],
};
const provider: ProviderKeyInfo = {
id: "openai", envVar: "OPENAI_API_KEY", label: "OpenAI",
};
export interface HarnessState {
keyPresent: boolean;
settings: UserSettings | null;
writes: { path: string; body: unknown }[];
blocked: string[];
generation: string[];
catalogFailure: boolean;
}
const json = (route: Route, body: unknown, status = 200) => route.fulfill({ status, json: body });
export const test = base.extend<{ harness: HarnessState }>({
harness: async ({ context }, use) => {
const state: HarnessState = {
keyPresent: false, settings: null, writes: [], blocked: [], generation: [], catalogFailure: false,
};
await context.routeWebSocket("**/*", (socket) => {
const url = new URL(socket.url());
if (url.origin === "ws://127.0.0.1:3005" && url.pathname.startsWith("/_next/")) {
socket.connectToServer(); // Next dev HMR only; no provider sockets.
return;
}
state.blocked.push(`WS ${url.origin}${url.pathname}`);
socket.close({ code: 1008, reason: "Offline test boundary" });
});
// Installed on the CONTEXT before a page navigates, including popups.
// Never let a missed provider fixture fall through to a live endpoint.
await context.route("**/*", async (route) => {
const request = route.request();
const url = new URL(request.url());
const method = request.method();
const p = url.pathname;
const block = async () => {
state.blocked.push(`${method} ${url.origin}${p}`); // no query strings/payloads in diagnostics
await route.abort("blockedbyclient");
};
if (url.origin !== "http://127.0.0.1:3005") return block();
if (!p.startsWith("/api/")) {
if (method === "GET" || method === "HEAD") return route.continue();
return block();
}
if (/\/(turn|generate|transcribe)(\/|$)/.test(p) || (p.startsWith("/api/chats") && method !== "GET")) {
state.generation.push(`${method} ${p}`);
return block();
}
if (p === "/api/keys" && method === "GET") return json(route, {
keys: state.keyPresent ? [{ id: provider.envVar, info: null, present: true, source: "file", masked: "synthetic…only (41 chars)", shadowedByEnv: false }] : [],
providers: [provider],
});
if (p === "/api/keys/OPENAI_API_KEY" && method === "PUT") {
const body: unknown = request.postDataJSON();
state.writes.push({ path: p, body });
if (JSON.stringify(body) !== JSON.stringify({ value: "synthetic-smoke-key-not-a-credential" })) return json(route, { error: "Only the synthetic fixture key is accepted" }, 400);
state.keyPresent = true;
return json(route, { status: { id: provider.envVar, info: null, present: true, source: "file", masked: "synthetic…only (41 chars)", shadowedByEnv: false } });
}
if (p === "/api/models" && method === "GET") {
if (state.catalogFailure) return json(route, { error: "Synthetic catalog unavailable" }, 503);
const slots = {
strong: state.settings?.models?.strong?.model ? { ...state.settings.models.strong, model: state.settings.models.strong.model } : null,
cheap: state.settings?.models?.cheap?.model ? { ...state.settings.models.cheap, model: state.settings.models.cheap.model } : null,
vision: state.settings?.models?.vision?.model ? { ...state.settings.models.vision, model: state.settings.models.vision.model } : null,
};
const catalog: ModelCatalogResponse = {
models: state.keyPresent ? [syntheticModel] : [], defaults: slots.strong, slots,
rolePins: { researcher: slots.cheap, demoBuilder: slots.cheap, visualMaker: slots.vision, extractor: slots.cheap, auditor: slots.cheap },
providers: [{ id: "openai", envVar: provider.envVar, keyPresent: state.keyPresent }],
};
return json(route, catalog);
}
if (p === "/api/settings" && method === "GET") return json(route, { settings: state.settings });
if (p === "/api/settings/models" && method === "PUT") {
const body = request.postDataJSON() as NonNullable<UserSettings["models"]>;
state.writes.push({ path: p, body });
state.settings = { ...state.settings, version: 1, models: body };
return json(route, { settings: state.settings });
}
if (p === "/api/subscriptions" && method === "GET") {
const chatgpt: ChatGPTStatus = { available: false, pending: false, allowBackgroundUse: false, activeId: null, accounts: [], error: null };
const native: NativeSubscriptionStatus[] = ["github-copilot", "xai"].map((providerId) => ({
providerId: providerId as NativeSubscriptionStatus["providerId"], available: false, connected: false,
pending: false, allowBackgroundUse: false, connectionId: null, epoch: 0, authorityRevision: 0, revision: 0,
}));
return json(route, { chatgpt, native });
}
if (p === "/api/custom-provider" && method === "GET") return json(route, {
baseUrl: "", authMode: "keyless", models: [], overrides: {}, manualModels: [],
keyPresent: false, keySource: "none", keyReuseRequired: false,
});
if (p === "/api/stt/token" && method === "GET") return json(route, { message: "No DEEPGRAM_API_KEY — synthetic offline fixture." }, 503);
// Real, read-only application contracts against empty fixture-owned data.
if (method === "GET" && ["/api/activity", "/api/chats", "/api/events", "/api/graph", "/api/graph/status", "/api/graph/costs", "/api/graph/render-settings"].includes(p)) return route.continue();
return block();
});
await use(state);
expect(state.blocked, "Unexpected network request (blocked before dispatch)").toEqual([]);
expect(state.generation, "Setup/settings must not generate or send").toEqual([]);
},
page: async ({ page, harness }, use) => {
void harness; // force interception before the test receives its page
await use(page);
},
});
export { expect };
e2e/smoke.spec.ts
+74
View File
@@ -0,0 +1,74 @@
import { test, expect, syntheticModel } from "./fixtures";
test("first run configures all model slots and opens settings without sending the draft", async ({ page, harness }) => {
await page.goto("/");
const draft = page.getByRole("textbox", { name: "What do you want to learn?", exact: true });
await expect(page.getByTestId("setup-cta-keys")).toBeVisible();
await draft.fill("Synthetic learning topic — do not send");
await page.getByRole("button", { name: "Start session", exact: true }).click();
await expect(page.getByTestId("setup-stage-keys")).toBeVisible();
await expect(page.getByTestId("setup-stage-pickers")).toHaveCount(0);
await page.getByTestId("setup-key-provider").click();
await page.getByTestId("provider-option-openai").click();
await page.getByTestId("setup-key-input").fill("synthetic-smoke-key-not-a-credential");
await page.getByTestId("setup-key-save").click();
await expect(page.getByTestId("setup-stage-pickers")).toBeVisible();
await expect(page.getByTestId("setup-save")).toBeDisabled();
for (const slot of ["strong", "cheap", "vision"]) {
await page.getByTestId(`setup-trigger-${slot}`).click();
await page.getByTestId(`model-option-${syntheticModel.id}`).click();
}
await expect(page.getByTestId("setup-save")).toBeEnabled();
await page.getByTestId("setup-save").click();
await expect(page.getByTestId("setup-dialog")).toHaveCount(0);
await expect(page.getByTestId("setup-cta")).toHaveCount(0);
await expect(draft).toHaveText("Synthetic learning topic — do not send");
const pin = { model: { provider: "openai", id: syntheticModel.id }, thinkingLevel: "off" };
expect(harness.writes).toEqual([
{ path: "/api/keys/OPENAI_API_KEY", body: { value: "synthetic-smoke-key-not-a-credential" } },
{ path: "/api/settings/models", body: { strong: pin, cheap: pin, vision: pin } },
]);
await page.getByRole("button", { name: "Settings", exact: true }).click();
await expect(page.getByTestId("settings-dialog")).toBeVisible();
await expect(page.getByTestId("settings-nav-keys")).toBeVisible();
await page.getByTestId("settings-close").click();
await expect(draft).toHaveText("Synthetic learning topic — do not send");
expect(harness.generation).toEqual([]);
await expect(page).toHaveURL("http://127.0.0.1:3005/");
});
test("catalog failure is visible and cannot save incomplete slots", async ({ page, harness }) => {
harness.catalogFailure = true;
await page.goto("/");
await page.getByTestId("setup-cta").click();
await expect(page.getByTestId("setup-catalog-error")).toContainText("Model catalog could not be loaded. Check the connection and retry.");
await expect(page.getByTestId("setup-stage-keys")).toHaveCount(0);
await expect(page.getByTestId("setup-save")).toHaveCount(0);
harness.catalogFailure = false;
await page.getByTestId("setup-retry").click();
await expect(page.getByTestId("setup-stage-keys")).toBeVisible();
await expect(page.getByTestId("setup-catalog-error")).toHaveCount(0);
expect(harness.writes).toEqual([]);
});
test("network boundary rejects external requests and unknown API writes", async ({ page, harness }) => {
await page.goto("/");
await page.evaluate(async () => {
await Promise.all([
fetch("https://example.invalid/must-not-leave-host").catch(() => undefined),
fetch("/api/unexpected-write", { method: "POST", body: "synthetic" }).catch(() => undefined),
new Promise<void>((resolve) => {
const socket = new WebSocket("wss://example.invalid/must-not-connect");
socket.onclose = () => resolve();
socket.onerror = () => resolve();
}),
]);
});
expect(harness.blocked.sort()).toEqual([
"GET https://example.invalid/must-not-leave-host",
"POST http://127.0.0.1:3005/api/unexpected-write",
"WS wss://example.invalid/must-not-connect",
]);
// Only these deliberately asserted denials are consumed; every other denial fails teardown.
harness.blocked.length = 0;
});