learnharness/tests/agent-image-paths.test.ts

/** Offline integration contracts at the provider boundary: real shared image
 * processing, faux model streams, synthetic rasters, and fake headless I/O.
 * No real browser, provider, learner data, or ambient credentials are used. */
import assert from "node:assert/strict";
import * as fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { after, test } from "node:test";
import { createCanvas } from "@napi-rs/canvas";
import * as photon from "@silvia-odwyer/photon-node";
import {
	createFauxCore,
	fauxAssistantMessage,
	fauxText,
	fauxToolCall,
	type FauxResponseStep,
	type ImageContent,
	type TranscriptContext,
} from "@earendil-works/pi-ai";
import type { PiEngine } from "../src/agent/models.ts";
import type { ModelImageResizeOptions } from "../src/agent/image-processing/index.ts";
import type { HeadlessApi } from "../src/agent/subagents/headless.ts";

// Install storage/auth seams BEFORE importing the agent composition root.
const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "lh-agent-image-paths-"));
const originalEnv = process.env;
const originalFetch = globalThis.fetch;
fs.mkdirSync(path.join(scratch, "home"));
fs.writeFileSync(path.join(scratch, "auth.json"), "{}");
process.env = {
	...(originalEnv.NODE_TEST_CONTEXT ? { NODE_TEST_CONTEXT: originalEnv.NODE_TEST_CONTEXT } : {}),
	...(originalEnv.NODE_OPTIONS ? { NODE_OPTIONS: originalEnv.NODE_OPTIONS } : {}),
	NODE_ENV: "test",
	HOME: path.join(scratch, "home"),
	LH_DATA_DIR: path.join(scratch, "data"),
	LH_AUTH_FILE: path.join(scratch, "auth.json"),
};
globalThis.fetch = async () => { throw new Error("CLOSED: unexpected network request"); };
after(() => {
	process.env = originalEnv;
	globalThis.fetch = originalFetch;
	fs.rmSync(scratch, { recursive: true, force: true });
});

const { readTool } = await import("../src/agent/tools/read.ts");
const { createTeacherAgent, createProberAgent, createPracticeAgent } = await import("../src/agent/teacher-agent.ts");
const { runPdfReaderAgent } = await import("../src/agent/subagents/pdf-reader.ts");
const { runVisualMaker } = await import("../src/agent/subagents/visual-maker.ts");

const PDF = path.join(import.meta.dirname, "fixtures", "sample-3page.pdf");
const SVG = '<svg xmlns="http://www.w3.org/2000/svg" width="4800" height="120" viewBox="0 0 4800 120"><rect width="4800" height="120" fill="white"/></svg>';
const CUSTOM_RESIZE = { maxWidth: 320, maxHeight: 200 };

function png(width: number, height: number): Buffer {
	const canvas = createCanvas(width, height);
	const context = canvas.getContext("2d");
	context.fillStyle = "#336699";
	context.fillRect(0, 0, width, height);
	return canvas.toBuffer("image/png");
}

function dimensions(image: ImageContent): { width: number; height: number } {
	const decoded = photon.PhotonImage.new_from_byteslice(Buffer.from(image.data, "base64"));
	try {
		return { width: decoded.get_width(), height: decoded.get_height() };
	} finally { decoded.free(); }
}

function images(context: TranscriptContext): ImageContent[] {
	return context.messages.flatMap((message) => Array.isArray(message.content)
		? message.content.filter((part): part is ImageContent => part.type === "image")
		: []);
}

function text(context: TranscriptContext): string {
	return context.messages.map((message) => typeof message.content === "string"
		? message.content
		: message.content.filter((part) => part.type === "text").map((part) => part.text).join("\n")).join("\n");
}

function scripted(steps: FauxResponseStep[], resize?: ModelImageResizeOptions, vision = true) {
	const faux = createFauxCore({ tokenSize: { min: 64, max: 64 } });
	faux.setResponses(steps);
	const base = faux.getModel();
	const model: PiEngine["model"] = {
		...base,
		input: vision ? ["text", "image"] : ["text"],
		...(resize ? { inputLimits: { ...base.inputLimits, images: { ...base.inputLimits?.images, resize } } } : {}),
	};
	const requests: TranscriptContext[] = [];
	const engine: PiEngine = {
		model,
		streamFn: (model, context, options) => {
			// Snapshot the actual provider-facing context, not mutable agent state.
			requests.push(structuredClone(context));
			return faux.streamSimple(model, context, options);
		},
	};
	return { engine, requests };
}

function readSteps(file: string): FauxResponseStep[] {
	return [
		fauxAssistantMessage([fauxToolCall("read", { path: file }, { id: "read-1" })], { stopReason: "toolUse" }),
		fauxAssistantMessage([fauxText("Inspected the supplied source.")]),
	];
}

function fakeHeadless(rendered: Buffer | ((page: number) => Buffer)) {
	const svgCalls: string[] = [];
	const pdfCalls: number[][] = [];
	const headless: HeadlessApi = {
		available: () => true,
		unavailableReason: () => null,
		runDemo: async () => { throw new Error("unexpected browser execution"); },
		renderSvg: async (svg) => {
			svgCalls.push(svg);
			return typeof rendered === "function" ? rendered(1) : rendered;
		},
		renderPdfPages: async (_file, pages) => {
			pdfCalls.push([...pages]);
			return pages.map((page) => ({ page, png: typeof rendered === "function" ? rendered(page) : rendered }));
		},
	};
	return { headless, svgCalls, pdfCalls };
}

function assertProcessedImage(context: TranscriptContext, expected: { width: number; height: number }, raw: Buffer): void {
	const parts = images(context);
	assert.equal(parts.length, 1, "exactly one successful image reaches the faux provider");
	assert.deepEqual(dimensions(parts[0]!), expected);
	assert.ok(Buffer.byteLength(parts[0]!.data, "utf8") < 4.5 * 1024 * 1024);
	assert.notEqual(parts[0]!.data, raw.toString("base64"), "the oversized raw raster is never sent");
	assert.match(text(context), /\[Image: original \d+x\d+, displayed at \d+x\d+\. Multiply coordinates by/);
}

test("shared teacher read detects valid image content under a non-image extension", async () => {
	const bytes = png(24, 16);
	const file = path.join(scratch, "lecture-notes.txt");
	fs.writeFileSync(file, bytes);
	const { engine, requests } = scripted(readSteps(file));
	// Pass the actual shared read tool, exercising the teacher's native clone
	// of read.execute rather than a pre-bound test-only implementation.
	const agent = createTeacherAgent(undefined, { engine, tools: { read: readTool } });
	await agent.prompt("Inspect the supplied local file.");
	assert.equal(requests.length, 2);
	const visible = requests[1]!;
	assert.equal(images(visible).length, 1);
	assert.equal(images(visible)[0]!.mimeType, "image/png");
	assert.equal(images(visible)[0]!.data, bytes.toString("base64"));
	assert.deepEqual(dimensions(images(visible)[0]!), { width: 24, height: 16 });
	assert.match(text(visible), /Read image file \[image\/png\]/);
});

test("shared read omits corrupt recognized images without exposing a raw image fallback", async () => {
	// Valid signature/IHDR, missing pixel data: sniff succeeds, decode fails.
	const bytes = png(24, 16).subarray(0, 33);
	const file = path.join(scratch, "corrupt.png");
	fs.writeFileSync(file, bytes);
	const { engine, requests } = scripted(readSteps(file));
	await createTeacherAgent(undefined, { engine, tools: { read: readTool } }).prompt("Inspect the file.");
	assert.equal(requests.length, 2);
	assert.deepEqual(images(requests[1]!), []);
	assert.match(text(requests[1]!), /Image omitted: could not be resized below the inline image size limit/);
	assert.ok(!JSON.stringify(requests[1]).includes(bytes.toString("base64")));
});

test("shared read supplies the native nonvision-model note", async () => {
	const file = path.join(scratch, "nonvision.png");
	fs.writeFileSync(file, png(24, 16));
	const { engine, requests } = scripted(readSteps(file), undefined, false);
	await createTeacherAgent(undefined, { engine, tools: { read: readTool } }).prompt("Inspect the file.");
	assert.equal(requests.length, 2);
	assert.match(text(requests[1]!), /\[Current model does not support images\. The image will be omitted from this request\.\]/);
});

test("teacher read binds independent active-model resize limits without mutating the shared tool", async () => {
	const bytes = png(640, 320);
	const file = path.join(scratch, "model-bound.png");
	fs.writeFileSync(file, bytes);
	const originalExecute = readTool.execute;
	const a = scripted(readSteps(file), { maxWidth: 96, maxHeight: 64 });
	const b = scripted(readSteps(file), { maxWidth: 80, maxHeight: 30 });
	const agentA = createTeacherAgent(undefined, { engine: a.engine, tools: { read: readTool } });
	const agentB = createTeacherAgent(undefined, { engine: b.engine, tools: { read: readTool } });
	await agentB.prompt("Inspect with model B.");
	await agentA.prompt("Inspect with model A.");
	assertProcessedImage(a.requests[1]!, { width: 96, height: 48 }, bytes);
	assertProcessedImage(b.requests[1]!, { width: 60, height: 30 }, bytes);
	assert.equal(readTool.execute, originalExecute);
});

for (const [role, resize, expected] of [
	["prober", { maxWidth: 96, maxHeight: 64 }, { width: 96, height: 48 }],
	["practice", { maxWidth: 80, maxHeight: 30 }, { width: 60, height: 30 }],
] as const) {
	test(`${role}'s actual registry retains argument healing and text reads while binding image limits`, async () => {
		const bytes = png(640, 320);
		const imageFile = path.join(scratch, `${role}-actual-registry.png`);
		const textFile = path.join(scratch, `${role}-actual-registry.txt`);
		fs.writeFileSync(imageFile, bytes);
		fs.writeFileSync(textFile, "first\nsecond\nthird\nfourth\nfifth");
		const { engine, requests } = scripted([
			fauxAssistantMessage([fauxToolCall("read", { path: imageFile }, { id: "registry-image" })], { stopReason: "toolUse" }),
			// Exact existing heal contract: schema-integer fields encoded as
			// JSON number strings heal before validation. The path stays intact.
			fauxAssistantMessage([fauxToolCall("read", { path: textFile, offset: "2", limit: "2" }, { id: "registry-text" })], { stopReason: "toolUse" }),
			fauxAssistantMessage([fauxText("Inspected the image and selected text lines.")]),
		], resize);
		const originalExecute = readTool.execute;
		const forbidden = () => { throw new Error("unexpected job dispatch or handoff"); };
		const chatStore = { getChat: () => null };
		// Only context ports are mocked; no app/store imports or explicit
		// tool overrides. Job/handoff operations must never execute here.
		const agent = role === "prober"
			? createProberAgent({
				chatId: "offline-read-prober", chatStore,
				jobStore: new Proxy({}, { get: forbidden }),
				spawn: forbidden, deliver: forbidden,
			} as never, { engine })
			: createPracticeAgent({ chatId: "offline-read-practice", chatStore }, { engine });
		const names = agent.state.tools.map((tool) => tool.name);
		assert.ok(names.includes(role === "prober" ? "probe_done" : "problem"), "real role registry was assembled");
		const boundRead = agent.state.tools.find((tool) => tool.name === "read");
		assert.ok(boundRead?.prepareArguments, "model binding preserves the registry's prepareArguments wrapper");
		assert.notEqual(boundRead.execute, originalExecute, "shared read execute is rebound to the active model");
		assert.deepEqual(boundRead.prepareArguments({ path: textFile, offset: "2", limit: "2" }), { path: textFile, offset: 2, limit: 2 });
		await agent.prompt("Inspect the local image, then read the selected text lines.");
		assert.equal(requests.length, 3);
		assertProcessedImage(requests[1]!, expected, bytes);
		const selected = requests[2]!.messages.find((message) => message.role === "toolResult" && message.toolCallId === "registry-text");
		assert.ok(selected && selected.role === "toolResult");
		assert.equal(selected.isError, false);
		assert.deepEqual(selected.content, [{ type: "text", text: "second\nthird\n\n[2 more lines in file. Use offset=4 to continue.]" }]);
		assert.equal(readTool.execute, originalExecute, "role construction did not mutate the shared tool");
	});
}

test("PDF reader preserves both processed image outputs from multiple render calls in one batch", async () => {
	const wide = png(4800, 120);
	const tall = png(120, 4800);
	const { headless, pdfCalls } = fakeHeadless((page) => page === 1 ? wide : tall);
	const { engine, requests } = scripted([
		fauxAssistantMessage([
			fauxToolCall("pdf_render", { pages: "1", dpi: 150 }, { id: "batch-render-1" }),
			fauxToolCall("pdf_render", { pages: "2", dpi: 150 }, { id: "batch-render-2" }),
		], { stopReason: "toolUse" }),
		fauxAssistantMessage([fauxText("Both rendered pages are visible (p. 1, p. 2). Pages consulted: 1, 2")]),
	]);
	const result = await runPdfReaderAgent({ path: PDF, question: "Compare the two figures." }, { engine, headless });
	assert.equal(requests.length, 2);
	assert.deepEqual(pdfCalls, [[1], [2]]);
	const visible = requests[1]!;
	const parts = images(visible);
	assert.equal(parts.length, 2, "the second render call must not overwrite the first call's images");
	assert.deepEqual(parts.map(dimensions), [{ width: 2000, height: 50 }, { width: 50, height: 2000 }]);
	for (let i = 0; i < parts.length; i++) {
		assert.ok(parts[i]!.data.length < 4.5 * 1024 * 1024);
		assert.notEqual(parts[i]!.data, [wide, tall][i]!.toString("base64"));
	}
	assert.deepEqual(visible.messages.slice(-3).map((message) => message.role), ["toolResult", "toolResult", "user"]);
	const pageMessage = visible.messages.at(-1)!;
	assert.ok(Array.isArray(pageMessage.content));
	assert.deepEqual(pageMessage.content.filter((part) => part.type === "text").map((part) => part.text.split("\n")[0]), [
		"The rendered page(s), in order (p1, p2):", "Page 1", "Page 2",
	]);
	assert.deepEqual(result.pagesRead, [1, 2]);
	assert.deepEqual(result.pagesConsulted, [1, 2]);
});

for (const [label, resize, expected] of [
	["native defaults", undefined, { width: 2000, height: 50 }],
	["active-model overrides", CUSTOM_RESIZE, { width: 320, height: 8 }],
] as const) {
	test(`PDF reader sends processed renders using ${label}; failed page is not counted`, async () => {
		const bytes = png(4800, 120);
		const { headless, pdfCalls } = fakeHeadless((page) => page === 1 ? bytes : Buffer.from("invalid rendered page"));
		const { engine, requests } = scripted([
			fauxAssistantMessage([fauxToolCall("pdf_render", { pages: "1-2", dpi: 150 }, { id: "render-1" })], { stopReason: "toolUse" }),
			fauxAssistantMessage([fauxText("The visible figure is on page 1 (p. 1). Pages consulted: 1")]),
		], resize);
		const result = await runPdfReaderAgent({ path: PDF, question: "Inspect the page figure." }, { engine, headless });
		assert.equal(requests.length, 2);
		assert.deepEqual(pdfCalls, [[1, 2]]);
		assertProcessedImage(requests[1]!, expected, bytes);
		assert.match(text(requests[1]!), /Page 2: \[Image omitted:/);
		assert.deepEqual(result.pagesRead, [1]);
		assert.deepEqual(result.pagesConsulted, [1]);
		assert.match(result.brief, /visible figure/);
		const visible = requests[1]!.messages;
		assert.equal(visible.at(-2)!.role, "toolResult");
		assert.equal(visible.at(-1)!.role, "user", "processed page images ride the user message after tool results");
	});

	for (const forced of [false, true]) {
		test(`visual maker ${forced ? "forced-final" : "explicit render_svg"} path processes images using ${label}`, async () => {
			const bytes = png(4800, 120);
			const { headless, svgCalls } = fakeHeadless(bytes);
			const steps: FauxResponseStep[] = forced ? [
				fauxAssistantMessage([fauxToolCall("write_svg", { svg: SVG }, { id: "write-1" })], { stopReason: "toolUse" }),
				fauxAssistantMessage([fauxText("Premature notes without inspection.")]),
				fauxAssistantMessage([fauxText("Confirmed after inspecting the final render.")]),
			] : [
				fauxAssistantMessage([
					fauxToolCall("write_svg", { svg: SVG }, { id: "write-1" }),
					fauxToolCall("render_svg", {}, { id: "render-1" }),
				], { stopReason: "toolUse" }),
				fauxAssistantMessage([fauxText("Confirmed after inspecting the render.")]),
			];
			const { engine, requests } = scripted(steps, resize);
			const result = await runVisualMaker({ brief: "One precise synthetic figure." }, { engine, headless });
			assert.equal(requests.length, forced ? 3 : 2);
			assert.deepEqual(svgCalls, [SVG]);
			assertProcessedImage(requests.at(-1)!, expected, bytes);
			assert.equal(result.svg, SVG);
			assert.match(result.notes!, /^Confirmed after inspecting/);
			assert.equal(requests.at(-1)!.messages.at(-1)!.role, "user");
			if (forced) {
				assert.deepEqual(images(requests[1]!), [], "premature finish did not see an image yet");
				assert.match(text(requests[2]!), /You finished without looking at a render of your final SVG/);
			} else {
				assert.match(text(requests[1]!), /Here is the current render of your SVG/);
				assert.equal(requests[1]!.messages.at(-2)!.role, "toolResult");
			}
		});
	}
}

for (const forced of [false, true]) {
	test(`visual maker ${forced ? "forced-final" : "explicit render_svg"} processing failure never verifies or delivers the SVG`, async () => {
		const { headless, svgCalls } = fakeHeadless(Buffer.from("corrupt render"));
		const steps: FauxResponseStep[] = [
			fauxAssistantMessage([
				fauxToolCall("write_svg", { svg: SVG }, { id: "write-1" }),
				...(!forced ? [fauxToolCall("render_svg", {}, { id: "render-1" })] : []),
			], { stopReason: "toolUse" }),
			fauxAssistantMessage([fauxText("Unverified claim that the picture is correct.")]),
		];
		const { engine, requests } = scripted(steps);
		const activity: string[] = [];
		const result = await runVisualMaker({ brief: "One synthetic figure." }, { engine, headless, onActivity: (line) => activity.push(line) });
		assert.equal(requests.length, 2);
		assert.ok(requests.every((request) => images(request).length === 0));
		assert.equal(result.svg, "");
		assert.match(result.notes!, /^FAILED: inspecting the final SVG failed/);
		assert.match(result.notes!, /Image omitted:/);
		assert.ok(!activity.some((line) => line.includes("inspecting")), "a failed image is never marked inspected");
		assert.equal(svgCalls.length, forced ? 1 : 2, "an explicit processing failure leaves the final-SVG gate unsatisfied");
		if (!forced) assert.match(text(requests[1]!), /RENDER ERROR.*Image omitted:/);
	});
}