From 84f93999c54922d327d957beb4c4831fbb0ffbde Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Wed, 28 Oct 2026 18:16:04 +0200 Subject: [PATCH] fix: normalize model-visible images with Pi processing --- next.config.ts | 6 + package-lock.json | 7 + package.json | 1 + scripts/smoke-image-runtime.mjs | 228 +++++++++++ src/agent/image-processing/LICENSE | 21 ++ src/agent/image-processing/convert.d.mts | 1 + src/agent/image-processing/convert.mjs | 25 ++ .../image-processing/exif-orientation.mjs | 157 ++++++++ src/agent/image-processing/index.ts | 59 +++ src/agent/image-processing/mime.d.mts | 3 + src/agent/image-processing/mime.mjs | 110 ++++++ src/agent/image-processing/photon.mjs | 21 ++ src/agent/image-processing/resize-core.d.mts | 10 + src/agent/image-processing/resize-core.mjs | 130 +++++++ src/agent/image-processing/resize-worker.mjs | 32 ++ src/agent/image-processing/resize.ts | 65 ++++ src/agent/image-processing/types.ts | 35 ++ src/agent/subagents/headless.ts | 57 ++- src/agent/subagents/pdf-reader.ts | 29 +- src/agent/subagents/visual-maker.ts | 21 +- src/agent/teacher-agent.ts | 5 +- src/agent/tools/read.ts | 73 ++-- tests/agent-image-paths.test.ts | 351 +++++++++++++++++ tests/demos.test.tsx | 7 +- tests/image-processing.test.ts | 357 ++++++++++++++++++ tests/pdf-native-render.test.ts | 106 +++++- tests/provider-tracing.test.ts | 3 + tests/read-tool.test.ts | 10 +- 28 files changed, 1870 insertions(+), 60 deletions(-) create mode 100644 scripts/smoke-image-runtime.mjs create mode 100644 src/agent/image-processing/LICENSE create mode 100644 src/agent/image-processing/convert.d.mts create mode 100644 src/agent/image-processing/convert.mjs create mode 100644 src/agent/image-processing/exif-orientation.mjs create mode 100644 src/agent/image-processing/index.ts create mode 100644 src/agent/image-processing/mime.d.mts create mode 100644 src/agent/image-processing/mime.mjs create mode 100644 src/agent/image-processing/photon.mjs create mode 100644 src/agent/image-processing/resize-core.d.mts create mode 100644 src/agent/image-processing/resize-core.mjs create mode 100644 src/agent/image-processing/resize-worker.mjs create mode 100644 src/agent/image-processing/resize.ts create mode 100644 src/agent/image-processing/types.ts create mode 100644 tests/agent-image-paths.test.ts create mode 100644 tests/image-processing.test.ts diff --git a/next.config.ts b/next.config.ts index a4236ad..43aef8d 100644 --- a/next.config.ts +++ b/next.config.ts @@ -78,6 +78,11 @@ const nextConfig: NextConfig = { // npm installs only the current platform's binary, so this glob // resolves to exactly one Skia binary per build. "node_modules/@napi-rs/canvas-*/**/*", + // Pi-compatible image resizing runs in an unbundled worker. Photon + // loads its colocated WASM through Node, also in packaged Electron. + "src/agent/image-processing/*.mjs", + "src/agent/image-processing/LICENSE", + "node_modules/@silvia-odwyer/photon-node/**/*", // Wave E stage 2: resvg rasterizes visual-maker SVGs natively; same // per-platform-binary story as the canvas above (one per build). "node_modules/@resvg/resvg-js-*/**/*", @@ -106,6 +111,7 @@ const nextConfig: NextConfig = { "@earendil-works/pi-agent-core", "pdfjs-dist", // sources v0: bundling breaks its node fake-worker path "@napi-rs/canvas", // native binding — bundling breaks the .node require + "@silvia-odwyer/photon-node", // CJS loads its adjacent WASM from disk "@resvg/resvg-js", // native binding — same ], }; diff --git a/package-lock.json b/package-lock.json index 638b775..9b359b6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@radix-ui/react-use-controllable-state": "^1.2.6", "@resvg/resvg-js": "^2.6.2", "@rive-app/react-webgl2": "^4.32.1", + "@silvia-odwyer/photon-node": "0.3.4", "@streamdown/cjk": "^1.0.3", "@streamdown/code": "^1.1.1", "@streamdown/math": "^1.0.2", @@ -4717,6 +4718,12 @@ "integrity": "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==", "license": "MIT" }, + "node_modules/@silvia-odwyer/photon-node": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/@silvia-odwyer/photon-node/-/photon-node-0.3.4.tgz", + "integrity": "sha512-bnly4BKB3KDTFxrUIcgCLbaeVVS8lrAkri1pEzskpmxu9MdfGQTy8b8EgcD83ywD3RPMsIulY8xJH5Awa+t9fA==", + "license": "Apache-2.0" + }, "node_modules/@sindresorhus/is": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-4.6.0.tgz", diff --git a/package.json b/package.json index 583eabc..2008ba4 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "@radix-ui/react-use-controllable-state": "^1.2.6", "@resvg/resvg-js": "^2.6.2", "@rive-app/react-webgl2": "^4.32.1", + "@silvia-odwyer/photon-node": "0.3.4", "@streamdown/cjk": "^1.0.3", "@streamdown/code": "^1.1.1", "@streamdown/math": "^1.0.2", diff --git a/scripts/smoke-image-runtime.mjs b/scripts/smoke-image-runtime.mjs new file mode 100644 index 0000000..c99e18d --- /dev/null +++ b/scripts/smoke-image-runtime.mjs @@ -0,0 +1,228 @@ +/** + * OFFLINE image runtime gate: exercise the shipped Photon/WASM resize core and + * actual worker, with file modules confined to the standalone payload. No app, + * provider, canvas, or network needed; a tiny synthetic PNG is generated here. + * + * node scripts/smoke-image-runtime.mjs [--standalone ] [--runtime ] + */ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import * as fs from "node:fs"; +import { createRequire } from "node:module"; +import * as os from "node:os"; +import * as path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { Worker } from "node:worker_threads"; +import { deflateSync } from "node:zlib"; +import { formatProbeError, probeEnv } from "./smoke-demo-runtime.mjs"; + +export function verifyImageRuntime(standalone, runtime = process.execPath) { + const home = fs.mkdtempSync(path.join(os.tmpdir(), "lh-image-runtime-")); + try { + fs.writeFileSync(path.join(home, "auth.json"), "{}\n", { mode: 0o600 }); + const result = spawnSync(runtime, [fileURLToPath(import.meta.url), "--probe", path.resolve(standalone)], { + cwd: home, + env: { ...probeEnv(process.env, home), ELECTRON_RUN_AS_NODE: "1" }, + encoding: "utf8", timeout: 30_000, killSignal: "SIGKILL", maxBuffer: 128 * 1024, + }); + if (result.status !== 0 || result.error) { + throw new Error(`image runtime probe failed (${result.error?.message ?? result.status}):\n${result.stdout ?? ""}${result.stderr ?? ""}`); + } + assert.match(result.stdout, /Photon in-process and worker resizing passed/, "probe exited without success marker"); + process.stdout.write(result.stdout); + } finally { fs.rmSync(home, { recursive: true, force: true }); } +} + +// Self-contained so the eval worker can install the SAME guards using only +// builtins, before importing any payload module. Module hooks do not otherwise +// propagate from the dedicated probe process into Worker threads. +function installGuards(require, standalone) { + const assert = require("node:assert/strict"); + const fs = require("node:fs"); + const path = require("node:path"); + const { fileURLToPath } = require("node:url"); + const { registerHooks, syncBuiltinESMExports } = require("node:module"); + const root = fs.realpathSync(standalone); + const inside = (file) => { + const real = fs.realpathSync(file); + const relative = path.relative(root, real); + assert.ok(relative !== ".." && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative), + `dependency escaped standalone payload: ${file}`); + return real; + }; + const photonRoot = path.join(root, "node_modules/@silvia-odwyer/photon-node"); + const manifest = JSON.parse(fs.readFileSync(inside(path.join(photonRoot, "package.json")), "utf8")); + assert.equal(manifest.version, "0.3.4", "unexpected Photon engine version"); + const photonEntry = inside(path.join(photonRoot, "photon_rs.js")); + const photonWasm = inside(path.join(photonRoot, "photon_rs_bg.wasm")); + assert.ok(fs.statSync(photonWasm).size > 0, "empty Photon WASM asset"); + let attempts = 0; + const deny = () => { attempts++; throw new Error("OFFLINE image gate: network forbidden"); }; + globalThis.fetch = deny; + globalThis.WebSocket = class { constructor() { deny(); } }; + for (const [name, methods] of [ + ["http", ["request", "get"]], ["https", ["request", "get"]], + ["net", ["connect", "createConnection"]], ["tls", ["connect"]], + ["dgram", ["createSocket"]], + ["dns", ["lookup", "resolve", "resolve4", "resolve6", "resolveAny", "reverse"]], + ["dns/promises", ["lookup", "resolve", "resolve4", "resolve6", "resolveAny", "reverse"]], + ]) { + const module = require(`node:${name}`); + for (const method of methods) module[method] = deny; + } + require("node:net").Socket.prototype.connect = deny; + const loaded = new Set(); + const wasmReads = new Set(); + const originalRead = fs.readFileSync; + fs.readFileSync = function (file, ...args) { + const filename = file instanceof URL ? fileURLToPath(file) : file; + const wasmFile = typeof filename === "string" && filename.endsWith(".wasm") ? inside(filename) : null; + const bytes = originalRead.call(this, file, ...args); + if (wasmFile) wasmReads.add(wasmFile); + return bytes; + }; + syncBuiltinESMExports(); + const hooks = registerHooks({ + resolve(specifier, context, nextResolve) { + const result = nextResolve(specifier, context); + if (result.url.startsWith("file:")) loaded.add(inside(fileURLToPath(result.url))); + else assert.ok(result.url.startsWith("node:"), `unexpected dependency URL: ${result.url}`); + return result; + }, + }); + return { + report() { + assert.equal(attempts, 0, "image runtime attempted network access"); + assert.ok(loaded.has(photonEntry), "Photon JS was not loaded from the payload"); + assert.ok(wasmReads.has(photonWasm), "Photon WASM was not loaded from the payload"); + return { modules: loaded.size, wasmFiles: [...wasmReads], networkAttempts: attempts }; + }, + cleanup() { + hooks.deregister(); + fs.readFileSync = originalRead; + syncBuiltinESMExports(); + }, + }; +} + +/** Pure Node 2×2 RGBA PNG, with valid PNG CRCs and zlib image data. */ +function fixturePng() { + const chunk = (type, data) => { + const buffer = Buffer.alloc(data.length + 12); + buffer.writeUInt32BE(data.length); + buffer.write(type, 4, "ascii"); + data.copy(buffer, 8); + let crc = ~0; + for (const byte of buffer.subarray(4, -4)) { + crc ^= byte; + for (let bit = 0; bit < 8; bit++) crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + buffer.writeUInt32BE(~crc >>> 0, buffer.length - 4); + return buffer; + }; + const header = Buffer.alloc(13); + header.writeUInt32BE(2, 0); + header.writeUInt32BE(2, 4); + header[8] = 8; // bit depth + header[9] = 6; // RGBA + const pixels = Buffer.from([ + 0, 255, 0, 0, 255, 0, 255, 0, 255, + 0, 0, 0, 255, 255, 255, 255, 255, 255, + ]); + return Buffer.concat([Buffer.from("89504e470d0a1a0a", "hex"), chunk("IHDR", header), chunk("IDAT", deflateSync(pixels)), chunk("IEND", Buffer.alloc(0))]); +} + +function assertResized(result) { + assert.ok(result, "resize returned null (missing/broken Photon must not silently pass)"); + assert.equal(result.originalWidth, 2); + assert.equal(result.originalHeight, 2); + assert.equal(result.width, 1); + assert.equal(result.height, 1); + assert.equal(result.wasResized, true); + assert.equal(result.mimeType, "image/png"); + const png = Buffer.from(result.data, "base64"); + assert.equal(png.subarray(0, 8).toString("hex"), "89504e470d0a1a0a"); + assert.equal(png.readUInt32BE(16), 1); + assert.equal(png.readUInt32BE(20), 1); +} + +async function probeWorker(root, inputBytes, options, responseType) { + const readyType = "lh:image-runtime-ready"; + const worker = new Worker(` + const { parentPort, workerData } = require("node:worker_threads"); + const guards = (${installGuards.toString()})(require, workerData.root); + const post = parentPort.postMessage.bind(parentPort); + parentPort.postMessage = (message, ...args) => { + if (message?.type === workerData.responseType) { + post({ ...message, runtimeProof: guards.report() }, ...args); + } else post(message, ...args); + }; + void import(workerData.entry).then(() => post({ type: workerData.readyType })); + `, { + eval: true, + workerData: { root, responseType, readyType, entry: pathToFileURL(path.join(root, "src/agent/image-processing/resize-worker.mjs")).href }, + }); + try { + return await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("image resize worker timed out")), 10_000); + const fail = (error) => { clearTimeout(timer); reject(error); }; + let ready = false; + worker.once("error", fail); + worker.once("exit", (code) => fail(new Error(`image resize worker exited before response: ${code}`))); + worker.on("message", (message) => { + try { + if (message?.type === readyType) { + assert.equal(ready, false, "duplicate worker READY"); + ready = true; + worker.postMessage({ inputBytes, mimeType: "image/png", options }); + } else if (message?.type === responseType) { + assert.equal(ready, true, "worker replied before its import completed"); + assert.equal(message.error, undefined, `worker resize failed: ${message.error}`); + assertResized(message.result); + assert.ok(message.runtimeProof?.modules > 0, "missing worker module-confinement proof"); + assert.equal(message.runtimeProof.networkAttempts, 0); + assert.equal(message.runtimeProof.wasmFiles.length, 1); + clearTimeout(timer); + resolve(message.runtimeProof); + } + } catch (error) { fail(error); } + }); + }); + } finally { await worker.terminate(); } +} + +async function probe(standalone) { + const root = fs.realpathSync(standalone); + const guards = installGuards(createRequire(import.meta.url), root); + try { + const entry = pathToFileURL(path.join(root, "src/agent/image-processing/resize-core.mjs")); + const { resizeImageInProcess, IMAGE_RESIZE_WORKER_RESPONSE_TYPE } = await import(entry.href); + assert.equal(typeof resizeImageInProcess, "function"); + assert.equal(IMAGE_RESIZE_WORKER_RESPONSE_TYPE, "pi:image-resize-response"); + const input = fixturePng(); + const options = { maxWidth: 1, maxHeight: 1, maxBytes: 1024 * 1024 }; + assertResized(await resizeImageInProcess(input, "image/png", options)); + const inProcess = guards.report(); + console.log(`[smoke-image-runtime] shipped resize core: PNG 2×2 → 1×1; ${inProcess.modules} modules and Photon WASM confined to ${root}`); + const worker = await probeWorker(root, input, options, IMAGE_RESIZE_WORKER_RESPONSE_TYPE); + guards.report(); + console.log(`[smoke-image-runtime] shipped resize worker: READY then tagged PNG 2×2 → 1×1; ${worker.modules} modules and Photon WASM confined to ${root}`); + console.log("[smoke-image-runtime] Photon in-process and worker resizing passed; zero network attempts"); + } finally { guards.cleanup(); } +} + +async function main() { + const args = process.argv.slice(2); + if (args[0] === "--probe") return probe(args[1]); + const value = (name, fallback) => { + const index = args.indexOf(name); + if (index < 0) return fallback; + assert.ok(args[index + 1] && !args[index + 1].startsWith("--"), `${name} requires a path`); + return path.resolve(args[index + 1]); + }; + verifyImageRuntime(value("--standalone", path.resolve(import.meta.dirname, "../.next/standalone")), value("--runtime", process.execPath)); +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error) => { console.error(`[smoke-image-runtime] FAIL: ${formatProbeError(error)}`); process.exitCode = 1; }); +} diff --git a/src/agent/image-processing/LICENSE b/src/agent/image-processing/LICENSE new file mode 100644 index 0000000..b0a8e9b --- /dev/null +++ b/src/agent/image-processing/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Mario Zechner + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/src/agent/image-processing/convert.d.mts b/src/agent/image-processing/convert.d.mts new file mode 100644 index 0000000..1826e6c --- /dev/null +++ b/src/agent/image-processing/convert.d.mts @@ -0,0 +1 @@ +export function convertImageBytesToPng(bytes: Uint8Array): Promise; diff --git a/src/agent/image-processing/convert.mjs b/src/agent/image-processing/convert.mjs new file mode 100644 index 0000000..f271633 --- /dev/null +++ b/src/agent/image-processing/convert.mjs @@ -0,0 +1,25 @@ +// Ported from @earendil-works/pi-coding-agent 1.1.0, dist/utils/image-convert.js (MIT). +// Only inline-image conversion is retained; terminal/Kitty registration is not used. +import { applyExifOrientation } from "./exif-orientation.mjs"; +import { loadPhoton } from "./photon.mjs"; + +function encodePng(photon, bytes) { + try { + const rawImage = photon.PhotonImage.new_from_byteslice(bytes); + const image = applyExifOrientation(photon, rawImage, bytes); + if (image !== rawImage) rawImage.free(); + try { + return new Uint8Array(image.get_bytes()); + } finally { + image.free(); + } + } catch { + return null; + } +} + +export async function convertImageBytesToPng(bytes) { + const photon = await loadPhoton(); + if (!photon) return null; + return encodePng(photon, bytes); +} diff --git a/src/agent/image-processing/exif-orientation.mjs b/src/agent/image-processing/exif-orientation.mjs new file mode 100644 index 0000000..1915c7f --- /dev/null +++ b/src/agent/image-processing/exif-orientation.mjs @@ -0,0 +1,157 @@ +// Ported from @earendil-works/pi-coding-agent 1.1.0, dist/utils/exif-orientation.js (MIT). +function readOrientationFromTiff(bytes, tiffStart) { + if (tiffStart + 8 > bytes.length) + return 1; + const byteOrder = (bytes[tiffStart] << 8) | bytes[tiffStart + 1]; + const le = byteOrder === 0x4949; + const read16 = (pos) => { + if (le) + return bytes[pos] | (bytes[pos + 1] << 8); + return (bytes[pos] << 8) | bytes[pos + 1]; + }; + const read32 = (pos) => { + if (le) + return bytes[pos] | (bytes[pos + 1] << 8) | (bytes[pos + 2] << 16) | (bytes[pos + 3] << 24); + return ((bytes[pos] << 24) | (bytes[pos + 1] << 16) | (bytes[pos + 2] << 8) | bytes[pos + 3]) >>> 0; + }; + const ifdOffset = read32(tiffStart + 4); + const ifdStart = tiffStart + ifdOffset; + if (ifdStart + 2 > bytes.length) + return 1; + const entryCount = read16(ifdStart); + for (let i = 0; i < entryCount; i++) { + const entryPos = ifdStart + 2 + i * 12; + if (entryPos + 12 > bytes.length) + return 1; + if (read16(entryPos) === 0x0112) { + const value = read16(entryPos + 8); + return value >= 1 && value <= 8 ? value : 1; + } + } + return 1; +} +function findJpegTiffOffset(bytes) { + let offset = 2; + while (offset < bytes.length - 1) { + if (bytes[offset] !== 0xff) + return -1; + const marker = bytes[offset + 1]; + if (marker === 0xff) { + offset++; + continue; + } + if (marker === 0xe1) { + if (offset + 4 >= bytes.length) + return -1; + const segmentStart = offset + 4; + if (segmentStart + 6 > bytes.length) + return -1; + if (hasExifHeader(bytes, segmentStart)) + return segmentStart + 6; + } + if (offset + 4 > bytes.length) + return -1; + const length = (bytes[offset + 2] << 8) | bytes[offset + 3]; + offset += 2 + length; + } + return -1; +} +function findWebpTiffOffset(bytes) { + let offset = 12; + while (offset + 8 <= bytes.length) { + const chunkId = String.fromCharCode(bytes[offset], bytes[offset + 1], bytes[offset + 2], bytes[offset + 3]); + const chunkSize = bytes[offset + 4] | (bytes[offset + 5] << 8) | (bytes[offset + 6] << 16) | (bytes[offset + 7] << 24); + const dataStart = offset + 8; + if (chunkId === "EXIF") { + if (dataStart + chunkSize > bytes.length) + return -1; + // Some WebP files have "Exif\0\0" prefix before the TIFF header + const tiffStart = chunkSize >= 6 && hasExifHeader(bytes, dataStart) ? dataStart + 6 : dataStart; + return tiffStart; + } + // RIFF chunks are padded to even size + offset = dataStart + chunkSize + (chunkSize % 2); + } + return -1; +} +function hasExifHeader(bytes, offset) { + return (bytes[offset] === 0x45 && + bytes[offset + 1] === 0x78 && + bytes[offset + 2] === 0x69 && + bytes[offset + 3] === 0x66 && + bytes[offset + 4] === 0x00 && + bytes[offset + 5] === 0x00); +} +function getExifOrientation(bytes) { + let tiffOffset = -1; + // JPEG: starts with FF D8 + if (bytes.length >= 2 && bytes[0] === 0xff && bytes[1] === 0xd8) { + tiffOffset = findJpegTiffOffset(bytes); + } + // WebP: starts with RIFF....WEBP + else if (bytes.length >= 12 && + bytes[0] === 0x52 && + bytes[1] === 0x49 && + bytes[2] === 0x46 && + bytes[3] === 0x46 && + bytes[8] === 0x57 && + bytes[9] === 0x45 && + bytes[10] === 0x42 && + bytes[11] === 0x50) { + tiffOffset = findWebpTiffOffset(bytes); + } + if (tiffOffset === -1) + return 1; + return readOrientationFromTiff(bytes, tiffOffset); +} +function rotate90(photon, image, dstIndex) { + const w = image.get_width(); + const h = image.get_height(); + const src = image.get_raw_pixels(); + const dst = new Uint8Array(src.length); + for (let y = 0; y < h; y++) { + for (let x = 0; x < w; x++) { + const srcIdx = (y * w + x) * 4; + const dstIdx = dstIndex(x, y, w, h) * 4; + dst[dstIdx] = src[srcIdx]; + dst[dstIdx + 1] = src[srcIdx + 1]; + dst[dstIdx + 2] = src[srcIdx + 2]; + dst[dstIdx + 3] = src[srcIdx + 3]; + } + } + return new photon.PhotonImage(dst, h, w); +} +// Flip orientations mutate in-place. Rotations return a new image (caller must free the old one if different). +export function applyExifOrientation(photon, image, originalBytes) { + const orientation = getExifOrientation(originalBytes); + if (orientation === 1) + return image; + switch (orientation) { + case 2: + photon.fliph(image); + return image; + case 3: + photon.fliph(image); + photon.flipv(image); + return image; + case 4: + photon.flipv(image); + return image; + case 5: { + const rotated = rotate90(photon, image, (x, y, _w, h) => x * h + (h - 1 - y)); + photon.fliph(rotated); + return rotated; + } + case 6: + return rotate90(photon, image, (x, y, _w, h) => x * h + (h - 1 - y)); + case 7: { + const rotated = rotate90(photon, image, (x, y, w, h) => (w - 1 - x) * h + y); + photon.fliph(rotated); + return rotated; + } + case 8: + return rotate90(photon, image, (x, y, w, h) => (w - 1 - x) * h + y); + default: + return image; + } +} diff --git a/src/agent/image-processing/index.ts b/src/agent/image-processing/index.ts new file mode 100644 index 0000000..ddeaf9b --- /dev/null +++ b/src/agent/image-processing/index.ts @@ -0,0 +1,59 @@ +// Ported from @earendil-works/pi-coding-agent 1.1.0, dist/utils/image-process.js (MIT). +import { convertImageBytesToPng } from "./convert.mjs"; +import { formatDimensionNote, resizeImage } from "./resize.ts"; +import type { ProcessImageOptions, ProcessImageResult } from "./types.ts"; + +export { detectSupportedImageMimeType, detectSupportedImageMimeTypeFromFile } from "./mime.mjs"; +export type { ModelImageResizeOptions, ProcessImageOptions, ProcessImageResult, ResizedImage, SupportedImageMimeType } from "./types.ts"; + +function baseMimeType(mimeType: string): string { + return mimeType.split(";")[0]?.trim().toLowerCase() ?? mimeType.toLowerCase(); +} + +function normalizeSupportedImageMimeType(mimeType: string): string | null { + switch (baseMimeType(mimeType)) { + case "image/png": return "image/png"; + case "image/jpeg": + case "image/jpg": return "image/jpeg"; + case "image/gif": return "image/gif"; + case "image/webp": return "image/webp"; + default: return null; + } +} + +async function normalizeImage(bytes: Uint8Array, mimeType: string): Promise<{ bytes: Uint8Array; mimeType: string; convertedFrom?: string } | null> { + const normalizedMimeType = normalizeSupportedImageMimeType(mimeType); + if (normalizedMimeType) return { bytes, mimeType: normalizedMimeType }; + const pngBytes = await convertImageBytesToPng(bytes); + if (!pngBytes) return null; + return { bytes: pngBytes, mimeType: "image/png", convertedFrom: baseMimeType(mimeType) }; +} + +function conversionHint(from: string | undefined, to: string): string | undefined { + if (!from || from === to) return undefined; + return `[Image converted from ${from} to ${to}.]`; +} + +export async function processImage(bytes: Uint8Array, mimeType: string, options?: ProcessImageOptions): Promise { + const autoResizeImages = options?.autoResizeImages ?? true; + const normalized = await normalizeImage(bytes, mimeType); + if (!normalized) { + return { ok: false, message: "[Image omitted: could not be converted to a supported inline image format.]" }; + } + if (autoResizeImages) { + const resized = await resizeImage(normalized.bytes, normalized.mimeType, options?.resizeOptions); + if (!resized) { + return { ok: false, message: "[Image omitted: could not be resized below the inline image size limit.]" }; + } + const hints: string[] = []; + const convertedHint = conversionHint(normalized.convertedFrom, resized.mimeType); + if (convertedHint) hints.push(convertedHint); + const dimensionNote = formatDimensionNote(resized); + if (dimensionNote) hints.push(dimensionNote); + return { ok: true, data: resized.data, mimeType: resized.mimeType, hints }; + } + const hints: string[] = []; + const convertedHint = conversionHint(normalized.convertedFrom, normalized.mimeType); + if (convertedHint) hints.push(convertedHint); + return { ok: true, data: Buffer.from(normalized.bytes).toString("base64"), mimeType: normalized.mimeType, hints }; +} diff --git a/src/agent/image-processing/mime.d.mts b/src/agent/image-processing/mime.d.mts new file mode 100644 index 0000000..6451c62 --- /dev/null +++ b/src/agent/image-processing/mime.d.mts @@ -0,0 +1,3 @@ +import type { SupportedImageMimeType } from "./types.ts"; +export function detectSupportedImageMimeType(buffer: Uint8Array): SupportedImageMimeType | null; +export function detectSupportedImageMimeTypeFromFile(filePath: string): Promise; diff --git a/src/agent/image-processing/mime.mjs b/src/agent/image-processing/mime.mjs new file mode 100644 index 0000000..c9930d6 --- /dev/null +++ b/src/agent/image-processing/mime.mjs @@ -0,0 +1,110 @@ +// Ported from @earendil-works/pi-coding-agent 1.1.0, dist/utils/mime.js (MIT). +import { open } from "node:fs/promises"; +const IMAGE_TYPE_SNIFF_BYTES = 4100; +const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; +export function detectSupportedImageMimeType(buffer) { + if (startsWith(buffer, [0xff, 0xd8, 0xff])) { + return buffer[3] === 0xf7 ? null : "image/jpeg"; + } + if (startsWith(buffer, PNG_SIGNATURE)) { + return isPng(buffer) && !isAnimatedPng(buffer) ? "image/png" : null; + } + if (startsWithAscii(buffer, 0, "GIF87a") || startsWithAscii(buffer, 0, "GIF89a")) { + return "image/gif"; + } + if (startsWithAscii(buffer, 0, "RIFF") && startsWithAscii(buffer, 8, "WEBP")) { + return "image/webp"; + } + if (startsWithAscii(buffer, 0, "BM") && isBmp(buffer)) { + return "image/bmp"; + } + return null; +} +export async function detectSupportedImageMimeTypeFromFile(filePath) { + const fileHandle = await open(filePath, "r"); + try { + const buffer = Buffer.alloc(IMAGE_TYPE_SNIFF_BYTES); + const { bytesRead } = await fileHandle.read(buffer, 0, IMAGE_TYPE_SNIFF_BYTES, 0); + return detectSupportedImageMimeType(buffer.subarray(0, bytesRead)); + } + finally { + await fileHandle.close(); + } +} +function isPng(buffer) { + return (buffer.length >= 16 && readUint32BE(buffer, PNG_SIGNATURE.length) === 13 && startsWithAscii(buffer, 12, "IHDR")); +} +function isAnimatedPng(buffer) { + let offset = PNG_SIGNATURE.length; + while (offset + 8 <= buffer.length) { + const chunkLength = readUint32BE(buffer, offset); + const chunkTypeOffset = offset + 4; + if (startsWithAscii(buffer, chunkTypeOffset, "acTL")) + return true; + if (startsWithAscii(buffer, chunkTypeOffset, "IDAT")) + return false; + const nextOffset = offset + 8 + chunkLength + 4; + if (nextOffset <= offset || nextOffset > buffer.length) + return false; + offset = nextOffset; + } + return false; +} +function isBmp(buffer) { + if (buffer.length < 26) + return false; + const declaredFileSize = readUint32LE(buffer, 2); + const pixelDataOffset = readUint32LE(buffer, 10); + const dibHeaderSize = readUint32LE(buffer, 14); + if (declaredFileSize !== 0 && declaredFileSize < 26) + return false; + if (pixelDataOffset < 14 + dibHeaderSize) + return false; + if (declaredFileSize !== 0 && pixelDataOffset >= declaredFileSize) + return false; + let colorPlanes; + let bitsPerPixel; + if (dibHeaderSize === 12) { + colorPlanes = readUint16LE(buffer, 22); + bitsPerPixel = readUint16LE(buffer, 24); + } + else if (dibHeaderSize >= 40 && dibHeaderSize <= 124) { + if (buffer.length < 30) + return false; + colorPlanes = readUint16LE(buffer, 26); + bitsPerPixel = readUint16LE(buffer, 28); + } + else { + return false; + } + return colorPlanes === 1 && [1, 4, 8, 16, 24, 32].includes(bitsPerPixel); +} +function readUint16LE(buffer, offset) { + return (buffer[offset] ?? 0) + ((buffer[offset + 1] ?? 0) << 8); +} +function readUint32BE(buffer, offset) { + return ((buffer[offset] ?? 0) * 0x1000000 + + ((buffer[offset + 1] ?? 0) << 16) + + ((buffer[offset + 2] ?? 0) << 8) + + (buffer[offset + 3] ?? 0)); +} +function readUint32LE(buffer, offset) { + return ((buffer[offset] ?? 0) + + ((buffer[offset + 1] ?? 0) << 8) + + ((buffer[offset + 2] ?? 0) << 16) + + (buffer[offset + 3] ?? 0) * 0x1000000); +} +function startsWith(buffer, bytes) { + if (buffer.length < bytes.length) + return false; + return bytes.every((byte, index) => buffer[index] === byte); +} +function startsWithAscii(buffer, offset, text) { + if (buffer.length < offset + text.length) + return false; + for (let index = 0; index < text.length; index++) { + if (buffer[offset + index] !== text.charCodeAt(index)) + return false; + } + return true; +} diff --git a/src/agent/image-processing/photon.mjs b/src/agent/image-processing/photon.mjs new file mode 100644 index 0000000..713d627 --- /dev/null +++ b/src/agent/image-processing/photon.mjs @@ -0,0 +1,21 @@ +// Adapted from @earendil-works/pi-coding-agent 1.1.0, dist/utils/photon.js (MIT). +// Source-vs-runtime adaptation: this app runs in Node/Electron, not a Bun +// compiled executable. Photon loads its colocated WASM normally; no global fs +// monkeypatch or executable-relative WASM fallback is needed. +let photonModule = null; +let loadPromise = null; + +export async function loadPhoton() { + if (photonModule) return photonModule; + if (loadPromise) return loadPromise; + loadPromise = (async () => { + try { + photonModule = await import("@silvia-odwyer/photon-node"); + return photonModule; + } catch { + photonModule = null; + return photonModule; + } + })(); + return loadPromise; +} diff --git a/src/agent/image-processing/resize-core.d.mts b/src/agent/image-processing/resize-core.d.mts new file mode 100644 index 0000000..8c2a194 --- /dev/null +++ b/src/agent/image-processing/resize-core.d.mts @@ -0,0 +1,10 @@ +import type { ModelImageResizeOptions, ResizedImage } from "./types.ts"; +export const IMAGE_RESIZE_WORKER_RESPONSE_TYPE: "pi:image-resize-response"; +export function resizeImageInProcess(inputBytes: Uint8Array, mimeType: string, options?: ModelImageResizeOptions): Promise; +export type ResizeImageWorkerResponse = { + type: typeof IMAGE_RESIZE_WORKER_RESPONSE_TYPE; + result: ResizedImage | null; +} | { + type: typeof IMAGE_RESIZE_WORKER_RESPONSE_TYPE; + error: string; +}; diff --git a/src/agent/image-processing/resize-core.mjs b/src/agent/image-processing/resize-core.mjs new file mode 100644 index 0000000..3e89f7c --- /dev/null +++ b/src/agent/image-processing/resize-core.mjs @@ -0,0 +1,130 @@ +// Ported from @earendil-works/pi-coding-agent 1.1.0, dist/utils/image-resize-core.js (MIT). +// Runtime .mjs resources stay unbundled for Next's Node/Electron worker packaging. +import { applyExifOrientation } from "./exif-orientation.mjs"; +import { loadPhoton } from "./photon.mjs"; +/** + * Tag on image resize worker replies. Node can post its own messages on the worker channel + * (for example `{ "watch:require": [...] }` under `node --watch`), so replies must be identifiable. + */ +export const IMAGE_RESIZE_WORKER_RESPONSE_TYPE = "pi:image-resize-response"; +// 4.5MB of base64 payload. Provides headroom below Anthropic's 5MB limit. +const DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024; +const DEFAULT_OPTIONS = { + maxWidth: 2000, + maxHeight: 2000, + maxBytes: DEFAULT_MAX_BYTES, + jpegQuality: 80, +}; +function encodeCandidate(buffer, mimeType) { + const data = Buffer.from(buffer).toString("base64"); + return { + data, + encodedSize: Buffer.byteLength(data, "utf-8"), + mimeType, + }; +} +/** + * Resize an image to fit within the specified max dimensions and encoded file size. + * Returns null if the image cannot be resized below maxBytes. + * + * Uses Photon (Rust/WASM) for image processing. If Photon is not available, + * returns null. + * + * Strategy for staying under maxBytes: + * 1. First resize to maxWidth/maxHeight + * 2. Try PNG first, then JPEG qualities in insertion order (first under cap wins) + * 3. If still too large, try the remaining JPEG qualities + * 4. If still too large, progressively reduce dimensions until 1x1 + */ +export async function resizeImageInProcess(inputBytes, mimeType, options) { + const opts = { ...DEFAULT_OPTIONS, ...options }; + const inputBase64Size = Math.ceil(inputBytes.byteLength / 3) * 4; + const photon = await loadPhoton(); + if (!photon) { + return null; + } + let image; + try { + const rawImage = photon.PhotonImage.new_from_byteslice(inputBytes); + image = applyExifOrientation(photon, rawImage, inputBytes); + if (image !== rawImage) + rawImage.free(); + const originalWidth = image.get_width(); + const originalHeight = image.get_height(); + const format = mimeType.split("/")[1] ?? "png"; + // Check if already within all limits (dimensions AND encoded size) + if (originalWidth <= opts.maxWidth && originalHeight <= opts.maxHeight && inputBase64Size < opts.maxBytes) { + return { + data: Buffer.from(inputBytes).toString("base64"), + mimeType: mimeType || `image/${format}`, + originalWidth, + originalHeight, + width: originalWidth, + height: originalHeight, + wasResized: false, + }; + } + // Calculate initial dimensions respecting max limits + let targetWidth = originalWidth; + let targetHeight = originalHeight; + if (targetWidth > opts.maxWidth) { + targetHeight = Math.round((targetHeight * opts.maxWidth) / targetWidth); + targetWidth = opts.maxWidth; + } + if (targetHeight > opts.maxHeight) { + targetWidth = Math.round((targetWidth * opts.maxHeight) / targetHeight); + targetHeight = opts.maxHeight; + } + function tryEncodings(width, height, jpegQualities) { + const resized = photon.resize(image, width, height, photon.SamplingFilter.Lanczos3); + try { + const candidates = [encodeCandidate(resized.get_bytes(), "image/png")]; + for (const quality of jpegQualities) { + candidates.push(encodeCandidate(resized.get_bytes_jpeg(quality), "image/jpeg")); + } + return candidates; + } + finally { + resized.free(); + } + } + const qualitySteps = Array.from(new Set([opts.jpegQuality, 85, 70, 55, 40])); + let currentWidth = targetWidth; + let currentHeight = targetHeight; + while (true) { + const candidates = tryEncodings(currentWidth, currentHeight, qualitySteps); + for (const candidate of candidates) { + if (candidate.encodedSize < opts.maxBytes) { + return { + data: candidate.data, + mimeType: candidate.mimeType, + originalWidth, + originalHeight, + width: currentWidth, + height: currentHeight, + wasResized: true, + }; + } + } + if (currentWidth === 1 && currentHeight === 1) { + break; + } + const nextWidth = currentWidth === 1 ? 1 : Math.max(1, Math.floor(currentWidth * 0.75)); + const nextHeight = currentHeight === 1 ? 1 : Math.max(1, Math.floor(currentHeight * 0.75)); + if (nextWidth === currentWidth && nextHeight === currentHeight) { + break; + } + currentWidth = nextWidth; + currentHeight = nextHeight; + } + return null; + } + catch { + return null; + } + finally { + if (image) { + image.free(); + } + } +} diff --git a/src/agent/image-processing/resize-worker.mjs b/src/agent/image-processing/resize-worker.mjs new file mode 100644 index 0000000..bbc9e88 --- /dev/null +++ b/src/agent/image-processing/resize-worker.mjs @@ -0,0 +1,32 @@ +// Ported from @earendil-works/pi-coding-agent 1.1.0, dist/utils/image-resize-worker.js (MIT). +import { parentPort } from "node:worker_threads"; +import { IMAGE_RESIZE_WORKER_RESPONSE_TYPE, resizeImageInProcess, } from "./resize-core.mjs"; +function isResizeImageWorkerRequest(value) { + if (!value || typeof value !== "object") + return false; + const record = value; + return record.inputBytes instanceof Uint8Array && typeof record.mimeType === "string"; +} +const port = parentPort; +if (!port) { + throw new Error("image resize worker requires parentPort"); +} +port.once("message", (message) => { + void (async () => { + try { + if (!isResizeImageWorkerRequest(message)) { + throw new Error("Invalid image resize worker request"); + } + const result = await resizeImageInProcess(message.inputBytes, message.mimeType, message.options); + const response = { type: IMAGE_RESIZE_WORKER_RESPONSE_TYPE, result }; + port.postMessage(response); + } + catch (error) { + const response = { + type: IMAGE_RESIZE_WORKER_RESPONSE_TYPE, + error: error instanceof Error ? error.message : String(error), + }; + port.postMessage(response); + } + })(); +}); diff --git a/src/agent/image-processing/resize.ts b/src/agent/image-processing/resize.ts new file mode 100644 index 0000000..09115c0 --- /dev/null +++ b/src/agent/image-processing/resize.ts @@ -0,0 +1,65 @@ +// Ported from @earendil-works/pi-coding-agent 1.1.0, dist/utils/image-resize.js (MIT). +import path from "node:path"; +import { Worker } from "node:worker_threads"; +import { + IMAGE_RESIZE_WORKER_RESPONSE_TYPE, + resizeImageInProcess, + type ResizeImageWorkerResponse, +} from "./resize-core.mjs"; +import type { ModelImageResizeOptions, ResizedImage } from "./types.ts"; + +function isResizeImageWorkerResponse(value: unknown): value is ResizeImageWorkerResponse { + return value !== null && typeof value === "object" && "type" in value && value.type === IMAGE_RESIZE_WORKER_RESPONSE_TYPE; +} + +async function resizeImageInWorker(workerPath: string, inputBytes: Uint8Array, mimeType: string, options?: ModelImageResizeOptions): Promise { + const worker = new Worker(workerPath); + try { + // Transfer a worker-owned copy so the caller's bytes remain intact. + const inputBytesForWorker = new Uint8Array(inputBytes); + return await new Promise((resolve, reject) => { + let settled = false; + const settle = (result: ResizedImage | null) => { + if (settled) return; + settled = true; + resolve(result); + }; + const fail = (error: Error) => { + if (settled) return; + settled = true; + reject(error); + }; + worker.on("message", (message: unknown) => { + // Ignore Node's own worker-channel messages (e.g. node --watch). + if (!isResizeImageWorkerResponse(message)) return; + if ("error" in message) fail(new Error(message.error)); + else settle(message.result); + }); + worker.once("error", fail); + worker.once("exit", (code) => { + if (!settled) fail(new Error(`Image resize worker exited with code ${code}`)); + }); + worker.postMessage({ inputBytes: inputBytesForWorker, mimeType, options }, [inputBytesForWorker.buffer]); + }); + } finally { + void worker.terminate().catch(() => undefined); + } +} + +export async function resizeImage(inputBytes: Uint8Array, mimeType: string, options?: ModelImageResizeOptions): Promise { + // Source-vs-runtime adaptation: Next bundles the TS orchestration, but traces + // unbundled .mjs worker/core/EXIF/Photon resources at this cwd-relative path. + // The packaged server's cwd is the standalone root. No Bun path is needed. + const workerPath = path.join(process.cwd(), "src/agent/image-processing/resize-worker.mjs"); + try { + return await resizeImageInWorker(workerPath, inputBytes, mimeType, options); + } catch { + return resizeImageInProcess(inputBytes, mimeType, options); + } +} + +export function formatDimensionNote(result: ResizedImage): string | undefined { + if (!result.wasResized) return undefined; + const scale = result.originalWidth / result.width; + return `[Image: original ${result.originalWidth}x${result.originalHeight}, displayed at ${result.width}x${result.height}. Multiply coordinates by ${scale.toFixed(2)} to map to original image.]`; +} diff --git a/src/agent/image-processing/types.ts b/src/agent/image-processing/types.ts new file mode 100644 index 0000000..34cc117 --- /dev/null +++ b/src/agent/image-processing/types.ts @@ -0,0 +1,35 @@ +// Port baseline: @earendil-works/pi-coding-agent 1.1.0 image utilities (MIT). +/** Native resize overrides, supplied by the caller for the selected model. */ +export interface ModelImageResizeOptions { + maxWidth?: number; + maxHeight?: number; + maxBytes?: number; + jpegQuality?: number; +} + +export interface ResizedImage { + data: string; + mimeType: string; + originalWidth: number; + originalHeight: number; + width: number; + height: number; + wasResized: boolean; +} + +export interface ProcessImageOptions { + autoResizeImages?: boolean; + resizeOptions?: ModelImageResizeOptions; +} + +export type ProcessImageResult = { + ok: true; + data: string; + mimeType: string; + hints: string[]; +} | { + ok: false; + message: string; +}; + +export type SupportedImageMimeType = "image/jpeg" | "image/png" | "image/gif" | "image/webp" | "image/bmp"; diff --git a/src/agent/subagents/headless.ts b/src/agent/subagents/headless.ts index 2325fe9..6277017 100644 --- a/src/agent/subagents/headless.ts +++ b/src/agent/subagents/headless.ts @@ -461,6 +461,54 @@ async function renderSvg(svg: string, opts: RenderSvgOptions = {}): Promise Number.isFinite(n) && n > 0)) { + throw new Error("PDF page dimensions must be finite and positive"); + } + if (!Number.isFinite(dpi) || dpi <= 0 || dpi / 72 === 0) { + throw new Error("PDF render DPI must be finite and positive"); + } + // Limit sides first, so the area calculation cannot overflow even when + // the unscaled page dimensions or requested DPI are enormous. + let scale = Math.min(dpi / 72, MAX_PDF_CANVAS_SIDE / baseWidth, MAX_PDF_CANVAS_SIDE / baseHeight); + const area = (baseWidth * scale) * (baseHeight * scale); + if (area > MAX_PDF_CANVAS_PIXELS) scale *= Math.sqrt(MAX_PDF_CANVAS_PIXELS / area); + const dimensions = (s: number) => ({ + width: Math.max(1, Math.ceil(baseWidth * s)), + height: Math.max(1, Math.ceil(baseHeight * s)), + }); + const fits = ({ width, height }: { width: number; height: number }) => + width <= MAX_PDF_CANVAS_SIDE && height <= MAX_PDF_CANVAS_SIDE && width * height <= MAX_PDF_CANVAS_PIXELS; + // Ceil can put an otherwise bounded floating-point viewport over budget. + // Find a safe scale rather than clipping the viewport or allocating first. + if (!fits(dimensions(scale))) { + let low = 0; + let high = scale; + for (let i = 0; i < 64; i++) { + const mid = low + (high - low) / 2; + if (fits(dimensions(mid))) low = mid; + else high = mid; + } + scale = low; + } + if (scale <= 0 || baseWidth * scale === 0 || baseHeight * scale === 0) { + throw new Error("PDF page dimensions cannot be rendered at a positive scale"); + } + return { scale, ...dimensions(scale) }; +} + /** Render PDF pages to PNGs — the sources-v0 pdf-reader's eyes. * * History: pdfjs ALWAYS painted the pages (scale = dpi/72, the skill's @@ -475,6 +523,9 @@ async function renderPdfPages( opts: RenderPdfOptions = {}, ): Promise { const dpi = opts.dpi ?? 150; + if (!Number.isFinite(dpi) || dpi <= 0 || dpi / 72 === 0) { + throw new Error("PDF render DPI must be finite and positive"); + } // NATIVE CANVAS (Wave E, owner 2026-10-10): pdfjs paints into // @napi-rs/canvas in this process — no browser involved. pdfjs-dist v6's // own Node integration IS @napi-rs/canvas (its optionalDependency: the @@ -495,8 +546,10 @@ async function renderPdfPages( for (const pageNo of pages) { if (pageNo < 1 || pageNo > doc.numPages) continue; const pg = await doc.getPage(pageNo); - const vp = pg.getViewport({ scale: dpi / 72 }); - const canvas = createCanvas(Math.ceil(vp.width), Math.ceil(vp.height)); + const base = pg.getViewport({ scale: 1 }); + const geometry = pdfCanvasGeometry(base.width, base.height, dpi); + const vp = pg.getViewport({ scale: geometry.scale }); + const canvas = createCanvas(geometry.width, geometry.height); const ctx = canvas.getContext("2d"); // pdfjs's render types speak the DOM context; @napi-rs/canvas's // context is the same 2D API implemented over Skia. diff --git a/src/agent/subagents/pdf-reader.ts b/src/agent/subagents/pdf-reader.ts index c3feae2..250c158 100644 --- a/src/agent/subagents/pdf-reader.ts +++ b/src/agent/subagents/pdf-reader.ts @@ -1,6 +1,7 @@ import { randomUUID } from "node:crypto"; import type { AssistantMessage, Message, ToolResultMessage, Usage, Tool } from "@earendil-works/pi-ai"; import { normalizeContext } from "@earendil-works/pi-ai"; +import { processImage, type ProcessImageResult } from "../image-processing/index.ts"; import { Type } from "typebox"; import { resolveSourcePath } from "../tools/fs-shared.ts"; import { piEngineFor, type PiEngine, type ThinkingLevel } from "../models.ts"; @@ -246,7 +247,7 @@ export async function runPdfReaderAgent( const toolResults: ToolResultMessage[] = []; /** Rendered pages this batch, appended as images AFTER the tool * results — the visual-maker's image-in-the-loop idiom. */ - let renderedImages: { page: number; png: Buffer }[] | null = null; + let renderedImages: { page: number; image: ProcessImageResult }[] | null = null; for (const call of toolCalls) { const args = (call.arguments ?? {}) as Record; @@ -287,19 +288,26 @@ export async function runPdfReaderAgent( const wanted = specNumbers(spec, RENDER_BATCH_CAP); opts.onActivity?.(`pdf_render: ${spec} @${dpi}dpi`); const pages = await headless.renderPdfPages(pdfPath, wanted, { dpi }); + const processed: { page: number; image: ProcessImageResult }[] = []; for (const p of pages) { - consulted.add(p.page); - read.add(p.page); + const image = await processImage(p.png, "image/png", { resizeOptions: engine.model.inputLimits?.images?.resize }); + opts.abortSignal?.throwIfAborted(); + processed.push({ page: p.page, image }); + // Only images actually delivered to the model count as read. + if (image.ok) { + consulted.add(p.page); + read.add(p.page); + } } if (pages.length === 0) { toolResults.push(toolText(call.id, call.name, "ERROR: no pages rendered (spec matched nothing?)")); } else { - renderedImages = pages; + renderedImages = [...(renderedImages ?? []), ...processed]; toolResults.push( toolText( call.id, call.name, - `Rendered ${pages.length} page(s) at ${dpi} DPI — the images follow below. Read them carefully (equations, diagrams, layout).`, + `Rendered ${pages.length} page(s) at ${dpi} DPI — processed images or explicit omission notes follow below. Read the images carefully (equations, diagrams, layout).`, ), ); } @@ -323,11 +331,12 @@ export async function runPdfReaderAgent( type: "text", text: `The rendered page(s), in order (${renderedImages.map((p) => `p${p.page}`).join(", ")}):`, }, - ...renderedImages.map((p) => ({ - type: "image" as const, - data: p.png.toString("base64"), - mimeType: "image/png", - })), + ...renderedImages.flatMap(({ page, image }) => image.ok + ? [ + { type: "text" as const, text: [`Page ${page}`, ...image.hints].join("\n") }, + { type: "image" as const, data: image.data, mimeType: image.mimeType }, + ] + : [{ type: "text" as const, text: `Page ${page}: ${image.message}` }]), ], timestamp: Date.now(), }, diff --git a/src/agent/subagents/visual-maker.ts b/src/agent/subagents/visual-maker.ts index 8daac95..d0707c3 100644 --- a/src/agent/subagents/visual-maker.ts +++ b/src/agent/subagents/visual-maker.ts @@ -1,6 +1,7 @@ import { randomUUID } from "node:crypto"; import type { AssistantMessage, Message, Tool, ToolResultMessage, Usage } from "@earendil-works/pi-ai"; import { normalizeContext } from "@earendil-works/pi-ai"; +import { processImage, type ProcessImageResult } from "../image-processing/index.ts"; import { Type } from "typebox"; import { piEngineFor, type PiEngine, type ThinkingLevel } from "../models.ts"; import { providerEventObserver, type ProviderEventObserver } from "../provider-events.ts"; @@ -193,6 +194,9 @@ export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerO const msg = err instanceof Error ? err.message : String(err); return { svg: "", notes: `FAILED: rendering the final SVG failed — ${msg}` }; } + const image = await processImage(png, "image/png", { resizeOptions: engine.model.inputLimits?.images?.resize }); + opts.abortSignal?.throwIfAborted(); + if (!image.ok) return { svg: "", notes: `FAILED: inspecting the final SVG failed — ${image.message}` }; opts.onActivity?.("render_svg: inspecting final render"); renderedSvg = currentSvg; messages = [ @@ -200,8 +204,8 @@ export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerO { role: "user", content: [ - { type: "text", text: UNLOOKED_FINISH_MESSAGE }, - { type: "image", data: png.toString("base64"), mimeType: "image/png" }, + { type: "text", text: [UNLOOKED_FINISH_MESSAGE, ...image.hints].join("\n") }, + { type: "image", data: image.data, mimeType: image.mimeType }, ], timestamp: Date.now(), }, @@ -212,7 +216,7 @@ export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerO } const toolResults: ToolResultMessage[] = []; - let renderPng: Buffer | null = null; // appended as the image message AFTER the tool results + let renderImage: Extract | null = null; // appended AFTER the tool results for (const call of toolCalls) { if (call.name === "write_svg") { const svg = (call.arguments as { svg?: unknown }).svg; @@ -260,7 +264,10 @@ export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerO } try { const png = await headless.renderSvg(currentSvg); - renderPng = png; + const image = await processImage(png, "image/png", { resizeOptions: engine.model.inputLimits?.images?.resize }); + opts.abortSignal?.throwIfAborted(); + if (!image.ok) throw new Error(image.message); + renderImage = image; renderedSvg = currentSvg; opts.onActivity?.("render_svg: inspecting render"); toolResults.push(toolText(call.id, call.name, "Rendered — the PNG follows in the next message. Look at it.")); @@ -275,14 +282,14 @@ export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerO } // Message order: assistant tool-calls → tool results → render image. messages = [...messages, ...toolResults]; - if (renderPng) { + if (renderImage) { messages = [ ...messages, { role: "user", content: [ - { type: "text", text: LOOK_MESSAGE }, - { type: "image", data: renderPng.toString("base64"), mimeType: "image/png" }, + { type: "text", text: [LOOK_MESSAGE, ...renderImage.hints].join("\n") }, + { type: "image", data: renderImage.data, mimeType: renderImage.mimeType }, ], timestamp: Date.now(), }, diff --git a/src/agent/teacher-agent.ts b/src/agent/teacher-agent.ts index 6fe6ac1..f2178c8 100644 --- a/src/agent/teacher-agent.ts +++ b/src/agent/teacher-agent.ts @@ -17,6 +17,7 @@ import { import { practiceTools, type PracticeToolContext } from "./tools/practice.ts"; import { healToolRecord, withArgumentHeal } from "./tools/heal.ts"; import { sendToUserTool } from "./tools/send-to-user.ts"; +import { createReadTool, readTool } from "./tools/read.ts"; import { quizInputObject, quizInputProblems } from "../core/question-core.ts"; import { QUESTION_TOOLS, type EnabledTools } from "../core/pending.ts"; import { toLlmMessages, nativeInputCause, PRACTICE_PROBLEM_TOOL } from "../core/native-messages.ts"; @@ -201,7 +202,9 @@ function baseAgentOptions(kind: AgentKind, opts: AgentBuildOptions, tools: Agent // Match Claude across providers (including OpenRouter), not the API // transport. Explicit test registries remain caller-owned. tools: [ - ...tools, + ...tools.map((tool) => tool.name === "read" && tool.execute === readTool.execute + ? { ...tool, execute: createReadTool(engine.model).execute as AgentTool["execute"] } + : tool), ...(opts.tools === undefined && engine.model.id.toLowerCase().includes("claude") ? [withArgumentHeal(sendToUserTool)] : []), diff --git a/src/agent/tools/read.ts b/src/agent/tools/read.ts index 7550d70..c7a6300 100644 --- a/src/agent/tools/read.ts +++ b/src/agent/tools/read.ts @@ -1,9 +1,10 @@ import * as fs from "node:fs/promises"; -import * as path from "node:path"; import { Type } from "typebox"; +import type { Api, Model } from "@earendil-works/pi-ai"; import type { AgentTool } from "@earendil-works/pi-agent-core"; import { serverExecute, type ImageFileResult } from "../tool-execute.ts"; import { resolveSourcePath } from "./fs-shared.ts"; +import { detectSupportedImageMimeTypeFromFile, processImage } from "../image-processing/index.ts"; /** * The read tool — sources v0 (2026-09): a Node port of pi's read semantics @@ -31,17 +32,6 @@ import { resolveSourcePath } from "./fs-shared.ts"; const MAX_LINES = 2000; const MAX_BYTES = 50 * 1024; -/** Extension map for v0 (pi sniffs magic bytes; extensions suffice here — - * learner files are pasted paths, not adversarial uploads). */ -const IMAGE_MIME: Record = { - ".jpg": "image/jpeg", - ".jpeg": "image/jpeg", - ".png": "image/png", - ".gif": "image/gif", - ".webp": "image/webp", - ".bmp": "image/bmp", -}; - const readInput = Type.Object({ path: Type.String({ minLength: 1, description: "Path to the file to read (relative or absolute)" }), offset: Type.Optional(Type.Integer({ minimum: 1, description: "Line number to start reading from (1-indexed)" })), @@ -95,26 +85,39 @@ function readTextSlice(text: string, offset: number | undefined, limit: number | * fragment (prompts/shared/). Relative paths resolve against the * learner's home (fs-shared.ts), the harness's anchor. */ -export const readTool: AgentTool = { - name: "read", - label: "read", - description: `Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50.0KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.`, - parameters: readInput, - execute: serverExecute("read", readInput, async ({ path: rawPath, offset, limit }): Promise => { - const absolute = resolveSourcePath(rawPath); - const ext = path.extname(absolute).toLowerCase(); - const mime = IMAGE_MIME[ext]; - const buffer = await fs.readFile(absolute); // ENOENT/EACCES → error result the model sees - if (mime) { - // NOTE v0: no resize — raw bytes as base64. Learner files are study - // material (screenshots, figures), not adversarial payloads; a resize - // step (sharp) can land later if large images prove wasteful. - return { - text: `Read image file [${mime}] — ${path.basename(absolute)} (${formatSize(buffer.byteLength)}), attached as an image.`, - images: [{ data: buffer.toString("base64"), mimeType: mime }], - }; - } - const text = buffer.toString("utf-8"); - return readTextSlice(text, offset, limit); - }), -}; +export type ReadModel = Pick, "input" | "inputLimits">; + +/** One implementation for every learner-file reader. Bind the active model + * per agent, rather than sharing mutable model state between conversations. + * Home-relative paths, text slicing and the app's abort guard stay unchanged; + * image detection/processing and its notes follow native Pi read behavior. */ +export function createReadTool(model?: ReadModel): AgentTool { + return { + name: "read", + label: "read", + description: `Read the contents of a file. Supports text files and images (jpg, png, gif, webp, bmp). Images are sent as attachments. For text files, output is truncated to 2000 lines or 50.0KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.`, + parameters: readInput, + execute: serverExecute("read", readInput, async ({ path: rawPath, offset, limit }, { signal }): Promise => { + const absolute = resolveSourcePath(rawPath); + const mime = await detectSupportedImageMimeTypeFromFile(absolute); + const buffer = await fs.readFile(absolute, { signal }); + if (mime) { + const processed = await processImage(buffer, mime, { resizeOptions: model?.inputLimits?.images?.resize }); + signal?.throwIfAborted(); + const note = processed.ok + ? [`Read image file [${processed.mimeType}]`, ...processed.hints] + : [`Read image file [${mime}]`, processed.message]; + if (model && !model.input.includes("image")) { + note.push("[Current model does not support images. The image will be omitted from this request.]"); + } + return { + text: note.join("\n"), + ...(processed.ok ? { images: [{ data: processed.data, mimeType: processed.mimeType }] } : {}), + }; + } + return readTextSlice(buffer.toString("utf-8"), offset, limit); + }), + }; +} + +export const readTool = createReadTool(); diff --git a/tests/agent-image-paths.test.ts b/tests/agent-image-paths.test.ts new file mode 100644 index 0000000..7c5eb62 --- /dev/null +++ b/tests/agent-image-paths.test.ts @@ -0,0 +1,351 @@ +/** Offline integration contracts at the provider boundary: real shared image + * processing, faux model streams, synthetic rasters, and fake headless I/O. + * No real browser, provider, learner data, or ambient credentials are used. */ +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { after, test } from "node:test"; +import { createCanvas } from "@napi-rs/canvas"; +import * as photon from "@silvia-odwyer/photon-node"; +import { + createFauxCore, + fauxAssistantMessage, + fauxText, + fauxToolCall, + type FauxResponseStep, + type ImageContent, + type TranscriptContext, +} from "@earendil-works/pi-ai"; +import type { PiEngine } from "../src/agent/models.ts"; +import type { ModelImageResizeOptions } from "../src/agent/image-processing/index.ts"; +import type { HeadlessApi } from "../src/agent/subagents/headless.ts"; + +// Install storage/auth seams BEFORE importing the agent composition root. +const scratch = fs.mkdtempSync(path.join(os.tmpdir(), "lh-agent-image-paths-")); +const originalEnv = process.env; +const originalFetch = globalThis.fetch; +fs.mkdirSync(path.join(scratch, "home")); +fs.writeFileSync(path.join(scratch, "auth.json"), "{}"); +process.env = { + ...(originalEnv.NODE_TEST_CONTEXT ? { NODE_TEST_CONTEXT: originalEnv.NODE_TEST_CONTEXT } : {}), + ...(originalEnv.NODE_OPTIONS ? { NODE_OPTIONS: originalEnv.NODE_OPTIONS } : {}), + NODE_ENV: "test", + HOME: path.join(scratch, "home"), + LH_DATA_DIR: path.join(scratch, "data"), + LH_AUTH_FILE: path.join(scratch, "auth.json"), +}; +globalThis.fetch = async () => { throw new Error("CLOSED: unexpected network request"); }; +after(() => { + process.env = originalEnv; + globalThis.fetch = originalFetch; + fs.rmSync(scratch, { recursive: true, force: true }); +}); + +const { readTool } = await import("../src/agent/tools/read.ts"); +const { createTeacherAgent, createProberAgent, createPracticeAgent } = await import("../src/agent/teacher-agent.ts"); +const { runPdfReaderAgent } = await import("../src/agent/subagents/pdf-reader.ts"); +const { runVisualMaker } = await import("../src/agent/subagents/visual-maker.ts"); + +const PDF = path.join(import.meta.dirname, "fixtures", "sample-3page.pdf"); +const SVG = ''; +const CUSTOM_RESIZE = { maxWidth: 320, maxHeight: 200 }; + +function png(width: number, height: number): Buffer { + const canvas = createCanvas(width, height); + const context = canvas.getContext("2d"); + context.fillStyle = "#336699"; + context.fillRect(0, 0, width, height); + return canvas.toBuffer("image/png"); +} + +function dimensions(image: ImageContent): { width: number; height: number } { + const decoded = photon.PhotonImage.new_from_byteslice(Buffer.from(image.data, "base64")); + try { + return { width: decoded.get_width(), height: decoded.get_height() }; + } finally { decoded.free(); } +} + +function images(context: TranscriptContext): ImageContent[] { + return context.messages.flatMap((message) => Array.isArray(message.content) + ? message.content.filter((part): part is ImageContent => part.type === "image") + : []); +} + +function text(context: TranscriptContext): string { + return context.messages.map((message) => typeof message.content === "string" + ? message.content + : message.content.filter((part) => part.type === "text").map((part) => part.text).join("\n")).join("\n"); +} + +function scripted(steps: FauxResponseStep[], resize?: ModelImageResizeOptions, vision = true) { + const faux = createFauxCore({ tokenSize: { min: 64, max: 64 } }); + faux.setResponses(steps); + const base = faux.getModel(); + const model: PiEngine["model"] = { + ...base, + input: vision ? ["text", "image"] : ["text"], + ...(resize ? { inputLimits: { ...base.inputLimits, images: { ...base.inputLimits?.images, resize } } } : {}), + }; + const requests: TranscriptContext[] = []; + const engine: PiEngine = { + model, + streamFn: (model, context, options) => { + // Snapshot the actual provider-facing context, not mutable agent state. + requests.push(structuredClone(context)); + return faux.streamSimple(model, context, options); + }, + }; + return { engine, requests }; +} + +function readSteps(file: string): FauxResponseStep[] { + return [ + fauxAssistantMessage([fauxToolCall("read", { path: file }, { id: "read-1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("Inspected the supplied source.")]), + ]; +} + +function fakeHeadless(rendered: Buffer | ((page: number) => Buffer)) { + const svgCalls: string[] = []; + const pdfCalls: number[][] = []; + const headless: HeadlessApi = { + available: () => true, + unavailableReason: () => null, + runDemo: async () => { throw new Error("unexpected browser execution"); }, + renderSvg: async (svg) => { + svgCalls.push(svg); + return typeof rendered === "function" ? rendered(1) : rendered; + }, + renderPdfPages: async (_file, pages) => { + pdfCalls.push([...pages]); + return pages.map((page) => ({ page, png: typeof rendered === "function" ? rendered(page) : rendered })); + }, + }; + return { headless, svgCalls, pdfCalls }; +} + +function assertProcessedImage(context: TranscriptContext, expected: { width: number; height: number }, raw: Buffer): void { + const parts = images(context); + assert.equal(parts.length, 1, "exactly one successful image reaches the faux provider"); + assert.deepEqual(dimensions(parts[0]!), expected); + assert.ok(Buffer.byteLength(parts[0]!.data, "utf8") < 4.5 * 1024 * 1024); + assert.notEqual(parts[0]!.data, raw.toString("base64"), "the oversized raw raster is never sent"); + assert.match(text(context), /\[Image: original \d+x\d+, displayed at \d+x\d+\. Multiply coordinates by/); +} + +test("shared teacher read detects valid image content under a non-image extension", async () => { + const bytes = png(24, 16); + const file = path.join(scratch, "lecture-notes.txt"); + fs.writeFileSync(file, bytes); + const { engine, requests } = scripted(readSteps(file)); + // Pass the actual shared read tool, exercising the teacher's native clone + // of read.execute rather than a pre-bound test-only implementation. + const agent = createTeacherAgent(undefined, { engine, tools: { read: readTool } }); + await agent.prompt("Inspect the supplied local file."); + assert.equal(requests.length, 2); + const visible = requests[1]!; + assert.equal(images(visible).length, 1); + assert.equal(images(visible)[0]!.mimeType, "image/png"); + assert.equal(images(visible)[0]!.data, bytes.toString("base64")); + assert.deepEqual(dimensions(images(visible)[0]!), { width: 24, height: 16 }); + assert.match(text(visible), /Read image file \[image\/png\]/); +}); + +test("shared read omits corrupt recognized images without exposing a raw image fallback", async () => { + // Valid signature/IHDR, missing pixel data: sniff succeeds, decode fails. + const bytes = png(24, 16).subarray(0, 33); + const file = path.join(scratch, "corrupt.png"); + fs.writeFileSync(file, bytes); + const { engine, requests } = scripted(readSteps(file)); + await createTeacherAgent(undefined, { engine, tools: { read: readTool } }).prompt("Inspect the file."); + assert.equal(requests.length, 2); + assert.deepEqual(images(requests[1]!), []); + assert.match(text(requests[1]!), /Image omitted: could not be resized below the inline image size limit/); + assert.ok(!JSON.stringify(requests[1]).includes(bytes.toString("base64"))); +}); + +test("shared read supplies the native nonvision-model note", async () => { + const file = path.join(scratch, "nonvision.png"); + fs.writeFileSync(file, png(24, 16)); + const { engine, requests } = scripted(readSteps(file), undefined, false); + await createTeacherAgent(undefined, { engine, tools: { read: readTool } }).prompt("Inspect the file."); + assert.equal(requests.length, 2); + assert.match(text(requests[1]!), /\[Current model does not support images\. The image will be omitted from this request\.\]/); +}); + +test("teacher read binds independent active-model resize limits without mutating the shared tool", async () => { + const bytes = png(640, 320); + const file = path.join(scratch, "model-bound.png"); + fs.writeFileSync(file, bytes); + const originalExecute = readTool.execute; + const a = scripted(readSteps(file), { maxWidth: 96, maxHeight: 64 }); + const b = scripted(readSteps(file), { maxWidth: 80, maxHeight: 30 }); + const agentA = createTeacherAgent(undefined, { engine: a.engine, tools: { read: readTool } }); + const agentB = createTeacherAgent(undefined, { engine: b.engine, tools: { read: readTool } }); + await agentB.prompt("Inspect with model B."); + await agentA.prompt("Inspect with model A."); + assertProcessedImage(a.requests[1]!, { width: 96, height: 48 }, bytes); + assertProcessedImage(b.requests[1]!, { width: 60, height: 30 }, bytes); + assert.equal(readTool.execute, originalExecute); +}); + +for (const [role, resize, expected] of [ + ["prober", { maxWidth: 96, maxHeight: 64 }, { width: 96, height: 48 }], + ["practice", { maxWidth: 80, maxHeight: 30 }, { width: 60, height: 30 }], +] as const) { + test(`${role}'s actual registry retains argument healing and text reads while binding image limits`, async () => { + const bytes = png(640, 320); + const imageFile = path.join(scratch, `${role}-actual-registry.png`); + const textFile = path.join(scratch, `${role}-actual-registry.txt`); + fs.writeFileSync(imageFile, bytes); + fs.writeFileSync(textFile, "first\nsecond\nthird\nfourth\nfifth"); + const { engine, requests } = scripted([ + fauxAssistantMessage([fauxToolCall("read", { path: imageFile }, { id: "registry-image" })], { stopReason: "toolUse" }), + // Exact existing heal contract: schema-integer fields encoded as + // JSON number strings heal before validation. The path stays intact. + fauxAssistantMessage([fauxToolCall("read", { path: textFile, offset: "2", limit: "2" }, { id: "registry-text" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("Inspected the image and selected text lines.")]), + ], resize); + const originalExecute = readTool.execute; + const forbidden = () => { throw new Error("unexpected job dispatch or handoff"); }; + const chatStore = { getChat: () => null }; + // Only context ports are mocked; no app/store imports or explicit + // tool overrides. Job/handoff operations must never execute here. + const agent = role === "prober" + ? createProberAgent({ + chatId: "offline-read-prober", chatStore, + jobStore: new Proxy({}, { get: forbidden }), + spawn: forbidden, deliver: forbidden, + } as never, { engine }) + : createPracticeAgent({ chatId: "offline-read-practice", chatStore }, { engine }); + const names = agent.state.tools.map((tool) => tool.name); + assert.ok(names.includes(role === "prober" ? "probe_done" : "problem"), "real role registry was assembled"); + const boundRead = agent.state.tools.find((tool) => tool.name === "read"); + assert.ok(boundRead?.prepareArguments, "model binding preserves the registry's prepareArguments wrapper"); + assert.notEqual(boundRead.execute, originalExecute, "shared read execute is rebound to the active model"); + assert.deepEqual(boundRead.prepareArguments({ path: textFile, offset: "2", limit: "2" }), { path: textFile, offset: 2, limit: 2 }); + await agent.prompt("Inspect the local image, then read the selected text lines."); + assert.equal(requests.length, 3); + assertProcessedImage(requests[1]!, expected, bytes); + const selected = requests[2]!.messages.find((message) => message.role === "toolResult" && message.toolCallId === "registry-text"); + assert.ok(selected && selected.role === "toolResult"); + assert.equal(selected.isError, false); + assert.deepEqual(selected.content, [{ type: "text", text: "second\nthird\n\n[2 more lines in file. Use offset=4 to continue.]" }]); + assert.equal(readTool.execute, originalExecute, "role construction did not mutate the shared tool"); + }); +} + +test("PDF reader preserves both processed image outputs from multiple render calls in one batch", async () => { + const wide = png(4800, 120); + const tall = png(120, 4800); + const { headless, pdfCalls } = fakeHeadless((page) => page === 1 ? wide : tall); + const { engine, requests } = scripted([ + fauxAssistantMessage([ + fauxToolCall("pdf_render", { pages: "1", dpi: 150 }, { id: "batch-render-1" }), + fauxToolCall("pdf_render", { pages: "2", dpi: 150 }, { id: "batch-render-2" }), + ], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("Both rendered pages are visible (p. 1, p. 2). Pages consulted: 1, 2")]), + ]); + const result = await runPdfReaderAgent({ path: PDF, question: "Compare the two figures." }, { engine, headless }); + assert.equal(requests.length, 2); + assert.deepEqual(pdfCalls, [[1], [2]]); + const visible = requests[1]!; + const parts = images(visible); + assert.equal(parts.length, 2, "the second render call must not overwrite the first call's images"); + assert.deepEqual(parts.map(dimensions), [{ width: 2000, height: 50 }, { width: 50, height: 2000 }]); + for (let i = 0; i < parts.length; i++) { + assert.ok(parts[i]!.data.length < 4.5 * 1024 * 1024); + assert.notEqual(parts[i]!.data, [wide, tall][i]!.toString("base64")); + } + assert.deepEqual(visible.messages.slice(-3).map((message) => message.role), ["toolResult", "toolResult", "user"]); + const pageMessage = visible.messages.at(-1)!; + assert.ok(Array.isArray(pageMessage.content)); + assert.deepEqual(pageMessage.content.filter((part) => part.type === "text").map((part) => part.text.split("\n")[0]), [ + "The rendered page(s), in order (p1, p2):", "Page 1", "Page 2", + ]); + assert.deepEqual(result.pagesRead, [1, 2]); + assert.deepEqual(result.pagesConsulted, [1, 2]); +}); + +for (const [label, resize, expected] of [ + ["native defaults", undefined, { width: 2000, height: 50 }], + ["active-model overrides", CUSTOM_RESIZE, { width: 320, height: 8 }], +] as const) { + test(`PDF reader sends processed renders using ${label}; failed page is not counted`, async () => { + const bytes = png(4800, 120); + const { headless, pdfCalls } = fakeHeadless((page) => page === 1 ? bytes : Buffer.from("invalid rendered page")); + const { engine, requests } = scripted([ + fauxAssistantMessage([fauxToolCall("pdf_render", { pages: "1-2", dpi: 150 }, { id: "render-1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("The visible figure is on page 1 (p. 1). Pages consulted: 1")]), + ], resize); + const result = await runPdfReaderAgent({ path: PDF, question: "Inspect the page figure." }, { engine, headless }); + assert.equal(requests.length, 2); + assert.deepEqual(pdfCalls, [[1, 2]]); + assertProcessedImage(requests[1]!, expected, bytes); + assert.match(text(requests[1]!), /Page 2: \[Image omitted:/); + assert.deepEqual(result.pagesRead, [1]); + assert.deepEqual(result.pagesConsulted, [1]); + assert.match(result.brief, /visible figure/); + const visible = requests[1]!.messages; + assert.equal(visible.at(-2)!.role, "toolResult"); + assert.equal(visible.at(-1)!.role, "user", "processed page images ride the user message after tool results"); + }); + + for (const forced of [false, true]) { + test(`visual maker ${forced ? "forced-final" : "explicit render_svg"} path processes images using ${label}`, async () => { + const bytes = png(4800, 120); + const { headless, svgCalls } = fakeHeadless(bytes); + const steps: FauxResponseStep[] = forced ? [ + fauxAssistantMessage([fauxToolCall("write_svg", { svg: SVG }, { id: "write-1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("Premature notes without inspection.")]), + fauxAssistantMessage([fauxText("Confirmed after inspecting the final render.")]), + ] : [ + fauxAssistantMessage([ + fauxToolCall("write_svg", { svg: SVG }, { id: "write-1" }), + fauxToolCall("render_svg", {}, { id: "render-1" }), + ], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("Confirmed after inspecting the render.")]), + ]; + const { engine, requests } = scripted(steps, resize); + const result = await runVisualMaker({ brief: "One precise synthetic figure." }, { engine, headless }); + assert.equal(requests.length, forced ? 3 : 2); + assert.deepEqual(svgCalls, [SVG]); + assertProcessedImage(requests.at(-1)!, expected, bytes); + assert.equal(result.svg, SVG); + assert.match(result.notes!, /^Confirmed after inspecting/); + assert.equal(requests.at(-1)!.messages.at(-1)!.role, "user"); + if (forced) { + assert.deepEqual(images(requests[1]!), [], "premature finish did not see an image yet"); + assert.match(text(requests[2]!), /You finished without looking at a render of your final SVG/); + } else { + assert.match(text(requests[1]!), /Here is the current render of your SVG/); + assert.equal(requests[1]!.messages.at(-2)!.role, "toolResult"); + } + }); + } +} + +for (const forced of [false, true]) { + test(`visual maker ${forced ? "forced-final" : "explicit render_svg"} processing failure never verifies or delivers the SVG`, async () => { + const { headless, svgCalls } = fakeHeadless(Buffer.from("corrupt render")); + const steps: FauxResponseStep[] = [ + fauxAssistantMessage([ + fauxToolCall("write_svg", { svg: SVG }, { id: "write-1" }), + ...(!forced ? [fauxToolCall("render_svg", {}, { id: "render-1" })] : []), + ], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("Unverified claim that the picture is correct.")]), + ]; + const { engine, requests } = scripted(steps); + const activity: string[] = []; + const result = await runVisualMaker({ brief: "One synthetic figure." }, { engine, headless, onActivity: (line) => activity.push(line) }); + assert.equal(requests.length, 2); + assert.ok(requests.every((request) => images(request).length === 0)); + assert.equal(result.svg, ""); + assert.match(result.notes!, /^FAILED: inspecting the final SVG failed/); + assert.match(result.notes!, /Image omitted:/); + assert.ok(!activity.some((line) => line.includes("inspecting")), "a failed image is never marked inspected"); + assert.equal(svgCalls.length, forced ? 1 : 2, "an explicit processing failure leaves the final-SVG gate unsatisfied"); + if (!forced) assert.match(text(requests[1]!), /RENDER ERROR.*Image omitted:/); + }); +} diff --git a/tests/demos.test.tsx b/tests/demos.test.tsx index d3e1665..48b3098 100644 --- a/tests/demos.test.tsx +++ b/tests/demos.test.tsx @@ -28,6 +28,7 @@ import assert from "node:assert/strict"; import * as fs from "node:fs"; import * as path from "node:path"; import { after, describe, it } from "node:test"; +import { createCanvas } from "@napi-rs/canvas"; import { Value } from "typebox/value"; import { createFauxCore, fauxAssistantMessage, fauxToolCall, type FauxResponseStep, type ToolCall } from "@earendil-works/pi-ai"; import { renderToStaticMarkup } from "react-dom/server"; @@ -171,7 +172,11 @@ function fakeHeadless(opts: { if (opts.renderFailures && api.svgRenders.length <= opts.renderFailures) { throw new Error("XML parse error"); } - return Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); // PNG magic + // The shared native image processor decodes renders, so a signature + // alone is no longer a successful fake raster. + const canvas = createCanvas(1, 1); + canvas.getContext("2d").fillRect(0, 0, 1, 1); + return canvas.toBuffer("image/png"); }, }; return api as HeadlessApi & { demoRuns: string[]; svgRenders: string[] }; diff --git a/tests/image-processing.test.ts b/tests/image-processing.test.ts new file mode 100644 index 0000000..3fb0478 --- /dev/null +++ b/tests/image-processing.test.ts @@ -0,0 +1,357 @@ +import assert from "node:assert/strict"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { Worker } from "node:worker_threads"; +import { createCanvas } from "@napi-rs/canvas"; +import * as photon from "@silvia-odwyer/photon-node"; +import { + detectSupportedImageMimeType, + detectSupportedImageMimeTypeFromFile, + processImage, +} from "../src/agent/image-processing/index.ts"; +import { applyExifOrientation } from "../src/agent/image-processing/exif-orientation.mjs"; +import { resizeImageInProcess } from "../src/agent/image-processing/resize-core.mjs"; +import { resizeImage } from "../src/agent/image-processing/resize.ts"; + +// Synthetic, offline fixtures only. Canvas generates inputs; Photon is the +// actual processing implementation and supplies independent encoding oracles. +function png(width = 96, height = 96, noise = false): Buffer { + const canvas = createCanvas(width, height); + const context = canvas.getContext("2d"); + if (noise) { + const pixels = context.createImageData(width, height); + let state = 0x12345678; + for (let i = 0; i < pixels.data.length; i += 4) { + for (let channel = 0; channel < 3; channel++) { + state = (Math.imul(state, 1664525) + 1013904223) >>> 0; + pixels.data[i + channel] = state >>> 24; + } + pixels.data[i + 3] = 255; + } + context.putImageData(pixels, 0, 0); + } else { + context.fillStyle = "#33aadd"; + context.fillRect(0, 0, width, height); + } + return canvas.toBuffer("image/png"); +} + +function encodings(bytes: Uint8Array, width: number, height: number, qualities = [80, 85, 70, 55, 40]) { + const image = photon.PhotonImage.new_from_byteslice(bytes); + const resized = photon.resize(image, width, height, photon.SamplingFilter.Lanczos3); + try { + return [ + { mimeType: "image/png", data: Buffer.from(resized.get_bytes()).toString("base64") }, + ...qualities.map((quality) => ({ mimeType: "image/jpeg", data: Buffer.from(resized.get_bytes_jpeg(quality)).toString("base64") })), + ]; + } finally { + resized.free(); + image.free(); + } +} + +function bmp(): Buffer { + // A 2x1, uncompressed, 24-bit Windows BMP (padded scanline). + const bytes = Buffer.alloc(62); + bytes.write("BM"); + bytes.writeUInt32LE(bytes.length, 2); + bytes.writeUInt32LE(54, 10); + bytes.writeUInt32LE(40, 14); + bytes.writeInt32LE(2, 18); + bytes.writeInt32LE(1, 22); + bytes.writeUInt16LE(1, 26); + bytes.writeUInt16LE(24, 28); + bytes.writeUInt32LE(8, 34); + bytes.set([0, 0, 255, 0, 255, 0], 54); + return bytes; +} + +function exifJpeg(orientation: number): Buffer { + const image = photon.PhotonImage.new_from_byteslice(png(12, 8)); + let jpeg: Buffer; + try { + jpeg = Buffer.from(image.get_bytes_jpeg(80)); + } finally { + image.free(); + } + const exif = Buffer.alloc(36); + exif.set([0xff, 0xe1, 0, 34]); + exif.write("Exif\0\0", 4, "binary"); + exif.write("II", 10); + exif.writeUInt16LE(42, 12); + exif.writeUInt32LE(8, 14); + exif.writeUInt16LE(1, 18); + exif.writeUInt16LE(0x0112, 20); + exif.writeUInt16LE(3, 22); + exif.writeUInt32LE(1, 24); + exif.writeUInt16LE(orientation, 28); + return Buffer.concat([jpeg.subarray(0, 2), exif, jpeg.subarray(2)]); +} + +test("already-fit image bytes are unchanged, MIME aliases normalize, and transfer leaves input intact", async () => { + const bytes = png(); + const original = Buffer.from(bytes); + const result = await processImage(bytes, " Image/PNG; charset=binary "); + assert.deepEqual(result, { ok: true, data: bytes.toString("base64"), mimeType: "image/png", hints: [] }); + assert.deepEqual(bytes, original); + const jpeg = exifJpeg(1); + assert.deepEqual(await processImage(jpeg, "image/jpg"), { ok: true, data: jpeg.toString("base64"), mimeType: "image/jpeg", hints: [] }); +}); + +test("native default dimensions are 2000x2000 and dimension hints use the native mapping", async () => { + const bytes = png(12000, 24); + const result = await processImage(bytes, "image/png"); + assert.ok(result.ok); + assert.equal(result.mimeType, "image/png"); + assert.deepEqual(result.hints, ["[Image: original 12000x24, displayed at 2000x4. Multiply coordinates by 6.00 to map to original image.]"]); + const decoded = photon.PhotonImage.new_from_byteslice(Buffer.from(result.data, "base64")); + try { + assert.equal(decoded.get_width(), 2000); + assert.equal(decoded.get_height(), 4); + } finally { decoded.free(); } + const tall = await resizeImage(png(24, 12000), "image/png"); + assert.ok(tall); + assert.equal(tall.width, 4); + assert.equal(tall.height, 2000); +}); + +test("model dimensions use sequential rounding, not a one-step scale", async () => { + const result = await resizeImage(png(301, 503), "image/png", { maxWidth: 101, maxHeight: 100 }); + assert.ok(result); + // 301x503 -> 101x169 -> 60x100. + assert.equal(result.width, 60); + assert.equal(result.height, 100); + assert.equal(result.data, encodings(png(301, 503), 60, 100)[0]!.data); + const rounding = await resizeImage(png(50, 77), "image/png", { maxWidth: 10, maxHeight: 10 }); + assert.ok(rounding); + assert.equal(rounding.width, 7); // round(100 / round(77 * 10 / 50)); one-step gives 6. + assert.equal(rounding.height, 10); +}); + +test("native cap is strict and measures base64 bytes, not binary bytes", async () => { + const source = png(24, 24); + // A valid PNG with ignored trailing bytes: under 4.5 MiB binary, exactly + // 4.5 MiB base64. At equality the unchanged fast path must NOT run. + const padded = Buffer.alloc((4.5 * 1024 * 1024 / 4) * 3); + source.copy(padded); + assert.ok(padded.length < 4.5 * 1024 * 1024); + assert.equal(padded.toString("base64").length, 4.5 * 1024 * 1024); + const exact = await processImage(padded, "image/png"); + assert.ok(exact.ok); + assert.notEqual(exact.data, padded.toString("base64")); + assert.ok(exact.data.length < 4.5 * 1024 * 1024); + assert.equal(exact.hints.length, 1, "native re-encoding is marked resized even at identical dimensions"); + const justBelow = padded.subarray(0, padded.length - 3); + const below = await processImage(justBelow, "image/png"); + assert.ok(below.ok); + assert.equal(below.data, justBelow.toString("base64")); + assert.deepEqual(below.hints, []); +}); + +test("PNG wins first under cap even when JPEG is smaller", async () => { + const bytes = png(96, 96, true); + const candidates = encodings(bytes, 95, 95); + assert.ok(candidates[0]!.data.length > candidates[1]!.data.length); + const result = await resizeImage(bytes, "image/png", { maxWidth: 95, maxBytes: candidates[0]!.data.length + 1 }); + assert.ok(result); + assert.equal(result.mimeType, "image/png"); + assert.equal(result.data, candidates[0]!.data); +}); + +test("JPEG quality selection is native first-fit [80,85,70,55,40], not smallest output", async () => { + const bytes = png(96, 96, true); + const candidates = encodings(bytes, 96, 96); + assert.ok(candidates[0]!.data.length > candidates[2]!.data.length); + assert.ok(candidates[2]!.data.length > candidates[1]!.data.length); + assert.ok(candidates[1]!.data.length > candidates[3]!.data.length); + const at80 = await resizeImage(bytes, "image/png", { maxBytes: candidates[1]!.data.length + 1 }); + assert.ok(at80); + assert.equal(at80.data, candidates[1]!.data); + // Strict equality excludes 80; 85 is larger, so 70 is the next fit. + const at70 = await resizeImage(bytes, "image/png", { maxBytes: candidates[1]!.data.length }); + assert.ok(at70); + assert.equal(at70.data, candidates[3]!.data); + assert.equal(at70.width, 96); + assert.equal(at70.height, 96); + for (const [index, quality] of [[4, 55], [5, 40]] as const) { + const candidate = candidates[index]!; + const cap = candidate.data.length + 1; + assert.ok(candidates.slice(0, index).every((earlier) => earlier.data.length >= cap)); + const lowerQuality = await resizeImage(bytes, "image/png", { maxBytes: cap }); + assert.ok(lowerQuality); + assert.equal(lowerQuality.data, candidate.data, `native quality ${quality} wins when preceding candidates are over cap`); + assert.equal(lowerQuality.width, 96); + assert.equal(lowerQuality.height, 96); + } + const override = await resizeImage(bytes, "image/png", { jpegQuality: 85, maxBytes: candidates[2]!.data.length + 1 }); + assert.ok(override); + assert.equal(override.data, candidates[2]!.data, "caller quality precedes the native remaining quality steps"); +}); + +test("all encodings over cap reduce each dimension by floor(0.75), eventually omitting at 1x1", async () => { + const bytes = png(97, 95, true); + const full = encodings(bytes, 97, 95); + const next = encodings(bytes, 72, 71); + const maxBytes = Math.min(...full.map((candidate) => candidate.data.length)); + const expected = next.find((candidate) => candidate.data.length < maxBytes); + assert.ok(expected, "synthetic fixture fits on the first reduced dimensions"); + const result = await resizeImage(bytes, "image/png", { maxBytes }); + assert.ok(result); + assert.equal(result.width, 72); + assert.equal(result.height, 71); + assert.equal(result.mimeType, expected.mimeType); + assert.equal(result.data, expected.data); + assert.equal(await resizeImage(bytes, "image/png", { maxBytes: 1 }), null); + const omitted = await processImage(bytes, "image/png", { resizeOptions: { maxBytes: 1 } }); + assert.deepEqual(omitted, { ok: false, message: "[Image omitted: could not be resized below the inline image size limit.]" }); +}); + +test("BMP converts to PNG and reports the native conversion hint", async () => { + const bytes = bmp(); + assert.equal(detectSupportedImageMimeType(bytes), "image/bmp"); + const result = await processImage(bytes, "IMAGE/BMP"); + assert.ok(result.ok); + assert.equal(result.mimeType, "image/png"); + assert.equal(detectSupportedImageMimeType(Buffer.from(result.data, "base64")), "image/png"); + assert.deepEqual(result.hints, ["[Image converted from image/bmp to image/png.]"]); + const decoded = photon.PhotonImage.new_from_byteslice(Buffer.from(result.data, "base64")); + try { + assert.equal(decoded.get_width(), 2); + assert.equal(decoded.get_height(), 1); + } finally { decoded.free(); } +}); + +test("EXIF-oriented dimensions govern resizing, with native unchanged-byte behavior when already fit", async () => { + for (const orientation of [2, 3, 4, 5, 6, 7, 8]) { + const bytes = exifJpeg(orientation); + const untouched = await resizeImage(bytes, "image/jpeg"); + assert.ok(untouched); + assert.equal(untouched.data, bytes.toString("base64")); + assert.equal(untouched.originalWidth, orientation >= 5 ? 8 : 12); + assert.equal(untouched.originalHeight, orientation >= 5 ? 12 : 8); + const resized = await resizeImage(bytes, "image/jpeg", { maxWidth: 6, maxHeight: 6 }); + assert.ok(resized); + assert.equal(resized.width, orientation >= 5 ? 4 : 6); + assert.equal(resized.height, orientation >= 5 ? 6 : 4); + } +}); + +test("EXIF mirrors and rotations preserve exact pixel placement for JPEG and both WebP EXIF layouts", () => { + const expected = [ + [1, 2, 3, 4, 5, 6], // unchanged + [3, 2, 1, 6, 5, 4], // horizontal mirror + [6, 5, 4, 3, 2, 1], // 180 degrees + [4, 5, 6, 1, 2, 3], // vertical mirror + [1, 4, 2, 5, 3, 6], // transpose + [4, 1, 5, 2, 6, 3], // 90 degrees clockwise + [6, 3, 5, 2, 4, 1], // transverse + [3, 6, 2, 5, 1, 4], // 90 degrees counterclockwise + ]; + for (let orientation = 1; orientation <= 8; orientation++) { + const jpeg = exifJpeg(orientation); + // Only metadata is needed here: decoding is covered by the resize test. + const exif = jpeg.subarray(6, 38); + for (const metadata of [jpeg, ...[exif, exif.subarray(6)].map((data) => { + // Odd-size RIFF chunk before EXIF verifies native even-byte padding. + const webp = Buffer.alloc(12 + 10 + 8 + data.length); + webp.write("RIFF", 0); + webp.writeUInt32LE(webp.length - 8, 4); + webp.write("WEBP", 8); + webp.write("JUNK", 12); + webp.writeUInt32LE(1, 16); + webp.write("EXIF", 22); + webp.writeUInt32LE(data.length, 26); + data.copy(webp, 30); + return webp; + })]) { + const pixels = Uint8Array.from([1, 2, 3, 4, 5, 6].flatMap((value) => [value, 0, 0, 255])); + const raw = new photon.PhotonImage(pixels, 3, 2); + const oriented = applyExifOrientation(photon, raw, metadata); + try { + assert.equal(oriented.get_width(), orientation >= 5 ? 2 : 3); + assert.equal(oriented.get_height(), orientation >= 5 ? 3 : 2); + assert.deepEqual(Array.from(oriented.get_raw_pixels()).filter((_, index) => index % 4 === 0), expected[orientation - 1]); + } finally { + if (oriented !== raw) oriented.free(); + raw.free(); + } + } + } +}); + +test("corrupt supported input is omitted, unsupported conversion fails, and explicit autoResizeImages=false retains native bypass", async () => { + const bytes = Buffer.from("not an image"); + assert.deepEqual(await processImage(bytes, "image/png"), { ok: false, message: "[Image omitted: could not be resized below the inline image size limit.]" }); + assert.deepEqual(await processImage(bytes, "image/bmp"), { ok: false, message: "[Image omitted: could not be converted to a supported inline image format.]" }); + assert.deepEqual(await processImage(bytes, "image/png", { autoResizeImages: false }), { ok: true, data: bytes.toString("base64"), mimeType: "image/png", hints: [] }); + const wide = png(12000, 24); + const bypass = await processImage(wide, "image/png", { autoResizeImages: false }); + assert.ok(bypass.ok); + assert.equal(bypass.data, wide.toString("base64")); + const converted = await processImage(bmp(), "image/bmp", { autoResizeImages: false }); + assert.ok(converted.ok); + assert.equal(converted.mimeType, "image/png"); +}); + +test("MIME sniffing matches native supported signatures and rejects APNG/JPEG-LS/malformed BMP", async () => { + assert.equal(detectSupportedImageMimeType(png()), "image/png"); + assert.equal(detectSupportedImageMimeType(Buffer.from([0xff, 0xd8, 0xff, 0xe0])), "image/jpeg"); + assert.equal(detectSupportedImageMimeType(Buffer.from([0xff, 0xd8, 0xff, 0xf7])), null); + assert.equal(detectSupportedImageMimeType(Buffer.from("GIF89a")), "image/gif"); + assert.equal(detectSupportedImageMimeType(Buffer.from("GIF87a")), "image/gif"); + assert.equal(detectSupportedImageMimeType(Buffer.from("RIFFxxxxWEBP")), "image/webp"); + assert.equal(detectSupportedImageMimeType(Buffer.from("BM")), null); + assert.equal(detectSupportedImageMimeType(Buffer.from("not an image")), null); + const truncatedPng = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); + assert.equal(detectSupportedImageMimeType(truncatedPng), null); + // MIME sniffing, like Pi, walks chunk headers without checking CRCs. + const source = png(2, 2); + const acTL = Buffer.alloc(20); + acTL.writeUInt32BE(8, 0); + acTL.write("acTL", 4); + const apng = Buffer.concat([source.subarray(0, 33), acTL, source.subarray(33)]); + assert.equal(detectSupportedImageMimeType(apng), null); + const badBmp = bmp(); + badBmp.writeUInt16LE(2, 26); + assert.equal(detectSupportedImageMimeType(badBmp), null); + const dir = await mkdtemp(path.join(os.tmpdir(), "lh-image-processing-")); + try { + await writeFile(path.join(dir, "renamed.txt"), source); + await writeFile(path.join(dir, "animated.png"), apng); + assert.equal(await detectSupportedImageMimeTypeFromFile(path.join(dir, "renamed.txt")), "image/png"); + assert.equal(await detectSupportedImageMimeTypeFromFile(path.join(dir, "animated.png")), null); + await assert.rejects(detectSupportedImageMimeTypeFromFile(path.join(dir, "missing")), { code: "ENOENT" }); + } finally { await rm(dir, { recursive: true, force: true }); } +}); + +test("unbundled Node worker responds with tagged output and matches the in-process fallback", async () => { + const bytes = png(96, 96, true); + const options = { maxWidth: 64, maxHeight: 64, maxBytes: 4000 }; + const worker = new Worker(path.join(process.cwd(), "src/agent/image-processing/resize-worker.mjs")); + try { + const reply = new Promise<{ type: string; result: unknown }>((resolve, reject) => { + worker.once("message", resolve); + worker.once("error", reject); + }); + worker.postMessage({ inputBytes: new Uint8Array(bytes), mimeType: "image/png", options }); + assert.deepEqual(await reply, { type: "pi:image-resize-response", result: await resizeImageInProcess(bytes, "image/png", options) }); + assert.deepEqual(await resizeImage(bytes, "image/png", options), await resizeImageInProcess(bytes, "image/png", options)); + } finally { await worker.terminate(); } +}); + +test("missing cwd-relative worker falls back in-process without altering the caller bytes", async () => { + const cwd = process.cwd(); + const dir = await mkdtemp(path.join(os.tmpdir(), "lh-image-worker-fallback-")); + const bytes = png(); + const options = { maxWidth: 64, maxHeight: 64 }; + const expected = await resizeImageInProcess(bytes, "image/png", options); + try { + process.chdir(dir); + assert.deepEqual(await resizeImage(bytes, "image/png", options), expected); + } finally { + process.chdir(cwd); + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/tests/pdf-native-render.test.ts b/tests/pdf-native-render.test.ts index 017aa8c..2675dc6 100644 --- a/tests/pdf-native-render.test.ts +++ b/tests/pdf-native-render.test.ts @@ -8,9 +8,10 @@ */ import assert from "node:assert/strict"; import * as fs from "node:fs"; +import * as os from "node:os"; import * as path from "node:path"; import { describe, it } from "node:test"; -import { realHeadless } from "../src/agent/subagents/headless.ts"; +import { MAX_PDF_CANVAS_PIXELS, MAX_PDF_CANVAS_SIDE, pdfCanvasGeometry, realHeadless } from "../src/agent/subagents/headless.ts"; import { standardFontsDir } from "../src/agent/subagents/pdf-ops.ts"; const FIXTURE = path.join(import.meta.dirname, "fixtures", "sample-3page.pdf"); @@ -31,6 +32,83 @@ function pngFacts(buf: Buffer): { w: number; h: number; idats: number } { return { w, h, idats }; } +/** Synthetic page only — no personal PDF or external fixture generator. */ +function oversizedPdf(): Buffer { + const content = "0 0 1 rg 0 0 20000 10000 re f\n"; + const objects = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 20000 10000] /Rotate 90 /Resources << >> /Contents 4 0 R >>", + `<< /Length ${Buffer.byteLength(content)} >>\nstream\n${content}endstream`, + ]; + let pdf = "%PDF-1.7\n"; + const offsets = [0]; + for (const [index, object] of objects.entries()) { + offsets.push(Buffer.byteLength(pdf)); + pdf += `${index + 1} 0 obj\n${object}\nendobj\n`; + } + const xref = Buffer.byteLength(pdf); + pdf += `xref\n0 ${offsets.length}\n0000000000 65535 f \n`; + for (const offset of offsets.slice(1)) pdf += `${String(offset).padStart(10, "0")} 00000 n \n`; + pdf += `trailer\n<< /Size ${offsets.length} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`; + return Buffer.from(pdf); +} + +describe("PDF pre-allocation canvas geometry", () => { + it("uses decimal 16 megapixels and an 8192-pixel side limit", () => { + assert.equal(MAX_PDF_CANVAS_PIXELS, 16_000_000); + assert.equal(MAX_PDF_CANVAS_SIDE, 8192); + }); + + it("preserves ordinary requested DPI, including fractional ceil dimensions and low DPI", () => { + for (const dpi of [11, 72, 144, 150, 300]) { + const geometry = pdfCanvasGeometry(595.28, 841.89, dpi); + assert.deepEqual(geometry, { + scale: dpi / 72, + width: Math.ceil(595.28 * dpi / 72), + height: Math.ceil(841.89 * dpi / 72), + }); + } + }); + + it("bounds extreme aspect ratios, huge geometry, sides, and rounded pixel area without upscaling", () => { + const cases: readonly (readonly [number, number, number])[] = [ + [1e12, 0.000001, 300], + [0.000001, 1e12, 300], + [1e308, 1e308, Number.MAX_VALUE], + [8192.000001, 10, 72], + [6100, 3900, 72], // ceil at the area-limited scale exceeds 16MP + [4000.000001, 4000.000001, 72], + [4000, 4000, 72], // exactly 16MP stays unchanged + ]; + for (const [width, height, dpi] of cases) { + const result = pdfCanvasGeometry(width, height, dpi); + assert.ok(result.scale > 0 && result.scale <= dpi / 72); + assert.equal(result.width, Math.max(1, Math.ceil(width * result.scale))); + assert.equal(result.height, Math.max(1, Math.ceil(height * result.scale))); + assert.ok(result.width <= MAX_PDF_CANVAS_SIDE && result.height <= MAX_PDF_CANVAS_SIDE); + assert.ok(result.width * result.height <= MAX_PDF_CANVAS_PIXELS); + } + assert.deepEqual(pdfCanvasGeometry(4000, 4000, 72), { scale: 1, width: 4000, height: 4000 }); + }); + + it("uses rotated viewport dimensions symmetrically", () => { + const normal = pdfCanvasGeometry(20000, 10000, 144); + const rotated = pdfCanvasGeometry(10000, 20000, 144); + assert.equal(rotated.scale, normal.scale); + assert.equal(rotated.width, normal.height); + assert.equal(rotated.height, normal.width); + }); + + it("rejects nonfinite or nonpositive dimensions and DPI cleanly", () => { + for (const invalid of [0, -1, NaN, Infinity, -Infinity]) { + assert.throws(() => pdfCanvasGeometry(invalid, 792, 72), /dimensions must be finite and positive/); + assert.throws(() => pdfCanvasGeometry(612, invalid, 72), /dimensions must be finite and positive/); + assert.throws(() => pdfCanvasGeometry(612, 792, invalid), /DPI must be finite and positive/); + } + }); +}); + describe("native pdf_render (pdfjs paints, @napi-rs/canvas hosts)", () => { it("the shared font directory ends in pdfjs's required slash on every OS", () => { const fonts = standardFontsDir(); @@ -63,6 +141,32 @@ describe("native pdf_render (pdfjs paints, @napi-rs/canvas hosts)", () => { assert.ok(Math.abs(fb.h - 2 * fa.h) <= 2, `144dpi height ≈ 2× 72dpi (${fa.h} → ${fb.h})`); }); + it("bounds a synthetic oversized rotated PDF before native canvas allocation", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "lh-pdf-canvas-test-")); + try { + const fixture = path.join(dir, "oversized.pdf"); + fs.writeFileSync(fixture, oversizedPdf()); + const [page] = await realHeadless.renderPdfPages(fixture, [1], { dpi: 144 }); + assert.ok(page); + const facts = pngFacts(page.png); + const expected = pdfCanvasGeometry(10000, 20000, 144); + assert.equal(facts.w, expected.width); + assert.equal(facts.h, expected.height); + assert.ok(facts.h > facts.w, "PDF rotation is included before scaling"); + assert.ok(facts.w <= 8192 && facts.h <= 8192); + assert.ok(facts.w * facts.h <= 16_000_000); + assert.equal(facts.idats, 1, "raw renderer still returns canonical PNG"); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects invalid DPI before opening the PDF", async () => { + for (const dpi of [0, -1, NaN, Infinity, -Infinity, Number.MIN_VALUE]) { + await assert.rejects(realHeadless.renderPdfPages("unused.pdf", [1], { dpi }), /DPI must be finite and positive/); + } + }); + it("skips out-of-range page numbers silently", async () => { const pages = await realHeadless.renderPdfPages(FIXTURE, [0, 2, 99], { dpi: 72 }); assert.deepEqual( diff --git a/tests/provider-tracing.test.ts b/tests/provider-tracing.test.ts index 0c255e4..f9ad601 100644 --- a/tests/provider-tracing.test.ts +++ b/tests/provider-tracing.test.ts @@ -45,6 +45,9 @@ describe("standalone JavaScript tracing exclusions and native workaround policy" "node_modules/pdfjs-dist/legacy/**/*", "node_modules/pdfjs-dist/standard_fonts/**/*", "node_modules/@napi-rs/canvas-*/**/*", + "src/agent/image-processing/*.mjs", + "src/agent/image-processing/LICENSE", + "node_modules/@silvia-odwyer/photon-node/**/*", "node_modules/@resvg/resvg-js-*/**/*", "node_modules/@earendil-works/pi-ai/dist/providers/data/**/*", "node_modules/@earendil-works/pi-ai/dist/auth/oauth/{load,github-copilot,xai,device-code}.js", diff --git a/tests/read-tool.test.ts b/tests/read-tool.test.ts index 6b000d3..b15d70f 100644 --- a/tests/read-tool.test.ts +++ b/tests/read-tool.test.ts @@ -10,10 +10,12 @@ import assert from "node:assert/strict"; import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; -import { describe, it } from "node:test"; +import { after, describe, it } from "node:test"; +import { createCanvas } from "@napi-rs/canvas"; import { readTool } from "../src/agent/tools/read.ts"; const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "lh-read-test-")); +after(() => fs.rmSync(tmp, { recursive: true, force: true })); function write(name: string, content: string): string { const file = path.join(tmp, name); @@ -100,8 +102,10 @@ describe("read tool — text semantics", () => { describe("read tool — image semantics (native content parts)", () => { it("returns native text + image parts for image files, details null (output-less part)", async () => { const file = path.join(tmp, "dot.png"); - // minimal valid PNG header bytes are enough — detection is by extension - fs.writeFileSync(file, Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])); + // Detection now sniffs bytes and the native processor decodes the image. + const canvas = createCanvas(1, 1); + canvas.getContext("2d").fillRect(0, 0, 1, 1); + fs.writeFileSync(file, canvas.toBuffer("image/png")); const result = await readTool.execute!("test", { path: file }); assert.equal(result.content.length, 2); const parts = result.content as Array<{ type: string; text?: string; data?: string; mimeType?: string }>;