fix(pdf): make pdfjs survive bundling + real-PDF font fidelity (live smoke)

Live dispatch smoke against the Turbopack dev server (master's ask,
2026-09-19) caught two things unit tests cannot - they run unbundled:

- Bundling pdfjs-dist broke its node fake-worker: the chunk-relative
  pdf.worker.mjs import 404s inside .next/dev/server/chunks. Fix:
  serverExternalPackages + tracing includes for legacy/ + standard_fonts/
  (next.config.ts - a clearly-marked seam, same precedent as bcdcd51; the
  standalone output now ships the worker and font files, verified).
- Real-world PDFs warned about missing standardFontDataUrl (non-embedded
  standard fonts silently fall back) and flooded the log with JBIG2
  warnings on scanned files. Fix: standardFontsDir() + verbosity 0 in
  both the node ops and the headless canvas shim.

Verified live end-to-end on GEB (802pp, text layer): teacher dispatch,
12-step read with vision renders, brief with accurate page citations,
wake turn folding the resolution (Chapter IX, pp. 268-270) into the
lesson. Scanned/DRM Polya correctly refuses to fabricate citations.
This commit is contained in:
amosblomqvist
2026-09-19 16:23:20 +03:00
parent 31f600a365
commit 2780166904
3 changed files with 28 additions and 3 deletions
+9
View File
@@ -19,8 +19,16 @@ const nextConfig: NextConfig = {
// pi-ai's provider catalog (39 JSON files) vanished from the trace and
// the server 500s on module load. Include the whole data dirs of the
// runtime-loaded engine packages.
// pdfjs-dist (sources v0 live smoke, 2026-09-19): its legacy build loads
// the node fake worker (pdf.worker.mjs) and our standard_fonts dir via
// runtime-resolved paths — invisible to the tracer. Bundling (without
// serverExternalPackages below) additionally breaks the worker's
// import.meta.url resolution inside .next/dev chunks — the live smoke
// caught it; unit tests run unbundled and never see it.
outputFileTracingIncludes: {
"*": [
"node_modules/pdfjs-dist/legacy/**/*",
"node_modules/pdfjs-dist/standard_fonts/**/*",
"node_modules/@earendil-works/pi-ai/dist/providers/data/**/*",
"node_modules/@earendil-works/pi-agent-core/dist/**/*.json",
// onnxruntime-node: the .node binding was traced, but its
@@ -38,6 +46,7 @@ const nextConfig: NextConfig = {
"@earendil-works/pi-ai",
"@earendil-works/pi-agent-core",
"@huggingface/transformers",
"pdfjs-dist", // sources v0: bundling breaks its node fake-worker path
],
};
+7 -2
View File
@@ -326,8 +326,12 @@ async function renderPdfPages(
const page = await browser.newPage();
await page.setViewport({ width: 1200, height: 1600, deviceScaleFactor: 1 });
await page.goto(pathToFileURL(file).href, { waitUntil: "load", timeout: 30_000 });
// Standard-font data as a file URL (live-smoke finding 2026-09-19:
// without it pdfjs renders non-embedded standard fonts via fallback);
// verbosity 0 = errors only (JBIG2 noise on scanned PDFs).
const fontsUrl = pathToFileURL(path.join(PATHS_nodeModules(), "pdfjs-dist", "standard_fonts") + path.sep).href;
const results = await page.evaluate(
async (b64pdf, workerUrl, pageNos, dpi) => {
async (b64pdf, workerUrl, pageNos, dpi, fontsUrl) => {
// window-scope dynamic import (Chrome evaluates this string in the
// shim page, not under Node)
const pdfjs = (await import(workerUrl)) as typeof import("pdfjs-dist");
@@ -335,7 +339,7 @@ async function renderPdfPages(
const bin = atob(b64pdf);
const bytes = new Uint8Array(bin.length);
for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i);
const doc = await pdfjs.getDocument({ data: bytes }).promise;
const doc = await pdfjs.getDocument({ data: bytes, standardFontDataUrl: fontsUrl, verbosity: 0 }).promise;
const out: Array<{ page: number; dataUrl: string; w: number; h: number }> = [];
for (const pageNo of pageNos) {
if (pageNo < 1 || pageNo > doc.numPages) continue;
@@ -355,6 +359,7 @@ async function renderPdfPages(
pdfjsUrl,
pages,
dpi,
fontsUrl,
);
return results.map((r) => ({
page: r.page,
+12 -1
View File
@@ -1,5 +1,6 @@
import * as fs from "node:fs/promises";
import * as path from "node:path";
import { fileURLToPath } from "node:url";
/**
* PDF plumbing for the pdf-reader sub-agent — sources v0 (2026-09): Node
@@ -42,12 +43,22 @@ async function loadPdfjs() {
return pdfjs;
}
/** Standard-font data dir (live-smoke finding 2026-09-19: without it pdfjs
* warns and falls back on non-embedded standard fonts — Helvetica, Times…).
* The node side takes a plain filesystem path. Resolved from this module's
* location, the headless.ts PATHS_nodeModules idiom. */
export function standardFontsDir(): string {
return path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "..", "..", "node_modules", "pdfjs-dist", "standard_fonts") + path.sep;
}
async function openDocument(filePath: string) {
const pdfjs = await loadPdfjs();
const data = new Uint8Array(await fs.readFile(filePath));
// The legacy build in plain node runs on a main-thread fake worker (no
// workerSrc needed); teardown is the loading task's destroy().
const task = pdfjs.getDocument({ data });
// verbosity 0 = errors only — scanned PDFs (JBIG2 XObjects) otherwise
// flood the log with per-page warnings (live-smoke finding 2026-09-19).
const task = pdfjs.getDocument({ data, standardFontDataUrl: standardFontsDir(), verbosity: 0 });
return { doc: await task.promise, task };
}