From 8cb5e1469103232377babd451dc8a21c8c3388e6 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Thu, 8 Oct 2026 17:35:15 +0300 Subject: [PATCH] =?UTF-8?q?desktop:=20the=20payload=20fixed=20=E2=80=94=20?= =?UTF-8?q?7.4GB=E2=86=92144MB,=20dev=20keys=20out,=20boot-smoke=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - desktop:dist cleans dist/ + .next/standalone before next build: the trace sweep was shipping the PREVIOUS build's DMG inside every new one (standalone/dist = 8.0GB inside the app; measured 7.4GB DMG → 144MB). - the same sweep shipped the repo-root auth.json (dev BYOK keys, byte-identical) in every DMG ever built — excluded in the trace and REFUSED by a new build-server.mjs guard (.env* belt-and-braces). - WARNING (next.config.ts): dir-style trace excludes (dist/** etc., "./"-anchored or not) strip server-external packages to bare package.jsons — pi-ai/pi-agent-core lost their dist/, the packaged server 500'd on ERR_MODULE_NOT_FOUND. Only the three verified-safe entries stay. - new scripts/smoke-server.mjs gates desktop:dist between payload curation and packaging: boots the standalone exactly as the Electron shell does (temp LH_DATA_DIR, empty auth) and requires 200s on / + graph/models/settings. Caught both defects above on its first day. - entitlements: allow-unsigned-executable-memory dropped (2019-era relic per current electron-builder docs; weakens hardened runtime). - pi-ai + pi-agent-core pinned exact 1.0.2 — persisted chat records carry their wire shapes; post-release a bump is a migration event. Gates: typecheck, lint, 1459/1459 tests, full desktop:dist + smoke PASS. --- build/entitlements.mac.plist | 12 ++- next.config.ts | 24 +++++ package-lock.json | 4 +- package.json | 6 +- scripts/build-server.mjs | 13 +++ scripts/smoke-server.mjs | 177 +++++++++++++++++++++++++++++++++++ 6 files changed, 226 insertions(+), 10 deletions(-) create mode 100644 scripts/smoke-server.mjs diff --git a/build/entitlements.mac.plist b/build/entitlements.mac.plist index f854f57..c52640a 100644 --- a/build/entitlements.mac.plist +++ b/build/entitlements.mac.plist @@ -3,16 +3,18 @@ com.apple.security.cs.allow-jit - com.apple.security.cs.allow-unsigned-executable-memory - com.apple.security.device.audio-input diff --git a/next.config.ts b/next.config.ts index 6fef519..01d7d88 100644 --- a/next.config.ts +++ b/next.config.ts @@ -11,9 +11,33 @@ const nextConfig: NextConfig = { // tracing sweeps the repo's real data/ — chat records, graph ledgers — // INTO the standalone bundle. A shipped desktop app would carry the // developer's session data inside it. + // + // SECRETS (2026-10-08, caught in the payload audit): the same sweep + // ships the repo-root auth.json — the DEV BYOK key store — byte-identical + // into the DMG (the packaged app never reads it: the shell passes + // LH_AUTH_FILE → userData). Excluded here; build-server.mjs also REFUSES + // to package any standalone still containing it. .env* belt-and-braces + // the same class (not observed swept, same seam). + // + // ⚠ EXCLUDE PATTERNS ARE LOAD-BEARING IN WAYS THE DOCS DON'T DESCRIBE + // (empirically established 2026-10-08): adding dir excludes like + // "dist/**/*" / "./dist/**/*" for repo-root output dirs STRIPPED THE + // server-external packages (pi-ai / pi-agent-core lost everything + // under their dist/ — the traced copy collapsed to a bare package.json + // and the packaged server died on ERR_MODULE_NOT_FOUND; openai/ + // @anthropic-ai — pi-ai's deps — vanished from the trace entirely). + // Anchored "./"-prefixed forms behave the same. Only the three entries + // below are verified safe (complete externals trace + clean payload). + // The 7.4 GB recursive-DMG payload (repo dist/ swept into the bundle, + // shipping the previous build inside the next) is instead prevented by + // the `rm -rf dist .next/standalone` clean step in desktop:dist — + // verified sufficient, since the sweep can only copy what exists at + // trace time. scripts/build-server.mjs guards the rest. outputFileTracingExcludes: { "*": [ "data/**/*", + "auth.json", + ".env*", ], }, // Empirically verified gaps (2026-09-17 standalone smoke): the tracer diff --git a/package-lock.json b/package-lock.json index fbab39d..460fd7d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,8 +8,8 @@ "name": "learning-harness", "version": "0.2.0", "dependencies": { - "@earendil-works/pi-agent-core": "^1.0.2", - "@earendil-works/pi-ai": "^1.0.2", + "@earendil-works/pi-agent-core": "1.0.2", + "@earendil-works/pi-ai": "1.0.2", "@mozilla/readability": "^0.6.0", "@radix-ui/react-use-controllable-state": "^1.2.6", "@rive-app/react-webgl2": "^4.32.1", diff --git a/package.json b/package.json index 4a991f1..5149919 100644 --- a/package.json +++ b/package.json @@ -14,11 +14,11 @@ "typecheck": "tsc --noEmit", "desktop:dev": "concurrently -k \"next dev\" \"LH_DEV_URL=http://localhost:3000 electron .\"", "dev:fresh": "bash scripts/dev-fresh.sh", - "desktop:dist": "next build && node scripts/build-server.mjs && electron-builder --mac --arm64" + "desktop:dist": "rm -rf dist .next/standalone && next build && node scripts/build-server.mjs && node scripts/smoke-server.mjs && electron-builder --mac --arm64" }, "dependencies": { - "@earendil-works/pi-agent-core": "^1.0.2", - "@earendil-works/pi-ai": "^1.0.2", + "@earendil-works/pi-agent-core": "1.0.2", + "@earendil-works/pi-ai": "1.0.2", "@mozilla/readability": "^0.6.0", "@radix-ui/react-use-controllable-state": "^1.2.6", "@rive-app/react-webgl2": "^4.32.1", diff --git a/scripts/build-server.mjs b/scripts/build-server.mjs index b720846..cae13ec 100644 --- a/scripts/build-server.mjs +++ b/scripts/build-server.mjs @@ -45,6 +45,19 @@ if (exists(leakedData)) { process.exit(1); } +// 2b) Safety: the repo's dev SECRETS must never ride along either — the +// same trace sweep that catches data/ shipped the repo-root auth.json +// (the DEV BYOK key store, 0600) byte-identical into the DMG (caught in +// the 2026-10-08 payload audit; excluded in next.config.ts — this guard +// makes a regression loud instead of silent). +for (const secret of ["auth.json", ".env", ".env.local"]) { + if (exists(path.join(STANDALONE, secret))) { + console.error(`[build-server] REFUSING: standalone contains ${secret} — dev secrets would ship in the app.`); + console.error(" Check outputFileTracingExcludes in next.config.ts."); + process.exit(1); + } +} + // 3) Sanity: the runtime-read prompt assets must be present (the teacher // agent reads them from disk at request time). for (const asset of [ diff --git a/scripts/smoke-server.mjs b/scripts/smoke-server.mjs new file mode 100644 index 0000000..023873f --- /dev/null +++ b/scripts/smoke-server.mjs @@ -0,0 +1,177 @@ +/** + * smoke-server.mjs — boot the packaged standalone server and prove it serves. + * + * The payload gate that pays for itself (2026-10-08): `next build` success + * and a completed electron-builder run say NOTHING about whether the + * packaged server actually starts. On its first day this smoke caught two + * real defects nothing else saw: (1) a trace-exclude regression that + * stripped the pi engine packages to bare package.jsons — the server + * answered 500 on every route with ERR_MODULE_NOT_FOUND; (2) before that, + * the dev auth.json (BYOK keys) shipping inside the bundle. It runs + * between build-server.mjs and electron-builder so a broken payload fails + * in seconds, not after a full packaging pass. + * + * What it does: spawns .next/standalone/server.js exactly the way the + * Electron shell does (electron/main.cjs) — 127.0.0.1 on a kernel-assigned + * free port, NODE_ENV=production, a THROWAWAY LH_DATA_DIR and an EMPTY + * LH_AUTH_FILE under os.tmpdir() (no real keys or data ever touch it) — + * polls GET / until 200, then probes the GET-only API surface (graph / + * models / settings — no side effects). Any non-200, an early server + * exit, or a 30s boot timeout fails the gate and prints the server's own + * log tail for diagnosis. The child is ALWAYS killed before exit — a + * leaked server holds its port against the next run (learned the hard + * way, same day). + */ +import { spawn } from "node:child_process"; +import * as fs from "node:fs"; +import * as http from "node:http"; +import * as net from "node:net"; +import * as os from "node:os"; +import * as path from "node:path"; + +const ROOT = path.resolve(import.meta.dirname, ".."); +const STANDALONE = path.join(ROOT, ".next", "standalone"); +const SERVER = path.join(STANDALONE, "server.js"); + +const POLL_MS = 500; +const BOOT_TIMEOUT_MS = 30_000; +/** GET-only probes — the routes that exercised today's failure class. */ +const PROBES = ["/api/graph", "/api/models", "/api/settings"]; + +if (!fs.existsSync(SERVER)) { + console.error("[smoke-server] .next/standalone/server.js missing — run next build first."); + process.exit(1); +} + +/** A free TCP port (listen(0) → the kernel assigns one) — mirrors main.cjs. */ +function freePort() { + return new Promise((resolve, reject) => { + const srv = net.createServer(); + srv.listen(0, "127.0.0.1", () => { + const { port } = srv.address(); + srv.close(() => resolve(port)); + }); + srv.on("error", reject); + }); +} + +/** One GET; resolves the status code (0 = connection refused/failed). */ +function get(url) { + return new Promise((resolve) => { + const req = http.get(url, (res) => { + res.resume(); + resolve(res.statusCode ?? 0); + }); + req.on("error", () => resolve(0)); + req.setTimeout(5000, () => { + req.destroy(); + resolve(0); + }); + }); +} + +async function main() { + const port = await freePort(); + const url = `http://127.0.0.1:${port}`; + const home = fs.mkdtempSync(path.join(os.tmpdir(), "lh-smoke-")); + const dataDir = path.join(home, "data"); + const authFile = path.join(home, "auth.json"); + fs.writeFileSync(authFile, "{}\n", "utf-8"); + + const tail = []; // rolling log tail for failure diagnosis + let tailBytes = 0; + const keep = (buf) => { + for (const line of buf.toString().split("\n")) { + tail.push(line); + tailBytes += line.length + 1; + } + while (tailBytes > 16_000 && tail.length > 2) { + tailBytes -= tail[0].length + 1; + tail.shift(); + } + }; + const dumpTail = () => { + console.error("--- server log tail " + "-".repeat(20)); + for (const line of tail.slice(-40)) if (line.trim()) console.error(` ${line}`); + }; + + const child = spawn(process.execPath, [SERVER], { + cwd: STANDALONE, + env: { + ...process.env, + NODE_ENV: "production", + PORT: String(port), + HOSTNAME: "127.0.0.1", + LH_DATA_DIR: dataDir, + LH_AUTH_FILE: authFile, + }, + stdio: ["ignore", "pipe", "pipe"], + }); + let exited = false; + child.stdout.on("data", keep); + child.stderr.on("data", keep); + child.on("exit", () => { + exited = true; + }); + + const kill = () => + new Promise((resolve) => { + if (exited) return resolve(); + const done = () => resolve(); + child.once("exit", done); + child.kill("SIGTERM"); + setTimeout(() => { + if (!exited) { + try { + child.kill("SIGKILL"); + } catch { + /* already gone */ + } + } + done(); + }, 3000); + }); + + let failed = null; + try { + // 1) Boot: poll GET / until 200 (or timeout / early exit). + let boot = 0; + const deadline = Date.now() + BOOT_TIMEOUT_MS; + while (Date.now() < deadline) { + if (exited) throw new Error(`server exited before answering on ${url}`); + boot = await get(url); + if (boot === 200) break; + await new Promise((r) => setTimeout(r, POLL_MS)); + } + if (boot !== 200) throw new Error(`server never answered 200 on ${url} (last status: ${boot})`); + console.log(`[smoke-server] boot ok: GET / → 200 (${url})`); + + // 2) API surface: every probe must be 200. + for (const route of PROBES) { + const status = await get(url + route); + if (status !== 200) throw new Error(`GET ${route} → ${status} (expected 200)`); + console.log(`[smoke-server] GET ${route} → 200`); + } + } catch (err) { + failed = err; + } finally { + await kill(); + try { + fs.rmSync(home, { recursive: true, force: true }); + } catch { + /* best effort — it's under os.tmpdir() */ + } + } + + if (failed) { + console.error(`[smoke-server] FAIL: ${failed.message}`); + dumpTail(); + process.exit(1); + } + console.log("[smoke-server] smoke passed — the packaged server boots and serves."); +} + +main().catch((err) => { + console.error(`[smoke-server] FAIL: ${err.message}`); + process.exit(1); +});