From df167505606a5b0c22511a58b03f8e42954511c2 Mon Sep 17 00:00:00 2001 From: Amos Date: Sat, 22 Aug 2026 09:36:15 +0300 Subject: [PATCH] W2: async job infra + researcher sub-agent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - src/store/jobs.ts: per-job JSON store in data/jobs (create/get/update/list, dedup by toolCallId) - src/agent/tools/jobs.ts: research tool — client-side (no execute), turn ends at the call; input per contracts ResearchToolInput - src/agent/tools/web.ts: server-side web_search (Google CSE) + web_fetch (Readability/Turndown/linkedom, Jina fallback) — errors return as strings, never thrown into the agent loop. Deps: @mozilla/readability, linkedom, turndown (+@types/turndown dev) - src/agent/subagents/researcher.ts: isolated ToolLoopAgent (web tools only, 12-step cap), researcher prompt ported from pi - src/agent/subagents/runner.ts: spawnPendingJobs hook + runResearchJob — fire-and-forget, activity line per tool call, fulfills the part in the LATEST chat record (output-available) before flipping job status; failure still fulfills the part so the teacher learns the outcome - src/server/chat-handler.ts: onFinish spawn hook after saveMessages - app/api/jobs/route.ts: GET ?chatId= polling endpoint - components/artifacts/job-card.tsx + registry entry: polling activity card (2s) → syncFromServer on finish; research-brief card (markdown + sources) - src/agent/prompts/20-jobs.md: research doctrine for the teacher - tests/jobs.test.ts: store CRUD, spawn hook via scripted mock teacher, runner completion/failure with fake model + stubbed fetch --- app/api/jobs/route.ts | 18 ++ components/artifacts/index.ts | 3 +- components/artifacts/job-card.tsx | 133 +++++++++++ package-lock.json | 351 ++++++++++++++++++++++++++++++ package.json | 6 + src/agent/prompts/20-jobs.md | 27 +++ src/agent/subagents/researcher.ts | 71 ++++++ src/agent/subagents/runner.ts | 182 ++++++++++++++++ src/agent/tools/jobs.ts | 45 +++- src/agent/tools/web.ts | 185 ++++++++++++++++ src/server/chat-handler.ts | 9 + src/store/jobs.ts | 88 ++++++++ tests/jobs.test.ts | 345 +++++++++++++++++++++++++++++ 13 files changed, 1456 insertions(+), 7 deletions(-) create mode 100644 app/api/jobs/route.ts create mode 100644 components/artifacts/job-card.tsx create mode 100644 src/agent/prompts/20-jobs.md create mode 100644 src/agent/subagents/researcher.ts create mode 100644 src/agent/subagents/runner.ts create mode 100644 src/agent/tools/web.ts create mode 100644 src/store/jobs.ts create mode 100644 tests/jobs.test.ts diff --git a/app/api/jobs/route.ts b/app/api/jobs/route.ts new file mode 100644 index 0000000..ba9766c --- /dev/null +++ b/app/api/jobs/route.ts @@ -0,0 +1,18 @@ +import { listJobs } from "@/src/store/jobs"; + +/** + * Job status polling endpoint (contracts.ts "THE ASYNC PROTOCOL" §4): the + * client polls GET /api/jobs?chatId=… while any job tool part is pending, + * finds the job by toolCallId, renders its activity line, and re-syncs the + * chat when the job flips to done/error. + */ +export async function GET(req: Request) { + const chatId = new URL(req.url).searchParams.get("chatId"); + if (!chatId) { + return Response.json({ error: "chatId query parameter is required" }, { status: 400 }); + } + if (!/^[A-Za-z0-9_-]+$/.test(chatId)) { + return Response.json({ error: "Invalid chatId" }, { status: 400 }); + } + return Response.json({ jobs: listJobs(chatId) }); +} diff --git a/components/artifacts/index.ts b/components/artifacts/index.ts index c8feafc..ccbd599 100644 --- a/components/artifacts/index.ts +++ b/components/artifacts/index.ts @@ -1,5 +1,6 @@ import type { ComponentType } from "react"; import type { ArtifactPartProps } from "@/src/core/contracts"; +import { JobCard } from "@/components/artifacts/job-card"; /** * Artifact renderer registry — the extension point for tool-part UI. @@ -14,6 +15,6 @@ import type { ArtifactPartProps } from "@/src/core/contracts"; * artifact itself from part.output. */ export const artifactRenderers: Record> = { - // W2: "research" → job activity card + research: JobCard, // W6: "demo" → sandboxed iframe card, "visual" → svg card }; diff --git a/components/artifacts/job-card.tsx b/components/artifacts/job-card.tsx new file mode 100644 index 0000000..3d4d7c6 --- /dev/null +++ b/components/artifacts/job-card.tsx @@ -0,0 +1,133 @@ +"use client"; + +import { useContext, useEffect, useRef, useState } from "react"; +import { MessageResponse } from "@/components/ai-elements/message"; +import { ChatActionsContext } from "@/components/chat"; +import type { ArtifactPartProps, JobRecord, ResearchResult } from "@/src/core/contracts"; + +/** + * The research job card (contracts.ts "THE ASYNC PROTOCOL" §4). + * + * While the part is input-available: an activity card — poll /api/jobs every + * 2s, find the job by toolCallId, show a pulsing dot + "Researching…" + the + * job's live activity line. When the job flips to done/error, call + * syncFromServer() ONCE (fresh messages replace local state; the part + * re-renders as the brief card and the sendAutomaticallyWhen continuation + * resumes the teacher) and stop polling. + * + * When output-available: the research brief — markdown body + a small source + * link list, in the visual language of answered-card.tsx. + */ +export function JobCard({ part }: ArtifactPartProps) { + if (part.state === "output-available") { + return ; + } + return ; +} + +function ResearchActivityCard({ toolCallId }: { toolCallId: string }) { + const actions = useContext(ChatActionsContext); + const [job, setJob] = useState(null); + const finishedRef = useRef(false); + + useEffect(() => { + if (!actions) return; + let cancelled = false; + + const poll = async () => { + if (finishedRef.current) return true; + try { + const res = await fetch(`/api/jobs?chatId=${encodeURIComponent(actions.chatId)}`); + if (!res.ok) return false; + const { jobs } = (await res.json()) as { jobs?: JobRecord[] }; + const match = jobs?.find((j) => j.toolCallId === toolCallId) ?? null; + if (cancelled) return false; + setJob(match); + if (match && match.status !== "running") { + // The part has (or is about to be) fulfilled server-side — pull the + // fresh messages once; the continuation resumes the teacher. + finishedRef.current = true; + void actions.syncFromServer(); + return true; + } + } catch { + /* transient fetch failure — keep polling */ + } + return false; + }; + + const timer = setInterval(() => { + if (finishedRef.current) { + clearInterval(timer); + return; + } + void poll().then((stop) => { + if (stop) clearInterval(timer); + }); + }, 2000); + void poll().then((stop) => { + if (stop) clearInterval(timer); + }); + + return () => { + cancelled = true; + clearInterval(timer); + }; + }, [actions, toolCallId]); + + return ( +
+
+ + Researching… +
+ {job?.activity && ( +
+ {job.activity} +
+ )} +
+ ); +} + +function ResearchBriefCard({ output }: { output: unknown }) { + const result = output as ResearchResult | undefined; + if (!result || typeof result.brief !== "string") { + return ( +
+ Research completed, but the result could not be displayed. +
+ ); + } + const failed = result.brief.startsWith("Research failed"); + return ( +
+
+ Research brief + {failed && failed} +
+
+ {result.brief} +
+ {result.sources.length > 0 && ( +
+
Sources
+ +
+ )} +
+ ); +} diff --git a/package-lock.json b/package-lock.json index b0d0fba..1e8e020 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.2.0", "dependencies": { "@ai-sdk/react": "^4.0.79", + "@mozilla/readability": "^0.6.0", "@openrouter/ai-sdk-provider": "^3.0.0", "@radix-ui/react-use-controllable-state": "^1.2.6", "@rive-app/react-webgl2": "^4.32.1", @@ -24,8 +25,10 @@ "cmdk": "^1.1.1", "embla-carousel-react": "^8.6.0", "katex": "^0.16.47", + "linkedom": "^0.18.13", "lucide-react": "^1.33.0", "media-chrome": "^4.19.2", + "mermaid": "^11.17.0", "motion": "^13.1.1", "nanoid": "^6.0.1", "next": "^16.3.2", @@ -34,9 +37,11 @@ "react": "^19.2.8", "react-dom": "^19.2.8", "react-jsx-parser": "^2.4.1", + "remark-breaks": "^4.0.0", "streamdown": "^2.5.0", "tailwind-merge": "^3.6.0", "tokenlens": "^1.3.1", + "turndown": "^7.2.4", "use-stick-to-bottom": "^1.1.6", "zod": "^4.4.3" }, @@ -45,6 +50,7 @@ "@types/node": "^24.0.0", "@types/react": "^19.2.18", "@types/react-dom": "^19.2.4", + "@types/turndown": "^5.0.6", "tailwindcss": "^4.3.3", "tsx": "^4.19.0", "typescript": "^5.6.0" @@ -1240,6 +1246,21 @@ "@chevrotain/types": "~11.1.2" } }, + "node_modules/@mixmark-io/domino": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@mixmark-io/domino/-/domino-2.2.0.tgz", + "integrity": "sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==", + "license": "BSD-2-Clause" + }, + "node_modules/@mozilla/readability": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/@mozilla/readability/-/readability-0.6.0.tgz", + "integrity": "sha512-juG5VWh4qAivzTAeMzvY9xs9HY5rAcr2E4I7tiSSCokRFi7XIZCAu92ZkSTsIj1OPceCifL3cpfteP3pDT9/QQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/@next/env": { "version": "16.3.2", "resolved": "https://registry.npmjs.org/@next/env/-/env-16.3.2.tgz", @@ -3713,6 +3734,13 @@ "license": "MIT", "optional": true }, + "node_modules/@types/turndown": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@types/turndown/-/turndown-5.0.6.tgz", + "integrity": "sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", @@ -3885,6 +3913,19 @@ "node": ">=6.0.0" } }, + "node_modules/boolbase": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-2.0.0.tgz", + "integrity": "sha512-DkVaaQHymRhpYEYo9x1oo7Q7B0Y6KJUsjm3c9eTyFDby4MHLBTwZ6ZDWBel5zrYxj1WsZgC5oLpiz+93MluXeA==", + "license": "ISC", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/caniuse-lite": { "version": "1.0.30001809", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001809.tgz", @@ -4041,6 +4082,45 @@ "layout-base": "^1.0.0" } }, + "node_modules/css-select": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-7.0.0.tgz", + "integrity": "sha512-snmjEVXy+1LnwXdxhYvTMj1d9tOh4HxkA1YmoayVBeeyR2C14Pum7fcxJIm4SswYspVy866eYNwlH6xC3/VH5g==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^2.0.0", + "css-what": "^8.0.0", + "domhandler": "^6.0.1", + "domutils": "^4.0.2", + "nth-check": "^3.0.1" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-8.0.0.tgz", + "integrity": "sha512-DH0Bqq3DNp5tdOReuNyAA+Ev4Y2GS5FMbZpeTLP6C4CDi0h5nL0BmUPChXw3o/qbHLDWHl49sbNqQVY7bMSDdw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/cssom": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==", + "license": "MIT" + }, "node_modules/csstype": { "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", @@ -4639,6 +4719,67 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/dom-serializer": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-3.1.1.tgz", + "integrity": "sha512-4MEa38/QexBob6gFNwu+EGdWvhJ1OKuNwdYY3Y3NyeWDQfnGeDYQUDfIRzWu5B5gsv03so2Uxd28YC6zrsx3Lw==", + "license": "MIT", + "dependencies": { + "domelementtype": "^3.0.0", + "domhandler": "^6.0.0", + "entities": "^8.0.0" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/dom-serializer/node_modules/entities": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-8.0.0.tgz", + "integrity": "sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-3.0.0.tgz", + "integrity": "sha512-umCQid3jKbDmVjx8jGaW7uUykm4DEUeyV21hPxNMo2nV955DhUThwqyOIDtreepP31hl84X7G5U9ZfsWvIB3Pg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause", + "engines": { + "node": ">=20.19.0" + } + }, + "node_modules/domhandler": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-6.0.1.tgz", + "integrity": "sha512-gYzvtM72ZtxQO0T048kd6HWSbbGCNOUwcnfQ01cqIJ4X2IYKFFHZ5mKvrQETcFXxsRObZulDaKmy//R7TPtsBg==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^3.0.0" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, "node_modules/dompurify": { "version": "3.4.14", "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.4.14.tgz", @@ -4648,6 +4789,24 @@ "@types/trusted-types": "^2.0.7" } }, + "node_modules/domutils": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-4.0.2.tgz", + "integrity": "sha512-qI4JLRKnSzqFqr7hAlS5xQDusBCjKSEG4t4+7aNrIQMHBcsC2TGEhuyABJdYkgSewL57PNLYEiibY2iPKhKpaA==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^3.0.0", + "domelementtype": "^3.0.0", + "domhandler": "^6.0.0" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/embla-carousel": { "version": "8.6.0", "resolved": "https://registry.npmjs.org/embla-carousel/-/embla-carousel-8.6.0.tgz", @@ -5130,6 +5289,12 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/html-escaper": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz", + "integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==", + "license": "MIT" + }, "node_modules/html-url-attributes": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/html-url-attributes/-/html-url-attributes-3.0.1.tgz", @@ -5150,6 +5315,104 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/htmlparser2": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", + "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "entities": "^7.0.1" + } + }, + "node_modules/htmlparser2/node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/htmlparser2/node_modules/dom-serializer/node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, + "node_modules/htmlparser2/node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/htmlparser2/node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/htmlparser2/node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, + "node_modules/htmlparser2/node_modules/entities": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", + "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/iconv-lite": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", @@ -5547,6 +5810,30 @@ "url": "https://opencollective.com/parcel" } }, + "node_modules/linkedom": { + "version": "0.18.13", + "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.18.13.tgz", + "integrity": "sha512-ES/o9qotMpzpN2MHs+Iq/JcVoOj8Fa5wiQYrTdFpvAnwXL0g66XHHUc9WUMk6nAlBtGsFQ24ne+SYnvnaQ2FSw==", + "license": "ISC", + "dependencies": { + "css-select": "^7.0.0", + "cssom": "^0.5.0", + "html-escaper": "^3.0.3", + "htmlparser2": "^10.1.0", + "uhyphen": "^0.2.0" + }, + "engines": { + "node": ">=16" + }, + "peerDependencies": { + "canvas": ">= 2" + }, + "peerDependenciesMeta": { + "canvas": { + "optional": true + } + } + }, "node_modules/linkify-it": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/linkify-it/-/linkify-it-3.0.3.tgz", @@ -5833,6 +6120,20 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/mdast-util-newline-to-break": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-newline-to-break/-/mdast-util-newline-to-break-2.0.0.tgz", + "integrity": "sha512-MbgeFca0hLYIEx/2zGsszCSEJJ1JSCdiY5xQxRcLDDGa8EPvlLPupJ4DSajbMPAnC0je8jfb9TiUATnxxrHUog==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-find-and-replace": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/mdast-util-phrasing": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", @@ -6809,6 +7110,22 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/nth-check": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-3.0.1.tgz", + "integrity": "sha512-GX0gsdbGVCgnRgbeGaubfjpBXyYRWOOCVeYh08bSQvDZqxz5ndXs1OTfAt/h36G1xvI94YIspsI0sVFqAV9+RQ==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^2.0.0" + }, + "engines": { + "node": ">=20.19.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/oniguruma-parser": { "version": "0.12.2", "resolved": "https://registry.npmjs.org/oniguruma-parser/-/oniguruma-parser-0.12.2.tgz", @@ -7254,6 +7571,21 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/remark-breaks": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/remark-breaks/-/remark-breaks-4.0.0.tgz", + "integrity": "sha512-IjEjJOkH4FuJvHZVIW0QCDWxcG96kCq7An/KVH2NfJe6rKZU2AsHeB3OEjPNRxi4QC34Xdx7I2KGYn6IpT7gxQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-newline-to-break": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/remark-cjk-friendly": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/remark-cjk-friendly/-/remark-cjk-friendly-2.3.1.tgz", @@ -7742,6 +8074,19 @@ "fsevents": "~2.3.3" } }, + "node_modules/turndown": { + "version": "7.2.4", + "resolved": "https://registry.npmjs.org/turndown/-/turndown-7.2.4.tgz", + "integrity": "sha512-I8yFsfRzmzK0WV1pNNOA4A7y4RDfFxPRxb3t+e3ui14qSGOxGtiSP6GjeX+Y6CHb7HYaFj7ECUD7VE5kQMZWGQ==", + "license": "MIT", + "dependencies": { + "@mixmark-io/domino": "^2.2.0" + }, + "engines": { + "node": ">=18", + "npm": ">=9" + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -7762,6 +8107,12 @@ "integrity": "sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==", "license": "MIT" }, + "node_modules/uhyphen": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/uhyphen/-/uhyphen-0.2.0.tgz", + "integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==", + "license": "ISC" + }, "node_modules/undici": { "version": "7.29.0", "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.0.tgz", diff --git a/package.json b/package.json index 941ae65..b850cbd 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ }, "dependencies": { "@ai-sdk/react": "^4.0.79", + "@mozilla/readability": "^0.6.0", "@openrouter/ai-sdk-provider": "^3.0.0", "@radix-ui/react-use-controllable-state": "^1.2.6", "@rive-app/react-webgl2": "^4.32.1", @@ -31,8 +32,10 @@ "cmdk": "^1.1.1", "embla-carousel-react": "^8.6.0", "katex": "^0.16.47", + "linkedom": "^0.18.13", "lucide-react": "^1.33.0", "media-chrome": "^4.19.2", + "mermaid": "^11.17.0", "motion": "^13.1.1", "nanoid": "^6.0.1", "next": "^16.3.2", @@ -41,9 +44,11 @@ "react": "^19.2.8", "react-dom": "^19.2.8", "react-jsx-parser": "^2.4.1", + "remark-breaks": "^4.0.0", "streamdown": "^2.5.0", "tailwind-merge": "^3.6.0", "tokenlens": "^1.3.1", + "turndown": "^7.2.4", "use-stick-to-bottom": "^1.1.6", "zod": "^4.4.3" }, @@ -52,6 +57,7 @@ "@types/node": "^24.0.0", "@types/react": "^19.2.18", "@types/react-dom": "^19.2.4", + "@types/turndown": "^5.0.6", "tailwindcss": "^4.3.3", "tsx": "^4.19.0", "typescript": "^5.6.0" diff --git a/src/agent/prompts/20-jobs.md b/src/agent/prompts/20-jobs.md new file mode 100644 index 0000000..cf0995b --- /dev/null +++ b/src/agent/prompts/20-jobs.md @@ -0,0 +1,27 @@ +## Background research + +You have a `research` tool that dispatches a background web-research agent. Use it to +outsource what you cannot reliably know yourself: + +- **Use it for:** facts you are uncertain about, current or fast-moving data (library versions, + release dates, recent events), niche specifics outside your training, and anything where + being wrong would poison the lesson. +- **Never use it for:** anything you know cold, internal reasoning, or math. Do not offload + thinking — only facts you genuinely need verified. + +**The call is asynchronous and your turn ends at it.** The discipline: + +1. **Say your piece first.** Explain the concept, set up what you're doing and why, and tell + the learner roughly what you're checking — the research brief arrives later as the tool's + output, and the lesson resumes from there. A bare tool call with no surrounding text is a + failed turn. +2. **Call with a single, specific question.** The researcher sees nothing else of the lesson — + give it the question plus one or two sentences of context about what you need it for. +3. **Resume from the brief.** When the result returns, weave it into the lesson. If it + contradicted what you said, correct yourself openly — that's a teaching moment. +4. **Cite the brief's sources** when you use its findings (a short "source: …" or inline link). + Never present researched facts as your own certainty. + +At most one research call per turn — don't fan out; the researcher searches multiple angles on +its own. If research fails or is unconfigured, you'll get a failure brief: say so plainly and +continue with what you can establish, flagging the uncertainty. diff --git a/src/agent/subagents/researcher.ts b/src/agent/subagents/researcher.ts new file mode 100644 index 0000000..6741b9f --- /dev/null +++ b/src/agent/subagents/researcher.ts @@ -0,0 +1,71 @@ +import { ToolLoopAgent, stepCountIs, type LanguageModel } from "ai"; +import { openrouter } from "@openrouter/ai-sdk-provider"; +import { webTools } from "../tools/web.ts"; + +/** + * The researcher sub-agent (vision §2.5): a SEPARATE ToolLoopAgent instance + * from the teacher — an isolated loop with only web_search + web_fetch, so + * raw search/fetch churn never touches the teacher's context window. + * + * System prompt adapted from the pi researcher agent + * (~/.pi/agent/extensions/interactive-subagents/agents/researcher.md). + * The model is injectable so tests run the loop with a MockLanguageModelV4. + */ + +const DEFAULT_RESEARCHER_MODEL = + process.env.LH_RESEARCH_MODEL ?? process.env.LH_MODEL ?? "anthropic/claude-sonnet-4.5"; + +const INSTRUCTIONS = `You are a research specialist. You work for a teacher agent inside a learning +application: given a research question, conduct thorough web research and produce a focused, +well-sourced brief that the teacher can teach from. + +You operate in an isolated context with no knowledge of the lesson or any prior conversation. +All necessary context is in the task description. + +Process: +1. Break the question into 2-4 searchable facets. +2. Search with web_search using varied angles: + - the direct answer query (the obvious one) + - an authoritative-source query (official docs, specs, primary sources) + - a practical-experience query (case studies, benchmarks, real-world usage) + - a recent-developments query (only if the topic is time-sensitive) +3. Read the results. Identify what's well-covered and what has gaps. If the first round + doesn't fully answer the question, search again with refined queries targeting the gaps. +4. For the 2-3 most promising source URLs, use web_fetch to get the full page content. +5. Synthesize everything into a brief that directly answers the question. + +Evaluation — what to keep vs drop: +- Official docs and primary sources outweigh blog posts and forum threads. +- Recent sources outweigh stale ones. +- Sources that directly address the question outweigh tangentially related ones. +- Drop SEO filler, outdated info, and beginner tutorials (unless that's the audience). + +Tool outputs may be error strings (e.g. search unconfigured, a page that failed to fetch). +Adapt: try another URL or another search angle; if research is genuinely impossible, say so +in the brief instead of inventing facts. + +Your FINAL assistant message is your entire deliverable — it must stand alone, using this format: + +## Summary +2-3 sentence direct answer. + +## Findings +Numbered findings with inline source citations: +1. **Finding** — explanation. [Source](url) + +## Sources +- Source Title (url) — why it's relevant + +## Gaps +What couldn't be answered. Suggested next steps.`; + +export function createResearcherAgent(model?: LanguageModel) { + return new ToolLoopAgent({ + model: model ?? openrouter(DEFAULT_RESEARCHER_MODEL), + instructions: INSTRUCTIONS, + tools: webTools, + stopWhen: stepCountIs(12), + }); +} + +export type ResearcherAgent = ReturnType; diff --git a/src/agent/subagents/runner.ts b/src/agent/subagents/runner.ts new file mode 100644 index 0000000..319a897 --- /dev/null +++ b/src/agent/subagents/runner.ts @@ -0,0 +1,182 @@ +import { isToolUIPart, type LanguageModel, type UIMessage } from "ai"; +import type { ResearchResult, ResearchToolInput } from "../../core/contracts.ts"; +import { JOB_TOOLS } from "../../core/pending.ts"; +import { getChat, saveMessages } from "../../store/chats.ts"; +import { createJob, findJobByToolCallId, getJob, updateJob } from "../../store/jobs.ts"; +import { createResearcherAgent } from "./researcher.ts"; + +/** + * The background job runner + the chat-handler spawn hook (contracts.ts + * "THE ASYNC PROTOCOL"). + * + * spawnPendingJobs — called from onFinish AFTER saveMessages: finds + * input-available JOB_TOOLS parts in the last assistant message with no + * existing job and fires runResearchJob WITHOUT awaiting (the HTTP + * response is long gone). + * + * runResearchJob — runs the researcher agent loop in the background, + * updating the job's activity line on every tool call, then rewrites the + * tool part in the LATEST chat record to output-available (read-modify- + * write; no awaits between the read and the write). The part is fulfilled + * BEFORE the job status flips, so a client that polls the job and then + * re-syncs always sees the finished part. + * + * On failure the part is still fulfilled ({brief: "Research failed: …"}) — + * the teacher must learn the outcome — and the job is marked error. + */ + +/** + * Test seam: the spawn hook creates runners with no injection point, so + * tests swap in a MockLanguageModelV4 here to keep job tests off the + * network. Undefined in production. + */ +export const researchModelOverride: { current?: LanguageModel } = {}; + +export interface RunResearchJobOptions { + model?: LanguageModel; +} + +export function spawnPendingJobs(chatId: string, messages: UIMessage[]): void { + const last = [...messages].reverse().find((m) => m.role === "assistant"); + if (!last) return; + for (const part of last.parts) { + if (!isToolUIPart(part) || part.state !== "input-available") continue; + const tool = part.type.slice("tool-".length); + if (!(JOB_TOOLS as readonly string[]).includes(tool)) continue; + if (tool !== "research") continue; // demo/visual runners arrive in W6 + if (findJobByToolCallId(chatId, part.toolCallId)) continue; // already has a job + const job = createJob({ kind: "research", chatId, toolCallId: part.toolCallId }); + void runResearchJob(job.id, { model: researchModelOverride.current }).catch((err) => { + console.error(`[job ${job.id}] research runner crashed:`, err); + updateJob(job.id, { status: "error", error: String(err), activity: "failed" }); + }); + } +} + +export async function runResearchJob(jobId: string, opts: RunResearchJobOptions = {}): Promise { + const job = getJob(jobId); + if (!job || job.status !== "running") return; + + const chat = getChat(job.chatId); + if (!chat) { + updateJob(jobId, { status: "error", error: `Chat not found: ${job.chatId}`, activity: "failed" }); + return; + } + const input = findToolPart(chat.messages, job.toolCallId)?.input as ResearchToolInput | undefined; + const question = typeof input?.question === "string" && input.question.trim() ? input.question : "(no question)"; + const context = typeof input?.context === "string" && input.context.trim() ? input.context : undefined; + + const prompt = [ + `Research question: ${question}`, + context ? `Context (what this is needed for): ${context}` : null, + "Produce your final brief in the required format.", + ] + .filter(Boolean) + .join("\n\n"); + + const fetchedUrls: string[] = []; + const agent = createResearcherAgent(opts.model); + + try { + updateJob(jobId, { activity: "researching" }); + const result = await agent.generate({ + prompt, + onToolExecutionStart: ({ toolCall }) => { + const input = toolCall.input as { query?: string; url?: string } | undefined; + if (toolCall.toolName === "web_fetch" && typeof input?.url === "string") { + fetchedUrls.push(input.url); + } + updateJob(jobId, { activity: activityLine(toolCall.toolName, input) }); + }, + }); + + const brief = result.text.trim(); + if (!brief) throw new Error("the researcher produced no text"); + const research: ResearchResult = { brief, sources: parseSources(brief, fetchedUrls) }; + + fulfillPart(job.chatId, job.toolCallId, research); + updateJob(jobId, { status: "done", result: research }); // activity line kept: the last tool call is the record + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + fulfillPart(job.chatId, job.toolCallId, { brief: `Research failed: ${reason}`, sources: [] }); + updateJob(jobId, { status: "error", activity: "failed", error: reason }); + } +} + +// ─── helpers ───────────────────────────────────────────────────────────────── + +function findToolPart(messages: UIMessage[], toolCallId: string) { + for (const m of messages) { + for (const part of m.parts) { + if (isToolUIPart(part) && part.toolCallId === toolCallId) return part; + } + } + return null; +} + +/** The activity line shown in the UI: `web_search: "query"` / `web_fetch: host`. */ +function activityLine(toolName: string, input: { query?: string; url?: string } | undefined): string { + if (toolName === "web_search") return `web_search: "${input?.query ?? ""}"`; + if (toolName === "web_fetch") return `web_fetch: ${hostOf(input?.url)}`; + return toolName; +} + +function hostOf(url: string | undefined): string { + if (!url) return "(no url)"; + try { + return new URL(url).hostname; + } catch { + return url.slice(0, 40); + } +} + +/** + * Rewrite the tool part in the LATEST chat record to output-available. + * Synchronous read-modify-write (no awaits) — atomic within the event loop. + */ +function fulfillPart(chatId: string, toolCallId: string, output: ResearchResult): void { + const chat = getChat(chatId); + if (!chat) return; + let found = false; + const messages = chat.messages.map((m) => + m.role !== "assistant" + ? m + : { + ...m, + parts: m.parts.map((part) => { + if (isToolUIPart(part) && part.toolCallId === toolCallId && part.state === "input-available") { + found = true; + return { ...part, state: "output-available" as const, output }; + } + return part; + }), + }, + ); + if (!found) return; + try { + saveMessages(chatId, messages); + } catch (err) { + console.error(`[chat ${chatId}] fulfilling job part ${toolCallId} failed:`, err); + } +} + +/** Sources for the result: markdown links cited in the brief, else fetched URLs. */ +function parseSources(brief: string, fetchedUrls: string[]): ResearchResult["sources"] { + const sources: ResearchResult["sources"] = []; + const seen = new Set(); + for (const match of brief.matchAll(/\[([^\]]+)\]\((https?:\/\/[^)\s]+)\)/g)) { + const title = (match[1] ?? "").trim(); + const url = match[2]!; + if (!title || seen.has(url)) continue; + seen.add(url); + sources.push({ title, url }); + } + if (sources.length === 0) { + for (const url of fetchedUrls) { + if (seen.has(url)) continue; + seen.add(url); + sources.push({ title: hostOf(url), url }); + } + } + return sources.slice(0, 10); +} diff --git a/src/agent/tools/jobs.ts b/src/agent/tools/jobs.ts index ba7fa02..33d5308 100644 --- a/src/agent/tools/jobs.ts +++ b/src/agent/tools/jobs.ts @@ -1,13 +1,46 @@ -import type { ToolSet } from "ai"; +import { tool, type ToolSet } from "ai"; +import { z } from "zod"; import type { TeacherToolContext } from "../tools.ts"; /** - * Async sub-agent job tools — W2 workstream (research; W6 adds demo + visual). - * CLIENT-SIDE semantics (no execute): the turn ends with the part pending, a - * background runner fulfills it, and the client continuation resumes the - * teacher. See src/core/contracts.ts "THE ASYNC PROTOCOL" — read it first. + * Async sub-agent job tools (W2: research; W6 adds demo + visual). + * CLIENT-SIDE semantics (no execute) — exactly like quiz/ask: the tool call + * streams to the browser in `input-available` state and the server turn ends + * there. chat-handler's onFinish spawns a background runner that fulfills + * the part server-side; the client polls /api/jobs and the continuation + * resumes the teacher with the result as a normal tool output. + * See src/core/contracts.ts "THE ASYNC PROTOCOL". + */ + +export const researchInputSchema = z.object({ + question: z + .string() + .describe("The single research question to answer. Make it specific — the researcher sees nothing else of the lesson."), + context: z + .string() + .optional() + .describe("Why you need this / what the lesson needs it for — focuses the brief. One or two sentences."), +}); + +export const researchTool = tool({ + description: + "Dispatch a background web-research agent: fact verification, current data, or niche specifics you are not certain about. " + + "The call is ASYNC — your turn ENDS here, so say everything you want to say first; the research brief arrives later as " + + "this tool's output and the lesson resumes from it. Cite the brief's sources when you use them. " + + "NEVER use it for things you know cold, internal reasoning, or math.", + inputSchema: researchInputSchema, +}); + +/** + * Per-request job tool registry. Context-free for now (the runner resolves + * everything from the chat record by toolCallId) — the ctx parameter keeps + * the createTeacherTools factory shape stable for W6's demo/visual tools. */ // eslint-disable-next-line @typescript-eslint/no-unused-vars export function jobTools(ctx: TeacherToolContext): ToolSet { - return {}; + return { + research: researchTool, + }; } + +export type ResearchInput = z.infer; diff --git a/src/agent/tools/web.ts b/src/agent/tools/web.ts new file mode 100644 index 0000000..03ac795 --- /dev/null +++ b/src/agent/tools/web.ts @@ -0,0 +1,185 @@ +import { tool } from "ai"; +import { z } from "zod"; +import { Readability } from "@mozilla/readability"; +import { parseHTML } from "linkedom"; +import TurndownService from "turndown"; + +/** + * Server-side web tools for the researcher sub-agent — a minimal port of the + * pi extensions (~/.pi/agent/extensions/web-search + web-fetch): + * + * web_search(query, count?) → {title, url, snippet}[] (Google Custom Search) + * web_fetch(url) → readable markdown text (Readability + Turndown, + * Jina Reader fallback for + * JS-rendered pages) + * + * Credentials come from env (the pi extension used auth.json): + * GOOGLE_SEARCH_API_KEY, GOOGLE_CSE_ID + * + * Tool contract: NEVER throw into the agent loop — missing credentials and + * every failure path return a clear error STRING as the tool output, so the + * researcher can adapt (or report) instead of crashing. + */ + +// ── web_search ─────────────────────────────────────────────────────────────── + +export interface WebSearchResult { + title: string; + url: string; + snippet: string; +} + +function searchCredentials(): { apiKey: string; cseId: string } | null { + const apiKey = process.env.GOOGLE_SEARCH_API_KEY; + const cseId = process.env.GOOGLE_CSE_ID; + return apiKey && cseId ? { apiKey, cseId } : null; +} + +async function googleSearch(query: string, count: number, creds: { apiKey: string; cseId: string }): Promise { + const url = new URL("https://www.googleapis.com/customsearch/v1"); + url.searchParams.set("key", creds.apiKey); + url.searchParams.set("cx", creds.cseId); + url.searchParams.set("q", query); + url.searchParams.set("num", String(Math.min(count, 10))); + + const resp = await fetch(url, { signal: AbortSignal.timeout(30_000) }); + if (!resp.ok) { + const body = await resp.text(); + throw new Error(`Google Custom Search API ${resp.status}: ${body.slice(0, 200)}`); + } + const data = (await resp.json()) as { items?: Array<{ title: string; link: string; snippet?: string }> }; + return (data.items ?? []).map((item) => ({ + title: item.title, + url: item.link, + snippet: item.snippet?.replace(/\n/g, " ") ?? "", + })); +} + +// ── web_fetch ──────────────────────────────────────────────────────────────── + +const USER_AGENT = + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"; +const FETCH_TIMEOUT_MS = 30_000; +const MAX_RESPONSE_BYTES = 5 * 1024 * 1024; +/** Keep the researcher's context sane — long pages truncate with a marker. */ +const MAX_CONTENT_CHARS = 40_000; +const MIN_USEFUL_CONTENT = 200; + +const turndown = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" }); + +function truncate(text: string): string { + if (text.length <= MAX_CONTENT_CHARS) return text; + return `${text.slice(0, MAX_CONTENT_CHARS)}\n\n[Truncated at ${MAX_CONTENT_CHARS} characters]`; +} + +/** Jina Reader fallback for JS-rendered pages (no deps — plain fetch). */ +async function fetchViaJina(url: string): Promise { + try { + const res = await fetch(`https://r.jina.ai/${url}`, { + headers: { Accept: "text/markdown", "X-No-Cache": "true" }, + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + }); + if (!res.ok) return null; + const content = await res.text(); + const marker = "Markdown Content:"; + const start = content.indexOf(marker); + if (start < 0) return null; + const markdown = content.slice(start + marker.length).trim(); + if (markdown.length < MIN_USEFUL_CONTENT || markdown.startsWith("Loading...")) return null; + return markdown; + } catch { + return null; + } +} + +function htmlToMarkdown(html: string): { title: string; content: string } | null { + const { document } = parseHTML(html); + const article = new Readability(document as unknown as Document).parse(); + if (!article?.content) return null; + const markdown = turndown.turndown(article.content); + if (markdown.trim().length < MIN_USEFUL_CONTENT) return null; + return { title: article.title ?? "", content: markdown }; +} + +async function fetchAndExtract(url: string): Promise { + const resp = await fetch(url, { + headers: { + "User-Agent": USER_AGENT, + Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + }, + signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), + }); + if (!resp.ok) throw new Error(`HTTP ${resp.status} ${resp.statusText}`); + + const contentType = resp.headers.get("content-type") ?? ""; + if (contentType.includes("application/pdf")) throw new Error("PDF documents are not supported"); + if (/image\/|audio\/|video\/|zip|octet-stream/.test(contentType)) { + throw new Error(`Unsupported content type: ${contentType.split(";")[0]}`); + } + const length = Number(resp.headers.get("content-length") ?? 0); + if (length > MAX_RESPONSE_BYTES) throw new Error("Response too large"); + + const text = await resp.text(); + if (!contentType.includes("html")) return truncate(text); // plain text / markdown / code + + const extracted = htmlToMarkdown(text); + if (extracted) { + const header = extracted.title ? `# ${extracted.title}\n\nSource: ${url}\n\n---\n\n` : ""; + return header + truncate(extracted.content); + } + // Readability failed — likely JS-rendered. Try Jina Reader. + const jina = await fetchViaJina(url); + if (jina) return truncate(jina); + throw new Error("Could not extract readable content (the page may be JavaScript-rendered)"); +} + +// ── tool registry ──────────────────────────────────────────────────────────── + +export const webTools = { + web_search: tool({ + description: + "Search the web via Google Custom Search. One search per call — use one call per search angle. " + + "Returns an array of results with title, url, and snippet.", + inputSchema: z.object({ + query: z.string().describe("The search query (a normal search string, phrases included)"), + count: z.number().int().min(1).max(10).optional().describe("Number of results (default 5, max 10)"), + }), + execute: async ({ query, count }): Promise => { + const creds = searchCredentials(); + if (!creds) { + return ( + "web_search is not configured on this server: GOOGLE_SEARCH_API_KEY and/or GOOGLE_CSE_ID " + + "are missing. Report in your brief that web search is unavailable." + ); + } + try { + return await googleSearch(query, count ?? 5, creds); + } catch (err) { + return `web_search failed: ${err instanceof Error ? err.message : String(err)}`; + } + }, + }), + + web_fetch: tool({ + description: + "Fetch a web page and extract its readable content as markdown (Readability). " + + "Handles HTML pages and plain text; JS-rendered pages fall back to a reader service.", + inputSchema: z.object({ + url: z.string().describe("The URL to fetch"), + }), + execute: async ({ url }): Promise => { + try { + const parsed = new URL(url); + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") throw new Error("Only http(s) URLs are supported"); + } catch { + return `web_fetch failed: invalid URL "${url}"`; + } + try { + return await fetchAndExtract(url); + } catch (err) { + return `web_fetch failed for ${url}: ${err instanceof Error ? err.message : String(err)}`; + } + }, + }), +}; diff --git a/src/server/chat-handler.ts b/src/server/chat-handler.ts index ab0d261..4de985c 100644 --- a/src/server/chat-handler.ts +++ b/src/server/chat-handler.ts @@ -1,6 +1,7 @@ import { createAgentUIStreamResponse, type UIMessage } from "ai"; import { createTeacherAgent, type TeacherAgent } from "../agent/teacher-agent.ts"; import { createTeacherTools } from "../agent/tools.ts"; +import { spawnPendingJobs } from "../agent/subagents/runner.ts"; import { getChat, saveMessages } from "../store/chats.ts"; /** @@ -38,6 +39,14 @@ export async function handleChatRequest(req: Request, agent?: TeacherAgent): Pro } catch (err) { console.error(`[chat ${chatId}] persist failed:`, err, { isAborted }); } + // Async sub-agent protocol (contracts.ts): scan the last assistant + // message for pending job tool parts and fire the background runners — + // fire-and-forget, never awaited; the HTTP response is already gone. + try { + spawnPendingJobs(chatId, messages as UIMessage[]); + } catch (err) { + console.error(`[chat ${chatId}] job spawn failed:`, err); + } }, }); } diff --git a/src/store/jobs.ts b/src/store/jobs.ts new file mode 100644 index 0000000..d230202 --- /dev/null +++ b/src/store/jobs.ts @@ -0,0 +1,88 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import { PATHS } from "../../config.ts"; +import type { JobKind, JobRecord } from "../core/contracts.ts"; + +/** + * Job store — one JSON file per background sub-agent job in data/jobs/ + * (contracts.ts "THE ASYNC PROTOCOL"). Keyed by toolCallId for dedup: the + * chat-handler spawn hook never creates two jobs for the same tool part. + * + * Follows src/store/chats.ts idioms: sync fs, id-guarded filenames, JSON + * with tab indentation. Writes are small and rare (activity line updates); + * the read-modify-write sections in the runner contain no awaits, so they + * are atomic within the single-threaded event loop. + */ + +/** Background jobs (data/jobs). Not in config.ts PATHS (orchestrator-owned). */ +const JOBS_DIR = path.join(PATHS.data, "jobs"); + +function jobFile(id: string): string { + if (!/^[A-Za-z0-9_-]+$/.test(id)) throw new Error(`Invalid job id: ${id}`); + return path.join(JOBS_DIR, `${id}.json`); +} + +export interface CreateJobInit { + kind: JobKind; + chatId: string; + toolCallId: string; + /** First activity line (defaults to "starting"). */ + activity?: string; +} + +export function createJob(init: CreateJobInit): JobRecord { + const now = new Date().toISOString(); + const record: JobRecord = { + id: `job_${Date.now().toString(36)}_${crypto.randomUUID().slice(0, 8)}`, + kind: init.kind, + chatId: init.chatId, + toolCallId: init.toolCallId, + status: "running", + activity: init.activity ?? "starting", + createdAt: now, + updatedAt: now, + }; + fs.mkdirSync(JOBS_DIR, { recursive: true }); + fs.writeFileSync(jobFile(record.id), JSON.stringify(record, null, "\t"), "utf-8"); + return record; +} + +export function getJob(id: string): JobRecord | null { + try { + return JSON.parse(fs.readFileSync(jobFile(id), "utf-8")) as JobRecord; + } catch { + return null; + } +} + +/** Merge a partial patch into the record (identity fields are immutable). */ +export function updateJob( + id: string, + patch: Partial>, +): JobRecord | null { + const record = getJob(id); + if (!record) return null; + const next: JobRecord = { ...record, ...patch, updatedAt: new Date().toISOString() }; + fs.writeFileSync(jobFile(id), JSON.stringify(next, null, "\t"), "utf-8"); + return next; +} + +/** All jobs for a chat, newest activity first. */ +export function listJobs(chatId: string): JobRecord[] { + try { + fs.mkdirSync(JOBS_DIR, { recursive: true }); + return fs + .readdirSync(JOBS_DIR) + .filter((f) => f.endsWith(".json")) + .map((f) => getJob(f.slice(0, -5))) + .filter((j): j is JobRecord => j !== null && j.chatId === chatId) + .sort((a, b) => b.updatedAt.localeCompare(a.updatedAt)); + } catch { + return []; + } +} + +/** The job fulfilling a given tool part, if any (spawn-hook dedup). */ +export function findJobByToolCallId(chatId: string, toolCallId: string): JobRecord | null { + return listJobs(chatId).find((j) => j.toolCallId === toolCallId) ?? null; +} diff --git a/tests/jobs.test.ts b/tests/jobs.test.ts new file mode 100644 index 0000000..992b4ab --- /dev/null +++ b/tests/jobs.test.ts @@ -0,0 +1,345 @@ +/** + * Job infrastructure tests — deterministic, zero network: + * + * 1. job store CRUD (create → get → update → list → dedup by toolCallId) + * 2. the spawn hook: drive handleChatRequest with a scripted mock teacher + * that emits a research tool call → a job record appears (runner fires + * with the mock override, so no network) + * 3. runner completion: a fake researcher model (web_search call → final + * text brief, globalThis.fetch stubbed) → the stored chat's part flips + * to output-available with a ResearchResult-shaped output, the job is + * done with the activity recorded + * 4. runner failure: a model that throws → the part is still fulfilled + * with a failure brief and the job is marked error + * + * Cleanup follows tests/chat-flow.test.ts: created chat/job files removed in + * finally blocks / the after() hook. + */ + +import assert from "node:assert/strict"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { after, describe, it } from "node:test"; +import { MockLanguageModelV4, simulateReadableStream } from "ai/test"; +import type { LanguageModelV4StreamPart } from "@ai-sdk/provider"; +import { createTeacherAgent } from "../src/agent/teacher-agent.ts"; +import { handleChatRequest } from "../src/server/chat-handler.ts"; +import { createChat, getChat } from "../src/store/chats.ts"; +import { createJob, findJobByToolCallId, getJob, listJobs, updateJob } from "../src/store/jobs.ts"; +import { researchModelOverride, runResearchJob, spawnPendingJobs } from "../src/agent/subagents/runner.ts"; +import { createResearcherAgent } from "../src/agent/subagents/researcher.ts"; +import { PATHS } from "../config.ts"; + +const JOBS_DIR = path.join(PATHS.data, "jobs"); + +const createdChats: string[] = []; +const createdJobs: string[] = []; + +function trackChat(id: string) { + createdChats.push(id); +} + +function trackJob(id: string) { + createdJobs.push(id); +} + +after(() => { + for (const id of createdChats) { + try { + fs.unlinkSync(path.join(PATHS.chats, `${id}.json`)); + } catch { + /* already gone */ + } + } + for (const id of createdJobs) { + try { + fs.unlinkSync(path.join(JOBS_DIR, `${id}.json`)); + } catch { + /* already gone */ + } + } +}); + +function usage() { + return { + inputTokens: { total: 10, noCache: 10, cacheRead: 0, cacheWrite: 0 }, + outputTokens: { total: 5, text: 5, reasoning: 0 }, + }; +} + +async function drain(res: Response): Promise { + if (res.status !== 200) { + assert.fail(`expected 200, got ${res.status}: ${await res.text()}`); + } + return res.text(); +} + +function post(chatId: string, messages: unknown[]): Request { + return new Request("http://test/api/chat", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ chatId, messages }), + }); +} + +// ─── 1. job store CRUD ─────────────────────────────────────────────────────── + +describe("job store", () => { + it("create → get → update → list round-trips", () => { + const job = createJob({ kind: "research", chatId: "chat_storetest", toolCallId: "call-store" }); + trackJob(job.id); + + assert.match(job.id, /^job_/); + assert.equal(job.status, "running"); + assert.equal(job.activity, "starting"); + + const loaded = getJob(job.id)!; + assert.equal(loaded.toolCallId, "call-store"); + assert.equal(loaded.kind, "research"); + + const updated = updateJob(job.id, { activity: 'web_search: "test"', status: "done", result: { brief: "b" } })!; + assert.equal(updated.activity, 'web_search: "test"'); + assert.equal(updated.status, "done"); + assert.ok(updated.updatedAt >= job.updatedAt); + // immutable identity fields survive a patch + assert.equal(updated.toolCallId, "call-store"); + assert.equal(getJob(updated.id)!.toolCallId, "call-store"); + + const listed = listJobs("chat_storetest"); + assert.ok(listed.some((j) => j.id === job.id)); + assert.equal(listJobs("chat_other").some((j) => j.id === job.id), false, "jobs are scoped to their chat"); + + const byTool = findJobByToolCallId("chat_storetest", "call-store"); + assert.equal(byTool?.id, job.id); + assert.equal(findJobByToolCallId("chat_storetest", "call-missing"), null); + }); +}); + +// ─── scripted teacher: emits a research tool call ──────────────────────────── + +const RESEARCH_INPUT = { + question: "What is the latest stable sigma.js version?", + context: "Choosing a graph rendering library for the lesson.", +}; + +/** A teacher that calls research on turn 1; plain text thereafter. */ +function researchCallingTeacher(): MockLanguageModelV4 { + let turn = 0; + return new MockLanguageModelV4({ + doStream: async () => { + turn++; + const chunks: LanguageModelV4StreamPart[] = + turn === 1 + ? [ + { type: "stream-start", warnings: [] }, + { type: "text-start", id: "t1" }, + { type: "text-delta", id: "t1", delta: "Let me check the current state of the library." }, + { type: "text-end", id: "t1" }, + { type: "tool-input-start", id: "call-r1", toolName: "research" }, + { type: "tool-input-delta", id: "call-r1", delta: JSON.stringify(RESEARCH_INPUT) }, + { type: "tool-input-end", id: "call-r1" }, + { type: "tool-call", toolCallId: "call-r1", toolName: "research", input: JSON.stringify(RESEARCH_INPUT) }, + { type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() }, + ] + : [ + { type: "stream-start", warnings: [] }, + { type: "text-start", id: "t2" }, + { type: "text-delta", id: "t2", delta: "Back with the brief." }, + { type: "text-end", id: "t2" }, + { type: "finish", finishReason: { unified: "stop", raw: "stop" }, usage: usage() }, + ]; + return { stream: simulateReadableStream({ chunks, initialDelayInMs: 0, chunkDelayInMs: 0 }) }; + }, + }); +} + +describe("spawn hook (async protocol §2)", () => { + it("a pending research part in the final assistant message spawns exactly one job", async () => { + const chat = createChat("Test: research spawn"); + trackChat(chat.id); + + const agent = createTeacherAgent(researchCallingTeacher()); + const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "teach me graph rendering" }] }; + await drain(await handleChatRequest(post(chat.id, [firstUser]), agent)); + + // The research part is pending (input-available) in the stored chat. + const saved = getChat(chat.id)!; + const part = saved.messages[1]!.parts.find((p) => p.type === "tool-research"); + assert.ok(part && part.type === "tool-research", "research tool part persisted"); + assert.equal(part.state, "input-available", "job tool part stays pending until the runner fulfills it"); + + // A job record exists for it. + const job = findJobByToolCallId(chat.id, "call-r1"); + assert.ok(job, "spawn hook created a job record"); + assert.equal(job!.kind, "research"); + assert.equal(job!.status, "running"); + trackJob(job!.id); + + // Re-running the spawn hook (idempotency) does not create a second job. + spawnPendingJobs(chat.id, saved.messages); + assert.equal(listJobs(chat.id).length, 1, "no duplicate job for the same tool part"); + }); +}); + +// ─── the researcher loop with a fake model + stubbed fetch ────────────────── + +const BRIEF = `## Summary +sigma.js v3 is current. + +## Findings +1. **v3 is the latest major.** [sigma.js docs](https://www.sigmajs.org/docs) + +## Sources +- sigma.js docs (https://www.sigmajs.org/docs)`; + +/** + * The researcher loop runs via agent.generate (non-streaming) — the mock + * therefore scripts doGenerate. Turn 1: a web_search tool call; turn 2: the + * final text brief. + */ +function researcherModel(): MockLanguageModelV4 { + let turn = 0; + return new MockLanguageModelV4({ + doGenerate: async () => { + turn++; + return turn === 1 + ? { + content: [ + { + type: "tool-call" as const, + toolCallId: "ws1", + toolName: "web_search", + input: JSON.stringify({ query: "sigma.js latest version" }), + }, + ], + finishReason: { unified: "tool-calls" as const, raw: "tool-calls" }, + usage: usage(), + warnings: [], + } + : { + content: [{ type: "text" as const, text: BRIEF }], + finishReason: { unified: "stop" as const, raw: "stop" }, + usage: usage(), + warnings: [], + }; + }, + }); +} + +function stubFetch() { + const original = globalThis.fetch; + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = String(input); + if (url.startsWith("https://www.googleapis.com/")) { + return new Response( + JSON.stringify({ + items: [{ title: "sigma.js", link: "https://www.sigmajs.org/", snippet: "v3 released" }], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + } + return new Response("unexpected fetch: " + url, { status: 500 }); + }) as typeof fetch; + return () => { + globalThis.fetch = original; + }; +} + +describe("research runner (async protocol §3)", () => { + it("fulfills the part with a ResearchResult and marks the job done with activity recorded", async () => { + const restoreFetch = stubFetch(); + try { + const chat = createChat("Test: runner completion"); + trackChat(chat.id); + + // A pending research part, as the spawn hook would find it. + const messages = [ + { id: "u1", role: "user", parts: [{ type: "text", text: "which sigma version?" }] }, + { + id: "a1", + role: "assistant", + parts: [ + { type: "text", text: "Checking." }, + { type: "tool-research", toolCallId: "call-run1", state: "input-available", input: RESEARCH_INPUT }, + ], + }, + ] as never; + // saveMessages requires the chat to exist; use the store directly. + const { saveMessages } = await import("../src/store/chats.ts"); + saveMessages(chat.id, messages); + + const job = createJob({ kind: "research", chatId: chat.id, toolCallId: "call-run1" }); + trackJob(job.id); + + await runResearchJob(job.id, { model: researcherModel() }); + + // The job is done, with the activity line from the tool call. + const done = getJob(job.id)!; + assert.equal(done.status, "done"); + assert.equal(done.activity, 'web_search: "sigma.js latest version"', "activity line recorded"); + assert.ok(done.result, "result stored on the job"); + + // The stored chat's part flipped to output-available with a + // ResearchResult-shaped output. + const saved = getChat(chat.id)!; + const part = saved.messages[1]!.parts.find((p) => p.type === "tool-research"); + assert.ok(part && part.type === "tool-research"); + assert.equal(part.state, "output-available"); + const output = part.output as { brief: string; sources: { title: string; url: string }[] }; + assert.equal(output.brief, BRIEF); + assert.ok(Array.isArray(output.sources) && output.sources.length > 0, "sources parsed from the brief"); + assert.equal(output.sources[0]!.url, "https://www.sigmajs.org/docs"); + } finally { + restoreFetch(); + } + }); + + it("a failing runner still fulfills the part (the teacher learns the outcome) and marks the job error", async () => { + const failing = new MockLanguageModelV4({ + doGenerate: async () => { + throw new Error("model exploded"); + }, + }); + const chat = createChat("Test: runner failure"); + trackChat(chat.id); + const { saveMessages } = await import("../src/store/chats.ts"); + saveMessages(chat.id, [ + { id: "u1", role: "user", parts: [{ type: "text", text: "hi" }] }, + { + id: "a1", + role: "assistant", + parts: [{ type: "tool-research", toolCallId: "call-err1", state: "input-available", input: RESEARCH_INPUT }], + }, + ] as never); + + const job = createJob({ kind: "research", chatId: chat.id, toolCallId: "call-err1" }); + trackJob(job.id); + + await runResearchJob(job.id, { model: failing }); + + const errored = getJob(job.id)!; + assert.equal(errored.status, "error"); + assert.match(errored.error ?? "", /model exploded/); + + const saved = getChat(chat.id)!; + const part = saved.messages[1]!.parts.find((p) => p.type === "tool-research"); + assert.ok(part && part.type === "tool-research"); + assert.equal(part.state, "output-available", "failure still fulfills the part"); + const output = part.output as { brief: string; sources: unknown[] }; + assert.match(output.brief, /^Research failed: /); + assert.deepEqual(output.sources, []); + }); +}); + +// ─── the researcher agent construction (model resolution seam) ────────────── + +describe("researcher agent", () => { + it("builds with the web tool set (no network touched)", () => { + const agent = createResearcherAgent(researcherModel()); + assert.ok(agent.tools.web_search, "web_search available"); + assert.ok(agent.tools.web_fetch, "web_fetch available"); + assert.equal(Object.keys(agent.tools).length, 2, "ONLY the web tools — no question/plan tools"); + // reset any accidental global override + researchModelOverride.current = undefined; + }); +});