From 4baf8385be36a237449cf6ec22b598c69930c4ff Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Sat, 12 Sep 2026 04:37:05 +0300 Subject: [PATCH] fix: parallel question calls are asked, saved, and answered in the model's call order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live observation (modal-realism run): the teacher batched two quizzes in one turn; the dock showed the SECOND question first, so the answering order was reversed vs the saved record and the transcript cards. Cause: findPending scanned parts BACKWARD within a message, so with two input-available quizzes it picked the last one first. No overwriting was involved — parts are created in call order on both wire and record; the scan direction alone reversed the interaction order. (pi executes a parallel batch with starts + result messages in call order; the execution-end order is completion-racy, which also allowed a millisecond dock-subject flip mid-turn.) Fix A — src/core/pending.ts: parts now scan FORWARD within a message (messages stay newest-first): the FIRST unanswered question is the dock subject; the second appears only after the first is answered. Fix B — src/agent/lh-tool.ts: question tools opt into sequential execution (pi runs the batch in call order when any called tool opts in), so tool-input-available chunks always arrive in call order and the dock subject can never flip mid-turn. Question executes are instant. Pinned in chat-flow.test.ts: a parallel two-quiz batch asserts wire order (A's available chunk before B's), part persistence order, and the dock selection sequence (A first, then B after A is answered). Gates: tsc, lint, build, 307/307. --- src/agent/lh-tool.ts | 6 +++++ src/core/pending.ts | 9 ++++++- tests/chat-flow.test.ts | 56 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/agent/lh-tool.ts b/src/agent/lh-tool.ts index 5e1c81e..3a2a14d 100644 --- a/src/agent/lh-tool.ts +++ b/src/agent/lh-tool.ts @@ -67,6 +67,12 @@ export function toPiTool(def: LhTool): AgentTool { label: def.name, description: def.description, parameters: z.toJSONSchema(def.schema as z.ZodType) as never, + // Question tools opt into SEQUENTIAL execution: pi runs the whole + // batch in call order when any called tool opts in, so the + // tool-input-available chunks arrive in the model's call order — the + // dock's subject can never flip mid-turn between two parallel + // questions. Question executes are instant; no latency is lost. + executionMode: def.clientQuestion ? "sequential" : "parallel", execute: async (toolCallId, args) => { // The authoring check: structural JSON Schema validation already // ran in the loop's prepare step; this re-run adds the refinements diff --git a/src/core/pending.ts b/src/core/pending.ts index 1c65b15..078e320 100644 --- a/src/core/pending.ts +++ b/src/core/pending.ts @@ -5,6 +5,13 @@ import { isToolUIPart, type UIMessage } from "ai"; * blocks the composer — one question at a time, mirroring the teacher's * prompt contract. Derived client state; the server enforces nothing. * + * Scan order: MESSAGES newest-first (a later turn's live question supersedes + * an older dangling one), but PARTS forward within a message — the model's + * FIRST unanswered question is the dock subject. Parallel question calls + * (pi executes them in one batch) are thus answered in the same order they + * were asked, saved, and rendered as cards; a backward part scan reversed + * that order (the live modal-realism observation). + * * Job tool parts (research/demo/visual) never pend client-side — the async * protocol resolves them server-side with an immediate ack (see * contracts/jobs.ts "THE ASYNC PROTOCOL"), so job tools are NOT returned @@ -26,7 +33,7 @@ export const JOB_TOOLS = ["research", "demo", "visual"] as const; function findPending(messages: UIMessage[], names: readonly string[]): PendingToolCall | null { for (let m = messages.length - 1; m >= 0; m--) { const msg = messages[m]!; - for (let p = msg.parts.length - 1; p >= 0; p--) { + for (let p = 0; p < msg.parts.length; p++) { const part = msg.parts[p]!; if (isToolUIPart(part) && part.state === "input-available") { const tool = part.type.slice("tool-".length); diff --git a/tests/chat-flow.test.ts b/tests/chat-flow.test.ts index 5a3c3f9..25824c1 100644 --- a/tests/chat-flow.test.ts +++ b/tests/chat-flow.test.ts @@ -355,6 +355,62 @@ describe("teacher chat flow (scripted model)", () => { const pending = findPendingToolCall(saved.messages); assert.equal(pending?.toolCallId, "call-clean", "the retried quiz is the pending one"); }); + + it("parallel quiz calls are asked, saved, and answered in the model's call order", async () => { + // The live modal-realism observation: the teacher batched two quizzes + // in one turn, and the dock showed the SECOND one first (the backward + // part scan), so the interaction order was reversed vs the record. + // Pins: (A) findPendingToolCall picks the FIRST unanswered question in + // part order; (B) question tools execute sequentially, so their + // tool-input-available chunks arrive in call order on the wire. + const QUIZ_A = { ...QUIZ_INPUT, question: "Quiz A: which planet is closest to the sun?" }; + const QUIZ_B = { ...QUIZ_INPUT, question: "Quiz B: which planet is largest?" }; + + const { engine } = scriptedTeacher(() => [ + () => + fauxAssistantMessage( + [fauxToolCall("quiz", QUIZ_A, { id: "call-a" }), fauxToolCall("quiz", QUIZ_B, { id: "call-b" })], + { stopReason: "toolUse" }, + ), + ]); + + const chat = chatStore.createChat("Test: parallel quizzes"); + createdChats.push(chat.id); + const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "quiz me twice" }] }; + const body = await drain(await handleChatRequest(post(chat.id, [firstUser]), agentFactory(engine))); + + // (B) wire determinism: the batches' input-available chunks in call order. + const availA = body.indexOf('"type":"tool-input-available","toolCallId":"call-a"'); + const availB = body.indexOf('"type":"tool-input-available","toolCallId":"call-b"'); + assert.ok(availA !== -1 && availB !== -1, "both quizzes became available"); + assert.ok(availA < availB, "call A's input became available before call B's (sequential execution)"); + + // Saved in call order. + const saved = chatStore.getChat(chat.id)!; + const quizIds = saved.messages[1]!.parts + .filter((p) => p.type === "tool-quiz") + .map((p) => (p as { toolCallId: string }).toolCallId); + assert.deepEqual(quizIds, ["call-a", "call-b"], "parts persisted in call order"); + + // (A) the dock serializes answering in the same order: A first, then B. + const pending = findPendingToolCall(saved.messages); + assert.equal(pending?.toolCallId, "call-a", "the FIRST asked question is the dock subject"); + // …the learner answers A (client-side grading seam — state flip only here)… + const answered = saved.messages.map((m) => + (m as { id?: string }).id === saved.messages[1]!.id + ? { + ...m, + parts: (m as { parts: unknown[] }).parts.map((p) => + (p as { toolCallId?: string }).toolCallId === "call-a" + ? { ...(p as object), state: "output-available", output: { correct: true } } + : p, + ), + } + : m, + ); + const next = findPendingToolCall(answered as UIMessage[]); + assert.equal(next?.toolCallId, "call-b", "the SECOND question becomes the dock subject only after the first is answered"); + }); }); describe("chat store", () => {