From d1542cdcbf90481f92df0541dc64e79c880b7b3f Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Fri, 25 Sep 2026 03:31:53 +0300 Subject: [PATCH] =?UTF-8?q?quiz=20argument=20order:=20explanation=20before?= =?UTF-8?q?=20correctAnswer=20=E2=80=94=20reasoning=20precedes=20the=20pic?= =?UTF-8?q?k=20(owner=20ruling,=202026-09-25)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/agent/prompts/shared/20-question-tools.md | 2 +- src/core/question-core.ts | 17 ++++++---- tests/question-core.test.ts | 32 +++++++++++++++++++ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/src/agent/prompts/shared/20-question-tools.md b/src/agent/prompts/shared/20-question-tools.md index 0cbb6c1..c323b1f 100644 --- a/src/agent/prompts/shared/20-question-tools.md +++ b/src/agent/prompts/shared/20-question-tools.md @@ -11,7 +11,7 @@ - `quiz` is GRADED; `ask_user_question` is not. If the question has a correct answer, use `quiz`. If you just need a preference, decision, or open-ended input, use `ask_user_question`. - `correctAnswer` is REQUIRED and is the option value, not a position number. Single-select: one string (e.g. "mercury"). Multi-select: an array of strings (e.g. ["belize", "niue"]). - Always pass the option's `value` string as `correctAnswer` — it is self-checking and prevents miscounting positions. A value that matches no option is a hard error. -- `explanation` is REQUIRED — always say why the correct answer is correct. +- `explanation` is REQUIRED — always say why the correct answer is correct. It comes BEFORE `correctAnswer` in the call by design: reason it out first, then commit the answer the reasoning supports. - Multi-select is graded as an exact-set match: he is correct only if he selects every correct option and no incorrect ones. - There is no free-text mode. An "I don't know" choice is ALWAYS added automatically — provide ONLY the real, gradable options (at least two). Never add your own uncertainty/opt-out option like "I don't know", "I'm not sure", or "Not sure"; that is handled for you and a manual one would be redundant or gradable-as-wrong. - If a result comes back as dontKnow, he honestly did not know and did NOT guess — treat it as a genuine knowledge gap, not as a wrong answer. diff --git a/src/core/question-core.ts b/src/core/question-core.ts index 611a630..3693a1f 100644 --- a/src/core/question-core.ts +++ b/src/core/question-core.ts @@ -127,6 +127,17 @@ export const quizInputBase = z.object({ .boolean() .optional() .describe("Set to true when more than one option is correct and the learner must select all of them."), + // Argument-order ruling (owner, 2026-09-25): the explanation is REASONING, + // and reasoning happens BEFORE the answer is chosen — models emit tool + // args in schema order, so explanation sits immediately BEFORE correctAnswer. + // (All consumers read by key; the order is authoring guidance, nothing else.) + explanation: z + .string() + .describe( + "REQUIRED. Explanation revealed AFTER the learner answers (shown whether they got it right or wrong). " + + "Use it to reinforce why the correct answer is correct. It is written BEFORE correctAnswer by design: " + + "reason it out first, then commit the answer the reasoning supports.", + ), correctAnswer: z .union([z.string(), z.array(z.string())]) .describe( @@ -136,12 +147,6 @@ export const quizInputBase = z.object({ "Always pass the value, not a position number — this is self-checking and prevents miscounting. " + "A value that matches no option fails validation and the call is returned to you to fix.", ), - explanation: z - .string() - .describe( - "REQUIRED. Explanation revealed AFTER the learner answers (shown whether they got it right or wrong). " + - "Use it to reinforce why the correct answer is correct.", - ), details: z.string().optional().describe("Optional extra context or instructions shown under the question."), }); diff --git a/tests/question-core.test.ts b/tests/question-core.test.ts index e3dfc57..c7c8d95 100644 --- a/tests/question-core.test.ts +++ b/tests/question-core.test.ts @@ -4,6 +4,7 @@ import assert from "node:assert/strict"; import { describe, it } from "node:test"; +import { z } from "zod"; import { askInputSchema, askOptionProblems, @@ -18,6 +19,7 @@ import { QUESTION_ENTRY, quizInputProblems, quizInputSchema, + probeQuizInputSchema, resolveCorrect, toggleOption, toggleOther, @@ -32,6 +34,36 @@ import { // dies at the tool-call boundary, and the SDK returns the error to the // teacher to retry — it must never reach the learner's dock. +describe("the schema's argument order (reasoning before the pick)", () => { + // Argument-order ruling (owner, 2026-09-25): the explanation is + // reasoning, and reasoning happens BEFORE the answer is chosen — models + // emit tool args in schema order, so explanation sits immediately before + // correctAnswer. z.toJSONSchema preserves zod's declaration order into + // the model-facing properties, so this pin guards the emission end to + // end. All consumers read by key — the order is authoring guidance. + it("emits explanation immediately before correctAnswer (both agents' schemas)", () => { + for (const schema of [quizInputSchema, probeQuizInputSchema]) { + const json = z.toJSONSchema(schema as z.ZodType, { io: "input" }) as { + properties: Record; + }; + const keys = Object.keys(json.properties); + assert.ok( + keys.indexOf("explanation") < keys.indexOf("correctAnswer"), + `explanation must precede correctAnswer in the emitted schema (got: ${keys.join(", ")})`, + ); + assert.deepEqual(keys, [ + "question", + "options", + "multiSelect", + "explanation", + "correctAnswer", + "details", + ...(keys.includes("nodeId") ? ["nodeId"] : []), + ]); + } + }); +}); + describe("quizInputSchema / quizInputProblems (the authoring contract)", () => { const valid = { question: "Which planet is closest to the Sun?",