Quiz hardening: cross-field schema validation rejects ungradeable quizzes

quizInputSchema now enforces (via pure quizInputProblems + superRefine) that
correctAnswer resolves against option values, options are non-empty and
unique, and multi-answer requires multiSelect — schema-valid now implies
gradeable. The AI SDK already returns invalid tool input to the model as a
tool-error part (verified in ai@7 run loop), so the teacher retries in the
same turn. chat-handler onError passes tool-input validation messages
through to the client (masked 'An error occurred.' for everything else).
Covers the Gate 3 GLM failure: correctAnswer slug matching no option.

src/agent/tools.ts

@@ -26,6 +26,7 @@ export const quizTool = tool({
 		"Ask the learner a GRADED question with a known correct answer, then instantly grade and give feedback. " +
 		"Use it to (1) assess what the learner already understands before teaching, and (2) run tight practice/retrieval loops after explaining. " +
 		"Options-only: single-select or multi-select, plus an automatic 'I don't know' choice so the learner can signal a genuine gap instead of guessing. " +
+		"correctAnswer must be the EXACT value (or label) of one of the listed options — a correctAnswer that matches no option fails validation and the call is returned to you to fix. " +
 		"No free-text answers — for non-graded questions use ask_user_question instead. " +
 		"Ask exactly one question per tool call; after calling, end your turn and wait for the graded result.",
 	inputSchema: quizInputSchema,

src/core/question-core.ts

@@ -39,7 +39,76 @@ const optionSchema = z.object({
 	description: z.string().optional().describe("Optional extra detail shown below the option"),
 });
 
-export const quizInputSchema = z.object({
+// ─── cross-field authoring validation (tool-schema refinements) ──────────
+// Enforced at the tool-call boundary (see the superRefines below): an invalid
+// question NEVER reaches the learner — the SDK returns the validation error
+// to the teacher as a tool result and the teacher retries. The rules mirror
+// the grading path exactly (normalizeOptions / coerceCorrectAnswer /
+// resolveCorrect), so schema-valid ⇔ gradeable.
+
+/** Option sanity shared by quiz + ask: labels non-empty, resolved values
+ *  (trimmed `value`, falling back to the trimmed label — exactly how
+ *  normalizeOptions keys them) distinct. Returns the problems plus the
+ *  resolved keys for the correctAnswer check. */
+function optionProblems(
+	options: Array<{ label: string; value?: string }>,
+): { problems: string[]; keys: string[] } {
+	const problems: string[] = [];
+	const keys: string[] = [];
+	options.forEach((option, i) => {
+		const label = option.label.trim();
+		if (label.length === 0) {
+			problems.push(`options[${i}].label is empty — every option needs a label.`);
+			return;
+		}
+		const key = option.value?.trim() || label;
+		if (keys.includes(key)) {
+			problems.push(`Two options resolve to the same value "${key}" — options must be distinct.`);
+		}
+		keys.push(key);
+	});
+	return { problems, keys };
+}
+
+/** ask options: the shared sanity, nothing more (ask is not graded). */
+export function askOptionProblems(options: Array<{ label: string; value?: string }>): string[] {
+	return optionProblems(options).problems;
+}
+
+/** The full quiz authoring contract: option sanity + correctAnswer must be
+ *  gradeable. Empty result = a quiz the dock can arm and grade faithfully. */
+export function quizInputProblems(input: {
+	options: Array<{ label: string; value?: string }>;
+	correctAnswer: string | string[];
+	multiSelect?: boolean;
+}): string[] {
+	const { problems, keys } = optionProblems(input.options);
+
+	const entries = coerceCorrectAnswer(input.correctAnswer).map((v) => v.trim());
+	const unique = [...new Set(entries)];
+	if (unique.length === 0 || unique.some((v) => v.length === 0)) {
+		problems.push("correctAnswer is required — provide the value (or label) of the correct option(s).");
+		return problems;
+	}
+	for (const v of unique) {
+		if (!keys.includes(v)) {
+			problems.push(
+				`correctAnswer "${v}" does not match any option value. Valid values: ${keys.map((k) => `"${k}"`).join(", ")}. ` +
+					"Copy the exact value (or label) of the intended option — never a paraphrase or slug.",
+			);
+		}
+	}
+	if (unique.length > 1 && input.multiSelect !== true) {
+		problems.push(
+				`correctAnswer lists ${unique.length} values but multiSelect is not true — a single-select question with several ` +
+					"correct answers can never be answered correctly. Set multiSelect: true, or make exactly one option correct.",
+		);
+	}
+	return problems;
+}
+
+export const quizInputSchema = z
+	.object({
 		question: z.string().describe("The single quiz question to ask"),
 		options: z
 			.array(optionSchema)
@@ -50,7 +119,8 @@ export const quizInputSchema = z.object({
 		correctAnswer: z
 			.union([z.string(), z.array(z.string())])
 			.describe(
-			"The correct answer as the option value(s) — the `value` field of the option you intend. " +
+				"The correct answer as the option value(s): copy the EXACT `value` — or `label` when `value` is omitted — " +
+					"of the intended option(s), never a paraphrase or slug; a mismatch fails validation. " +
 					"Single-select: one string. Multi-select: an array; the learner is only correct if their selection matches this set exactly.",
 			),
 		explanation: z
@@ -65,9 +135,21 @@ export const quizInputSchema = z.object({
 				"Optional plan-tree node id this question assesses. Set whenever the question targets a tree node — " +
 					"the outcome feeds the learner's knowledge graph. Omit for ad-hoc probes.",
 			),
-});
+	})
+	// The cross-field authoring contract (the Gate-3 hardening): a quiz whose
+	// correctAnswer can't be graded NEVER reaches the learner. The SDK validates
+	// client-side tool inputs against this schema at the call boundary and, on
+	// failure, returns the validation error to the teacher as a tool result —
+	// the teacher retries with a fixed quiz in the same turn. Schema-valid ⇔
+	// gradeable: the rules are exactly what the grading path accepts.
+	.superRefine((input, ctx) => {
+		for (const problem of quizInputProblems(input)) {
+			ctx.addIssue({ code: "custom", message: problem });
+		}
+	});
 
-export const askInputSchema = z.object({
+export const askInputSchema = z
+	.object({
 		question: z.string().describe("The single question to ask"),
 		mode: z
 			.enum(["single-select", "multi-select", "text"])
@@ -84,7 +166,14 @@ export const askInputSchema = z.object({
 			),
 		multiSelect: z.boolean().optional().describe("Set true to allow multiple answers to be selected"),
 		details: z.string().optional().describe("Optional extra context or instructions shown under the question"),
-});
+	})
+	// Same option sanity as the quiz (labels non-empty, values distinct — ask
+	// is not graded, but duplicate values break the dock's value keying).
+	.superRefine((input, ctx) => {
+		for (const problem of askOptionProblems(input.options ?? [])) {
+			ctx.addIssue({ code: "custom", message: problem });
+		}
+	});
 
 export type QuizToolInput = z.infer<typeof quizInputSchema>;
 export type AskToolInput = z.infer<typeof askInputSchema>;

src/server/chat-handler.ts

@@ -1,4 +1,4 @@
-import { createAgentUIStreamResponse, type UIMessage } from "ai";
+import { createAgentUIStreamResponse, InvalidToolInputError, NoSuchToolError, type UIMessage } from "ai";
 import { createTeacherAgent, type TeacherAgent } from "../agent/teacher-agent.ts";
 import { createJobRunner } from "../agent/subagents/runner.ts";
 import { createExtractionPipeline } from "../agent/subagents/extraction.ts";
@@ -45,6 +45,17 @@ export async function handleChatRequest(req: Request, agent?: TeacherAgent): Pro
 				buildTeacherContext(stores.graphStore, chatId),
 			),
 		uiMessages: body.messages as UIMessage[],
+		// Tool-call validation failures are messages OUR schemas authored (the
+		// quiz authoring contract, unknown tools) — safe and useful in the
+		// transcript: they are exactly what the teacher saw and retried on. The
+		// tool loop delivers them as plain strings (what the model received) or
+		// as the original error object; expose both, and keep everything else
+		// (provider/transport internals) masked, as the SDK defaults.
+		onError: (error) => {
+			if (typeof error === "string") return error;
+			if (InvalidToolInputError.isInstance(error) || NoSuchToolError.isInstance(error)) return error.message;
+			return "An error occurred.";
+		},
 		onFinish: ({ messages, isAborted }) => {
 			persistTurn(stores.chatStore, chatId, messages as UIMessage[], isAborted);
 			spawnTurnJobs(jobRunner, chatId, messages as UIMessage[]);

tests/chat-flow.test.ts

@@ -229,6 +229,91 @@ describe("teacher chat flow (scripted model)", () => {
 		const saved = chatStore.getChat(chat.id)!;
 		assert.ok(saved.messages.length >= 1, "partial transcript persisted");
 	});
+
+	it("an ungradeable quiz is rejected at the call boundary, the error reaches the model, and the model retries", async () => {
+		// The authoring contract (question-core superRefine): a quiz whose
+		// correctAnswer matches no option value must NEVER reach the dock —
+		// the SDK validates client-side tool inputs against the schema, marks
+		// the call invalid, and returns the validation error to the teacher as
+		// a tool result. The loop re-prompts the model, which retries in the
+		// same turn. This pins that whole mechanism (the live Nietzsche bug).
+		const BAD_QUIZ_INPUT = {
+			question: "What is Nietzsche's genealogical claim about religion?",
+			options: [{ label: "A cultural diagnosis" }, { label: "Atheism as a conclusion" }],
+			correctAnswer: "cultural-diagnosis", // the exact live failure shape: a slug, no match
+			explanation: "Genealogy diagnoses, it does not convert.",
+		};
+
+		const calls: string[][] = [];
+		let step = 0;
+		const retryModel = new MockLanguageModelV4({
+			doStream: async (options) => {
+				calls.push(options.prompt.map((m) => JSON.stringify(m.content)));
+				step++;
+				const chunks: LanguageModelV4StreamPart[] =
+					step === 1
+						? [
+								{ type: "stream-start", warnings: [] },
+								{ type: "tool-input-start", id: "call-bad", toolName: "quiz" },
+								{ type: "tool-input-delta", id: "call-bad", delta: JSON.stringify(BAD_QUIZ_INPUT) },
+								{ type: "tool-input-end", id: "call-bad" },
+								{
+										type: "tool-call",
+										toolCallId: "call-bad",
+										toolName: "quiz",
+										input: JSON.stringify(BAD_QUIZ_INPUT),
+									},
+								{ type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() },
+							]
+						: [
+								{ type: "stream-start", warnings: [] },
+								{ type: "tool-input-start", id: "call-good", toolName: "quiz" },
+								{ type: "tool-input-delta", id: "call-good", delta: JSON.stringify(QUIZ_INPUT) },
+								{ type: "tool-input-end", id: "call-good" },
+								{
+										type: "tool-call",
+										toolCallId: "call-good",
+										toolName: "quiz",
+										input: JSON.stringify(QUIZ_INPUT),
+									},
+								{ type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() },
+							];
+				return { stream: simulateReadableStream({ chunks, initialDelayInMs: 0, chunkDelayInMs: 0 }) };
+			},
+		});
+
+		const agent = createTeacherAgent(retryModel);
+		const chat = chatStore.createChat("Test: bad quiz retry");
+		createdChats.push(chat.id);
+		const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "quiz me" }] };
+		await drain(await handleChatRequest(post(chat.id, [firstUser]), agent));
+
+		// The model was re-prompted (the error did not end the turn)…
+		assert.equal(calls.length, 2, "the invalid call triggered a second model prompt");
+		// …and the second prompt carried the validation error back as a tool result.
+		assert.match(
+			calls[1]!.join(" "),
+			/does not match any option value/,
+			"the validation error reached the model",
+		);
+
+		// The transcript: the failed call persisted as an error part (renders
+		// nothing client-side), the retry as the pending quiz.
+		const saved = chatStore.getChat(chat.id)!;
+		const assistantMsg = saved.messages[1]!;
+		const quizParts = assistantMsg.parts.filter((p) => p.type === "tool-quiz");
+		const failed = quizParts.find(
+				(p) => p.type === "tool-quiz" && (p as { state?: string }).state === "output-error",
+			);
+		assert.ok(failed, "the invalid call persisted as an error-state part");
+		assert.match(
+			String((failed as { errorText?: string }).errorText),
+			/does not match any option value/,
+		);
+		const pending = findPendingToolCall(saved.messages);
+		assert.equal(pending?.toolCallId, "call-good", "the RETRIED quiz is the pending one");
+		assert.deepEqual(pending?.input, QUIZ_INPUT);
+	});
 });
 
 describe("chat store", () => {

tests/question-core.test.ts

@@ -5,6 +5,8 @@
 import assert from "node:assert/strict";
 import { describe, it } from "node:test";
 import {
+	askInputSchema,
+	askOptionProblems,
 	buildAnswer,
 	coerceCorrectAnswer,
 	findPendingQuestion,
@@ -14,12 +16,124 @@ import {
 	normalizeOptions,
 	QUESTION_ANSWERED_ENTRY,
 	QUESTION_ENTRY,
+	quizInputProblems,
+	quizInputSchema,
 	resolveCorrect,
 	shuffleOptions,
 	type QuestionData,
 	toClientQuestion,
 } from "../src/core/question-core.ts";
 
+// ─── the authoring contract (tool-schema refinements) ───────────────────
+// Schema-valid ⇔ gradeable: a quiz whose correctAnswer can't be resolved
+// dies at the tool-call boundary, and the SDK returns the error to the
+// teacher to retry — it must never reach the learner's dock.
+
+describe("quizInputSchema / quizInputProblems (the authoring contract)", () => {
+	const valid = {
+		question: "Which planet is closest to the Sun?",
+		options: [{ label: "Venus" }, { label: "Mercury" }, { label: "Mars" }],
+		correctAnswer: "Mercury",
+		explanation: "Mercury is the innermost planet.",
+	};
+
+	it("accepts a valid single-select quiz (label as correctAnswer)", () => {
+		const parsed = quizInputSchema.safeParse(valid);
+		assert.equal(parsed.success, true);
+		assert.deepEqual(quizInputProblems(valid), []);
+	});
+
+	it("accepts explicit values and multi-select arrays (incl. JSON-stringified)", () => {
+		const multi = {
+			...valid,
+			options: [
+				{ label: "A", value: "a" },
+				{ label: "B", value: "b" },
+				{ label: "C", value: "c" },
+			],
+			correctAnswer: ["a", "b"],
+			multiSelect: true,
+		};
+		assert.equal(quizInputSchema.safeParse(multi).success, true);
+		// The union lists String first, so providers sometimes stringify arrays.
+		const stringified = { ...multi, correctAnswer: JSON.stringify(multi.correctAnswer) };
+		assert.equal(quizInputSchema.safeParse(stringified).success, true);
+		assert.deepEqual(quizInputProblems(stringified), []);
+	});
+
+	it("rejects the observed failure: a correctAnswer slug matching no option value, listing the valid values", () => {
+		// The exact class GLM produced live (Gate 3 + live verification).
+		const bad = {
+			question: "What is Nietzsche's genealogical claim about religion?",
+			options: [
+				{ label: "A cultural diagnosis" },
+			{ label: "Atheism as a conclusion" },
+			{ label: "A call to convert people" },
+		],
+			correctAnswer: "cultural-diagnosis",
+			explanation: "...",
+		};
+		const parsed = quizInputSchema.safeParse(bad);
+		assert.equal(parsed.success, false);
+		const problems = quizInputProblems(bad);
+		assert.equal(problems.length, 1);
+		assert.match(problems[0]!, /does not match any option value/);
+		assert.match(problems[0]!, /"A cultural diagnosis"/); // the valid values are listed for the retry
+		// The schema's error message carries the problems (the SDK returns them to the teacher).
+		const msg = JSON.stringify(parsed.error?.issues);
+		assert.match(msg, /does not match any option value/);
+	});
+
+	it("rejects multiple correct answers without multiSelect (ungradeable single-select)", () => {
+		const bad = { ...valid, correctAnswer: ["Venus", "Mercury"] };
+		assert.equal(quizInputSchema.safeParse(bad).success, false);
+		assert.match(quizInputProblems(bad).join(" "), /multiSelect/);
+	});
+
+	it("rejects duplicate option values", () => {
+		const bad = {
+			...valid,
+			options: [
+				{ label: "Mercury", value: "same" },
+				{ label: "Venus", value: "same" },
+			],
+		};
+		assert.equal(quizInputSchema.safeParse(bad).success, false);
+		assert.match(quizInputProblems(bad).join(" "), /same value/);
+	});
+
+	it("rejects an empty option label", () => {
+		const bad = { ...valid, options: [{ label: "  " }, { label: "Mercury" }, { label: "Mars" }] };
+		assert.equal(quizInputSchema.safeParse(bad).success, false);
+		assert.match(quizInputProblems(bad).join(" "), /label is empty/);
+	});
+
+	it("requires a resolvable correctAnswer", () => {
+		assert.equal(quizInputSchema.safeParse({ ...valid, correctAnswer: "" }).success, false);
+		assert.equal(quizInputSchema.safeParse({ ...valid, correctAnswer: "[]" }).success, false);
+		assert.match(quizInputProblems({ ...valid, correctAnswer: "" }).join(" "), /required/);
+	});
+});
+
+describe("askInputSchema / askOptionProblems", () => {
+	it("accepts a valid ask", () => {
+		const parsed = askInputSchema.safeParse({
+			question: "Which topic first?",
+			options: [{ label: "Limits" }, { label: "Chain rule" }],
+		});
+		assert.equal(parsed.success, true);
+	});
+
+	it("rejects duplicate option values and empty labels (they break the dock's keying)", () => {
+		const bad = { question: "?", options: [{ label: "A", value: "x" }, { label: "B", value: "x" }] };
+		assert.equal(askInputSchema.safeParse(bad).success, false);
+		assert.match(askOptionProblems(bad.options).join(" "), /same value/);
+		const emptyLabel = { question: "?", options: [{ label: "" }, { label: "B" }] };
+		assert.equal(askInputSchema.safeParse(emptyLabel).success, false);
+		assert.match(askOptionProblems(emptyLabel.options).join(" "), /label is empty/);
+	});
+});
+
 // ─── normalizeOptions ───────────────────────────────────────────────────────
 
 describe("normalizeOptions", () => {