Quiz hardening: cross-field schema validation rejects ungradeable quizzes
quizInputSchema now enforces (via pure quizInputProblems + superRefine) that correctAnswer resolves against option values, options are non-empty and unique, and multi-answer requires multiSelect — schema-valid now implies gradeable. The AI SDK already returns invalid tool input to the model as a tool-error part (verified in ai@7 run loop), so the teacher retries in the same turn. chat-handler onError passes tool-input validation messages through to the client (masked 'An error occurred.' for everything else). Covers the Gate 3 GLM failure: correctAnswer slug matching no option.
This commit is contained in:
Amos
@@ -229,6 +229,91 @@ describe("teacher chat flow (scripted model)", () => {
|
||||
const saved = chatStore.getChat(chat.id)!;
|
||||
assert.ok(saved.messages.length >= 1, "partial transcript persisted");
|
||||
});
|
||||
|
||||
it("an ungradeable quiz is rejected at the call boundary, the error reaches the model, and the model retries", async () => {
|
||||
// The authoring contract (question-core superRefine): a quiz whose
|
||||
// correctAnswer matches no option value must NEVER reach the dock —
|
||||
// the SDK validates client-side tool inputs against the schema, marks
|
||||
// the call invalid, and returns the validation error to the teacher as
|
||||
// a tool result. The loop re-prompts the model, which retries in the
|
||||
// same turn. This pins that whole mechanism (the live Nietzsche bug).
|
||||
const BAD_QUIZ_INPUT = {
|
||||
question: "What is Nietzsche's genealogical claim about religion?",
|
||||
options: [{ label: "A cultural diagnosis" }, { label: "Atheism as a conclusion" }],
|
||||
correctAnswer: "cultural-diagnosis", // the exact live failure shape: a slug, no match
|
||||
explanation: "Genealogy diagnoses, it does not convert.",
|
||||
};
|
||||
|
||||
const calls: string[][] = [];
|
||||
let step = 0;
|
||||
const retryModel = new MockLanguageModelV4({
|
||||
doStream: async (options) => {
|
||||
calls.push(options.prompt.map((m) => JSON.stringify(m.content)));
|
||||
step++;
|
||||
const chunks: LanguageModelV4StreamPart[] =
|
||||
step === 1
|
||||
? [
|
||||
{ type: "stream-start", warnings: [] },
|
||||
{ type: "tool-input-start", id: "call-bad", toolName: "quiz" },
|
||||
{ type: "tool-input-delta", id: "call-bad", delta: JSON.stringify(BAD_QUIZ_INPUT) },
|
||||
{ type: "tool-input-end", id: "call-bad" },
|
||||
{
|
||||
type: "tool-call",
|
||||
toolCallId: "call-bad",
|
||||
toolName: "quiz",
|
||||
input: JSON.stringify(BAD_QUIZ_INPUT),
|
||||
},
|
||||
{ type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() },
|
||||
]
|
||||
: [
|
||||
{ type: "stream-start", warnings: [] },
|
||||
{ type: "tool-input-start", id: "call-good", toolName: "quiz" },
|
||||
{ type: "tool-input-delta", id: "call-good", delta: JSON.stringify(QUIZ_INPUT) },
|
||||
{ type: "tool-input-end", id: "call-good" },
|
||||
{
|
||||
type: "tool-call",
|
||||
toolCallId: "call-good",
|
||||
toolName: "quiz",
|
||||
input: JSON.stringify(QUIZ_INPUT),
|
||||
},
|
||||
{ type: "finish", finishReason: { unified: "tool-calls", raw: "tool_calls" }, usage: usage() },
|
||||
];
|
||||
return { stream: simulateReadableStream({ chunks, initialDelayInMs: 0, chunkDelayInMs: 0 }) };
|
||||
},
|
||||
});
|
||||
|
||||
const agent = createTeacherAgent(retryModel);
|
||||
const chat = chatStore.createChat("Test: bad quiz retry");
|
||||
createdChats.push(chat.id);
|
||||
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "quiz me" }] };
|
||||
await drain(await handleChatRequest(post(chat.id, [firstUser]), agent));
|
||||
|
||||
// The model was re-prompted (the error did not end the turn)…
|
||||
assert.equal(calls.length, 2, "the invalid call triggered a second model prompt");
|
||||
// …and the second prompt carried the validation error back as a tool result.
|
||||
assert.match(
|
||||
calls[1]!.join(" "),
|
||||
/does not match any option value/,
|
||||
"the validation error reached the model",
|
||||
);
|
||||
|
||||
// The transcript: the failed call persisted as an error part (renders
|
||||
// nothing client-side), the retry as the pending quiz.
|
||||
const saved = chatStore.getChat(chat.id)!;
|
||||
const assistantMsg = saved.messages[1]!;
|
||||
const quizParts = assistantMsg.parts.filter((p) => p.type === "tool-quiz");
|
||||
const failed = quizParts.find(
|
||||
(p) => p.type === "tool-quiz" && (p as { state?: string }).state === "output-error",
|
||||
);
|
||||
assert.ok(failed, "the invalid call persisted as an error-state part");
|
||||
assert.match(
|
||||
String((failed as { errorText?: string }).errorText),
|
||||
/does not match any option value/,
|
||||
);
|
||||
const pending = findPendingToolCall(saved.messages);
|
||||
assert.equal(pending?.toolCallId, "call-good", "the RETRIED quiz is the pending one");
|
||||
assert.deepEqual(pending?.input, QUIZ_INPUT);
|
||||
});
|
||||
});
|
||||
|
||||
describe("chat store", () => {
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import {
|
||||
askInputSchema,
|
||||
askOptionProblems,
|
||||
buildAnswer,
|
||||
coerceCorrectAnswer,
|
||||
findPendingQuestion,
|
||||
@@ -14,12 +16,124 @@ import {
|
||||
normalizeOptions,
|
||||
QUESTION_ANSWERED_ENTRY,
|
||||
QUESTION_ENTRY,
|
||||
quizInputProblems,
|
||||
quizInputSchema,
|
||||
resolveCorrect,
|
||||
shuffleOptions,
|
||||
type QuestionData,
|
||||
toClientQuestion,
|
||||
} from "../src/core/question-core.ts";
|
||||
|
||||
// ─── the authoring contract (tool-schema refinements) ───────────────────
|
||||
// Schema-valid ⇔ gradeable: a quiz whose correctAnswer can't be resolved
|
||||
// dies at the tool-call boundary, and the SDK returns the error to the
|
||||
// teacher to retry — it must never reach the learner's dock.
|
||||
|
||||
describe("quizInputSchema / quizInputProblems (the authoring contract)", () => {
|
||||
const valid = {
|
||||
question: "Which planet is closest to the Sun?",
|
||||
options: [{ label: "Venus" }, { label: "Mercury" }, { label: "Mars" }],
|
||||
correctAnswer: "Mercury",
|
||||
explanation: "Mercury is the innermost planet.",
|
||||
};
|
||||
|
||||
it("accepts a valid single-select quiz (label as correctAnswer)", () => {
|
||||
const parsed = quizInputSchema.safeParse(valid);
|
||||
assert.equal(parsed.success, true);
|
||||
assert.deepEqual(quizInputProblems(valid), []);
|
||||
});
|
||||
|
||||
it("accepts explicit values and multi-select arrays (incl. JSON-stringified)", () => {
|
||||
const multi = {
|
||||
...valid,
|
||||
options: [
|
||||
{ label: "A", value: "a" },
|
||||
{ label: "B", value: "b" },
|
||||
{ label: "C", value: "c" },
|
||||
],
|
||||
correctAnswer: ["a", "b"],
|
||||
multiSelect: true,
|
||||
};
|
||||
assert.equal(quizInputSchema.safeParse(multi).success, true);
|
||||
// The union lists String first, so providers sometimes stringify arrays.
|
||||
const stringified = { ...multi, correctAnswer: JSON.stringify(multi.correctAnswer) };
|
||||
assert.equal(quizInputSchema.safeParse(stringified).success, true);
|
||||
assert.deepEqual(quizInputProblems(stringified), []);
|
||||
});
|
||||
|
||||
it("rejects the observed failure: a correctAnswer slug matching no option value, listing the valid values", () => {
|
||||
// The exact class GLM produced live (Gate 3 + live verification).
|
||||
const bad = {
|
||||
question: "What is Nietzsche's genealogical claim about religion?",
|
||||
options: [
|
||||
{ label: "A cultural diagnosis" },
|
||||
{ label: "Atheism as a conclusion" },
|
||||
{ label: "A call to convert people" },
|
||||
],
|
||||
correctAnswer: "cultural-diagnosis",
|
||||
explanation: "...",
|
||||
};
|
||||
const parsed = quizInputSchema.safeParse(bad);
|
||||
assert.equal(parsed.success, false);
|
||||
const problems = quizInputProblems(bad);
|
||||
assert.equal(problems.length, 1);
|
||||
assert.match(problems[0]!, /does not match any option value/);
|
||||
assert.match(problems[0]!, /"A cultural diagnosis"/); // the valid values are listed for the retry
|
||||
// The schema's error message carries the problems (the SDK returns them to the teacher).
|
||||
const msg = JSON.stringify(parsed.error?.issues);
|
||||
assert.match(msg, /does not match any option value/);
|
||||
});
|
||||
|
||||
it("rejects multiple correct answers without multiSelect (ungradeable single-select)", () => {
|
||||
const bad = { ...valid, correctAnswer: ["Venus", "Mercury"] };
|
||||
assert.equal(quizInputSchema.safeParse(bad).success, false);
|
||||
assert.match(quizInputProblems(bad).join(" "), /multiSelect/);
|
||||
});
|
||||
|
||||
it("rejects duplicate option values", () => {
|
||||
const bad = {
|
||||
...valid,
|
||||
options: [
|
||||
{ label: "Mercury", value: "same" },
|
||||
{ label: "Venus", value: "same" },
|
||||
],
|
||||
};
|
||||
assert.equal(quizInputSchema.safeParse(bad).success, false);
|
||||
assert.match(quizInputProblems(bad).join(" "), /same value/);
|
||||
});
|
||||
|
||||
it("rejects an empty option label", () => {
|
||||
const bad = { ...valid, options: [{ label: " " }, { label: "Mercury" }, { label: "Mars" }] };
|
||||
assert.equal(quizInputSchema.safeParse(bad).success, false);
|
||||
assert.match(quizInputProblems(bad).join(" "), /label is empty/);
|
||||
});
|
||||
|
||||
it("requires a resolvable correctAnswer", () => {
|
||||
assert.equal(quizInputSchema.safeParse({ ...valid, correctAnswer: "" }).success, false);
|
||||
assert.equal(quizInputSchema.safeParse({ ...valid, correctAnswer: "[]" }).success, false);
|
||||
assert.match(quizInputProblems({ ...valid, correctAnswer: "" }).join(" "), /required/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("askInputSchema / askOptionProblems", () => {
|
||||
it("accepts a valid ask", () => {
|
||||
const parsed = askInputSchema.safeParse({
|
||||
question: "Which topic first?",
|
||||
options: [{ label: "Limits" }, { label: "Chain rule" }],
|
||||
});
|
||||
assert.equal(parsed.success, true);
|
||||
});
|
||||
|
||||
it("rejects duplicate option values and empty labels (they break the dock's keying)", () => {
|
||||
const bad = { question: "?", options: [{ label: "A", value: "x" }, { label: "B", value: "x" }] };
|
||||
assert.equal(askInputSchema.safeParse(bad).success, false);
|
||||
assert.match(askOptionProblems(bad.options).join(" "), /same value/);
|
||||
const emptyLabel = { question: "?", options: [{ label: "" }, { label: "B" }] };
|
||||
assert.equal(askInputSchema.safeParse(emptyLabel).success, false);
|
||||
assert.match(askOptionProblems(emptyLabel.options).join(" "), /label is empty/);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── normalizeOptions ───────────────────────────────────────────────────────
|
||||
|
||||
describe("normalizeOptions", () => {
|
||||
|
||||
Reference in New Issue
Block a user