two questions per teaching unit: predict → teach → check (owner 2026-10-03)

The teaching cycle gains its motivation question. The teach quiz schema
now carries a REQUIRED type field — predict (posed before the teaching,
arrives open, the learner predicts the next step) or check (posed after,
arrives collapsed behind the reveal). The prober's quiz stays role-less.

- core: quizInputObject.type union + normalizeQuizType + the reveal law
  as pure quizNeedsReveal; QuestionData.questionType; note → thinking
  rename (legacy reads fold back); readingThinking channel on answers;
  compactAnswerForModel emits thinking / thinkingWhileReading
- tools: descriptions stripped to mechanics — QUIZ_OPTION_CRAFT and
  EXPLAIN_DOCTRINE deleted, doctrine moved to the prompt layer
  (reverses the 2026-09-27 placement, owner 2026-10-03); shared
  quizDescription/explainDescription builders differ only in deltas
- prompts: shared/20-question-tools.md rewritten (option craft,
  transcribed-speech leniency, the two thinking channels); new
  teacher/28-predictions.md (cycle, roles, skip rule, resolution law);
  20-teaching + 30-question-principles scoped to the new taxonomy
- UI: predictions render open with a Prediction eyebrow, verdict shows
  but the explanation tucks behind a toggle; the reading-thinking field
  mounts while a check waits (dictation claims it until reveal);
  answered card separates thinking vs thinking-while-reading blocks
This commit is contained in:
amosblomqvist
2026-10-04 00:53:29 +03:00
parent 571cc94053
commit 360d564a6c
19 changed files with 664 additions and 106 deletions
+3
View File
@@ -62,6 +62,7 @@ const QUIZ_INPUT = {
],
correctAnswer: "Mercury",
explanation: "Mercury is the innermost planet.",
type: "check",
nodeId: "planet-order",
};
@@ -619,6 +620,7 @@ describe("teacher chat flow (scripted model)", () => {
// same turn. This pins that whole mechanism (the live Nietzsche bug).
const BAD_QUIZ_INPUT = {
question: "What is Nietzsche's genealogical claim about religion?",
type: "check",
nodeId: "genealogy",
options: [{ label: "A cultural diagnosis" }, { label: "Atheism as a conclusion" }],
correctAnswer: "cultural-diagnosis", // the exact live failure shape: a slug, no match
@@ -695,6 +697,7 @@ describe("teacher chat flow (scripted model)", () => {
// question, and the model's retry must become the pending quiz.
const STRINGY_QUIZ_INPUT = {
question: "Lewis's indexical analysis of 'actual' — which is it?",
type: "check",
nodeId: "indexicals",
options: JSON.stringify([
{ label: "Like 'I'", value: "indexical" },
+1 -1
View File
@@ -185,7 +185,7 @@ function fixtureMessages(): LhMessage[] {
quizPart(
"call-3",
{ question: "Q3", nodeId: "alpha", options: [{ label: "a" }, { label: "b" }], correctAnswer: "a", explanation: "e" },
answer({ questionText: "Q3 text", correct: true, note: "recalled the diagram", answeredAt: "2026-08-22T02:00:00.000Z" }),
answer({ questionText: "Q3 text", correct: true, thinking: "recalled the diagram", answeredAt: "2026-08-22T02:00:00.000Z" }),
),
]),
assistant("a4", [
+21
View File
@@ -1280,6 +1280,27 @@ describe("prompt composition", () => {
assert.ok(!prober.includes("Question principles"), "the teach-phase fragment never composes into the prober");
});
it("the predict-check cycle composes into the teacher; the craft chunk reaches BOTH agents (owner 2026-10-03)", () => {
const teacher = composeInstructions("teacher");
// the cycle doctrine (28-predictions.md) — teacher-only
assert.ok(teacher.includes("predict → teach → check"), "the cycle line is in");
assert.ok(teacher.includes('The prediction (`type: "predict"`)'), "the prediction role section is in");
assert.ok(teacher.includes("Skip the prediction"), "the skip rule is in");
assert.ok(teacher.includes('The check (`type: "check"`)'), "the check role section is in");
// the craft + thinking channels (shared/20-question-tools.md) — BOTH agents
const prober = composeInstructions("prober");
for (const [name, prompt] of [
["teacher", teacher],
["prober", prober],
] as const) {
assert.ok(prompt.includes("asymmetry"), `${name} carries the option craft (moved out of tool descriptions)`);
assert.ok(prompt.includes("TRANSCRIBED SPEECH"), `${name} reads dictated thinking leniently`);
assert.ok(prompt.includes("thinkingWhileReading"), `${name} knows the reading channel`);
}
// the cycle is teach-phase doctrine — it never composes into the prober
assert.ok(!prober.includes("predict → teach → check"), "the teaching cycle stays out of the prober");
});
it("the sources-v0 prompt lines compose: read/pdf doctrine in the teacher, the read line in the prober", () => {
const teacher = composeInstructions("teacher");
assert.ok(teacher.includes("Learner source files"), "the teacher carries the source-files section");
+105 -4
View File
@@ -24,6 +24,9 @@ import {
explainInputProblems,
quizInputSchema,
probeQuizInputSchema,
quizDisplayInputSchema,
normalizeQuizType,
quizNeedsReveal,
resolveCorrect,
toggleOption,
toggleOther,
@@ -64,6 +67,7 @@ describe("the schema's argument order (reasoning before the pick)", () => {
"explanation",
"correctAnswer",
"details",
...(keys.includes("type") ? ["type"] : []),
...(keys.includes("nodeId") ? ["nodeId"] : []),
]);
}
@@ -76,6 +80,7 @@ describe("quizInputSchema / quizInputProblems (the authoring contract)", () => {
options: [{ label: "Venus" }, { label: "Mercury" }, { label: "Mars" }],
correctAnswer: "Mercury",
explanation: "Mercury is the innermost planet.",
type: "check",
nodeId: "planet-order",
};
@@ -366,8 +371,8 @@ describe("buildAnswer", () => {
});
it("keeps the note only when non-empty", () => {
assert.equal(buildAnswer(quizData(), { values: ["mercury"], note: " " }).note, undefined);
assert.equal(buildAnswer(quizData(), { values: ["mercury"], note: " x " }).note, "x");
assert.equal(buildAnswer(quizData(), { values: ["mercury"], thinking: " " }).thinking, undefined);
assert.equal(buildAnswer(quizData(), { values: ["mercury"], thinking: " x " }).thinking, "x");
});
it("ignores unknown option values", () => {
@@ -418,14 +423,14 @@ describe("buildAnswer", () => {
describe("compactAnswerForModel", () => {
it("quiz: the learner's selection, verdict, confidence, note — NOTHING the model authored", () => {
const ad = buildAnswer(quizData(), { values: ["mercury"], confidence: 70, note: "guessed" });
const ad = buildAnswer(quizData(), { values: ["mercury"], confidence: 70, thinking: "guessed" });
const compact = compactAnswerForModel(ad);
assert.deepEqual(compact, {
kind: "quiz",
answers: [{ index: 1, label: "Mercury" }],
correct: true,
confidence: 70,
note: "guessed",
thinking: "guessed",
});
const raw = JSON.stringify(compact);
// every one of these lives in the tool CALL's arguments already
@@ -615,3 +620,99 @@ describe("buildAnswer / formatAnswerMessage: explain", () => {
});
});
// ─── the question's role: predict vs check (owner 2026-10-03) ─────────────
// Teaching units are a predict → teach → check cycle. The teach quiz schema
// carries a REQUIRED `type` declaring the role; the prober's quiz never does
// (probing questions are role-less). The reveal law keys off it: only CHECKS
// collapse behind "Check your understanding" — predictions arrive open.
describe("the quiz role field (predict | check)", () => {
const valid = {
question: "Which planet is closest to the Sun?",
options: [{ label: "Venus" }, { label: "Mercury" }, { label: "Mars" }],
correctAnswer: "Mercury",
explanation: "Mercury is the innermost planet.",
type: "check",
nodeId: "planet-order",
};
it("the teach schema REQUIRES type; the probe schema has no such field", () => {
assert.equal(Value.Check(quizInputSchema, valid), true, "fixture (type: check) is boundary-valid");
assert.equal(
Value.Check(quizInputSchema, { ...valid, type: undefined }),
false,
"a teach quiz without a role is rejected at the boundary",
);
assert.equal(
"nodeId" in (probeQuizInputSchema as { properties: Record<string, unknown> }).properties === false,
true,
);
assert.equal(
"type" in (probeQuizInputSchema as { properties: Record<string, unknown> }).properties,
false,
"the probe quiz stays role-less",
);
});
it("normalizeQuizType: exact literals only — old records and wobble degrade to check-style", () => {
assert.equal(normalizeQuizType("predict"), "predict");
assert.equal(normalizeQuizType("check"), "check");
assert.equal(normalizeQuizType(undefined), undefined);
assert.equal(normalizeQuizType("prediction"), undefined);
assert.equal(normalizeQuizType(42), undefined);
});
it("quizNeedsReveal: checks collapse, predictions never do", () => {
assert.equal(quizNeedsReveal("quiz", { type: "check" }, "teach"), true);
assert.equal(quizNeedsReveal("quiz", { type: "predict" }, "teach"), false, "predictions arrive OPEN");
assert.equal(
quizNeedsReveal("quiz", undefined, "teach"),
true,
"old records (no type) render check-style",
);
assert.equal(quizNeedsReveal("quiz", { type: "predict" }, "probe"), false);
assert.equal(quizNeedsReveal("quiz", { type: "check" }, "probe"), false);
assert.equal(quizNeedsReveal("ask_user_question", {}, "teach"), false);
assert.equal(quizNeedsReveal("explain", {}, "teach"), false);
});
function quizQd(): QuestionData {
const options = normalizeOptions(valid.options);
return {
id: "q1",
kind: "quiz",
question: valid.question,
mode: "single-select",
options,
correctIndices: resolveCorrect(coerceCorrectAnswer(valid.correctAnswer), options).indices,
explanation: valid.explanation,
createdAt: new Date().toISOString(),
};
}
it("buildAnswer snapshots the question's role and carries both thinking channels", () => {
const predictQd: QuestionData = { ...quizQd(), questionType: "predict" };
const a = buildAnswer(predictQd, { values: ["Mercury"], thinking: " guessed ", readingThinking: " reading note " });
assert.equal(a.questionType, "predict");
assert.equal(a.thinking, "guessed", "thinking is trimmed");
assert.equal(a.readingThinking, "reading note");
const b = buildAnswer(quizQd(), { values: ["Mercury"] });
assert.equal(b.thinking, undefined, "empty channels stay absent, never empty strings");
assert.equal(b.readingThinking, undefined);
assert.equal(b.questionType, undefined, "a role-less QuestionData (probe / old record) answers role-less");
});
it("compactAnswerForModel: distinct keys for the two channels; legacy `note` folds into thinking", () => {
const a = buildAnswer(quizQd(), { values: ["Mercury"], thinking: "h", readingThinking: "r" });
const compact = compactAnswerForModel(a) as Record<string, unknown>;
assert.equal(compact.thinking, "h");
assert.equal(compact.thinkingWhileReading, "r");
const legacy = compactAnswerForModel({ ...a, thinking: undefined, note: "old note" }) as Record<string, unknown>;
assert.equal(legacy.thinking, "old note", "records written before the rename still reach the model");
assert.equal("note" in legacy, false, "the legacy key never re-crosses the wire");
});
it("the display parse KEEPS the role: quizDisplayInputSchema retains type where the base strips it", () => {
const cleaned = Value.Clean(quizDisplayInputSchema, { ...valid }) as Record<string, unknown>;
assert.equal(cleaned.type, "check", "role survives the display parse (the dock gates collapse + labels on it)");
});
});
+47
View File
@@ -59,6 +59,7 @@ const validQuiz = {
],
correctAnswer: "4",
explanation: "Arithmetic.",
type: "check",
nodeId: "arith",
};
@@ -383,3 +384,49 @@ describe("the prober's explain: no nodeId field at all", () => {
assert.equal(empty.success, false);
});
});
// ─── descriptions are MECHANICS (owner 2026-10-03) ────────────────────────
// Question-design doctrine lives in the system prompt (shared craft chunk +
// teacher fragments), NOT in tool descriptions. A description that teaches
// the model how to write questions drifts the moment a new variant (prober,
// future adaptive practice) reuses the tool with a different doctrine.
describe("tool descriptions carry mechanics only, no question-craft", () => {
const teachQuiz = teachQuizToolFor({ chatId: "chat_x", chatStore: undefined as never });
for (const [name, tool] of [
["teach quiz", teachQuiz],
["probe quiz", probeQuizTool],
] as const) {
it(`${name}: no craft doctrine, mechanics intact`, () => {
const d = tool.description;
for (const banned of ["asymmetry", "distractor", "believable", "misconception"]) {
assert.ok(!d.toLowerCase().includes(banned), `${name} description leaks craft: "${banned}"`);
}
for (const mechanical of ["GRADED", "ONE question per turn", "ask_user_question"]) {
assert.ok(d.includes(mechanical), `${name} description lost mechanics: "${mechanical}"`);
}
});
}
it("the teach quiz declares the `type` field and `nodeId`; the probe quiz declares neither", () => {
assert.ok(teachQuiz.description.includes("`type`"), "teach quiz points at the role field");
assert.ok(teachQuiz.description.includes("`nodeId`"), "teach quiz points at the plan-map validation");
assert.ok(!probeQuizTool.description.includes("`type`"), "the probe quiz has no roles");
assert.ok(!probeQuizTool.description.includes("`nodeId`"), "the probe quiz has no plan map");
});
it("the explain tools keep the COMMITMENT rule (the grading contract) and drop the softer doctrine", () => {
for (const [name, tool] of [
["teach explain", teachExplainToolFor({ chatId: "chat_x", chatStore: undefined as never })],
["probe explain", probeExplainTool],
] as const) {
assert.ok(tool.description.includes("COMMITTED"), `${name} keeps the commitment rule`);
assert.ok(!tool.description.includes("near-miss"), `${name} drops the doctrine (now in the shared prompt)`);
}
});
it("the craft now lives in the shared prompt chunk (both agents compose it)", async () => {
const shared = await fs.promises.readFile(
path.join(PATHS.root, "src/agent/prompts/shared/20-question-tools.md"),
"utf-8",
);
assert.ok(shared.includes("asymmetry"), "the option craft moved here, not deleted");
assert.ok(shared.includes("TRANSCRIBED SPEECH"), "the dictation-leniency note lives here");
});
});
+80 -2
View File
@@ -23,7 +23,14 @@ import { JobsContext } from "../components/jobs-context.ts";
import { ChatActionsContext } from "../components/chat.tsx";
import type { JobRecord } from "../src/core/contracts.ts";
import { AnsweredCard } from "../components/answered-card.tsx";
import { buildAnswer, coerceCorrectAnswer, normalizeOptions, resolveCorrect, type QuestionData } from "../src/core/question-core.ts";
import {
buildAnswer,
coerceCorrectAnswer,
normalizeOptions,
resolveCorrect,
quizNeedsReveal,
type QuestionData,
} from "../src/core/question-core.ts";
const quizInput = {
question: "Which planet is closest to the sun?",
@@ -276,7 +283,7 @@ describe("render smoke tests", () => {
it("AnsweredCard promotes the learner's thinking to a first-class block", () => {
const answer = buildAnswer(quizQD(), {
values: ["Venus"],
note: "I picked Venus because it looks brightest — but now I realize brightness isn't distance.",
thinking: "I picked Venus because it looks brightest — but now I realize brightness isn't distance.",
});
const html = renderToStaticMarkup(<AnsweredCard answer={answer} />);
assert.ok(html.includes('data-testid="your-thinking"'), "the thinking block exists");
@@ -509,3 +516,74 @@ describe("AnsweredCard: the explain variant", () => {
});
});
// ─── the two roles in the UI (predict vs check, owner 2026-10-03) ─────────
// Predictions arrive OPEN before the teaching; checks arrive collapsed after
// it. The answered card tells them apart (eyebrow) and tucks a prediction's
// explanation (the teaching that follows IS the explanation) while a check's
// stays inline.
describe("predict vs check rendering", () => {
it("a predict quiz renders the 'Prediction' eyebrow in the dock", () => {
const html = renderToStaticMarkup(
<QuestionDock
{...{
toolName: "quiz" as const,
toolCallId: "call-p1",
input: { ...quizInput, type: "predict", nodeId: "planet-order" },
}}
expanded={true}
onExpand={() => {}}
onSubmit={() => {}}
/>,
);
assert.ok(html.includes("Prediction"), `eyebrow missing: ${html.slice(0, 200)}`);
});
it("a check quiz keeps the 'Quiz' eyebrow (old records unchanged)", () => {
const html = renderToStaticMarkup(
<QuestionDock
{...{
toolName: "quiz" as const,
toolCallId: "call-c1",
input: { ...quizInput, type: "check", nodeId: "planet-order" },
}}
expanded={true}
onExpand={() => {}}
onSubmit={() => {}}
/>,
);
assert.ok(html.includes("Quiz"));
});
it("a prediction answer SHOWS the verdict but TUCKS the explanation behind a toggle", () => {
const qd: QuestionData = { ...quizQD(), questionType: "predict" };
const answer = buildAnswer(qd, { values: ["Venus"] }); // wrong on purpose
const html = renderToStaticMarkup(<AnsweredCard answer={answer} />);
assert.ok(html.includes("Prediction"), "the eyebrow distinguishes the card");
assert.ok(/incorrect|wrong|&times;|✗/i.test(html), "the verdict still shows (owner: predictions are graded)");
assert.ok(!html.includes("Why:"), "the explanation is not center stage");
assert.ok(html.includes("tucked-explanation"), "it sits behind the toggle");
});
it("a check answer keeps the inline explanation", () => {
const answer = buildAnswer(quizQD(), { values: ["Venus"] });
const html = renderToStaticMarkup(<AnsweredCard answer={answer} />);
assert.ok(html.includes("Why:"), "checks explain inline (current behavior)");
assert.ok(!html.includes("tucked-explanation"));
});
it("the reading-thinking channel renders as its own block; legacy `note` still renders", () => {
const withReading = buildAnswer(quizQD(), {
values: ["Mercury"],
thinking: "answering thought",
readingThinking: "was unsure mid-paragraph",
});
const html = renderToStaticMarkup(<AnsweredCard answer={withReading} />);
assert.ok(html.includes("Your thinking while reading"), "the reading channel is labeled distinctly");
assert.ok(html.includes("answering thought"));
assert.ok(html.includes("was unsure mid-paragraph"));
const legacy = { ...buildAnswer(quizQD(), { values: ["Mercury"] }), note: "a 2026-09 answer" };
const legacyHtml = renderToStaticMarkup(<AnsweredCard answer={legacy} />);
assert.ok(legacyHtml.includes("a 2026-09 answer"), "records from before the rename still render");
});
});
+2 -2
View File
@@ -923,7 +923,7 @@ describe("newAnswerResult two audiences", () => {
explanation: "Mercury orbits closest.",
createdAt: "2026-01-01T00:00:00Z",
};
const full = buildAnswer(qd, { values: ["mercury"], confidence: 70, note: "guessed" });
const full = buildAnswer(qd, { values: ["mercury"], confidence: 70, thinking: "guessed" });
const msg = newAnswerResult("call-1", "quiz", full);
const modelSees = JSON.parse((msg.content as { type: "text"; text: string }[])[0]!.text);
assert.deepEqual(modelSees, {
@@ -931,7 +931,7 @@ describe("newAnswerResult two audiences", () => {
answers: [{ index: 1, label: "Mercury" }],
correct: true,
confidence: 70,
note: "guessed",
thinking: "guessed",
});
assert.equal(msg.details, full, "details is the full QuestionAnswerData verbatim");
assert.equal((msg.details as { questionText?: string }).questionText, "Which planet is closest to the Sun?");
+2
View File
@@ -29,6 +29,7 @@ const TEACHER_QUIZ = {
],
correctAnswer: "truth",
explanation: "Convention-T schematizes truth for the object language.",
type: "check", // teach-only role field — the display parse KEEPS it (the dock gates by it)
nodeId: "convention-t", // teacher-only field — the display parse must accept and strip it
};
@@ -55,6 +56,7 @@ describe("parseQuizInput", () => {
assert.ok(parsed);
assert.equal(parsed.question, TEACHER_QUIZ.question);
assert.equal((parsed as Record<string, unknown>).nodeId, undefined, "unknown keys are stripped, not trusted");
assert.equal((parsed as { type?: unknown }).type, "check", "the role field survives the display parse (owner 2026-10-03)");
});
it("parses a boundary-valid PROBER quiz (no nodeId at all)", () => {