free-recall explain tool: the quizzing-mechanic switch (owner 2026-09-27)
Sycophancy mitigation as a structural commitment: the model writes its full correctAnswer (plus explanation, in that order) BEFORE the learner sees the question; the commitment rides the persisted call args, is revealed on submit, and grades his free text in the model's next reply — the client never grades free recall. - tools.explain setting (segmented 'Quizzing — Multiple choice | Free recall' in Agent settings' Tools group): free arms explain and drops quiz for BOTH agents (prober variant carries no nodeId, same pattern as the probe quiz); mechanics keys never pass filterEnabledTools — quiz stays unwritable (the 2026-09-19 wedge ruling stands) - doctrine lives pi-natively in the tool descriptions (owner ruling — no conditional system-prompt chunks): explain carries the commitment rule; the MC option-building craft moved out of the prompt into the quiz description; shared/20-question-tools.md slims to the quiz-agnostic core - display seams: parseExplainInput (LaTeX-debris restoration included), dock text-mode with an 'I don't know' row (kbd row 0), AnsweredCard reveal (The answer / Why, no verdict chips), [Explain answer] format - phases/plan seams recognize both quizzing tools' nodeId tags; explain answers deliberately feed no ledger evidence (no client grading to key on — a future ruling can parse the model's grade) 1049/1049 tests, typecheck/lint/build green.
This commit is contained in:
amosblomqvist
@@ -85,7 +85,9 @@ export function derivePlanStatuses(tree: PlanTree, messages: readonly PartsCarri
|
||||
// duplicates collapsed (several checks on one node = one stay on it)
|
||||
for (const m of messages) {
|
||||
for (const part of m.parts) {
|
||||
if (part.type !== "tool-quiz") continue;
|
||||
// Either quizzing tool (quiz/explain) tags position — the mechanic
|
||||
// is a settings switch; the fold recognizes both (owner 2026-09-27).
|
||||
if (part.type !== "tool-quiz" && part.type !== "tool-explain") continue;
|
||||
const nodeId = (part.input as { nodeId?: unknown } | undefined)?.nodeId;
|
||||
if (typeof nodeId !== "string" || !ids.has(nodeId)) continue; // untagged/unknown — moves nothing
|
||||
if (runs.at(-1) !== nodeId) runs.push(nodeId);
|
||||
@@ -130,7 +132,7 @@ export function derivePlanStatusesLh(tree: PlanTree, messages: readonly LhMessag
|
||||
const content = (m as { content?: unknown }).content;
|
||||
if (!Array.isArray(content)) continue;
|
||||
for (const block of content as Array<{ type?: string; name?: string; arguments?: unknown }>) {
|
||||
if (block.type !== "toolCall" || block.name !== "quiz") continue;
|
||||
if (block.type !== "toolCall" || (block.name !== "quiz" && block.name !== "explain")) continue;
|
||||
const nodeId = (block.arguments as { nodeId?: unknown } | undefined)?.nodeId;
|
||||
if (typeof nodeId !== "string" || !ids.has(nodeId)) continue;
|
||||
if (runs.at(-1) !== nodeId) runs.push(nodeId);
|
||||
|
||||
+8
-1
@@ -39,7 +39,7 @@ export interface PendingToolCall {
|
||||
}
|
||||
|
||||
/** Tools that render in the question dock and gate the composer. */
|
||||
export const QUESTION_TOOLS = ["quiz", "ask_user_question"] as const;
|
||||
export const QUESTION_TOOLS = ["quiz", "ask_user_question", "explain"] as const;
|
||||
|
||||
/** Tools fulfilled by background sub-agent jobs (W2/W6). */
|
||||
export const JOB_TOOLS = ["research", "demo", "visual"] as const;
|
||||
@@ -62,6 +62,13 @@ export const JOB_TOOLS = ["research", "demo", "visual"] as const;
|
||||
* 2026-09). */
|
||||
export const TOGGLEABLE_TOOLS = ["research", "demo", "visual", "pdf"] as const;
|
||||
|
||||
/** The keys the settings WRITE side accepts in agent.tools: the plain
|
||||
* toggleables PLUS the quizzing-mechanics switch (`explain: true` arms
|
||||
* free-recall quizzing and drops the MC quiz — see quizMechanics in
|
||||
* agent/tools.ts; owner 2026-09-27). `quiz` itself stays unwritable:
|
||||
* mechanics-managed, never individually toggleable. */
|
||||
export const WRITABLE_TOOL_KEYS = [...TOGGLEABLE_TOOLS, "explain"] as const;
|
||||
|
||||
/** The resolved settings → factory seam shape (Platform wires chat-handler
|
||||
* to pass this through from the settings record; storage and defaults are
|
||||
* Platform's lane). JSON-native: a partial record keyed by tool name.
|
||||
|
||||
+8
-2
@@ -83,7 +83,13 @@ function advance(
|
||||
if (phase === "probe" && part.type === "tool-probe_done" && part.state === "output-available") {
|
||||
phase = "plan";
|
||||
}
|
||||
if (phase === "plan" && part.type === "tool-quiz" && taggedNodeId(part.input)) {
|
||||
// plan → teach via the first nodeId-tagged quizzing CALL — quiz OR
|
||||
// explain, whichever mechanic the settings armed (both carry nodeId).
|
||||
if (
|
||||
phase === "plan" &&
|
||||
(part.type === "tool-quiz" || part.type === "tool-explain") &&
|
||||
taggedNodeId(part.input)
|
||||
) {
|
||||
phase = "teach";
|
||||
}
|
||||
}
|
||||
@@ -136,7 +142,7 @@ export function phaseAt3Lh(messages: readonly LhMessage[], upToIndex?: number):
|
||||
for (const block of content) {
|
||||
if (
|
||||
block.type === "toolCall" &&
|
||||
block.name === "quiz" &&
|
||||
(block.name === "quiz" || block.name === "explain") &&
|
||||
taggedNodeId(block.arguments)
|
||||
) {
|
||||
phase = "teach";
|
||||
|
||||
@@ -217,6 +217,68 @@ export const quizInputObject = Type.Object({
|
||||
* it ran at the call boundary, where quizInputProblems runs now). */
|
||||
export const probeQuizInputSchema = quizInputBase;
|
||||
|
||||
// ─── the explain tool (free-recall quizzing, owner 2026-09-27) ─────────────
|
||||
//
|
||||
// Quiz mechanics WITHOUT options: the learner answers in his own words.
|
||||
// The SYCOPHANCY MITIGATION is structural — the model commits to the full
|
||||
// correct answer BEFORE posing (the commitment rides the persisted call
|
||||
// args, is revealed to the learner on submit, and is restated in the answer
|
||||
// result the model grades against; the client never grades free text).
|
||||
// Mechanics doctrine lives in the TOOL DESCRIPTIONS (owner ruling
|
||||
// 2026-09-27: pi-native — doctrine travels with the armed tool), not in
|
||||
// the system prompt.
|
||||
|
||||
export const explainInputBase = Type.Object({
|
||||
question: Type.String({
|
||||
description:
|
||||
"The single question to ask. ONE question per turn: ask, then end your turn — the answer determines the next.",
|
||||
}),
|
||||
explanation: Type.String({
|
||||
description:
|
||||
"REQUIRED. Explanation revealed AFTER the learner answers (shown however he did). It is written BEFORE " +
|
||||
"correctAnswer by design: reason it out first, then commit to the answer the reasoning supports.",
|
||||
}),
|
||||
correctAnswer: Type.String({
|
||||
description:
|
||||
"REQUIRED. The full correct answer in your own words — COMMITTED before the learner ever sees the question. " +
|
||||
"The commitment is immutable: when his answer returns, grade it strictly against this text — never revise the " +
|
||||
"commitment after seeing his words, and never mark a wrong answer right to be kind. Both this and the explanation " +
|
||||
"are shown to him on submit; your next reply grades the nuance of what he wrote.",
|
||||
}),
|
||||
details: Type.Optional(
|
||||
Type.String({ description: "Optional extra context or instructions shown under the question." }),
|
||||
),
|
||||
});
|
||||
|
||||
/** The TEACH-side explain object: base + the plan-position field (same
|
||||
* semantics as the quiz's nodeId — the seam recognizes BOTH tools). */
|
||||
export const explainInputObject = Type.Object({
|
||||
...explainInputBase.properties,
|
||||
nodeId: Type.String({
|
||||
description:
|
||||
"REQUIRED plan-tree node id this question is about — must EXACTLY match one of this session's " +
|
||||
"plan_init node slugs (validated at call time; a mismatched id is rejected back to you). " +
|
||||
"The lesson's position on the map derives from these tags (the node the latest tagged question is about is " +
|
||||
"where the map stands; earlier ones count as visited). A node takes as many questions as it deserves: stay on " +
|
||||
"one node for as many turns as the understanding needs — one reasoning step at a time, never rush forward. " +
|
||||
"For off-map checks use ask_user_question.",
|
||||
}),
|
||||
});
|
||||
|
||||
/** The explain authoring contract: the commitment must exist. Empty
|
||||
* result = an explain the dock can arm and the model can grade against. */
|
||||
export function explainInputProblems(input: { correctAnswer: string }): string[] {
|
||||
if (input.correctAnswer.trim().length === 0) {
|
||||
return ["correctAnswer is required — commit to the full correct answer BEFORE posing the question."];
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/** The tool names that carry the quizzing mechanic — the teach seam
|
||||
* (phases), the plan-position fold, and the pending gate all recognize
|
||||
* whichever is armed. */
|
||||
export const QUIZ_TOOL_NAMES = ["quiz", "explain"] as const;
|
||||
|
||||
/** The TEACHER's quiz declaration (base + the position field). */
|
||||
export const quizInputSchema = quizInputObject;
|
||||
|
||||
@@ -249,6 +311,8 @@ export const askInputSchema = Type.Object({
|
||||
|
||||
export type QuizToolInput = Static<typeof quizInputObject>;
|
||||
export type AskToolInput = Static<typeof askInputSchema>;
|
||||
export type ExplainToolInput = Static<typeof explainInputObject>;
|
||||
export type ProbeExplainToolInput = Static<typeof explainInputBase>;
|
||||
|
||||
|
||||
/** A real selected answer, in the order the options were displayed. */
|
||||
@@ -258,7 +322,7 @@ export interface OptionAnswer {
|
||||
index: number; // 1-based display index
|
||||
}
|
||||
|
||||
export type QuestionKind = "quiz" | "ask";
|
||||
export type QuestionKind = "quiz" | "ask" | "explain";
|
||||
export type SelectMode = "single-select" | "multi-select";
|
||||
export type QuestionMode = SelectMode | "text";
|
||||
|
||||
@@ -272,6 +336,8 @@ export interface QuestionData {
|
||||
options: QuizOption[]; // display order (post-shuffle)
|
||||
// quiz only. NEVER send to the client before answering:
|
||||
correctIndices?: number[];
|
||||
// explain only. NEVER send to the client before answering:
|
||||
committedAnswer?: string;
|
||||
explanation?: string;
|
||||
createdAt: string;
|
||||
}
|
||||
@@ -285,11 +351,12 @@ export interface QuestionAnswerData {
|
||||
dontKnow: boolean;
|
||||
answers: OptionAnswer[];
|
||||
other?: string; // ask: "Other" free text
|
||||
text?: string; // ask: text-mode answer
|
||||
text?: string; // ask/explain: text-mode answer
|
||||
note?: string;
|
||||
correct?: boolean; // quiz only
|
||||
correctIndices?: number[]; // quiz only (rendering)
|
||||
explanation?: string; // quiz only (rendering)
|
||||
committed?: string; // explain only (rendering): the model's committed answer
|
||||
explanation?: string; // quiz/explain only (rendering)
|
||||
dismissed?: boolean;
|
||||
answeredAt: string;
|
||||
}
|
||||
@@ -526,12 +593,18 @@ export function buildAnswer(qd: QuestionData, req: AnswerRequest): QuestionAnswe
|
||||
base.explanation = qd.explanation;
|
||||
base.correct = base.dontKnow ? false : isCorrect(base.answers.map((a) => a.index), qd.correctIndices ?? []);
|
||||
}
|
||||
if (qd.kind === "explain") {
|
||||
// No client grading — the model grades in its reply, against the
|
||||
// commitment. Snapshots ride along for the reveal + rendering.
|
||||
base.committed = qd.committedAnswer;
|
||||
base.explanation = qd.explanation;
|
||||
}
|
||||
return base;
|
||||
}
|
||||
|
||||
/** The user message that carries the answer into the agent's context. */
|
||||
export function formatAnswerMessage(ad: QuestionAnswerData): string {
|
||||
const label = ad.kind === "quiz" ? "Quiz answer" : "Answer to your question";
|
||||
const label = ad.kind === "quiz" ? "Quiz answer" : ad.kind === "explain" ? "Explain answer" : "Answer to your question";
|
||||
const head = `[${label}]`;
|
||||
const lines: string[] = [head];
|
||||
|
||||
@@ -553,6 +626,18 @@ export function formatAnswerMessage(ad: QuestionAnswerData): string {
|
||||
lines.push(`Result: ${verdict}`);
|
||||
}
|
||||
lines.push(`Correct: ${correctStr}`);
|
||||
} else if (ad.kind === "explain") {
|
||||
if (ad.dontKnow) {
|
||||
lines.push(
|
||||
`The learner selected "I don't know" — they did NOT attempt an answer (a genuine knowledge gap, not a wrong guess).`,
|
||||
);
|
||||
} else {
|
||||
lines.push(`The learner answered: ${ad.text ?? "(no answer)"}`);
|
||||
}
|
||||
if (ad.committed) {
|
||||
lines.push(`Your committed correct answer (immutable — grade against it, never revise it): ${ad.committed}`);
|
||||
}
|
||||
lines.push("Grade his answer strictly against your committed answer in your reply: name what he got and what he missed.");
|
||||
} else {
|
||||
if (ad.text) {
|
||||
lines.push(`The learner answered: ${ad.text}`);
|
||||
|
||||
+12
-1
@@ -29,7 +29,7 @@
|
||||
|
||||
import { Value } from "typebox/value";
|
||||
import type { TSchema } from "typebox";
|
||||
import { quizInputBase, askInputSchema, askOptionProblems, quizInputProblems, type AskToolInput, type QuizToolInput } from "./question-core.ts";
|
||||
import { quizInputBase, askInputSchema, askOptionProblems, quizInputProblems, explainInputBase, explainInputProblems, type AskToolInput, type QuizToolInput, type ExplainToolInput } from "./question-core.ts";
|
||||
import { restoreMathEscapes } from "./math-delims.ts";
|
||||
import { probeDoneInputSchema, type ProbeDoneInput } from "./contracts/probe.ts";
|
||||
|
||||
@@ -103,6 +103,17 @@ export function parseAskInput(input: unknown): AskToolInput | undefined {
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/** Explain parse for DISPLAY: the shared authoring base (the teacher's
|
||||
* nodeId is an unknown key Value.Clean strips, same as the quiz seam) —
|
||||
* both explain variants share the same fields, and the commitment
|
||||
* refinement (explainInputProblems) mirrors the boundary chain. */
|
||||
export function parseExplainInput(input: unknown): ExplainToolInput | undefined {
|
||||
const cleaned = boundaryParse<ExplainToolInput>(explainInputBase, restoreMathEscapesDeep(input));
|
||||
if (!cleaned) return undefined;
|
||||
if (explainInputProblems(cleaned).length > 0) return undefined;
|
||||
return cleaned;
|
||||
}
|
||||
|
||||
/** probe_done parse — the core-owned schema the tool itself declares. */
|
||||
export function parseProbeDoneInput(input: unknown): ProbeDoneInput | undefined {
|
||||
return boundaryParse<ProbeDoneInput>(probeDoneInputSchema, input);
|
||||
|
||||
Reference in New Issue
Block a user