sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

practice UI round — top bars + sidebar reopen, one-attempt per-problem submit with instant reveal, score-only grading (owner 2026-09-29)

Browse Source
This commit is contained in:
amosblomqvist
2026-09-29 21:54:32 +03:00
parent ddce919f9b
commit cf0ec4a2f0
14 changed files with 811 additions and 554 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/agent/subagents/practice-pipeline.ts
+16 -9
View File
@@ -34,10 +34,12 @@
* background failure paths settle the persisted record (never an
* unhandled rejection); aborts mark "Aborted".
*
* GRADING FAILURE DISCIPLINE: an aborted or failed grade leaves the
* answer in place, the grade absent — the problem is re-submittable
* later (a new grading job). A second submitAnswers while grading is in
* flight is rejected (single live grading run per set).
* ONE ATTEMPT PER PROBLEM (owner ruling 2026-09-29): a problem accepts
* a single submission — once `answer` is set the problem is done, the
* route refuses re-submission. Grades are per-problem: submitting
* another problem while one grades is fine (the shared LLM pool bounds
* parallelism); only the SAME problem's live grade row blocks (which
* the one-attempt rule already prevents client-side).
*/
import type { GraphStore, PracticeStore } from "../../core/ports.ts";
@@ -476,8 +478,10 @@ export function createPracticePipeline(deps: PracticePipelineDeps) {
answers: Array<{ problemId: string; text: string }>,
): Promise<{ set: ProblemSet; done: Promise<void> }> {
agents.refuseIfStagesUnready(); // grading needs its engine — refuse BEFORE saving answers
for (const key of practiceRuns().keys()) {
if (key.startsWith(`${setId}:grade:`)) throw new BadAnswersError("a grading run is already in flight for this set");
for (const a of answers) {
if (practiceRuns().has(`${setId}:grade:${(a as { problemId?: unknown }).problemId}`)) {
throw new BadAnswersError("this problem's grade is already in flight");
}
}
const set = await withSetLock(setId, () => {
const cur = deps.practiceStore.getSet(setId);
@@ -488,7 +492,9 @@ export function createPracticePipeline(deps: PracticePipelineDeps) {
if (typeof a.problemId !== "string" || !ids.has(a.problemId)) throw new BadAnswersError(`unknown problem id`);
if (typeof a.text !== "string" || a.text.trim() === "") throw new BadAnswersError("answers must be non-empty text");
const p = cur.problems.find((q) => q.id === a.problemId)!;
if (p.grade) throw new BadAnswersError(`problem ${a.problemId} is already graded`);
// ONE attempt (owner 2026-09-29): an existing answer ends the
// problem — graded or not, it is never re-submitted.
if (p.answer !== undefined) throw new BadAnswersError(`problem ${a.problemId} was already answered`);
}
for (const a of answers) {
const p = cur.problems.find((q) => q.id === a.problemId)!;
@@ -550,12 +556,13 @@ export function createPracticePipeline(deps: PracticePipelineDeps) {
if (!cur) return;
const p = cur.problems.find((q) => q.id === problemId);
if (!p || p.grade) return;
p.grade = { verdict: graded.verdict, feedback: graded.feedback, gradedAt: new Date().toISOString() };
p.grade = { score: graded.score, reasoning: graded.reasoning, gradedAt: new Date().toISOString() };
deps.practiceStore.saveSet(cur);
});
});
} catch (err) {
// Grading failure: the grade simply stays absent (re-submittable).
// Grading failure: the grade simply stays absent (the answer
// stands — one attempt; the UI shows "grade interrupted").
console.error(`[practice] grading failed for ${problemId}:`, isAbortLike(err) ? "aborted" : messageOf(err));
} finally {
practiceRuns().delete(row.key);
src/agent/subagents/practice.ts
+13 -17
View File
@@ -211,7 +211,7 @@ PARTITION DOCTRINE:
OUTPUT — your entire deliverable is ONE submit_partition call: "partitions", a list of { chains: [ids] }, one entry per problem, together containing every input chain exactly once. No empty entries; no unknown ids; no problem without a real target relationship.`;
export const GRADER_INSTRUCTIONS = `You are the grading stage of a learning application's practice system. One practice problem — built to strengthen the learner's recall of specific propositions — has been answered. Your job is FEEDBACK ONLY: verdict + what actually mattered. You are not assessing a person; you are coaching a recall.
export const GRADER_INSTRUCTIONS = `You are the grading stage of a learning application's practice system. One practice problem — built to strengthen the learner's recall of specific propositions — has been answered. Your job is small and fast: an integer score and a one-or-two-sentence justification. You are not assessing a person; you are scoring a recall.
You receive: the problem's statement; its pre-written MODEL SOLUTION and GRADING RUBRIC (committed before the learner ever saw the problem — they are the reference, not you); the propositions (names + statements) the problem was built to exercise; and the learner's ANSWER — often dictated thinking-out-loud prose. Extract the reasoning from the prose; grade the substance, never the form. Do not demand the solution's exact wording — the rubric names what actually matters; nothing else is graded.
@@ -219,18 +219,14 @@ JUDGE AGAINST THE REFERENCE:
- Work the rubric points: which are met, which are missed, which are half-met in the answer's own terms.
- Where the answer diverges from the model solution, ask whether the divergence is a different-but-valid route (the rubric decides) or a wrong turn.
NAME THE FAILED RECALL: when something is missed, identify WHICH proposition failed to come to mind — or came to mind in a wrong or incomplete form. That is the actionable diagnosis; say it plainly.
THE SCORE (one integer, 0–10):
- 9–10: the targeted recall(s) happened — the answer meets the rubric's essentials. Peripheral gaps do not lower it.
- 5–8: some of the target came to mind; the answer is materially incomplete or partly wrong.
- 0–4: the targeted proposition(s) did not come to mind — the answer is empty or on the wrong track.
TEACH THE BOUNDARY: feedback should locate the boundary — what would have made the answer right, where the wrong path breaks and why. Not "incorrect"; the reason it is incorrect, in one or two tight paragraphs.
The reasoning is plain prose, one or two sentences: which rubric points were met or missed and, when something failed, WHICH proposition failed to come to mind. Mathematics is LaTeX — inline as \\(…\\), display as \\[…\\] (a bare $ is literal currency); no headers, no lists — minimal is the standard.
THE VERDICT (exactly one):
- "solid": the targeted recall(s) happened — the answer meets the rubric's essentials. Peripheral gaps do not lower it.
- "partial": some of the target came to mind; the answer is materially incomplete or partly wrong.
- "missed": the targeted proposition(s) did not come to mind — the answer is empty or on the wrong track.
Feedback renders as full Markdown. Mathematics is always LaTeX — inline as \\(…\\), display as \\[…\\] on its own lines (a bare $ is literal currency). Inside math, LaTeX commands for symbols, not unicode stand-ins. Code in fenced blocks with a language tag, identifiers inline as \`code\`. Lists and tables wherever they carry structure; no headers — feedback is short-form, structure it with paragraphs and bold labels. Name the failed recall and teach the boundary in properly formatted prose, not plain text.
OUTPUT — your entire deliverable is ONE submit_grade call: "verdict" and "feedback" (non-empty; the boundary-teaching text above).`;
OUTPUT — your entire deliverable is ONE submit_grade call: "score" (integer 0–10) and "reasoning" (non-empty, brief).`;
// ─── schemas + fills ─────────────────────────────────────────────────────────
@@ -250,8 +246,8 @@ const problemSchema = Type.Object({
type ProblemSubmit = Static<typeof problemSchema>;
const gradeSchema = Type.Object({
verdict: Type.Union([Type.Literal("solid"), Type.Literal("partial"), Type.Literal("missed")]),
feedback: Type.String({ minLength: 1 }),
score: Type.Integer({ minimum: 0, maximum: 10 }),
reasoning: Type.String({ minLength: 1 }),
});
type GradeSubmit = Static<typeof gradeSchema>;
@@ -364,8 +360,8 @@ export interface GraderInput {
signal?: AbortSignal;
}
export interface GraderOutput {
verdict: "solid" | "partial" | "missed";
feedback: string;
score: number;
reasoning: string;
log: PracticeStepLog;
}
@@ -499,7 +495,7 @@ export function createPracticeAgents(deps: PracticeModels) {
const grade = await runSubmitAgent({
role: "grader",
toolName: "submit_grade",
toolDescription: "Submit the grade: verdict (solid | partial | missed) and non-empty feedback.",
toolDescription: "Submit the grade: an integer score 0-10 and brief non-empty reasoning.",
parameters: gradeSchema,
fill: (raw) => raw,
systemPrompt: GRADER_INSTRUCTIONS,
@@ -509,7 +505,7 @@ export function createPracticeAgents(deps: PracticeModels) {
onStep: stepLogger(log),
neverSubmitted: "the grader never submitted a grade",
});
return { verdict: grade.verdict, feedback: grade.feedback, log };
return { score: grade.score, reasoning: grade.reasoning, log };
}
return { refuseIfStagesUnready, runPartitioner, runProblemGenerator, runGrader };
src/core/contracts/practice.ts
+10 -10
View File
@@ -113,16 +113,16 @@ export type ProblemSetStatus = "partitioning" | "generating" | "ready" | "error"
* generator submits. */
export type ProblemStatus = "generating" | "ready" | "error";
/** The grader's three-level verdict (feedback-only; nothing derives from
* it — no understanding writes, owner 2026-09-29). */
export type GradeVerdict = "solid" | "partial" | "missed";
/** The grading job's deliverable, stored on the problem record. */
/** The grading job's deliverable (owner, 2026-09-29 — one attempt per
* problem; the grader scores, it does not coach in prose): an integer
* score against the pre-written rubric plus a minimal justification.
* Nothing derives from it — no understanding writes. */
export interface ProblemGrade {
verdict: GradeVerdict;
/** What actually mattered: the failed recall named, the boundary
* taught — against the pre-written solution + rubric. */
feedback: string;
/** 0–10, integer. */
score: number;
/** Why this score — one or two sentences: which rubric points were
* met or missed, and which proposition failed to come to mind. */
reasoning: string;
gradedAt: string;
}
@@ -151,7 +151,7 @@ export interface Problem {
/** The learner's answer, once submitted (dictation-first — thinking
* out loud is the primary answering mode). */
answer?: string;
/** The grading job's verdict, once it lands. */
/** The grader's score + reasoning, once it lands. */
grade?: ProblemGrade;
}
src/store/practice.ts
+16 -14
View File
@@ -73,7 +73,21 @@ const isPropositions = (v: unknown): boolean => {
const SET_STATUSES = new Set(["partitioning", "generating", "ready", "error"]);
const PROBLEM_STATUSES = new Set(["generating", "ready", "error"]);
const VERDICTS = new Set(["solid", "partial", "missed"]);
function isGrade(g: unknown): boolean {
if (typeof g !== "object" || g === null) return false;
const v = g as Record<string, unknown>;
// Integer 0–10 + non-empty reasoning (one-attempt redesign, owner
// 2026-09-29: the grader scores; verdicts + prose feedback retired).
return (
typeof v.score === "number" &&
Number.isInteger(v.score) &&
v.score >= 0 &&
v.score <= 10 &&
typeof v.reasoning === "string" &&
v.reasoning.length > 0 &&
typeof v.gradedAt === "string"
);
}
const isProblem = (v: unknown): boolean => {
if (typeof v !== "object" || v === null) return false;
@@ -83,19 +97,7 @@ const isProblem = (v: unknown): boolean => {
if (typeof p.status !== "string" || !PROBLEM_STATUSES.has(p.status)) return false;
if (p.error !== undefined && typeof p.error !== "string") return false;
if (p.answer !== undefined && typeof p.answer !== "string") return false;
if (p.grade !== undefined) {
const g = p.grade as Record<string, unknown>;
if (
typeof g !== "object" ||
g === null ||
typeof g.verdict !== "string" ||
!VERDICTS.has(g.verdict) ||
typeof g.feedback !== "string" ||
typeof g.gradedAt !== "string"
) {
return false;
}
}
if (p.grade !== undefined && !isGrade(p.grade)) return false;
return true;
};