Merge accepted main probe baseline before maintenance checks

2026-10-27 21:13:59 +02:00
parent 27abb31219 2b1343ab38
commit 00a7d1437c
7 changed files with 178 additions and 32 deletions
@@ -103,6 +103,16 @@ test("phaseAt3: a mere probe_done CALL (input-available, not yet concluded) keep
 	assert.equal(still, "probe");
 });

+test("phaseAt3 and phaseTimeline: explicit probe refusals never advance the UI phase", () => {
+	const refused = assistantTool("probe_done", "output-available", { ok: false, error: "question open" });
+	const errored = assistantTool("probe_done", "output-error", { ok: true });
+	const tagged = assistantQuiz({ nodeId: "n1" });
+	const clean = assistantTool("probe_done", "output-available", { ok: true });
+	const messages = [refused, errored, tagged, clean, tagged];
+	assert.equal(phaseAt3(messages, 2), "probe");
+	assert.deepEqual(phaseTimeline(messages), ["probe", "probe", "probe", "plan", "teach"]);
+});
+
 test("phaseAt3: legacy transcripts without probe_done read as probe (no backwards compat — data wiped)", () => {
 	const legacy = phaseAt3([user("teach me lacan"), assistantTool("plan_init", "output-available", PLAN_OK)]);
 	assert.equal(legacy, "probe", "plan_init alone never flips a phase post-split");
@@ -158,23 +168,31 @@ function assistantQuizCall(arguments_: Record<string, unknown>): LhMessage {
 	} as unknown as LhMessage;
 }

+function assistantProbeCall(): LhMessage {
+	return {
+		...assistantQuizCall({}),
+		content: [{ type: "toolCall", id: "call_probe_done", name: "probe_done", arguments: {} }],
+	} as LhMessage;
+}
+
 test("phaseAt3Lh (native fold): probe→plan at the clean probe_done result; plan→teach at the first tagged quiz CALL", () => {
 	const done = toolResult("probe_done");
+	const call = assistantProbeCall();
 	const tagged = assistantQuizCall({ question: "q", nodeId: "gravity-basics", correctAnswer: "a" });
 	const untagged = assistantQuizCall({ question: "q", correctAnswer: "a" });

-	assert.equal(phaseAt3Lh([done, untagged]), "plan", "an untagged quiz call never starts teaching");
-	assert.equal(phaseAt3Lh([done, tagged]), "teach", "the tagged quiz CALL is the seam");
+	assert.equal(phaseAt3Lh([call, done, untagged]), "plan", "an untagged quiz call never starts teaching");
+	assert.equal(phaseAt3Lh([call, done, tagged]), "teach", "the tagged quiz CALL is the seam");
 	assert.equal(phaseAt3Lh([tagged]), "probe", "the chain never skips (tag before probe_done)");
 	assert.equal(
-		phaseAt3Lh([toolResult("probe_done", true), tagged]),
+		phaseAt3Lh([call, toolResult("probe_done", true), tagged]),
 		"probe",
 		"an ERRORED probe_done concludes nothing",
 	);
 	// The two folds agree on the same settled transcript (the view projection
 	// of the tagged call carries its arguments as the part input).
 	assert.equal(
-		phaseAt3Lh([done, tagged]),
+		phaseAt3Lh([call, done, tagged]),
 		phaseAt3([
 			{ parts: [{ type: "tool-probe_done", state: "output-available" } as never] },
 			{ parts: [{ type: "tool-quiz", input: { nodeId: "gravity-basics" } } as never] },
@@ -188,6 +206,7 @@ test("phaseAt3Lh (native fold): probe→plan at the clean probe_done result; pla

 test("phaseAt3 + phaseAt3Lh: a tagged explain CALL is the plan→teach seam too", () => {
 	const done = toolResult("probe_done");
+	const call = assistantProbeCall();
 	const explainCall = (args: Record<string, unknown>): LhMessage =>
 		({
 			id: "a_explain",
@@ -199,10 +218,10 @@ test("phaseAt3 + phaseAt3Lh: a tagged explain CALL is the plan→teach seam too"
 	const tagged = explainCall({ question: "q", correctAnswer: "a", explanation: "e", nodeId: "gravity-basics" });
 	const untagged = explainCall({ question: "q", correctAnswer: "a", explanation: "e" });

-	assert.equal(phaseAt3Lh([done, untagged]), "plan", "an untagged explain never starts teaching");
-	assert.equal(phaseAt3Lh([done, tagged]), "teach", "the tagged explain CALL is the seam");
+	assert.equal(phaseAt3Lh([call, done, untagged]), "plan", "an untagged explain never starts teaching");
+	assert.equal(phaseAt3Lh([call, done, tagged]), "teach", "the tagged explain CALL is the seam");
 	assert.equal(
-		phaseAt3Lh([done, tagged]),
+		phaseAt3Lh([call, done, tagged]),
 		phaseAt3([
 			{ parts: [{ type: "tool-probe_done", state: "output-available" } as never] },
 			{ parts: [{ type: "tool-explain", input: { nodeId: "gravity-basics" } } as never] },
@@ -13,12 +13,13 @@ import * as path from "node:path";
 import { after, describe, it } from "node:test";
 import { probeDoneTool } from "../src/agent/tools/probe.ts";
 import type { SteerMessage } from "../src/core/contracts.ts";
-import type { LhMessage } from "../src/core/native-messages.ts";
+import { hasProbeDone, type LhMessage } from "../src/core/native-messages.ts";
+import { phaseAt3Lh } from "../src/core/phases.ts";
 import { HANDOFF_TEXT } from "../src/core/contracts.ts";
 import { chatStore } from "../src/store/chats.ts";
 import { PATHS } from "../config.ts";

-const dataDir = path.join(PATHS.root, "data");
+const dataDir = PATHS.data;
 fs.mkdirSync(dataDir, { recursive: true });

 const createdChats: string[] = [];
@@ -142,3 +143,61 @@ describe("probe_done: the probe→planning seam", () => {
 		assert.equal(res.ok, true, "ok:true even when the queue is lost — the next client message constructs the teacher off the transcript seam");
 	});
 });
+
+/** A real record result, distinct from the calling assistant message. */
+function probeResult(overrides: Partial<Extract<LhMessage, { role: "toolResult" }>> = {}): LhMessage {
+	return {
+		id: "probe-result", role: "toolResult", toolCallId: "call_probe", toolName: "probe_done",
+		content: [{ type: "text", text: "ok" }], isError: false, timestamp: 1,
+		...overrides,
+	};
+}
+
+const probeCall = () => assistantWithCall("probe", "probe_done", { assessment: ASSESSMENT });
+const failedProbeCall = (stopReason: "error" | "aborted"): LhMessage => ({ ...probeCall(), stopReason } as LhMessage);
+const incompleteProbes: Array<[string, () => LhMessage[]]> = [
+	["unmatched call", () => [probeCall()]],
+	["errored result", () => [probeCall(), probeResult({ isError: true })]],
+	["explicit refusal without isError", () => [probeCall(), probeResult({ details: { ok: false, error: "refused" } })]],
+	["result without a call", () => [probeResult()]],
+	["wrong call id", () => [probeCall(), probeResult({ toolCallId: "other" })]],
+	["wrong result tool name", () => [probeCall(), probeResult({ toolName: "plan_init" })]],
+	["wrong call tool name", () => [assistantWithCall("probe", "plan_init", {}), probeResult()]],
+	["result before the call", () => [probeResult(), probeCall()]],
+	["failed calling generation", () => [failedProbeCall("error"), probeResult({ details: { ok: true } })]],
+	["aborted calling generation", () => [failedProbeCall("aborted"), probeResult({ details: { ok: true } })]],
+];
+
+describe("probe completion: matched successful results only (PR130 foundation)", () => {
+	for (const [name, messages] of incompleteProbes) {
+		it(`${name} keeps probing open and allows a fresh conclusion`, async () => {
+			const chatId = newChat(`Probe recovery: ${name}`);
+			const before = messages();
+			const snapshot = structuredClone(before);
+			chatStore.saveMessages(chatId, before);
+			assert.equal(hasProbeDone(before), false);
+			assert.equal(phaseAt3Lh(before), "probe");
+			const delivered: SteerMessage[] = [];
+			const tools = probeDoneTool({ chatId, chatStore, deliver: (s) => delivered.push(s) });
+			assert.equal((await execute(tools, { assessment: ASSESSMENT })).ok, true);
+			assert.equal(delivered.length, 1, "only the fresh invocation queues a handoff");
+			assert.deepEqual(before, snapshot, "the scans never mutate history");
+			assert.deepEqual(chatStore.getChat(chatId)!.messages, snapshot, "the tool does not delete or replay saved attempts");
+		});
+	}
+
+	for (const [name, result] of [
+		["current acknowledgment", probeResult({ details: { ok: true } })],
+		["older clean acknowledgment without details", probeResult()],
+		["older clean acknowledgment without ok", probeResult({ details: { assessment: ASSESSMENT } })],
+	] as const) {
+		it(`${name} concludes exactly at the matching result`, () => {
+			const messages = [probeCall(), result];
+			const before = structuredClone(messages);
+			assert.equal(hasProbeDone(messages), true);
+			assert.equal(phaseAt3Lh(messages, 0), "probe", "a future result cannot conclude an earlier prefix");
+			assert.equal(phaseAt3Lh(messages, 1), "plan");
+			assert.deepEqual(messages, before);
+		});
+	}
+});
@@ -17,6 +17,7 @@ import type { AgentBuildOptions, TeachingAgent } from "../src/agent/teacher-agen
 import type { PiEngine } from "../src/agent/models.ts";
 import type { LhMessage } from "../src/core/native-messages.ts";
 import type { QuestionData } from "../src/core/question-core.ts";
+import type { SteerMessage } from "../src/core/contracts.ts";

 // Set the storage/auth seams BEFORE any app/config-loading imports.
 assert.equal(process.env.NODE_ENV, "test", "Run with NODE_ENV=test; these tests must not invoke real providers.");
@@ -36,7 +37,8 @@ const { jobStore } = await import("../src/store/jobs.ts");
 const { subscribe } = await import("../src/core/events.ts");
 const { buildAnswer, normalizeOptions, resolveCorrect } = await import("../src/core/question-core.ts");
 const { ChatInputError, endsInFailedTurn, failedResponseAtEnd, inputAppends, parseChatInput, retryDrops } = await import("../src/core/input-events.ts");
-const { INTERRUPTED_TEXT, asAgentMessages, newHandoffMessage, newPlanBlockMessage, newUserMessage, reconstructRequestCause, toLlmMessages } = await import("../src/core/native-messages.ts");
+const { INTERRUPTED_TEXT, asAgentMessages, hasProbeDone, newHandoffMessage, newPlanBlockMessage, newUserMessage, reconstructRequestCause, toLlmMessages } = await import("../src/core/native-messages.ts");
+const { phaseAt3Lh } = await import("../src/core/phases.ts");
 const { newCompactionMarker } = await import("../src/core/compaction.ts");

 after(() => {
@@ -287,6 +289,47 @@ describe("failed-response Retry (issue #110)", () => {
 		assert.deepEqual(contexts.at(-1), ["user"]);
 	});

+	it("a retained failed probe retries with the prober, concludes once, then selects the teacher", async () => {
+		const input = { assessment: { strands: [{ strand: "planet order", assessment: "Names known; ordering needs work." }] } };
+		const { engine, histories } = scripted([
+			() => fauxAssistantMessage([fauxToolCall("probe_done", input, { id: "failed-probe" })], { stopReason: "error", errorMessage: "cut off" }),
+			() => fauxAssistantMessage([fauxToolCall("probe_done", input, { id: "fresh-probe" })], { stopReason: "toolUse" }),
+			() => fauxAssistantMessage("Assessment ready."),
+			() => fauxAssistantMessage("Now planning the lesson."),
+		]);
+		const chat = createConfiguredChat("Test: failed probe recovery");
+		const delivered: SteerMessage[] = [];
+		const selections: boolean[] = [];
+		const factory: AgentFactory = (opts) => {
+			selections.push(opts.teaching);
+			if (opts.teaching) return createTeacherAgent(undefined, { ...opts, engine });
+			return createProberAgent({
+				chatId: chat.id, chatStore, jobStore, deliver: (s) => delivered.push(s), spawn: () => {},
+			}, { ...opts, engine });
+		};
+		await turn(engine, chat.id, { type: "text", text: "teach me planet order" }, factory);
+		const before = chatStore.getChat(chat.id)!.messages;
+		assert.ok(endsInFailedTurn(before));
+		assert.equal(hasProbeDone(before), false);
+		assert.equal(phaseAt3Lh(before), "probe");
+		assert.equal(delivered.length, 0, "the stored failed call never ran");
+
+		assert.equal((await turn(engine, chat.id, { type: "retry" }, factory)).status, 200);
+		const recovered = chatStore.getChat(chat.id)!.messages;
+		assertPreserved(before, recovered);
+		assert.deepEqual(selections, [false, false], "Retry must still build the prober");
+		assert.equal(hasProbeDone(recovered), true);
+		assert.equal(phaseAt3Lh(recovered), "plan");
+		assert.equal(delivered.length, 1, "the full retained-store guard accepts the fresh conclusion");
+		assert.deepEqual(recovered.filter((m) => m.role === "toolResult").map((m) => m.role === "toolResult" && m.toolCallId), ["fresh-probe"]);
+		assert.doesNotMatch(histories[1]!, /failed-probe/, "the failed call is omitted only from temporary Retry history");
+
+		assert.equal((await turn(engine, chat.id, { type: "text", text: "go on" }, factory)).status, 200);
+		assert.deepEqual(selections, [false, false, true]);
+		assert.equal(delivered.length, 1, "later turns do not replay either saved call");
+		assertPreserved(recovered, chatStore.getChat(chat.id)!.messages);
+	});
+
 	for (const kind of ["prober", "teacher", "practice"] as const) {
 		it(`uses the same retained-history retry boundary for the ${kind}`, async () => {
 			const { engine, contexts } = scripted([() => fauxAssistantMessage("Recovered response.")]);