fix(tools): the abort cascade — kill running tool calls, honest activity labels
Root cause of the live-test bugs (2026-09-19): pi passes every run's
AbortSignal to tool execute, but toPiTool dropped it — agent.abort()
only landed at the next step boundary, so a long local grep held its
turn hostage until the dev server's memory restart killed the run
mid-tool, freezing the record at its write-ahead state (the 'went
back in time' rollback).
- lh-tool: execute ctx gains signal (parent + timeout combined); every
call races a 30s system-level timeout with a 250ms cooperative grace —
abort kills a stuck tool but lets a poll-the-signal tool deliver its
partial results. Pre-aborted signals never fire listeners: checked
upfront. toolTimeoutMs is the test seam (pdfModelOverride idiom).
- fs-shared: walkFiles returns {timedOut, aborted} instead of throwing —
abort yields partial results.
- grep: the scan phase was unbounded (the walk's 10s deadline only
bounded collection) — same budget now checked per file; find/grep
check the signal and return partials with a notice.
- pending/chat: streamingToolActivity names the ACTUAL running tool —
a part sits input-streaming through its whole execution, so the old
shimmer called minutes-long greps 'Writing a question…'.
Tests: toPiTool guard (forward, abort, timeout, grace, question
bypass, ToolInputError passthrough), walk mid-abort, find/grep
pre-abort + mid-run abort, activity labels. Gates: 617/617.
This commit is contained in:
amosblomqvist
+58
-1
@@ -4,7 +4,7 @@ import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import { describe, it } from "node:test";
|
||||
|
||||
import { expandBraces, globToRegex, matchGlob } from "../src/agent/tools/fs-shared.ts";
|
||||
import { expandBraces, globToRegex, matchGlob, walkFiles } from "../src/agent/tools/fs-shared.ts";
|
||||
import { findTool } from "../src/agent/tools/find.ts";
|
||||
import { grepTool } from "../src/agent/tools/grep.ts";
|
||||
import { lsTool } from "../src/agent/tools/ls.ts";
|
||||
@@ -186,3 +186,60 @@ describe("read (home-anchored re-port)", () => {
|
||||
assert.match(out, /Eigenvectors keep their direction/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("abort plumbing (the 2026-09-19 cascade fix)", () => {
|
||||
it("walkFiles stops promptly when the signal aborts mid-walk", async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "lh-fs-abort-"));
|
||||
for (let i = 0; i < 20; i++) await fs.writeFile(path.join(root, `f${i}.txt`), "needle\n");
|
||||
const ac = new AbortController();
|
||||
let visited = 0;
|
||||
const result = await walkFiles(
|
||||
root,
|
||||
() => {
|
||||
visited++;
|
||||
if (visited === 3) ac.abort(); // deterministic: abort from inside the walk
|
||||
return false;
|
||||
},
|
||||
ac.signal,
|
||||
);
|
||||
assert.equal(result.aborted, true);
|
||||
assert.equal(result.timedOut, false);
|
||||
assert.ok(visited < 20, "the walk stopped early");
|
||||
});
|
||||
|
||||
it("find honors a pre-aborted signal without walking", async () => {
|
||||
FIXTURE = await setup();
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
const out = (await findTool.execute!({ pattern: "*.md" }, { toolCallId: "c", signal: ac.signal })) as string;
|
||||
assert.equal(out, "Operation aborted");
|
||||
});
|
||||
|
||||
it("grep honors a pre-aborted signal without scanning", async () => {
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
const out = (await grepTool.execute!({ pattern: "eigenvector" }, { toolCallId: "c", signal: ac.signal })) as string;
|
||||
assert.equal(out, "Operation aborted");
|
||||
});
|
||||
|
||||
it("grep aborts mid-run with partial results, not a hang", async () => {
|
||||
// The exact shape of the 2.7-minute lesson: a tree big enough that
|
||||
// the walk+scan outlast an abort mid-flight. Whichever phase the
|
||||
// abort lands in, the tool returns promptly with an abort notice —
|
||||
// never "No matches found", never a hang.
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "lh-fs-midabort-"));
|
||||
await fs.mkdir(path.join(root, "deep"), { recursive: true });
|
||||
for (let i = 0; i < 200; i++) {
|
||||
await fs.writeFile(path.join(root, i % 2 ? `f${i}.txt` : `deep/f${i}.txt`), "needle line\n");
|
||||
}
|
||||
const ac = new AbortController();
|
||||
setTimeout(() => ac.abort(), 1);
|
||||
const started = Date.now();
|
||||
const out = (await grepTool.execute!(
|
||||
{ pattern: "needle", path: root },
|
||||
{ toolCallId: "c", signal: ac.signal },
|
||||
)) as string;
|
||||
assert.match(out, /aborted/);
|
||||
assert.ok(Date.now() - started < 5_000, "returned promptly");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
/**
|
||||
* streamingToolActivity (2026-09-19, the "Writing a question…" fix): a
|
||||
* tool part stays input-streaming for its WHOLE execution (the bridge
|
||||
* defers input-available to tool_execution_end), so the transcript
|
||||
* activity line must name the ACTUAL running tool — the old code shimmered
|
||||
* "Writing a question…" over minutes-long greps.
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import type { UIMessage } from "ai";
|
||||
import { streamingToolActivity } from "../src/core/pending.ts";
|
||||
|
||||
function msg(role: string, parts: unknown[]): UIMessage {
|
||||
return { id: "m1", role, parts } as unknown as UIMessage;
|
||||
}
|
||||
|
||||
function toolPart(tool: string, state: string) {
|
||||
return { type: `tool-${tool}`, toolCallId: "c1", state, input: {} };
|
||||
}
|
||||
|
||||
describe("streamingToolActivity (the turn-activity label)", () => {
|
||||
it("question tools keep their question shimmer", () => {
|
||||
assert.equal(streamingToolActivity([msg("assistant", [toolPart("quiz", "input-streaming")])]), "Writing a question…");
|
||||
assert.equal(
|
||||
streamingToolActivity([msg("assistant", [toolPart("ask_user_question", "input-streaming")])]),
|
||||
"Writing a question…",
|
||||
);
|
||||
});
|
||||
|
||||
it("local tools name their actual activity", () => {
|
||||
assert.equal(streamingToolActivity([msg("assistant", [toolPart("grep", "input-streaming")])]), "Searching file contents…");
|
||||
assert.equal(streamingToolActivity([msg("assistant", [toolPart("read", "input-streaming")])]), "Reading a file…");
|
||||
assert.equal(streamingToolActivity([msg("assistant", [toolPart("find", "input-streaming")])]), "Finding files…");
|
||||
assert.equal(streamingToolActivity([msg("assistant", [toolPart("ls", "input-streaming")])]), "Listing a directory…");
|
||||
});
|
||||
|
||||
it("job dispatch tools get their own labels; unknown tools stay generic", () => {
|
||||
assert.equal(streamingToolActivity([msg("assistant", [toolPart("pdf", "input-streaming")])]), "Reading a PDF…");
|
||||
assert.equal(
|
||||
streamingToolActivity([msg("assistant", [toolPart("mystery_tool", "input-streaming")])]),
|
||||
"Working…",
|
||||
);
|
||||
});
|
||||
|
||||
it("no activity once the call settles or the turn ends", () => {
|
||||
// output-available = done executing → no shimmer (the transcript
|
||||
// shows the settled card instead).
|
||||
assert.equal(streamingToolActivity([msg("assistant", [toolPart("grep", "output-available")])]), null);
|
||||
// the last message being the user's = nothing running.
|
||||
assert.equal(streamingToolActivity([msg("user", [{ type: "text", text: "hi" }])]), null);
|
||||
assert.equal(streamingToolActivity([]), null);
|
||||
});
|
||||
|
||||
it("the SECOND streaming part wins only if the first settles — parts scan forward", () => {
|
||||
// A settled call followed by a live one: the live one is the activity.
|
||||
const parts = [toolPart("read", "output-available"), toolPart("grep", "input-streaming")];
|
||||
assert.equal(streamingToolActivity([msg("assistant", parts)]), "Searching file contents…");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,132 @@
|
||||
/**
|
||||
* The abort-cascade fix (2026-09-19): pi passes every run's AbortSignal to
|
||||
* tool execute — the bridge used to drop it, so agent.abort() only landed
|
||||
* at the NEXT step boundary and a long-running local tool (the grep that
|
||||
* outlived its turn by minutes) held the turn hostage until the dev
|
||||
* server's memory restart killed the run mid-tool, freezing the record at
|
||||
* its write-ahead state. toPiTool now threads the signal and races every
|
||||
* execute against the system-level timeout; the fs tools honor it with
|
||||
* partial results (fs-tools.test.ts covers their side).
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import { afterEach, describe, it } from "node:test";
|
||||
import { z } from "zod";
|
||||
import { lhTool, toPiTool, toolTimeoutMs, ToolInputError } from "../src/agent/lh-tool.ts";
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
|
||||
afterEach(() => {
|
||||
toolTimeoutMs.current = DEFAULT_TIMEOUT_MS;
|
||||
});
|
||||
|
||||
/** A tool that captures ctx.signal, then either hangs or resolves `value`. */
|
||||
function probeTool(captured: { signal?: AbortSignal }, opts: { hang?: boolean; value?: string } = {}) {
|
||||
return lhTool({
|
||||
name: "probe",
|
||||
description: "test probe",
|
||||
schema: z.object({}),
|
||||
execute: (_input, ctx) => {
|
||||
captured.signal = ctx?.signal;
|
||||
if (opts.hang) return new Promise<string>(() => {}); // never settles, no timer
|
||||
return opts.value ?? "ok";
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
describe("toPiTool abort + timeout guard", () => {
|
||||
it("forwards a live abort signal to the tool's ctx", async () => {
|
||||
// Even with no parent signal, the tool still gets the guard's combined
|
||||
// signal (parent+timeout) — a stable reference tools can poll.
|
||||
const captured: { signal?: AbortSignal } = {};
|
||||
const pi = toPiTool(probeTool(captured));
|
||||
await pi.execute("c1", {}, undefined as never);
|
||||
assert.ok(captured.signal, "the tool received a signal");
|
||||
assert.equal(captured.signal!.aborted, false);
|
||||
// The forwarded signal is the guard's COMBINED one — aborting the
|
||||
// parent flips it, which is what long-running tools poll.
|
||||
const ac = new AbortController();
|
||||
const captured2: { signal?: AbortSignal } = {};
|
||||
const pi2 = toPiTool(probeTool(captured2, { value: "done" }));
|
||||
const pending = pi2.execute("c1b", {}, ac.signal);
|
||||
ac.abort();
|
||||
await pending;
|
||||
assert.equal(captured2.signal!.aborted, true);
|
||||
});
|
||||
|
||||
it("a signal-wired tool observes the run's abort", async () => {
|
||||
const ac = new AbortController();
|
||||
const captured: { signal?: AbortSignal } = {};
|
||||
const pi = toPiTool(probeTool(captured, { hang: true }));
|
||||
const pending = pi.execute("c2", {}, ac.signal).catch((e: Error) => e.message);
|
||||
ac.abort();
|
||||
assert.match((await pending) as string, /Operation aborted/);
|
||||
assert.equal(captured.signal!.aborted, true);
|
||||
});
|
||||
|
||||
it("rejects a hanging tool at the system-level timeout", async () => {
|
||||
toolTimeoutMs.current = 50;
|
||||
const pi = toPiTool(probeTool({}, { hang: true }));
|
||||
await assert.rejects(pi.execute("c3", {}, undefined as never), /timed out/);
|
||||
});
|
||||
|
||||
it("a pre-aborted run rejects immediately with pi's wording", async () => {
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
const pi = toPiTool(probeTool({}, { hang: true }));
|
||||
await assert.rejects(pi.execute("c4", {}, ac.signal), /Operation aborted/);
|
||||
});
|
||||
|
||||
it("a tool that honors the signal settles before the timeout wins", async () => {
|
||||
// The race resolves with the tool's own (partial) result — the fs
|
||||
// tools' contract: abort yields partial results, not an error throw.
|
||||
toolTimeoutMs.current = 2_000;
|
||||
const tool = lhTool({
|
||||
name: "partial",
|
||||
description: "test",
|
||||
schema: z.object({}),
|
||||
execute: (_input, ctx) =>
|
||||
new Promise((resolve) => {
|
||||
// The fs-tools contract: poll upfront (a listener on an
|
||||
// already-aborted signal never fires), else listen.
|
||||
const signal = ctx!.signal!;
|
||||
if (signal.aborted) return resolve("partial results");
|
||||
signal.addEventListener("abort", () => resolve("partial results"), { once: true });
|
||||
}),
|
||||
});
|
||||
const ac = new AbortController();
|
||||
const pi = toPiTool(tool);
|
||||
const pending = pi.execute("c5", {}, ac.signal);
|
||||
ac.abort();
|
||||
const result = (await pending) as { content: Array<{ text?: string }> };
|
||||
assert.equal(result.content[0]!.text, "partial results");
|
||||
});
|
||||
|
||||
it("question tools bypass the guard — the placeholder is instant", async () => {
|
||||
const ac = new AbortController();
|
||||
ac.abort();
|
||||
const tool = lhTool({
|
||||
name: "quiz",
|
||||
description: "test",
|
||||
schema: z.object({}),
|
||||
clientQuestion: true,
|
||||
});
|
||||
const pi = toPiTool(tool);
|
||||
const result = (await pi.execute("c6", {}, ac.signal)) as { content: Array<{ text?: string }> };
|
||||
assert.match(result.content[0]!.text!, /Question posed/);
|
||||
});
|
||||
|
||||
it("validation errors pass through the guard unchanged", async () => {
|
||||
const tool = lhTool({
|
||||
name: "strict",
|
||||
description: "test",
|
||||
schema: z.object({ n: z.number() }),
|
||||
execute: () => "never",
|
||||
});
|
||||
const pi = toPiTool(tool);
|
||||
await assert.rejects(pi.execute("c7", { n: "not a number" }, undefined as never), (e: unknown) => {
|
||||
assert.ok(e instanceof ToolInputError);
|
||||
assert.equal(e.toolName, "strict");
|
||||
return true;
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user