fix: use native Pi identity and refresh provider catalogs
This commit is contained in:
amosblomqvist
@@ -13,7 +13,7 @@
|
||||
* ALL the env vars a provider reads as API keys.
|
||||
*
|
||||
* Owner-spec exclusions (asserted ABSENT): github-copilot (OAuth),
|
||||
* amazon-bedrock (AWS creds), google-vertex (ADC), azure-openai-responses
|
||||
* amazon-bedrock (AWS creds), google-vertex (ADC), azure
|
||||
* (needs base URL), the cloudflare account-scoped pair; anthropic keeps
|
||||
* ANTHROPIC_API_KEY only (AUTH_TOKEN/OAUTH_TOKEN stay env-only).
|
||||
*/
|
||||
@@ -32,7 +32,7 @@ const EXCLUDED_PROVIDERS = new Set([
|
||||
"github-copilot",
|
||||
"amazon-bedrock",
|
||||
"google-vertex",
|
||||
"azure-openai-responses",
|
||||
"azure",
|
||||
"cloudflare-workers-ai",
|
||||
"cloudflare-ai-gateway",
|
||||
]);
|
||||
|
||||
@@ -25,7 +25,7 @@ import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import * as fs from "node:fs";
|
||||
import { after, describe, it } from "node:test";
|
||||
import { createFauxCore, fauxAssistantMessage, fauxToolCall, type FauxResponseStep, type ToolCall } from "@earendil-works/pi-ai";
|
||||
import { createFauxCore, fauxAssistantMessage, fauxToolCall, type FauxResponseStep, type ToolCall, type Usage } from "@earendil-works/pi-ai";
|
||||
import type { PiEngine } from "../src/agent/models.ts";
|
||||
import type { LhMessage } from "../src/core/native-messages.ts";
|
||||
import { newPlanBlockMessage } from "../src/core/native-messages.ts";
|
||||
@@ -873,21 +873,30 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
const verdict: AuditVerdicts = {
|
||||
links: [{ id: FINDINGS.propositions[0]!.id, linkTo: existingId }],
|
||||
};
|
||||
const nativeUsage: Record<"extractor" | "auditor", Usage[]> = { extractor: [], auditor: [] };
|
||||
const recordUsage = (stage: "extractor" | "auditor", engine: PiEngine): PiEngine => ({
|
||||
...engine,
|
||||
streamFn: (model, context, options) => {
|
||||
const stream = engine.streamFn(model, context, options);
|
||||
void stream.result().then((message) => nativeUsage[stage].push(message.usage));
|
||||
return stream;
|
||||
},
|
||||
});
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
// During the auditor's generate, the registry must already carry
|
||||
// the extractor's fully accumulated live usage (2 steps × 10/5).
|
||||
extractor: submitModel("submit_findings", FINDINGS),
|
||||
auditor: submitModel("submit_review", verdict, () =>
|
||||
extractor: recordUsage("extractor", submitModel("submit_findings", FINDINGS)),
|
||||
auditor: recordUsage("auditor", submitModel("submit_review", verdict, () =>
|
||||
usageSeen.push(extractionStatuses().find((r) => r.chatId === chatId)?.usage.extractor?.steps.at(-1)?.inputTokens),
|
||||
),
|
||||
)),
|
||||
});
|
||||
await pipeline.runExtraction(chatId);
|
||||
|
||||
// Live: mid-audit the extractor's usage was fully accumulated (faux
|
||||
// estimates tokens from the prompt/response text — deterministic,
|
||||
// positive, no cache).
|
||||
// positive, with native first-conversation cache writes in 1.4).
|
||||
assert.equal(usageSeen.length, 1);
|
||||
assert.ok(typeof usageSeen[0] === "number" && usageSeen[0]! > 0, "the extractor's step billed input tokens");
|
||||
|
||||
@@ -903,8 +912,16 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
const step = run.stages.extractor.steps[0]!;
|
||||
assert.ok(step.inputTokens! > 0 && step.outputTokens! > 0, "estimated usage captured");
|
||||
assert.equal(step.cacheReadTokens, 0);
|
||||
assert.equal(step.cacheWriteTokens, 0);
|
||||
assert.equal(nativeUsage.extractor.length, 1);
|
||||
assert.ok(nativeUsage.extractor[0]!.cacheWrite > 0, "fresh native conversation writes its cached prefix");
|
||||
assert.equal(step.cacheReadTokens, nativeUsage.extractor[0]!.cacheRead);
|
||||
assert.equal(step.cacheWriteTokens, nativeUsage.extractor[0]!.cacheWrite);
|
||||
assert.equal(step.reasoningTokens, 0);
|
||||
assert.equal(nativeUsage.auditor.length, 1);
|
||||
assert.equal(run.stages.auditor.steps[0]!.cacheReadTokens, 0, "auditor owns a fresh conversation too");
|
||||
assert.ok(nativeUsage.auditor[0]!.cacheWrite > 0);
|
||||
assert.equal(run.stages.auditor.steps[0]!.cacheReadTokens, nativeUsage.auditor[0]!.cacheRead);
|
||||
assert.equal(run.stages.auditor.steps[0]!.cacheWriteTokens, nativeUsage.auditor[0]!.cacheWrite);
|
||||
assert.equal(run.stages.auditor.steps.length, 1);
|
||||
assert.ok(!Number.isNaN(Date.parse(run.at)));
|
||||
});
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import { normalizeContext } from "@earendil-works/pi-ai";
|
||||
import { Type } from "typebox";
|
||||
import { piEngineForSpec, piModels } from "../src/agent/models.ts";
|
||||
import { modelCatalog, validateChatSettings, validateModelsSettings } from "../src/server/model-catalog.ts";
|
||||
import { callCost } from "../src/core/prices.ts";
|
||||
|
||||
const specimens = [
|
||||
{ id: "recursive/flint-v1.0:free", input: ["text"] },
|
||||
{ id: "xingchen-agi/xing4.0-29b-a4b:free", input: ["text"] },
|
||||
{ id: "nvidia/nemotron-3.5-super-vl:free", input: ["text", "image"] },
|
||||
{ id: "openrouter/free", input: ["text", "image"] },
|
||||
];
|
||||
|
||||
describe("native 1.4 OpenRouter free catalog (offline, no live discovery)", { concurrency: false }, () => {
|
||||
for (const { id, input } of specimens) {
|
||||
it(`${id}: shared listing, validation, tool-capable engine, native modality, and zero cost`, async () => {
|
||||
const before = process.env;
|
||||
const originalFetch = globalThis.fetch;
|
||||
process.env = { NODE_ENV: "test", HOME: "/nonexistent/free-catalog", OPENROUTER_API_KEY: "dummy-free-catalog" };
|
||||
try {
|
||||
const catalog = modelCatalog();
|
||||
const listed = catalog.models.find((m) => m.provider === "openrouter" && m.id === id);
|
||||
assert.ok(listed);
|
||||
assert.deepEqual(listed.input, input);
|
||||
assert.deepEqual(listed.cost, { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 });
|
||||
const model = { provider: "openrouter", id };
|
||||
assert.deepEqual(validateChatSettings({ model }), { ok: true });
|
||||
assert.deepEqual(validateModelsSettings({ strong: { model }, cheap: { model } }), { ok: true });
|
||||
assert.equal(validateModelsSettings({ vision: { model } }).ok, true, "server catalog validation stays provider/model based; vision filtering belongs to the picker");
|
||||
assert.equal(catalog.models.filter((m) => m.input.includes("image")).some((m) => m.provider === "openrouter" && m.id === id), input.includes("image"));
|
||||
const engine = piEngineForSpec(model);
|
||||
assert.equal(engine.model, piModels().getModel("openrouter", id));
|
||||
assert.equal(engine.model.api, "openai-completions");
|
||||
assert.deepEqual(engine.model.input, input);
|
||||
assert.equal(callCost({ inputTokens: 1_000_000, outputTokens: 500_000, cacheReadTokens: 100_000, cacheWriteTokens: 50_000 }, `openrouter:${id}`), 0);
|
||||
let payload: Record<string, unknown> | undefined;
|
||||
globalThis.fetch = async (input, init) => {
|
||||
const request = new Request(input, init);
|
||||
assert.equal(new URL(request.url).origin, "https://openrouter.ai");
|
||||
assert.equal(request.headers.get("authorization"), "Bearer dummy-free-catalog");
|
||||
payload = JSON.parse(await request.text());
|
||||
return new Response("{}", { status: 400, headers: { "content-type": "application/json" } });
|
||||
};
|
||||
await engine.streamFn(engine.model, normalizeContext({ messages: [{ role: "user", content: "offline", timestamp: 1 }], tools: [{ name: "offline_tool", description: "offline", parameters: Type.Object({}) }] }), { sessionId: "chat_free_offline" }).result();
|
||||
assert.equal(payload?.model, id);
|
||||
assert.match(JSON.stringify(payload?.tools), /offline_tool/);
|
||||
delete process.env.OPENROUTER_API_KEY;
|
||||
assert.ok(!modelCatalog().models.some((m) => m.provider === "openrouter"), "even free models retain the provider key gate");
|
||||
} finally { process.env = before; globalThis.fetch = originalFetch; }
|
||||
});
|
||||
}
|
||||
it("removed/unknown native models remain unpriced and unresolvable, never aliased to a paid fallback", () => {
|
||||
assert.equal(callCost({ inputTokens: 1, outputTokens: 1 }, "openrouter:removed/no-such-model"), null);
|
||||
assert.throws(() => piEngineForSpec({ provider: "openrouter", id: "removed/no-such-model" }), /catalog has no model/);
|
||||
});
|
||||
});
|
||||
@@ -14,7 +14,7 @@ import {
|
||||
type JsonObject, type TranscriptContext, type Usage,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import { builtinProviders } from "@earendil-works/pi-ai/providers/all";
|
||||
import { piEngineForSpec, type PiEngine } from "../src/agent/models.ts";
|
||||
import { APP_IDENTITY, piEngineForSpec, type PiEngine } from "../src/agent/models.ts";
|
||||
import { runVisualMaker } from "../src/agent/subagents/visual-maker.ts";
|
||||
import { runPdfReaderAgent } from "../src/agent/subagents/pdf-reader.ts";
|
||||
import type { HeadlessApi } from "../src/agent/subagents/headless.ts";
|
||||
@@ -35,6 +35,10 @@ interface Transport {
|
||||
origin: string; path: string; authHeader: string;
|
||||
}
|
||||
const TRANSPORTS: Transport[] = [
|
||||
{ provider: "opencode-go", id: "claude-haiku-5-5", envVar: "OPENCODE_API_KEY", api: "anthropic-messages", origin: "https://opencode.ai", path: "/inference/go/v1/messages", authHeader: "x-api-key" },
|
||||
{ provider: "opencode-go", id: "gemini-4-argon-contributor", envVar: "OPENCODE_API_KEY", api: "google-generative-ai", origin: "https://opencode.ai", path: "/inference/go/v1/models/gemini-4-argon-contributor:streamGenerateContent", authHeader: "x-goog-api-key" },
|
||||
{ provider: "opencode-go", id: "glm-5.3", envVar: "OPENCODE_API_KEY", api: "openai-completions", origin: "https://opencode.ai", path: "/inference/go/v1/chat/completions", authHeader: "authorization" },
|
||||
{ provider: "opencode-go", id: "gpt-6-luna", envVar: "OPENCODE_API_KEY", api: "openai-responses", origin: "https://opencode.ai", path: "/inference/go/v1/responses", authHeader: "authorization" },
|
||||
{ provider: "openrouter", id: "openai/gpt-4o-mini", envVar: "OPENROUTER_API_KEY", api: "openai-completions", origin: "https://openrouter.ai", path: "/api/v1/chat/completions", authHeader: "authorization" },
|
||||
{ provider: "openai", id: "gpt-4o-mini", envVar: "OPENAI_API_KEY", api: "openai-responses", origin: "https://api.openai.com", path: "/v1/responses", authHeader: "authorization" },
|
||||
{ provider: "anthropic", id: "claude-haiku-4-5", envVar: "ANTHROPIC_API_KEY", api: "anthropic-messages", origin: "https://api.anthropic.com", path: "/v1/messages", authHeader: "x-api-key" },
|
||||
@@ -131,7 +135,7 @@ describe("provider-neutral raw agent loops", { concurrency: false }, () => {
|
||||
it(`${loop}: every actual stream request has NO credential override (recording faux engine)`, async () => {
|
||||
await isolated({ OPENROUTER_API_KEY: OR_KEY, LH_PROVIDER_EVENTS: "1" }, async () => {
|
||||
const faux = createFauxCore({ tokenSize: { min: 64, max: 64 } });
|
||||
faux.setResponses([0, 1, 2].map((round) => {
|
||||
faux.setResponses([0, 1, 2, 0, 1, 2].map((round) => {
|
||||
const calls = callsFor(loop, round);
|
||||
return fauxAssistantMessage(calls.length
|
||||
? calls.map((c, i) => fauxToolCall(c.name, c.args, { id: `call_${round}_${i}` }))
|
||||
@@ -149,7 +153,11 @@ describe("provider-neutral raw agent loops", { concurrency: false }, () => {
|
||||
const run = await runLoop(loop, engine, signal);
|
||||
assertSuccess(loop, run);
|
||||
assert.equal(requests.length, 3);
|
||||
const sessionId = requests[0]!.options.sessionId;
|
||||
assert.match(String(sessionId), /^[0-9a-f-]{36}$/);
|
||||
assert.notEqual(sessionId, "provider-neutral-offline");
|
||||
for (const { options } of requests) {
|
||||
assert.equal(options.sessionId, sessionId);
|
||||
for (const key of ["apiKey", "getApiKey", "auth", "headers"]) {
|
||||
assert.ok(!Object.hasOwn(options, key), `${loop} stream must not supply ${key}`);
|
||||
}
|
||||
@@ -159,6 +167,12 @@ describe("provider-neutral raw agent loops", { concurrency: false }, () => {
|
||||
}
|
||||
assert.match(JSON.stringify(requests[2]!.context), /image\/png/);
|
||||
assert.match(JSON.stringify(requests[2]!.context), /toolResult/);
|
||||
const second = await runLoop(loop, engine, signal);
|
||||
assertSuccess(loop, second);
|
||||
assert.equal(requests.length, 6);
|
||||
const secondId = requests[3]!.options.sessionId;
|
||||
assert.notEqual(secondId, sessionId, "a separate unsaved operation must not reuse a conversation");
|
||||
for (const request of requests.slice(3)) assert.equal(request.options.sessionId, secondId);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -189,6 +203,12 @@ describe("provider-neutral raw agent loops", { concurrency: false }, () => {
|
||||
assertSuccess(loop, run);
|
||||
assert.equal(requests.length, 3);
|
||||
for (const [round, request] of requests.entries()) {
|
||||
assert.equal(request.headers.get("user-agent"), APP_IDENTITY.userAgent.custom);
|
||||
if (transport.provider === "opencode-go") {
|
||||
assert.match(request.headers.get("x-opencode-session") ?? "", /^[0-9a-f-]{36}$/);
|
||||
assert.equal(request.headers.get("x-opencode-session"), requests[0]!.headers.get("x-opencode-session"));
|
||||
assert.equal(request.headers.get("x-opencode-client"), "learnharness");
|
||||
}
|
||||
const url = new URL(request.url);
|
||||
assert.equal(url.origin, transport.origin);
|
||||
assert.equal(url.pathname, transport.path);
|
||||
@@ -263,29 +283,32 @@ function closedFetch(transport: Transport, loop: Loop, requests: RecordedRequest
|
||||
}
|
||||
|
||||
function assertPayload(transport: Transport, loop: Loop, round: number, body: Record<string, unknown>) {
|
||||
if (transport.provider !== "google") assert.equal(body.model, transport.id);
|
||||
const tools = JSON.stringify(body.tools);
|
||||
if (transport.api !== "google-generative-ai") assert.equal(body.model, transport.id);
|
||||
// Native Responses models may declare tools in additional_tools input
|
||||
// items rather than top-level tools; assert the SDK's complete payload.
|
||||
const tools = JSON.stringify(body);
|
||||
assert.match(tools, new RegExp(loop === "visual" ? "write_svg" : "pdf_extract"));
|
||||
assert.match(tools, new RegExp(loop === "visual" ? "render_svg" : "pdf_render"));
|
||||
if (transport.provider === "google") assert.ok(Array.isArray(body.contents));
|
||||
else if (transport.provider === "openai") assert.ok(Array.isArray(body.input));
|
||||
if (transport.api === "google-generative-ai") assert.ok(Array.isArray(body.contents));
|
||||
else if (transport.api === "openai-responses") assert.ok(Array.isArray(body.input));
|
||||
else assert.ok(Array.isArray(body.messages));
|
||||
const json = JSON.stringify(body);
|
||||
assert.match(json, new RegExp(loop === "visual" ? "white square" : "Read page 1"));
|
||||
if (round === 2) {
|
||||
assert.ok(json.includes(PNG.toString("base64")), "the actual rendered image must reach the selected provider");
|
||||
assert.match(json, new RegExp(transport.provider === "google" ? "functionResponse" : transport.provider === "openai" ? "function_call_output" : transport.provider === "anthropic" ? "tool_result" : '"role":"tool"'));
|
||||
const supportsImages = piEngineForSpec({ provider: transport.provider, id: transport.id }).model.input.includes("image");
|
||||
assert.equal(json.includes(PNG.toString("base64")), supportsImages, "native modality filtering: only vision models receive rendered images");
|
||||
assert.match(json, new RegExp(transport.api === "google-generative-ai" ? "functionResponse" : transport.api === "openai-responses" ? "function_call_output" : transport.api === "anthropic-messages" ? "tool_result" : '"role":"tool"'));
|
||||
assert.match(json, new RegExp(loop === "visual" ? "Rendered" : "Page 1"));
|
||||
}
|
||||
}
|
||||
|
||||
function sseResponse(transport: Transport, calls: Call[], round: number): Response {
|
||||
const events: Record<string, unknown>[] = [];
|
||||
if (transport.provider === "openrouter") {
|
||||
if (transport.api === "openai-completions") {
|
||||
events.push({ id: `chatcmpl_offline_${round}`, object: "chat.completion.chunk", created: 1, model: transport.id,
|
||||
choices: [{ index: 0, delta: calls.length ? { role: "assistant", tool_calls: calls.map((c, i) => ({ index: i, id: `call_${round}_${i}`, type: "function", function: { name: c.name, arguments: JSON.stringify(c.args) } })) } : { role: "assistant", content: FINAL }, finish_reason: null }] });
|
||||
events.push({ choices: [{ index: 0, delta: {}, finish_reason: calls.length ? "tool_calls" : "stop" }], usage: { prompt_tokens: 20, completion_tokens: 10, total_tokens: 30 } });
|
||||
} else if (transport.provider === "openai") {
|
||||
} else if (transport.api === "openai-responses") {
|
||||
const output = calls.length ? calls.map((c, i) => ({ type: "function_call", id: `fc_${round}_${i}`, call_id: `call_${round}_${i}`, name: c.name, arguments: JSON.stringify(c.args), status: "completed" }))
|
||||
: [{ type: "message", id: `msg_${round}`, role: "assistant", status: "completed", content: [{ type: "output_text", text: FINAL, annotations: [] }] }];
|
||||
events.push({ type: "response.created", response: { id: `resp_offline_${round}`, status: "in_progress" } });
|
||||
@@ -297,7 +320,7 @@ function sseResponse(transport: Transport, calls: Call[], round: number): Respon
|
||||
events.push({ type: "response.output_item.done", output_index: index, item });
|
||||
}
|
||||
events.push({ type: "response.completed", response: { id: `resp_offline_${round}`, status: "completed", output, usage: { input_tokens: 20, output_tokens: 10, total_tokens: 30 } } });
|
||||
} else if (transport.provider === "anthropic") {
|
||||
} else if (transport.api === "anthropic-messages") {
|
||||
events.push({ type: "message_start", message: { id: `msg_offline_${round}`, type: "message", role: "assistant", model: transport.id, content: [], stop_reason: null, usage: { input_tokens: 20, output_tokens: 0 } } });
|
||||
const blocks = calls.length ? calls.map((c, i) => ({ type: "tool_use", id: `call_${round}_${i}`, name: c.name, input: {} })) : [{ type: "text", text: "" }];
|
||||
for (const [index, block] of blocks.entries()) {
|
||||
@@ -310,6 +333,6 @@ function sseResponse(transport: Transport, calls: Call[], round: number): Respon
|
||||
} else {
|
||||
events.push({ responseId: `google_offline_${round}`, candidates: [{ index: 0, content: { role: "model", parts: calls.length ? calls.map((c, i) => ({ functionCall: { id: `call_${round}_${i}`, name: c.name, args: c.args } })) : [{ text: FINAL }] }, finishReason: "STOP" }], usageMetadata: { promptTokenCount: 20, candidatesTokenCount: 10, totalTokenCount: 30 } });
|
||||
}
|
||||
const data = events.map((event) => `${typeof event.type === "string" ? `event: ${event.type}\n` : ""}data: ${JSON.stringify(event)}\n\n`).join("") + (transport.provider === "openrouter" ? "data: [DONE]\n\n" : "");
|
||||
const data = events.map((event) => `${typeof event.type === "string" ? `event: ${event.type}\n` : ""}data: ${JSON.stringify(event)}\n\n`).join("") + (transport.api === "openai-completions" ? "data: [DONE]\n\n" : "");
|
||||
return new Response(data, { status: 200, headers: { "content-type": "text/event-stream" } });
|
||||
}
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { describe, it } from "node:test";
|
||||
import { Type } from "typebox";
|
||||
import { createFauxCore, fauxAssistantMessage, fauxText, fauxToolCall, normalizeContext, type Model, type SimpleStreamOptions } from "@earendil-works/pi-ai";
|
||||
import { streamSimple as streamCodex } from "@earendil-works/pi-ai/api/openai-codex-responses";
|
||||
import { APP_IDENTITY, piEngineForSpec, type PiEngine } from "../src/agent/models.ts";
|
||||
import { createProberAgent, createTeacherAgent, createPracticeAgent, type AgentBuildOptions } from "../src/agent/teacher-agent.ts";
|
||||
import { runSubagent } from "../src/agent/subagents/pi-run.ts";
|
||||
import { healToolRecord } from "../src/agent/tools/heal.ts";
|
||||
import metadata from "../package.json" with { type: "json" };
|
||||
|
||||
const factories = [
|
||||
(opts: AgentBuildOptions) => createProberAgent({ chatId: opts.chatId, searchAvailable: () => false } as never, opts),
|
||||
(opts: AgentBuildOptions) => createTeacherAgent(undefined, opts),
|
||||
(opts: AgentBuildOptions) => createPracticeAgent({} as never, opts),
|
||||
];
|
||||
|
||||
function recordingEngine() {
|
||||
const faux = createFauxCore({});
|
||||
faux.setResponses([fauxAssistantMessage([fauxToolCall("again", {}, { id: "call_0" })], { stopReason: "toolUse" }), { ...fauxAssistantMessage([fauxText("done")]), durationMs: 42 }]);
|
||||
const requests: SimpleStreamOptions[] = [];
|
||||
const engine: PiEngine = { model: faux.getModel(), streamFn: (model, context, options) => {
|
||||
requests.push(options ?? {});
|
||||
return faux.streamSimple(model, context, options);
|
||||
} };
|
||||
return { engine, requests };
|
||||
}
|
||||
const again = { name: "again", label: "again", description: "Continue", parameters: Type.Object({}), execute: async () => ({ content: [{ type: "text" as const, text: "ok" }], details: {} }) };
|
||||
|
||||
// Whole-process replacement isolates native auth/profile discovery; fetch never
|
||||
// delegates. No real credentials, .env, OAuth accounts, or model inference.
|
||||
async function closed<T>(run: () => Promise<T>): Promise<T> {
|
||||
const env = process.env;
|
||||
const fetch = globalThis.fetch;
|
||||
process.env = { NODE_ENV: "test", HOME: "/nonexistent/session-identity", OPENROUTER_API_KEY: "dummy-router", OPENCODE_API_KEY: "dummy-go", OPENAI_API_KEY: "dummy-openai" };
|
||||
globalThis.fetch = async () => { throw new Error("CLOSED: unexpected fetch"); };
|
||||
try { return await run(); } finally { process.env = env; globalThis.fetch = fetch; }
|
||||
}
|
||||
|
||||
describe("native app and conversation identity", { concurrency: false }, () => {
|
||||
it("shares one deeply frozen metadata-derived identity, without mutable conversation state", () => {
|
||||
assert.deepEqual(APP_IDENTITY, { userAgent: { custom: `learnharness/${metadata.version}` }, attribution: { name: metadata.name, url: metadata.homepage } });
|
||||
for (const value of [APP_IDENTITY, APP_IDENTITY.userAgent, APP_IDENTITY.attribution]) assert.ok(Object.isFrozen(value));
|
||||
assert.ok(!Object.hasOwn(APP_IDENTITY, "sessionId"));
|
||||
});
|
||||
|
||||
it("reconstructed prober/teacher/practice Agents reuse the exact chat id across tool rounds", async () => {
|
||||
for (const chatId of ["chat_exact_native_123", "chat_independent_456"]) {
|
||||
for (const factory of factories) {
|
||||
const { engine, requests } = recordingEngine();
|
||||
const agent = factory({ engine, chatId, tools: { again } });
|
||||
assert.equal(agent.identity, APP_IDENTITY);
|
||||
assert.equal(agent.sessionId, chatId);
|
||||
await agent.prompt("offline");
|
||||
assert.equal(requests.length, 2);
|
||||
for (const options of requests) {
|
||||
assert.equal(options.sessionId, chatId);
|
||||
assert.deepEqual(options.identity, APP_IDENTITY);
|
||||
assert.ok(options.signal instanceof AbortSignal);
|
||||
assert.equal(options.apiKey, undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("separate extractor/auditor/compaction invocations own distinct stable unsaved conversations", async () => {
|
||||
const ids = new Set<unknown>();
|
||||
// Compaction uses the shared runner with the researcher role.
|
||||
for (const role of ["extractor", "auditor", "researcher"] as const) {
|
||||
const { engine, requests } = recordingEngine();
|
||||
let usage = 0;
|
||||
const result = await runSubagent([again], { role, engine, chatId: "chat_shared_debug_only", systemPrompt: "offline", prompt: "offline", maxTurns: 3, onUsage: () => usage++ });
|
||||
assert.equal(requests.length, 2);
|
||||
assert.equal(usage, 2);
|
||||
const id = requests[0]!.sessionId;
|
||||
assert.match(id ?? "", /^[0-9a-f-]{36}$/);
|
||||
assert.notEqual(id, "chat_shared_debug_only");
|
||||
assert.ok(!ids.has(id));
|
||||
ids.add(id);
|
||||
for (const options of requests) { assert.equal(options.sessionId, id); assert.deepEqual(options.identity, APP_IDENTITY); }
|
||||
assert.ok(result.newMessages.some((m) => m.role === "toolResult" && typeof m.durationMs === "number"), "new native tool duration fields are preserved");
|
||||
assert.ok(result.newMessages.some((m) => m.role === "assistant" && typeof m.durationMs === "number"), "new native assistant duration fields are preserved");
|
||||
}
|
||||
});
|
||||
|
||||
for (const [provider, id, endpoint] of [
|
||||
["opencode-go", "claude-haiku-5-5", "/inference/go/v1/messages"],
|
||||
["opencode-go", "gemini-4-argon-contributor", "/inference/go/v1/models/gemini-4-argon-contributor:streamGenerateContent"],
|
||||
["opencode-go", "glm-5.3", "/inference/go/v1/chat/completions"],
|
||||
["opencode-go", "gpt-6-luna", "/inference/go/v1/responses"],
|
||||
["openrouter", "openrouter/free", "/api/v1/chat/completions"],
|
||||
] as const) {
|
||||
it(`complete native ${provider}/${id} foreground serialization carries app identity and exact sessions`, async () => closed(async () => {
|
||||
const engine = piEngineForSpec({ provider, id, thinkingLevel: "off" });
|
||||
const requests: Request[] = [];
|
||||
globalThis.fetch = async (input, init) => { const request = new Request(input, init); requests.push(request); await request.text(); return new Response('{"error":{"message":"closed fixture"}}', { status: 400, headers: { "content-type": "application/json" } }); };
|
||||
for (const factory of factories) await factory({ engine, chatId: "chat_wire_exact", tools: {} }).prompt("offline");
|
||||
assert.equal(requests.length, 3);
|
||||
for (const request of requests) {
|
||||
assert.equal(new URL(request.url).pathname, endpoint);
|
||||
assert.equal(request.headers.get("user-agent"), `learnharness/${metadata.version}`);
|
||||
if (provider === "opencode-go") {
|
||||
assert.equal(new URL(request.url).origin, "https://opencode.ai");
|
||||
assert.equal(request.headers.get("x-opencode-client"), metadata.name);
|
||||
assert.equal(request.headers.get("x-opencode-session"), "chat_wire_exact");
|
||||
assert.ok(!JSON.stringify(Object.fromEntries(request.headers)).includes("dummy-router"));
|
||||
} else {
|
||||
assert.equal(new URL(request.url).origin, "https://openrouter.ai");
|
||||
assert.equal(request.headers.get("x-openrouter-title"), metadata.name);
|
||||
assert.equal(request.headers.get("http-referer"), metadata.homepage);
|
||||
assert.equal(request.headers.get("x-session-id"), "chat_wire_exact");
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
it("native OpenAI cache keys clamp to 64 characters while affinity retains exact session", async () => closed(async () => {
|
||||
const engine = piEngineForSpec({ provider: "openai", id: "gpt-4o-mini", thinkingLevel: "off" });
|
||||
const sessionId = "chat_" + "a".repeat(70);
|
||||
let body: Record<string, unknown> = {};
|
||||
let headers = new Headers();
|
||||
globalThis.fetch = async (input, init) => { const request = new Request(input, init); headers = request.headers; body = JSON.parse(await request.text()); return new Response("{}", { status: 400, headers: { "content-type": "application/json" } }); };
|
||||
await createTeacherAgent(undefined, { engine, chatId: sessionId, tools: {} }).prompt("offline");
|
||||
assert.equal(body.prompt_cache_key, Array.from(sessionId).slice(0, 64).join(""));
|
||||
assert.equal(headers.get("session_id"), sessionId);
|
||||
}));
|
||||
|
||||
it("native OpenAI cache-key clamp counts Unicode code points, not UTF-16 units", async () => closed(async () => {
|
||||
const engine = piEngineForSpec({ provider: "openai", id: "gpt-4o-mini", thinkingLevel: "off" });
|
||||
const sessionId = "chat_" + "😀".repeat(70);
|
||||
let payload: unknown;
|
||||
// HTTP affinity headers cannot encode emoji. Observe native payload
|
||||
// serialization before transport, then stop deliberately (no accounts).
|
||||
await engine.streamFn(engine.model, normalizeContext({ messages: [{ role: "user", content: "offline", timestamp: 1 }] }), {
|
||||
sessionId,
|
||||
onPayload: (value) => { payload = value; throw new Error("CLOSED: payload captured"); },
|
||||
}).result();
|
||||
assert.equal((payload as { prompt_cache_key: string }).prompt_cache_key, Array.from(sessionId).slice(0, 64).join(""));
|
||||
}));
|
||||
|
||||
it("native Codex serialization shares the 64-code-point clamp without an account or OAuth flow", async () => closed(async () => {
|
||||
const engine = piEngineForSpec({ provider: "openai-codex", id: "gpt-6-luna", thinkingLevel: "off" });
|
||||
// Synthetic, unsigned fixture token solely satisfies native local claim
|
||||
// decoding. Stop at payload capture: no account, OAuth, or fetch.
|
||||
const claims = { "https://api.openai.com/auth": { chatgpt_account_id: "offline-fixture" } };
|
||||
const apiKey = `dummy.${Buffer.from(JSON.stringify(claims)).toString("base64url")}.dummy`;
|
||||
for (const sessionId of ["chat_" + "a".repeat(70), "chat_" + "😀".repeat(70)]) {
|
||||
let payload: unknown;
|
||||
// The app intentionally has no Codex auth row: exercise the native
|
||||
// serializer directly rather than adding an OAuth configuration path.
|
||||
assert.equal(engine.model.api, "openai-codex-responses");
|
||||
const result = await streamCodex(engine.model as Model<"openai-codex-responses">, normalizeContext({ messages: [{ role: "user", content: "offline", timestamp: 1 }] }), {
|
||||
apiKey, sessionId, identity: APP_IDENTITY,
|
||||
onPayload: (value) => { payload = value; throw new Error("CLOSED: payload captured"); },
|
||||
}).result();
|
||||
assert.ok(payload, result.errorMessage);
|
||||
assert.equal((payload as { prompt_cache_key: string }).prompt_cache_key, Array.from(sessionId).slice(0, 64).join(""));
|
||||
}
|
||||
}));
|
||||
|
||||
it("SDK native header precedence and cache-off behavior remain untouched", async () => closed(async () => {
|
||||
const engine = piEngineForSpec({ provider: "opencode-go", id: "gpt-6-luna", thinkingLevel: "off" });
|
||||
let headers = new Headers();
|
||||
let body: Record<string, unknown> = {};
|
||||
globalThis.fetch = async (input, init) => { const request = new Request(input, init); headers = request.headers; body = JSON.parse(await request.text()); return new Response("{}", { status: 400, headers: { "content-type": "application/json" } }); };
|
||||
await engine.streamFn(engine.model, normalizeContext({ messages: [{ role: "user", content: "offline", timestamp: 1 }] }), { sessionId: "chat_cache_off", cacheRetention: "none", headers: { "user-agent": "explicit-sdk-override", "x-opencode-client": null } }).result();
|
||||
assert.equal(headers.get("user-agent"), "explicit-sdk-override");
|
||||
assert.equal(headers.get("x-opencode-client"), null);
|
||||
// Native OpenCode routing identity remains even with cache retention off.
|
||||
assert.equal(headers.get("x-opencode-session"), "chat_cache_off");
|
||||
assert.equal(body.prompt_cache_key, undefined);
|
||||
}));
|
||||
|
||||
it("completed malformed native JSON stays rejected despite app prepareArguments healing", async () => closed(async () => {
|
||||
const engine = piEngineForSpec({ provider: "openrouter", id: "openai/gpt-4o-mini", thinkingLevel: "off" });
|
||||
const malformed = '{"value":';
|
||||
globalThis.fetch = async () => new Response(`data: ${JSON.stringify({ id: "offline", choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "call_bad", type: "function", function: { name: "check", arguments: malformed } }] }, finish_reason: null }] })}\n\ndata: ${JSON.stringify({ choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }] })}\n\ndata: [DONE]\n\n`, { headers: { "content-type": "text/event-stream" } });
|
||||
let executed = 0;
|
||||
let prepared = 0;
|
||||
const tools = healToolRecord({ check: { name: "check", label: "check", description: "check", parameters: Type.Object({ value: Type.Optional(Type.String()) }), execute: async () => { executed++; return { content: [], details: {} }; } } });
|
||||
const heal = tools.check!.prepareArguments!;
|
||||
tools.check!.prepareArguments = (args) => { prepared++; return heal(args); };
|
||||
const result = await runSubagent(Object.values(tools), { role: "extractor", engine, systemPrompt: "offline", prompt: "offline", maxTurns: 1 });
|
||||
const assistant = result.newMessages.find((m) => m.role === "assistant");
|
||||
assert.ok(assistant?.role === "assistant");
|
||||
const call = assistant.content.find((c) => c.type === "toolCall");
|
||||
assert.ok(call?.type === "toolCall" && call.argumentsError);
|
||||
assert.equal(executed, 0);
|
||||
assert.equal(prepared, 1, "SDK prepares arguments but preserves argumentsError for native validation");
|
||||
assert.ok(result.newMessages.some((m) => m.role === "toolResult" && m.isError));
|
||||
}));
|
||||
});
|
||||
+3
-3
@@ -51,11 +51,11 @@ test("callCost: glm-5.3-flash math — fresh input, cached read, output", () =>
|
||||
test("callCost: cache writes bill at the cacheWrite rate", () => {
|
||||
const cost = callCost(
|
||||
call({ inputTokens: 1_000_000, outputTokens: 0, cacheWriteTokens: 500_000 }),
|
||||
"openrouter:qwen/qwen3.8-max-0902",
|
||||
"anthropic:claude-haiku-4-5",
|
||||
);
|
||||
assert.ok(cost !== null);
|
||||
// fresh 1M @ $2/M + cache write 500k @ $2.5/M
|
||||
assert.ok(Math.abs(cost! - (2 + 1.25)) < 1e-9);
|
||||
// Native 1.4 catalog: fresh 1M @ $1/M + cache write 500k @ $1.25/M.
|
||||
assert.ok(Math.abs(cost! - (1 + 0.625)) < 1e-9);
|
||||
});
|
||||
|
||||
test("callCost: unknown model and absent fields are null, never zero", () => {
|
||||
|
||||
Reference in New Issue
Block a user