Observability contracts: usage accounting, plan blocks, price table (Phase 0)
The 2026-09 observability wave's type-level foundation, landed before the streams so nothing renders against guessed shapes: - contracts/chat.ts: CallUsage + TurnUsage (per-model-call token accounting, one entry per teacher/wake turn keyed by messageId) and ChatRecord.turns — a DELIBERATELY parallel append-only usage log, not per-message metadata: the client re-POSTs its message list every turn and its copies never carry server-stamped fields, so anything inside messages gets clobbered by the next persist (the wave-3 merge-rule lesson). - contracts/chat.ts: PlanBlockData — the persisted data-plan-block part (full DAG on plan_init, one-level neighborhood on plan updates, snapshot semantics so history renders what was true then). - contracts/jobs.ts: JobUsage + JobRecord.usage — a sub-agent run's accumulated accounting, written by the runner. - core/ports.ts + store/chats.ts: ChatStore.appendTurnUsage — idempotent per messageId (abort replays / wake retries append once); listChats no longer leaks turns into the list payload. - core/prices.ts: PRICES + CONTEXT_LIMITS (generated from OpenRouter's published pricing via scripts/sync-pricing.ts) + pure folds: callCost, turnCost, sessionCost, contextWindow, jobCost. Unknown model or absent fields render as absent, never zero. - server/observability.ts: barrel-approved surface for the transport layer (fixes the import-law violation in the graph status route that slipped through the wave-3 merge verification). Capture wiring (agent onStepFinish -> persist seam, plan-block emission) lands next as master seam work; the streams start against these shapes.
This commit is contained in:
@@ -5,8 +5,7 @@
|
||||
* (extracting / auditing / merging) per chat, oldest first. Titles joined
|
||||
* from the chat store; a missing chat record still reports the run.
|
||||
*/
|
||||
import { extractionStatuses } from "@/src/agent/subagents/extraction.ts";
|
||||
import { chatStore } from "@/src/store/chats.ts";
|
||||
import { chatStore, extractionStatuses } from "@/src/server";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
|
||||
Vendored
+2
-2
@@ -1,7 +1,7 @@
|
||||
/// <reference types="next" />
|
||||
/// <reference types="next/image-types/global" />
|
||||
import "./.next/dev/types/routes.d.ts";
|
||||
import "./.next/dev/types/root-params.d.ts";
|
||||
import "./.next/types/routes.d.ts";
|
||||
import "./.next/types/root-params.d.ts";
|
||||
|
||||
// NOTE: This file should not be edited
|
||||
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* Regenerate src/core/prices.ts tables from OpenRouter's published pricing.
|
||||
*
|
||||
* Usage: npx tsx scripts/sync-pricing.ts
|
||||
*
|
||||
* Pulls GET https://openrouter.ai/api/v1/models, filters to the models in
|
||||
* config.ts MODELS (+ known spelling variants), and rewrites the PRICES +
|
||||
* CONTEXT_LIMITS tables in src/core/prices.ts — the tables are generated;
|
||||
* hand edits are overwritten. Per-token prices convert to per-1M-token USD;
|
||||
* fields OpenRouter doesn't publish (null) are omitted. A model missing from
|
||||
* OpenRouter's list gets no entry — cost renders as absent ("—"), never zero.
|
||||
*/
|
||||
|
||||
import { MODELS } from "../config.ts";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
|
||||
interface OpenRouterModel {
|
||||
id: string;
|
||||
context_length?: number;
|
||||
pricing?: Record<string, string | null>;
|
||||
}
|
||||
|
||||
/** Spelling variants worth keeping priced alongside a config id (the config's
|
||||
* id must always resolve even if OpenRouter renames the slug). */
|
||||
const VARIANTS: Record<string, string[]> = {
|
||||
"qwen/qwen3.8-max": ["qwen/qwen3.8-max-0902"],
|
||||
"qwen/qwen3.8-max-0902": ["qwen/qwen3.8-max"],
|
||||
};
|
||||
|
||||
const API = "https://openrouter.ai/api/v1/models";
|
||||
const OUT = path.join(import.meta.dirname, "..", "src", "core", "prices.ts");
|
||||
|
||||
const perM = (v: string | null | undefined): number | undefined => {
|
||||
if (v === null || v === undefined || v === "") return undefined;
|
||||
const n = Number(v) * 1e6;
|
||||
return Number.isFinite(n) ? Number(n.toFixed(4)) : undefined;
|
||||
};
|
||||
|
||||
function priceLine(id: string, model: OpenRouterModel): string {
|
||||
const p = model.pricing ?? {};
|
||||
const fields = [
|
||||
`input: ${perM(p.prompt)}`,
|
||||
`output: ${perM(p.completion)}`,
|
||||
`cacheRead: ${perM(p.input_cache_read)}`,
|
||||
...(perM(p.input_cache_write) !== undefined ? [`cacheWrite: ${perM(p.input_cache_write)}`] : []),
|
||||
];
|
||||
return `\t${JSON.stringify(id)}: { ${fields.join(", ")} },`;
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const res = await fetch(API);
|
||||
if (!res.ok) throw new Error(`OpenRouter models request failed: ${res.status} ${res.statusText}`);
|
||||
const { data } = (await res.json()) as { data: OpenRouterModel[] };
|
||||
const byId = new Map(data.map((m) => [m.id, m]));
|
||||
|
||||
const wanted = Object.values(MODELS).map((m) => m.model);
|
||||
const all = [...new Set([...wanted, ...wanted.flatMap((id) => VARIANTS[id] ?? [])])];
|
||||
|
||||
const priced = new Map<string, { price: string; limit: string }>();
|
||||
for (const id of all) {
|
||||
const model = byId.get(id);
|
||||
if (!model) {
|
||||
console.warn(`!! ${id}: not on OpenRouter — entry omitted (cost renders as "—")`);
|
||||
continue;
|
||||
}
|
||||
const limit = model.context_length;
|
||||
priced.set(id, {
|
||||
price: priceLine(id, model),
|
||||
limit: `\t${JSON.stringify(id)}: ${limit ?? "undefined"},`,
|
||||
});
|
||||
}
|
||||
|
||||
const tables = `/** Token prices, USD per 1M tokens. A missing model makes cost render
|
||||
* as absent ("—"), never zero. Refresh: npx tsx scripts/sync-pricing.ts */
|
||||
export const PRICES: Record<string, ModelPrice> = {
|
||||
${[...priced.values()].map((p) => p.price).join("\n")}
|
||||
};
|
||||
|
||||
/** Context-window limits (tokens) per model — generated alongside PRICES. */
|
||||
export const CONTEXT_LIMITS: Record<string, number> = {
|
||||
${[...priced.values()].map((p) => p.limit).join("\n")}
|
||||
};
|
||||
`;
|
||||
|
||||
const src = fs.readFileSync(OUT, "utf-8");
|
||||
const start = src.indexOf("/** Token prices");
|
||||
const end = src.indexOf("// ─── Pure helpers");
|
||||
if (start === -1 || end === -1) throw new Error("prices.ts markers not found — file structure changed");
|
||||
fs.writeFileSync(OUT, src.slice(0, start) + tables + "\n" + src.slice(end), "utf-8");
|
||||
|
||||
console.log(`prices.ts regenerated: ${all.length} models checked, ${priced.size} priced.`);
|
||||
for (const id of all) if (!priced.has(id)) console.warn(` unpriced: ${id}`);
|
||||
}
|
||||
|
||||
void main();
|
||||
@@ -8,7 +8,7 @@
|
||||
*/
|
||||
import type { UIMessage } from "ai";
|
||||
import type { Phase3 } from "../phases.ts";
|
||||
import type { PlanTree } from "./plan.ts";
|
||||
import type { PlanNodeStatus, PlanTree } from "./plan.ts";
|
||||
|
||||
// ─── Chat records (the session store's on-disk format) ───────────────────────
|
||||
|
||||
@@ -24,6 +24,74 @@ export interface ChatRecord {
|
||||
messages: UIMessage[];
|
||||
/** The session's plan tree (vision §2) — written by plan tools via saveTree. */
|
||||
tree?: PlanTree;
|
||||
/** Append-only usage log (observability, 2026-09): one entry per model
|
||||
* turn. DELIBERATELY a parallel log, not per-message metadata — the client
|
||||
* re-POSTs its message list every turn and its copies never carry
|
||||
* server-stamped fields, so anything inside message objects gets clobbered
|
||||
* by the next persist. Keyed by messageId so the transcript view can fold
|
||||
* each entry onto its message; aggregates are folds over this log
|
||||
* (core/prices.ts). */
|
||||
turns?: TurnUsage[];
|
||||
}
|
||||
|
||||
// ─── Usage accounting (the money channel) ─────────────────────────────────────
|
||||
|
||||
/** One model call's token accounting — the SDK's LanguageModelUsage
|
||||
* flattened to the fields we persist (undefined = provider didn't report;
|
||||
* renderers show absent, never zero). */
|
||||
export interface CallUsage {
|
||||
/** Total input (prompt) tokens, cached portion included. */
|
||||
inputTokens: number | undefined;
|
||||
outputTokens: number | undefined;
|
||||
/** Cached input tokens READ (billed at the cache-read rate, if any). */
|
||||
cacheReadTokens: number | undefined;
|
||||
/** Input tokens written to the cache this call (OpenRouter bills writes
|
||||
* for some providers; undefined = not reported/not billed). */
|
||||
cacheWriteTokens: number | undefined;
|
||||
/** The reasoning portion of the output tokens. */
|
||||
reasoningTokens: number | undefined;
|
||||
}
|
||||
|
||||
/** One billing turn: one full invocation of the teacher (a client turn or a
|
||||
* wake turn — same handler, same log). A turn with tool calls spans several
|
||||
* agent steps, and each step bills separately. Capture lands at the persist
|
||||
* seam (server); cost is INFERRED at read time from core/prices.ts. */
|
||||
export interface TurnUsage {
|
||||
/** The assistant message the turn produced — keys this entry to the
|
||||
* transcript (the block this turn "belongs" to). */
|
||||
messageId: string;
|
||||
at: string;
|
||||
/** The OpenRouter model id that ran the turn. */
|
||||
model: string;
|
||||
/** One entry per agent step (model call) in this turn. */
|
||||
steps: CallUsage[];
|
||||
}
|
||||
|
||||
// ─── Plan blocks (the plan lives in the chat) ────────────────────────────────
|
||||
|
||||
/** Payload of the persisted `data-plan-block` UI part (2026-09 design): a
|
||||
* SNAPSHOT of the plan at the moment it changed, rendered as a block in the
|
||||
* chat flow. Attached to the assistant message that carries the plan_init
|
||||
* tool call (kind "full") or the answered check quiz part (kind
|
||||
* "neighborhood"). Data parts never reach the model (convertToModelMessages
|
||||
* drops them) — the block is pure UI history, reappearing identically on
|
||||
* reload because it lives in the record.
|
||||
*
|
||||
* Rendering contract: a "full" block renders the whole DAG borderless,
|
||||
* horizontally filling the chat column, natural height; a "neighborhood"
|
||||
* block renders current node + direct parents + direct children in its own
|
||||
* small container, the current node glowing — clearly a fragment, not the
|
||||
* whole plan. Clicking either opens the full plan in a popup. */
|
||||
export interface PlanBlockData {
|
||||
kind: "full" | "neighborhood";
|
||||
at: string;
|
||||
/** The current node at snapshot time (the glow target; null = none). */
|
||||
current: string | null;
|
||||
/** Included nodes: all of them (full) or the one-level-up/one-level-down
|
||||
* neighborhood of `current` (neighborhood). Statuses are the snapshot's,
|
||||
* derived at emission time — the block never re-derives at render. */
|
||||
nodes: Array<{ id: string; title: string; status: PlanNodeStatus }>;
|
||||
edges: Array<{ from: string; to: string }>;
|
||||
}
|
||||
|
||||
/** The list view's shape (messageCount instead of full messages) + the
|
||||
|
||||
@@ -27,6 +27,27 @@ export interface JobRecord {
|
||||
/** The sub-agent's result (steer-artifact payload). */
|
||||
result?: unknown;
|
||||
error?: string;
|
||||
/** Money channel (observability, 2026-09): accumulated token usage of
|
||||
* the sub-agent's run, written by the runner as steps complete. Cost is
|
||||
* inferred at read time (core/prices.ts). Absent on pre-2026-09 jobs and
|
||||
* until the runner's capture lands — renderers show "—", never zero. */
|
||||
usage?: JobUsage;
|
||||
}
|
||||
|
||||
/** A sub-agent run's accumulated usage — the same accounting shape as a
|
||||
* chat turn (contracts/chat.ts CallUsage), flattened to one running total
|
||||
* plus a step count. */
|
||||
export interface JobUsage {
|
||||
/** The OpenRouter model id that ran the sub-agent. */
|
||||
model: string;
|
||||
/** Running totals summed over the run's model calls so far. */
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
cacheReadTokens: number;
|
||||
cacheWriteTokens: number;
|
||||
reasoningTokens: number;
|
||||
/** How many model calls the run has made (each bills separately). */
|
||||
steps: number;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -25,3 +25,4 @@ export * from "./phases.ts";
|
||||
export * from "./pending.ts";
|
||||
export * from "./math-delims.ts";
|
||||
export * from "./ui-message.ts";
|
||||
export * from "./prices.ts";
|
||||
|
||||
@@ -21,6 +21,7 @@ import type {
|
||||
NodeMapping,
|
||||
PlanTree,
|
||||
SessionCheckpoint,
|
||||
TurnUsage,
|
||||
} from "./contracts.ts";
|
||||
|
||||
// ─── ChatStore — the session record's home (data/chats) ─────────────────────
|
||||
@@ -36,6 +37,10 @@ export interface ChatStore {
|
||||
saveMessages(id: string, messages: UIMessage[]): void;
|
||||
/** Persist the session plan tree (plan tools call this in execute). */
|
||||
saveTree(id: string, tree: PlanTree): void;
|
||||
/** Append one usage entry to the record's turn log (the money channel,
|
||||
* 2026-09): idempotent per messageId — a turn persisted twice (abort
|
||||
* replays, wake-turn retries) appends exactly once. */
|
||||
appendTurnUsage(id: string, entry: TurnUsage): void;
|
||||
}
|
||||
|
||||
// ─── GraphStore — the append-only evidence ledgers (data/graph) ──────────────
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* Price + context-limit registry — the money channel's lookup table (pure core).
|
||||
*
|
||||
* Every model the harness runs gets an entry: per-1M-token USD prices and
|
||||
* the model's context window. Usage is CAPTURED in the records (chat turns,
|
||||
* jobs); cost is INFERRED here at read time — nothing stores dollars, so a
|
||||
* price correction re-prices the whole history for free.
|
||||
*
|
||||
* The table is GENERATED, not hand-maintained: scripts/sync-pricing.ts pulls
|
||||
* OpenRouter's published per-token pricing and rewrites this file. Stale
|
||||
* prices are possible between syncs (they change rarely); a MISSING model
|
||||
* makes cost render as absent ("—"), never zero — the honest presentation.
|
||||
*
|
||||
* Client-safe: pure types + numbers, no node imports; exposed via the core
|
||||
* barrel (@/src/core) so shell components fold usage without cost math
|
||||
* living in components.
|
||||
*/
|
||||
|
||||
import type { CallUsage, ChatRecord, JobRecord, TurnUsage } from "./contracts.ts";
|
||||
|
||||
/** Per-1M-token USD prices for one model. Fields absent where the provider
|
||||
* doesn't bill them (cacheWrite is rare). */
|
||||
export interface ModelPrice {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead?: number;
|
||||
cacheWrite?: number;
|
||||
}
|
||||
|
||||
/** Token prices, USD per 1M tokens. A missing model makes cost render
|
||||
* as absent ("—"), never zero. Refresh: npx tsx scripts/sync-pricing.ts */
|
||||
export const PRICES: Record<string, ModelPrice> = {
|
||||
"z-ai/glm-5.3-flash": { input: 0.15, output: 0.5, cacheRead: 0.03 },
|
||||
"qwen/qwen3.8-max-0902": { input: 2, output: 6, cacheRead: 0.25, cacheWrite: 2.5 },
|
||||
};
|
||||
|
||||
/** Context-window limits (tokens) per model — generated alongside PRICES. */
|
||||
export const CONTEXT_LIMITS: Record<string, number> = {
|
||||
"z-ai/glm-5.3-flash": 1310720,
|
||||
"qwen/qwen3.8-max-0902": 1000000,
|
||||
};
|
||||
|
||||
// ─── Pure helpers (folds + cost inference) ───────────────────────────────────
|
||||
|
||||
/** Sum a list of per-step usages into one CallUsage. A field stays
|
||||
* undefined when EVERY step left it unreported — absent is not zero. */
|
||||
export function sumSteps(steps: readonly CallUsage[]): CallUsage {
|
||||
const sumIfAny = (vals: (number | undefined)[]): number | undefined =>
|
||||
vals.every((v) => v === undefined) ? undefined : vals.reduce<number>((a, v) => a + (v ?? 0), 0);
|
||||
return {
|
||||
inputTokens: sumIfAny(steps.map((s) => s.inputTokens)),
|
||||
outputTokens: sumIfAny(steps.map((s) => s.outputTokens)),
|
||||
cacheReadTokens: sumIfAny(steps.map((s) => s.cacheReadTokens)),
|
||||
cacheWriteTokens: sumIfAny(steps.map((s) => s.cacheWriteTokens)),
|
||||
reasoningTokens: sumIfAny(steps.map((s) => s.reasoningTokens)),
|
||||
};
|
||||
}
|
||||
|
||||
/** One model call's inferred cost in USD — null when the model is unknown
|
||||
* or the usage fields are absent (never guess a price). Cached input bills
|
||||
* at the cache-read rate; cache writes bill separately where priced. */
|
||||
export function callCost(
|
||||
usage: { inputTokens?: number; outputTokens?: number; cacheReadTokens?: number; cacheWriteTokens?: number },
|
||||
model: string,
|
||||
): number | null {
|
||||
const price = PRICES[model];
|
||||
if (!price) return null;
|
||||
if (usage.inputTokens === undefined || usage.outputTokens === undefined) return null;
|
||||
const cacheRead = usage.cacheReadTokens ?? 0;
|
||||
const freshInput = Math.max(0, usage.inputTokens - cacheRead);
|
||||
const cacheWriteCost = (usage.cacheWriteTokens ?? 0) / 1e6 * (price.cacheWrite ?? 0);
|
||||
return (
|
||||
(freshInput / 1e6) * price.input +
|
||||
(cacheRead / 1e6) * (price.cacheRead ?? price.input) +
|
||||
(usage.outputTokens / 1e6) * price.output +
|
||||
cacheWriteCost
|
||||
);
|
||||
}
|
||||
|
||||
/** One turn's inferred cost — the sum over its steps; null when unpriceable. */
|
||||
export function turnCost(turn: TurnUsage): number | null {
|
||||
let total = 0;
|
||||
for (const step of turn.steps) {
|
||||
const c = callCost(step, turn.model);
|
||||
if (c === null) return null;
|
||||
total += c;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
/** Fold a chat record's turn log: total inferred session cost. Unknown-model
|
||||
* or partial-usage turns don't poison the total — they count as unpriced
|
||||
* and are reported alongside, so the UI can show "n turns unpriced". */
|
||||
export function sessionCost(record: Pick<ChatRecord, "turns">): { cost: number; unpriced: number } {
|
||||
let cost = 0;
|
||||
let unpriced = 0;
|
||||
for (const turn of record.turns ?? []) {
|
||||
const c = turnCost(turn);
|
||||
if (c === null) unpriced++;
|
||||
else cost += c;
|
||||
}
|
||||
return { cost, unpriced };
|
||||
}
|
||||
|
||||
/** The session's context-window footprint: the LAST model call's input+output
|
||||
* tokens against the model's limit (the next turn re-sends the whole
|
||||
* conversation — the last call's input IS the running transcript size).
|
||||
* Null when the record has no usage yet or the model's limit is unknown. */
|
||||
export function contextWindow(
|
||||
record: Pick<ChatRecord, "turns">,
|
||||
): { model: string; usedTokens: number; limit: number } | null {
|
||||
const turns = record.turns ?? [];
|
||||
const last = turns[turns.length - 1];
|
||||
if (!last) return null;
|
||||
const limit = CONTEXT_LIMITS[last.model];
|
||||
if (limit === undefined) return null;
|
||||
const lastStep = last.steps[last.steps.length - 1];
|
||||
if (!lastStep || lastStep.inputTokens === undefined) return null;
|
||||
return {
|
||||
model: last.model,
|
||||
usedTokens: (lastStep.inputTokens ?? 0) + (lastStep.outputTokens ?? 0),
|
||||
limit,
|
||||
};
|
||||
}
|
||||
|
||||
/** A job's inferred cost — null until the runner's capture lands / for
|
||||
* unknown models (render "—", never zero). */
|
||||
export function jobCost(job: Pick<JobRecord, "usage">): number | null {
|
||||
if (!job.usage) return null;
|
||||
return callCost(job.usage, job.usage.model);
|
||||
}
|
||||
@@ -10,3 +10,4 @@ export * from "./graph.ts";
|
||||
export * from "./jobs.ts";
|
||||
export * from "./stores.ts";
|
||||
export * from "./turn-liveness.ts";
|
||||
export * from "./observability.ts";
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
/**
|
||||
* Server-side observability surface — the barrel-approved exports the
|
||||
* transport layer (app/api/*) reads for background-process status.
|
||||
*
|
||||
* The dock (2026-09 observability design) needs: extraction-pipeline runs
|
||||
* (from the pipeline's module-level registry) and the store handles for
|
||||
* title joins. app/ routes import ONLY from @/src/core and @/src/server —
|
||||
* this file is where the deep imports surface, so routes stay clean.
|
||||
*/
|
||||
|
||||
export { extractionStatuses, type ExtractionPhase } from "../agent/subagents/extraction.ts";
|
||||
export { chatStore } from "../store/chats.ts";
|
||||
+17
-2
@@ -11,7 +11,7 @@ import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { PATHS } from "../../config.ts";
|
||||
import type { ChatStore } from "../core/ports.ts";
|
||||
import type { ChatRecord, ChatSummary, PlanTree } from "../core/contracts.ts";
|
||||
import type { ChatRecord, ChatSummary, PlanTree, TurnUsage } from "../core/contracts.ts";
|
||||
import type { UIMessage } from "ai";
|
||||
import { findPendingToolCall } from "../core/pending.ts";
|
||||
import { phaseOfTree } from "../core/phases.ts";
|
||||
@@ -51,7 +51,9 @@ function listChats(): ChatSummary[] {
|
||||
.filter((f) => f.endsWith(".json"))
|
||||
.map((f) => getChat(f.slice(0, -5)))
|
||||
.filter((r): r is ChatRecord => r !== null)
|
||||
.map(({ messages, tree, ...summary }) => ({
|
||||
.map(({ messages, tree, turns, ...summary }) => ({
|
||||
// turns stay out of the list payload (list = summaries; usage
|
||||
// aggregates ride the full record, the picker never needs them).
|
||||
...summary,
|
||||
messageCount: messages.length,
|
||||
// Record-derived status for the session picker: phase from the
|
||||
@@ -76,6 +78,18 @@ function saveMessages(id: string, messages: UIMessage[]): void {
|
||||
fs.writeFileSync(chatFile(id), JSON.stringify(record, null, "\t"), "utf-8");
|
||||
}
|
||||
|
||||
/** Append one usage entry to the record's turn log — idempotent per
|
||||
* messageId: abort replays and wake-turn retries persist the same turn
|
||||
* twice; the second append is a no-op. */
|
||||
function appendTurnUsage(id: string, entry: TurnUsage): void {
|
||||
const record = getChat(id);
|
||||
if (!record) throw new Error(`Chat not found: ${id}`);
|
||||
record.turns = record.turns?.some((t) => t.messageId === entry.messageId)
|
||||
? record.turns
|
||||
: [...(record.turns ?? []), entry];
|
||||
fs.writeFileSync(chatFile(id), JSON.stringify(record, null, "\t"), "utf-8");
|
||||
}
|
||||
|
||||
/** Persist the session plan tree (plan tools call this in execute). */
|
||||
function saveTree(id: string, tree: PlanTree): void {
|
||||
const record = getChat(id);
|
||||
@@ -92,4 +106,5 @@ export const chatStore: ChatStore = {
|
||||
listChats,
|
||||
saveMessages,
|
||||
saveTree,
|
||||
appendTurnUsage,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
/**
|
||||
* Usage-accounting tests — the money channel's pure helpers (core/prices.ts)
|
||||
* and the ChatStore usage-log seam (appendTurnUsage).
|
||||
*
|
||||
* Pure helper tests are pure. The store test uses the real fs chatStore with
|
||||
* a fixture chat, cleaned up in after().
|
||||
*/
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import * as fs from "node:fs";
|
||||
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { PATHS } from "../config.ts";
|
||||
import { callCost, contextWindow, jobCost, sessionCost, sumSteps, turnCost } from "../src/core/prices.ts";
|
||||
import type { CallUsage, JobRecord, TurnUsage } from "../src/core/contracts.ts";
|
||||
|
||||
const call = (over: Partial<CallUsage> = {}): CallUsage => ({
|
||||
inputTokens: 0,
|
||||
outputTokens: 0,
|
||||
cacheReadTokens: undefined,
|
||||
cacheWriteTokens: undefined,
|
||||
reasoningTokens: undefined,
|
||||
...over,
|
||||
});
|
||||
|
||||
test("sumSteps folds steps; absent stays absent until a value appears", () => {
|
||||
const sum = sumSteps([
|
||||
call({ inputTokens: 100, outputTokens: 20 }),
|
||||
call({ inputTokens: 50, outputTokens: 10, cacheReadTokens: 30 }),
|
||||
]);
|
||||
assert.equal(sum.inputTokens, 150);
|
||||
assert.equal(sum.outputTokens, 30);
|
||||
assert.equal(sum.cacheReadTokens, 30);
|
||||
assert.equal(sum.cacheWriteTokens, undefined); // never reported → stays undefined
|
||||
// all-absent → all-undefined (helper defaults input/output to 0, so
|
||||
// explicit undefined here)
|
||||
const empty = sumSteps([call({ inputTokens: undefined, outputTokens: undefined }), call({ inputTokens: undefined, outputTokens: undefined })]);
|
||||
assert.equal(empty.inputTokens, undefined);
|
||||
});
|
||||
|
||||
test("callCost: glm-5.3-flash math — fresh input, cached read, output", () => {
|
||||
const cost = callCost(call({ inputTokens: 100_000, outputTokens: 20_000, cacheReadTokens: 10_000 }), "z-ai/glm-5.3-flash");
|
||||
assert.ok(cost !== null);
|
||||
// fresh 90k @ $0.15/M + cached 10k @ $0.03/M + output 20k @ $0.50/M
|
||||
assert.ok(Math.abs(cost! - (0.0135 + 0.0003 + 0.01)) < 1e-9);
|
||||
});
|
||||
|
||||
test("callCost: cache writes bill at the cacheWrite rate", () => {
|
||||
const cost = callCost(
|
||||
call({ inputTokens: 1_000_000, outputTokens: 0, cacheWriteTokens: 500_000 }),
|
||||
"qwen/qwen3.8-max-0902",
|
||||
);
|
||||
assert.ok(cost !== null);
|
||||
// fresh 1M @ $2/M + cache write 500k @ $2.5/M
|
||||
assert.ok(Math.abs(cost! - (2 + 1.25)) < 1e-9);
|
||||
});
|
||||
|
||||
test("callCost: unknown model and absent fields are null, never zero", () => {
|
||||
assert.equal(callCost(call({ inputTokens: 100, outputTokens: 5 }), "not/a-model"), null);
|
||||
assert.equal(callCost(call({ inputTokens: 100, outputTokens: undefined }), "z-ai/glm-5.3-flash"), null);
|
||||
});
|
||||
|
||||
test("turnCost sums steps; sessionCost reports unpriced turns", () => {
|
||||
const turn: TurnUsage = {
|
||||
messageId: "m1",
|
||||
at: "2026-09-09T00:00:00Z",
|
||||
model: "z-ai/glm-5.3-flash",
|
||||
steps: [call({ inputTokens: 10_000, outputTokens: 2_000 }), call({ inputTokens: 12_000, outputTokens: 1_000 })],
|
||||
};
|
||||
assert.ok(turnCost(turn) !== null);
|
||||
const totals = sessionCost({
|
||||
turns: [turn, { messageId: "m2", at: "x", model: "not/a-model", steps: [call({ inputTokens: 1, outputTokens: 1 })] }],
|
||||
});
|
||||
assert.equal(totals.unpriced, 1);
|
||||
assert.ok(totals.cost > 0);
|
||||
});
|
||||
|
||||
test("contextWindow: last call's footprint against the model's limit", () => {
|
||||
const cw = contextWindow({
|
||||
turns: [
|
||||
{ messageId: "m1", at: "x", model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 9_000, outputTokens: 1_000 })] },
|
||||
{ messageId: "m2", at: "x", model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 90_000, outputTokens: 10_000 })] },
|
||||
],
|
||||
});
|
||||
assert.ok(cw);
|
||||
assert.equal(cw.model, "z-ai/glm-5.3-flash");
|
||||
assert.equal(cw.usedTokens, 100_000);
|
||||
assert.equal(cw.limit, 1_310_720);
|
||||
assert.equal(contextWindow({ turns: [] }), null);
|
||||
});
|
||||
|
||||
test("jobCost: null without usage, computed with it", () => {
|
||||
assert.equal(jobCost({}), null);
|
||||
const job: Pick<JobRecord, "usage"> = {
|
||||
usage: {
|
||||
model: "z-ai/glm-5.3-flash",
|
||||
inputTokens: 1_000_000,
|
||||
outputTokens: 100_000,
|
||||
cacheReadTokens: 0,
|
||||
cacheWriteTokens: 0,
|
||||
reasoningTokens: 0,
|
||||
steps: 4,
|
||||
},
|
||||
};
|
||||
assert.ok(jobCost(job) !== null);
|
||||
assert.ok(jobCost(job)! > 0);
|
||||
});
|
||||
|
||||
// ─── the store seam ──────────────────────────────────────────────────────────
|
||||
|
||||
test("appendTurnUsage: appends, is idempotent per messageId, survives saveMessages", () => {
|
||||
const record = chatStore.createChat("usage test chat");
|
||||
const entry: TurnUsage = {
|
||||
messageId: "msg_a",
|
||||
at: "2026-09-09T00:00:00Z",
|
||||
model: "z-ai/glm-5.3-flash",
|
||||
steps: [call({ inputTokens: 100, outputTokens: 10 })],
|
||||
};
|
||||
chatStore.appendTurnUsage(record.id, entry);
|
||||
chatStore.appendTurnUsage(record.id, entry); // replay (abort/retry) → no-op
|
||||
chatStore.appendTurnUsage(record.id, { ...entry, messageId: "msg_b" });
|
||||
let reread = chatStore.getChat(record.id);
|
||||
assert.equal(reread?.turns?.length, 2);
|
||||
|
||||
// saveMessages must not clobber the log (read-mutate-write record)
|
||||
chatStore.saveMessages(record.id, [{ id: "msg_a", role: "user", parts: [] }]);
|
||||
reread = chatStore.getChat(record.id);
|
||||
assert.equal(reread?.turns?.length, 2);
|
||||
|
||||
// and the list payload must NOT leak turns
|
||||
const listed = chatStore.listChats().find((c) => c.id === record.id);
|
||||
assert.ok(listed);
|
||||
assert.equal("turns" in listed, false);
|
||||
|
||||
fs.unlinkSync(`${PATHS.chats}/${record.id}.json`);
|
||||
});
|
||||
Reference in New Issue
Block a user