Observability contracts: usage accounting, plan blocks, price table (Phase 0)
The 2026-09 observability wave's type-level foundation, landed before the streams so nothing renders against guessed shapes: - contracts/chat.ts: CallUsage + TurnUsage (per-model-call token accounting, one entry per teacher/wake turn keyed by messageId) and ChatRecord.turns — a DELIBERATELY parallel append-only usage log, not per-message metadata: the client re-POSTs its message list every turn and its copies never carry server-stamped fields, so anything inside messages gets clobbered by the next persist (the wave-3 merge-rule lesson). - contracts/chat.ts: PlanBlockData — the persisted data-plan-block part (full DAG on plan_init, one-level neighborhood on plan updates, snapshot semantics so history renders what was true then). - contracts/jobs.ts: JobUsage + JobRecord.usage — a sub-agent run's accumulated accounting, written by the runner. - core/ports.ts + store/chats.ts: ChatStore.appendTurnUsage — idempotent per messageId (abort replays / wake retries append once); listChats no longer leaks turns into the list payload. - core/prices.ts: PRICES + CONTEXT_LIMITS (generated from OpenRouter's published pricing via scripts/sync-pricing.ts) + pure folds: callCost, turnCost, sessionCost, contextWindow, jobCost. Unknown model or absent fields render as absent, never zero. - server/observability.ts: barrel-approved surface for the transport layer (fixes the import-law violation in the graph status route that slipped through the wave-3 merge verification). Capture wiring (agent onStepFinish -> persist seam, plan-block emission) lands next as master seam work; the streams start against these shapes.
This commit is contained in:
amosblomqvist
@@ -8,7 +8,7 @@
|
||||
*/
|
||||
import type { UIMessage } from "ai";
|
||||
import type { Phase3 } from "../phases.ts";
|
||||
import type { PlanTree } from "./plan.ts";
|
||||
import type { PlanNodeStatus, PlanTree } from "./plan.ts";
|
||||
|
||||
// ─── Chat records (the session store's on-disk format) ───────────────────────
|
||||
|
||||
@@ -24,6 +24,74 @@ export interface ChatRecord {
|
||||
messages: UIMessage[];
|
||||
/** The session's plan tree (vision §2) — written by plan tools via saveTree. */
|
||||
tree?: PlanTree;
|
||||
/** Append-only usage log (observability, 2026-09): one entry per model
|
||||
* turn. DELIBERATELY a parallel log, not per-message metadata — the client
|
||||
* re-POSTs its message list every turn and its copies never carry
|
||||
* server-stamped fields, so anything inside message objects gets clobbered
|
||||
* by the next persist. Keyed by messageId so the transcript view can fold
|
||||
* each entry onto its message; aggregates are folds over this log
|
||||
* (core/prices.ts). */
|
||||
turns?: TurnUsage[];
|
||||
}
|
||||
|
||||
// ─── Usage accounting (the money channel) ─────────────────────────────────────
|
||||
|
||||
/** One model call's token accounting — the SDK's LanguageModelUsage
|
||||
* flattened to the fields we persist (undefined = provider didn't report;
|
||||
* renderers show absent, never zero). */
|
||||
export interface CallUsage {
|
||||
/** Total input (prompt) tokens, cached portion included. */
|
||||
inputTokens: number | undefined;
|
||||
outputTokens: number | undefined;
|
||||
/** Cached input tokens READ (billed at the cache-read rate, if any). */
|
||||
cacheReadTokens: number | undefined;
|
||||
/** Input tokens written to the cache this call (OpenRouter bills writes
|
||||
* for some providers; undefined = not reported/not billed). */
|
||||
cacheWriteTokens: number | undefined;
|
||||
/** The reasoning portion of the output tokens. */
|
||||
reasoningTokens: number | undefined;
|
||||
}
|
||||
|
||||
/** One billing turn: one full invocation of the teacher (a client turn or a
|
||||
* wake turn — same handler, same log). A turn with tool calls spans several
|
||||
* agent steps, and each step bills separately. Capture lands at the persist
|
||||
* seam (server); cost is INFERRED at read time from core/prices.ts. */
|
||||
export interface TurnUsage {
|
||||
/** The assistant message the turn produced — keys this entry to the
|
||||
* transcript (the block this turn "belongs" to). */
|
||||
messageId: string;
|
||||
at: string;
|
||||
/** The OpenRouter model id that ran the turn. */
|
||||
model: string;
|
||||
/** One entry per agent step (model call) in this turn. */
|
||||
steps: CallUsage[];
|
||||
}
|
||||
|
||||
// ─── Plan blocks (the plan lives in the chat) ────────────────────────────────
|
||||
|
||||
/** Payload of the persisted `data-plan-block` UI part (2026-09 design): a
|
||||
* SNAPSHOT of the plan at the moment it changed, rendered as a block in the
|
||||
* chat flow. Attached to the assistant message that carries the plan_init
|
||||
* tool call (kind "full") or the answered check quiz part (kind
|
||||
* "neighborhood"). Data parts never reach the model (convertToModelMessages
|
||||
* drops them) — the block is pure UI history, reappearing identically on
|
||||
* reload because it lives in the record.
|
||||
*
|
||||
* Rendering contract: a "full" block renders the whole DAG borderless,
|
||||
* horizontally filling the chat column, natural height; a "neighborhood"
|
||||
* block renders current node + direct parents + direct children in its own
|
||||
* small container, the current node glowing — clearly a fragment, not the
|
||||
* whole plan. Clicking either opens the full plan in a popup. */
|
||||
export interface PlanBlockData {
|
||||
kind: "full" | "neighborhood";
|
||||
at: string;
|
||||
/** The current node at snapshot time (the glow target; null = none). */
|
||||
current: string | null;
|
||||
/** Included nodes: all of them (full) or the one-level-up/one-level-down
|
||||
* neighborhood of `current` (neighborhood). Statuses are the snapshot's,
|
||||
* derived at emission time — the block never re-derives at render. */
|
||||
nodes: Array<{ id: string; title: string; status: PlanNodeStatus }>;
|
||||
edges: Array<{ from: string; to: string }>;
|
||||
}
|
||||
|
||||
/** The list view's shape (messageCount instead of full messages) + the
|
||||
|
||||
@@ -27,6 +27,27 @@ export interface JobRecord {
|
||||
/** The sub-agent's result (steer-artifact payload). */
|
||||
result?: unknown;
|
||||
error?: string;
|
||||
/** Money channel (observability, 2026-09): accumulated token usage of
|
||||
* the sub-agent's run, written by the runner as steps complete. Cost is
|
||||
* inferred at read time (core/prices.ts). Absent on pre-2026-09 jobs and
|
||||
* until the runner's capture lands — renderers show "—", never zero. */
|
||||
usage?: JobUsage;
|
||||
}
|
||||
|
||||
/** A sub-agent run's accumulated usage — the same accounting shape as a
|
||||
* chat turn (contracts/chat.ts CallUsage), flattened to one running total
|
||||
* plus a step count. */
|
||||
export interface JobUsage {
|
||||
/** The OpenRouter model id that ran the sub-agent. */
|
||||
model: string;
|
||||
/** Running totals summed over the run's model calls so far. */
|
||||
inputTokens: number;
|
||||
outputTokens: number;
|
||||
cacheReadTokens: number;
|
||||
cacheWriteTokens: number;
|
||||
reasoningTokens: number;
|
||||
/** How many model calls the run has made (each bills separately). */
|
||||
steps: number;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -25,3 +25,4 @@ export * from "./phases.ts";
|
||||
export * from "./pending.ts";
|
||||
export * from "./math-delims.ts";
|
||||
export * from "./ui-message.ts";
|
||||
export * from "./prices.ts";
|
||||
|
||||
@@ -21,6 +21,7 @@ import type {
|
||||
NodeMapping,
|
||||
PlanTree,
|
||||
SessionCheckpoint,
|
||||
TurnUsage,
|
||||
} from "./contracts.ts";
|
||||
|
||||
// ─── ChatStore — the session record's home (data/chats) ─────────────────────
|
||||
@@ -36,6 +37,10 @@ export interface ChatStore {
|
||||
saveMessages(id: string, messages: UIMessage[]): void;
|
||||
/** Persist the session plan tree (plan tools call this in execute). */
|
||||
saveTree(id: string, tree: PlanTree): void;
|
||||
/** Append one usage entry to the record's turn log (the money channel,
|
||||
* 2026-09): idempotent per messageId — a turn persisted twice (abort
|
||||
* replays, wake-turn retries) appends exactly once. */
|
||||
appendTurnUsage(id: string, entry: TurnUsage): void;
|
||||
}
|
||||
|
||||
// ─── GraphStore — the append-only evidence ledgers (data/graph) ──────────────
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* Price + context-limit registry — the money channel's lookup table (pure core).
|
||||
*
|
||||
* Every model the harness runs gets an entry: per-1M-token USD prices and
|
||||
* the model's context window. Usage is CAPTURED in the records (chat turns,
|
||||
* jobs); cost is INFERRED here at read time — nothing stores dollars, so a
|
||||
* price correction re-prices the whole history for free.
|
||||
*
|
||||
* The table is GENERATED, not hand-maintained: scripts/sync-pricing.ts pulls
|
||||
* OpenRouter's published per-token pricing and rewrites this file. Stale
|
||||
* prices are possible between syncs (they change rarely); a MISSING model
|
||||
* makes cost render as absent ("—"), never zero — the honest presentation.
|
||||
*
|
||||
* Client-safe: pure types + numbers, no node imports; exposed via the core
|
||||
* barrel (@/src/core) so shell components fold usage without cost math
|
||||
* living in components.
|
||||
*/
|
||||
|
||||
import type { CallUsage, ChatRecord, JobRecord, TurnUsage } from "./contracts.ts";
|
||||
|
||||
/** Per-1M-token USD prices for one model. Fields absent where the provider
|
||||
* doesn't bill them (cacheWrite is rare). */
|
||||
export interface ModelPrice {
|
||||
input: number;
|
||||
output: number;
|
||||
cacheRead?: number;
|
||||
cacheWrite?: number;
|
||||
}
|
||||
|
||||
/** Token prices, USD per 1M tokens. A missing model makes cost render
|
||||
* as absent ("—"), never zero. Refresh: npx tsx scripts/sync-pricing.ts */
|
||||
export const PRICES: Record<string, ModelPrice> = {
|
||||
"z-ai/glm-5.3-flash": { input: 0.15, output: 0.5, cacheRead: 0.03 },
|
||||
"qwen/qwen3.8-max-0902": { input: 2, output: 6, cacheRead: 0.25, cacheWrite: 2.5 },
|
||||
};
|
||||
|
||||
/** Context-window limits (tokens) per model — generated alongside PRICES. */
|
||||
export const CONTEXT_LIMITS: Record<string, number> = {
|
||||
"z-ai/glm-5.3-flash": 1310720,
|
||||
"qwen/qwen3.8-max-0902": 1000000,
|
||||
};
|
||||
|
||||
// ─── Pure helpers (folds + cost inference) ───────────────────────────────────
|
||||
|
||||
/** Sum a list of per-step usages into one CallUsage. A field stays
|
||||
* undefined when EVERY step left it unreported — absent is not zero. */
|
||||
export function sumSteps(steps: readonly CallUsage[]): CallUsage {
|
||||
const sumIfAny = (vals: (number | undefined)[]): number | undefined =>
|
||||
vals.every((v) => v === undefined) ? undefined : vals.reduce<number>((a, v) => a + (v ?? 0), 0);
|
||||
return {
|
||||
inputTokens: sumIfAny(steps.map((s) => s.inputTokens)),
|
||||
outputTokens: sumIfAny(steps.map((s) => s.outputTokens)),
|
||||
cacheReadTokens: sumIfAny(steps.map((s) => s.cacheReadTokens)),
|
||||
cacheWriteTokens: sumIfAny(steps.map((s) => s.cacheWriteTokens)),
|
||||
reasoningTokens: sumIfAny(steps.map((s) => s.reasoningTokens)),
|
||||
};
|
||||
}
|
||||
|
||||
/** One model call's inferred cost in USD — null when the model is unknown
|
||||
* or the usage fields are absent (never guess a price). Cached input bills
|
||||
* at the cache-read rate; cache writes bill separately where priced. */
|
||||
export function callCost(
|
||||
usage: { inputTokens?: number; outputTokens?: number; cacheReadTokens?: number; cacheWriteTokens?: number },
|
||||
model: string,
|
||||
): number | null {
|
||||
const price = PRICES[model];
|
||||
if (!price) return null;
|
||||
if (usage.inputTokens === undefined || usage.outputTokens === undefined) return null;
|
||||
const cacheRead = usage.cacheReadTokens ?? 0;
|
||||
const freshInput = Math.max(0, usage.inputTokens - cacheRead);
|
||||
const cacheWriteCost = (usage.cacheWriteTokens ?? 0) / 1e6 * (price.cacheWrite ?? 0);
|
||||
return (
|
||||
(freshInput / 1e6) * price.input +
|
||||
(cacheRead / 1e6) * (price.cacheRead ?? price.input) +
|
||||
(usage.outputTokens / 1e6) * price.output +
|
||||
cacheWriteCost
|
||||
);
|
||||
}
|
||||
|
||||
/** One turn's inferred cost — the sum over its steps; null when unpriceable. */
|
||||
export function turnCost(turn: TurnUsage): number | null {
|
||||
let total = 0;
|
||||
for (const step of turn.steps) {
|
||||
const c = callCost(step, turn.model);
|
||||
if (c === null) return null;
|
||||
total += c;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
/** Fold a chat record's turn log: total inferred session cost. Unknown-model
|
||||
* or partial-usage turns don't poison the total — they count as unpriced
|
||||
* and are reported alongside, so the UI can show "n turns unpriced". */
|
||||
export function sessionCost(record: Pick<ChatRecord, "turns">): { cost: number; unpriced: number } {
|
||||
let cost = 0;
|
||||
let unpriced = 0;
|
||||
for (const turn of record.turns ?? []) {
|
||||
const c = turnCost(turn);
|
||||
if (c === null) unpriced++;
|
||||
else cost += c;
|
||||
}
|
||||
return { cost, unpriced };
|
||||
}
|
||||
|
||||
/** The session's context-window footprint: the LAST model call's input+output
|
||||
* tokens against the model's limit (the next turn re-sends the whole
|
||||
* conversation — the last call's input IS the running transcript size).
|
||||
* Null when the record has no usage yet or the model's limit is unknown. */
|
||||
export function contextWindow(
|
||||
record: Pick<ChatRecord, "turns">,
|
||||
): { model: string; usedTokens: number; limit: number } | null {
|
||||
const turns = record.turns ?? [];
|
||||
const last = turns[turns.length - 1];
|
||||
if (!last) return null;
|
||||
const limit = CONTEXT_LIMITS[last.model];
|
||||
if (limit === undefined) return null;
|
||||
const lastStep = last.steps[last.steps.length - 1];
|
||||
if (!lastStep || lastStep.inputTokens === undefined) return null;
|
||||
return {
|
||||
model: last.model,
|
||||
usedTokens: (lastStep.inputTokens ?? 0) + (lastStep.outputTokens ?? 0),
|
||||
limit,
|
||||
};
|
||||
}
|
||||
|
||||
/** A job's inferred cost — null until the runner's capture lands / for
|
||||
* unknown models (render "—", never zero). */
|
||||
export function jobCost(job: Pick<JobRecord, "usage">): number | null {
|
||||
if (!job.usage) return null;
|
||||
return callCost(job.usage, job.usage.model);
|
||||
}
|
||||
Reference in New Issue
Block a user