sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

feat(models): strong/cheap/vision model slots + multi-provider engine federation

Browse Source 
Model slots wave (2026-09-20), chunk B:

- settings: new `models` section {strong, cheap, vision} slots (each
  field independent, same shape as a role override); ROLE_SLOT maps all
  nine roles to tiers (strong=prober+teacher, cheap=researcher/
  demoBuilder/extractor/auditor/labeler, vision=visualMaker+pdfReader);
  resolution chain per role: override → slot → factory pin.
- config: ModelSpec gains provider (all pins openrouter, the BYOK
  default); MODEL_PINS in stores.ts now spans all nine roles (the strong
  slot's factory rep IS the prober pin).
- engine: piModels() registers EVERY builtin provider catalog;
  piEngineForSpec(resolved spec) is the new engine factory (pi-ai
  resolves per-provider keys from env itself); piEngineFor stays as the
  config-pin fallback. chat-handler resolves the conversation chain
  (chat override → strong slot → pin) and injects engineFor into the job
  runner, closing the old gap where per-role overrides never reached
  job engines.
- /api/models: models federate ONLY keyed providers (providers surface
  with keyPresent, env-wins, fresh per call); slots (resolved tier pins)
  join rolePins; catalogDefaultsFor inherits the strong slot; the models
  settings section gets the same catalog gate as chat.
- pricing: PRICES/CONTEXT_LIMITS uniformly keyed provider:id
  (regenerated from ALL pi-ai catalogs, 1354 keys); usage stamps
  (TurnUsage, JobUsage, extraction stages) switch to provider:id —
  owner decision: no legacy compat, old stamps render unpriced.
This commit is contained in:
amosblomqvist
2026-09-22 00:14:56 +03:00
parent f1373d5de5
commit f9e3a4c43d
26 changed files with 3529 additions and 912 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/usage.test.ts
+13 -13
View File
@@ -40,7 +40,7 @@ test("sumSteps folds steps; absent stays absent until a value appears", () => {
});
test("callCost: glm-5.3-flash math — fresh input, cached read, output", () => {
const cost = callCost(call({ inputTokens: 100_000, outputTokens: 20_000, cacheReadTokens: 10_000 }), "z-ai/glm-5.3-flash");
const cost = callCost(call({ inputTokens: 100_000, outputTokens: 20_000, cacheReadTokens: 10_000 }), "openrouter:z-ai/glm-5.3-flash");
assert.ok(cost !== null);
// fresh 90k @ $0.075/M + cached 10k @ $0.015/M + output 20k @ $0.25/M
// (pi-ai catalog prices — regenerated tables, scripts/sync-pricing)
@@ -50,7 +50,7 @@ test("callCost: glm-5.3-flash math — fresh input, cached read, output", () =>
test("callCost: cache writes bill at the cacheWrite rate", () => {
const cost = callCost(
call({ inputTokens: 1_000_000, outputTokens: 0, cacheWriteTokens: 500_000 }),
"qwen/qwen3.8-max-0902",
"openrouter:qwen/qwen3.8-max-0902",
);
assert.ok(cost !== null);
// fresh 1M @ $2/M + cache write 500k @ $2.5/M
@@ -59,7 +59,7 @@ test("callCost: cache writes bill at the cacheWrite rate", () => {
test("callCost: unknown model and absent fields are null, never zero", () => {
assert.equal(callCost(call({ inputTokens: 100, outputTokens: 5 }), "not/a-model"), null);
assert.equal(callCost(call({ inputTokens: 100, outputTokens: undefined }), "z-ai/glm-5.3-flash"), null);
assert.equal(callCost(call({ inputTokens: 100, outputTokens: undefined }), "openrouter:z-ai/glm-5.3-flash"), null);
});
test("turnCost sums steps; sessionCost reports unpriced turns", () => {
@@ -67,7 +67,7 @@ test("turnCost sums steps; sessionCost reports unpriced turns", () => {
id: "t1",
messageId: "m1",
at: "2026-09-09T00:00:00Z",
model: "z-ai/glm-5.3-flash",
model: "openrouter:z-ai/glm-5.3-flash",
steps: [call({ inputTokens: 10_000, outputTokens: 2_000 }), call({ inputTokens: 12_000, outputTokens: 1_000 })],
};
assert.ok(turnCost(turn) !== null);
@@ -81,12 +81,12 @@ test("turnCost sums steps; sessionCost reports unpriced turns", () => {
test("contextWindow: last call's footprint against the model's limit", () => {
const cw = contextWindow({
turns: [
{ id: "t1", messageId: "m1", at: "x", model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 9_000, outputTokens: 1_000 })] },
{ id: "t2", messageId: "m2", at: "x", model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 90_000, outputTokens: 10_000 })] },
{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 9_000, outputTokens: 1_000 })] },
{ id: "t2", messageId: "m2", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 90_000, outputTokens: 10_000 })] },
],
});
assert.ok(cw);
assert.equal(cw.model, "z-ai/glm-5.3-flash");
assert.equal(cw.model, "openrouter:z-ai/glm-5.3-flash");
assert.equal(cw.usedTokens, 100_000);
assert.equal(cw.limit, 1_048_576);
assert.equal(contextWindow({ turns: [] }), null);
@@ -96,7 +96,7 @@ test("jobCost: null without usage, computed with it", () => {
assert.equal(jobCost({}), null);
const job: Pick<JobRecord, "usage"> = {
usage: {
model: "z-ai/glm-5.3-flash",
model: "openrouter:z-ai/glm-5.3-flash",
inputTokens: 1_000_000,
outputTokens: 100_000,
cacheReadTokens: 0,
@@ -117,7 +117,7 @@ test("appendTurnUsage: appends, idempotent per turn id, continuation turns (same
id: "turn_1",
messageId: "msg_a",
at: "2026-09-09T00:00:00Z",
model: "z-ai/glm-5.3-flash",
model: "openrouter:z-ai/glm-5.3-flash",
steps: [call({ inputTokens: 100, outputTokens: 10 })],
};
chatStore.appendTurnUsage(record.id, entry);
@@ -149,8 +149,8 @@ test("appendExtractionUsage: appends, idempotent per startedAt, not in list payl
durationMs: 42_000,
stats: { nodes: 1, edges: 2, events: 3 },
stages: {
extractor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
auditor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
extractor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
auditor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
},
});
chatStore.appendExtractionUsage(record.id, runEntry(1));
@@ -186,8 +186,8 @@ test("appendSourceUsage: appends to the source record, idempotent per startedAt,
durationMs: 42_000,
stats: { nodes: 1, edges: 2, events: 3 },
stages: {
extractor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
auditor: { model: "z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
extractor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 100, outputTokens: 10 })] },
auditor: { model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 50, outputTokens: 5 })] },
},
});
const id = "src_usagetest01";