sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Context-meter amendment: CONTEXT_LIMITS dies — pi's live catalog is the only limit source

Browse Source 
Owner ruling (2026-09-23, amendment to 04e336d): no hardcoded fallback
exists anywhere. contextWindow(record, limits) resolves the denominator
ONLY from the live index (provider:id -> contextWindow straight from
pi's catalog); a model missing from it (historical session, package
update, failed fetch) reads limit null and the meter renders
usage-only — the honest-render precedent, never a fake window.

- src/core/prices.ts: the 1354-entry CONTEXT_LIMITS table (65KB) and its
  generation apparatus deleted; the fold returns limit: number | null.
  PRICES untouched (regen verified byte-identical via the reworked
  sync-pricing script).
- scripts/sync-pricing.ts: generates PRICES only; contextWindow leaves
  the priced map, the guard, and the template.
- The dock meter's owner-selected treatment: the activity feed carries
  the limit server-side. catalogContextLimits() (server/model-catalog.ts)
  builds the index over the same keyed providers the picker federates;
  activityFeed() joins it onto every running job (ActivityJob.contextLimit,
  null when the model is missing — the dock reads it off the feed, no
  client-side table, no regression while the UI's catalog-join chunk lands).
- components/activity-dock.tsx: toBlobs reads job.contextLimit (import
  of CONTEXT_LIMITS gone); stats-bar renders usage-only on null limit
  (mechanical interim — UI stream's join chunk follows).
- tests: usage.test.ts pins the no-fallback contract (index hit / absent
  / empty / missing-model -> null); activity-feed.test.ts pins the
  server join (catalog model -> 1048576, unknown -> null, no-usage ->
  null); render pins updated to the interim usage-only render.
This commit is contained in:
amosblomqvist
2026-09-24 02:10:06 +03:00
parent b68a16fdb4
commit 4ded547cb5
11 changed files with 200 additions and 1450 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/activity-feed.test.ts
+39
View File
@@ -59,3 +59,42 @@ test("activityFeed: running jobs across ALL chats, running-only, title-joined, o
}
}
});
test("activityFeed: contextLimit is the server's live-catalog join (owner amendment, 2026-09-23)", () => {
// keyed-provider federation — the same env-key pattern as the
// model-catalog tests; file-local (each test file is its own process)
process.env.OPENROUTER_API_KEY ??= "test-key-openrouter-catalog";
const chat = chatStore.createChat("limit join chat");
const createdJobs: string[] = [];
const usageFor = (model: string) => ({
model,
inputTokens: 1_000,
outputTokens: 100,
cacheReadTokens: 0,
cacheWriteTokens: 0,
reasoningTokens: 0,
steps: 1,
});
try {
const known = jobStore.createJob({ kind: "research", chatId: chat.id, toolCallId: "tc_known" });
const unknown = jobStore.createJob({ kind: "research", chatId: chat.id, toolCallId: "tc_unknown" });
const bare = jobStore.createJob({ kind: "research", chatId: chat.id, toolCallId: "tc_bare" });
createdJobs.push(known.id, unknown.id, bare.id);
jobStore.updateJob(known.id, { usage: usageFor("openrouter:z-ai/glm-5.3-flash") });
jobStore.updateJob(unknown.id, { usage: usageFor("nope/unknown") });
const feed = activityFeed();
const knownJob = feed.jobs.find((j) => j.id === known.id)!;
assert.equal(knownJob.contextLimit, 1_048_576, "pi's catalog, joined server-side onto the feed");
const unknownJob = feed.jobs.find((j) => j.id === unknown.id)!;
assert.equal(unknownJob.contextLimit, null, "missing from the live catalog → null, never a guessed window");
const bareJob = feed.jobs.find((j) => j.id === bare.id)!;
assert.equal(bareJob.contextLimit, null, "no usage yet → no context line");
} finally {
fs.unlinkSync(`${PATHS.chats}/${chat.id}.json`);
const jobsDir = path.join(PATHS.data, "jobs");
for (const id of createdJobs) {
fs.unlinkSync(path.join(jobsDir, `${id}.json`));
}
}
});
tests/render-activity-dock.test.tsx
+10 -5
View File
@@ -26,6 +26,7 @@ function job(over: Partial<ActivityFeed["jobs"][number]> = {}): ActivityFeed["jo
createdAt: new Date(NOW - 45_000).toISOString(),
updatedAt: new Date(NOW - 1000).toISOString(),
title: "Limits lesson",
contextLimit: null,
...over,
};
}
@@ -91,19 +92,23 @@ describe("activity dock pure layer", () => {
};
const withLast = job({
usage: { ...base, lastStep: step(90_000, 2_000) },
// The amendment (owner 2026-09-23): the limit is the server's
// live-catalog join riding the feed — the dock never resolves it.
contextLimit: 131_072,
});
const blob = toBlobs({ jobs: [withLast], extractions: [] })[0]!;
assert.equal(blob.context?.used, 92_000);
assert.equal(blob.context?.limit, 1048576);
assert.equal(blob.context?.limit, 131_072);
// No lastStep yet (capture pending) → "—", even though usage exists.
const stripped = toBlobs({ jobs: [job({ usage: { ...base } })], extractions: [] })[0]!;
assert.equal(stripped.context, null);
// Unknown model → no limit → "—".
const unknownModel = toBlobs({
jobs: [job({ usage: { ...base, model: "nope/unknown", lastStep: step(5, 5) } })],
// Model missing from the live catalog (historical session,
// package update) → the feed carries null → no context line.
const uncatalogued = toBlobs({
jobs: [job({ usage: { ...base, model: "nope/unknown", lastStep: step(5, 5), } })],
extractions: [],
})[0]!;
assert.equal(unknownModel.context, null);
assert.equal(uncatalogued.context, null);
});
it("extraction blobs carry no money/context (—, the doctrine)", () => {
tests/render-stats-bar.test.tsx
+9 -5
View File
@@ -50,9 +50,11 @@ describe("stats bar", () => {
const html = renderToStaticMarkup(
<StatsBar turns={[turn(), turn()]} />,
);
// last turn's last call: 10k in + 1k out = 11k; pi catalog limit
// 1_048_576 → "1M" (tables regenerated from pi-ai's catalog)
assert.match(html, /data-testid="stats-context">11k \/ 1M</);
// last turn's last call: 10k in + 1k out = 11k. The LIMIT is the
// live catalog's — and the bar doesn't hold the index yet (the
// catalog join lands with the UI chunk), so the meter renders
// USAGE-ONLY (owner amendment: never a fallback window).
assert.match(html, /data-testid="stats-context">11k</);
// pi catalog price 0.075/0.25 per 1M: each turn 10k×0.075 + 1k×0.25
// = $0.001 → two = $0.0020
assert.match(html, /data-testid="stats-cost">\$0\.0020</);
@@ -67,8 +69,10 @@ describe("stats bar", () => {
<StatsBar turns={[turn({ model: "nope/unknown" })]} />,
);
assert.match(html, /data-testid="stats-cost">—</);
// unknown model also kills the context limit
assert.match(html, /data-testid="stats-context">—</);
// unknown model: no catalog limit exists, but the footprint is
// still an honest reading — usage-only (never "—" theater over
// real data, never a fake window)
assert.match(html, /data-testid="stats-context">11k</);
});
it("prettyModel strips vendor + date suffix, capitalizes", () => {
tests/usage.test.ts
+70 -50
View File
@@ -78,18 +78,22 @@ test("turnCost sums steps; sessionCost reports unpriced turns", () => {
assert.ok(totals.cost > 0);
});
test("contextWindow: last call's footprint against the model's limit", () => {
const cw = contextWindow({
turns: [
{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 9_000, outputTokens: 1_000 })] },
{ id: "t2", messageId: "m2", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 90_000, outputTokens: 10_000 })] },
],
});
test("contextWindow: last call's footprint against the live index's limit", () => {
const limits = new Map([["openrouter:z-ai/glm-5.3-flash", 1_048_576]]);
const cw = contextWindow(
{
turns: [
{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 9_000, outputTokens: 1_000 })] },
{ id: "t2", messageId: "m2", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 90_000, outputTokens: 10_000 })] },
],
},
limits,
);
assert.ok(cw);
assert.equal(cw.model, "openrouter:z-ai/glm-5.3-flash");
assert.equal(cw.usedTokens, 100_000);
assert.equal(cw.limit, 1_048_576);
assert.equal(contextWindow({ turns: [] }), null);
assert.equal(contextWindow({ turns: [] }, limits), null);
});
test("contextWindow NUMERATOR: cache tokens count (owner audit, 2026-09-23)", () => {
@@ -97,70 +101,86 @@ test("contextWindow NUMERATOR: cache tokens count (owner audit, 2026-09-23)", ()
// while the real footprint was ~190k — provider `input` EXCLUDES the
// cached portion, so input+output alone is wildly low. The fold is
// pi's totalTokens semantics: all four fields.
const cw = contextWindow({
turns: [
{
id: "t1",
messageId: "m1",
at: "x",
model: "openrouter:z-ai/glm-5.3-flash",
steps: [call({ inputTokens: 40_000, outputTokens: 1_000, cacheReadTokens: 145_000, cacheWriteTokens: 4_000 })],
},
],
});
const limits = new Map([["openrouter:z-ai/glm-5.3-flash", 1_048_576]]);
const cw = contextWindow(
{
turns: [
{
id: "t1",
messageId: "m1",
at: "x",
model: "openrouter:z-ai/glm-5.3-flash",
steps: [call({ inputTokens: 40_000, outputTokens: 1_000, cacheReadTokens: 145_000, cacheWriteTokens: 4_000 })],
},
],
},
limits,
);
assert.ok(cw);
assert.equal(cw.usedTokens, 40_000 + 1_000 + 145_000 + 4_000, "input + output + cacheRead + cacheWrite");
// Cache fields absent (non-caching providers): the fold is unchanged.
const plain = contextWindow({
turns: [{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 9_000, outputTokens: 1_000 })] }],
});
const plain = contextWindow(
{ turns: [{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 9_000, outputTokens: 1_000 })] }] },
limits,
);
assert.ok(plain);
assert.equal(plain.usedTokens, 10_000);
});
test("contextWindow DENOMINATOR: the live-limits index wins; CONTEXT_LIMITS is the fallback", () => {
test("contextWindow DENOMINATOR: the live index is the ONLY source — no fallback (owner amendment, 2026-09-23)", () => {
const turns = [
{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 10_000, outputTokens: 1_000 })] },
];
// The live catalog (GET /api/models) wins even when it disagrees with
// the generated table.
const live = new Map([["openrouter:z-ai/glm-5.3-flash", 65_536]]);
const cw = contextWindow({ turns }, live);
assert.ok(cw);
assert.equal(cw.limit, 65_536, "the live index beats the static table");
// No live entry: the generated table serves the uncatalogued model.
assert.equal(contextWindow({ turns })!.limit, 1_048_576);
const emptyLive = new Map<string, number>();
assert.equal(contextWindow({ turns }, emptyLive)!.limit, 1_048_576, "an empty live index falls back too");
// Unknown to both: null (no meter, never a guessed limit).
assert.equal(contextWindow({ turns: [{ ...turns[0]!, model: "unknown/model" }] }), null);
assert.equal(contextWindow({ turns: [{ ...turns[0]!, model: "unknown/model" }] }, live), null);
assert.equal(cw.limit, 65_536, "pi's catalog, via the index, is the one source");
// No index at all (a client that hasn't fetched the catalog): the
// meter renders usage-only — limit null, never a guessed window.
const noIndex = contextWindow({ turns });
assert.ok(noIndex);
assert.equal(noIndex.usedTokens, 11_000);
assert.equal(noIndex.limit, null);
const emptyIndex = contextWindow({ turns }, new Map<string, number>());
assert.ok(emptyIndex);
assert.equal(emptyIndex.limit, null, "an empty index (failed fetch) is usage-only too");
// A model missing from the live catalog (historical session, package
// update removed it): usage-only, never a fake window.
const missing = contextWindow({ turns: [{ ...turns[0]!, model: "nope/gone-model" }] }, live);
assert.ok(missing);
assert.equal(missing.limit, null);
assert.equal(missing.usedTokens, 11_000, "the footprint still reads honestly");
});
test("contextWindow ride-along: a zeroed/aborted last step falls back to the previous step", () => {
// An aborted last call reports zeroed usage — that is not the session
// footprint. The honest reading is the previous step's usage.
const cw = contextWindow({
turns: [
{
id: "t1",
messageId: "m1",
at: "x",
model: "openrouter:z-ai/glm-5.3-flash",
steps: [
call({ inputTokens: 80_000, outputTokens: 5_000 }),
call({ inputTokens: 0, outputTokens: 0 }), // aborted — zeroed
],
},
],
});
const limits = new Map([["openrouter:z-ai/glm-5.3-flash", 1_048_576]]);
const cw = contextWindow(
{
turns: [
{
id: "t1",
messageId: "m1",
at: "x",
model: "openrouter:z-ai/glm-5.3-flash",
steps: [
call({ inputTokens: 80_000, outputTokens: 5_000 }),
call({ inputTokens: 0, outputTokens: 0 }), // aborted — zeroed
],
},
],
},
limits,
);
assert.ok(cw);
assert.equal(cw.usedTokens, 85_000, "the previous step's footprint, not a bogus tiny meter");
// ALL steps zeroed (a first-call abort): no reading at all — null.
assert.equal(
contextWindow({
turns: [{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 0, outputTokens: 0 })] }],
}),
contextWindow(
{ turns: [{ id: "t1", messageId: "m1", at: "x", model: "openrouter:z-ai/glm-5.3-flash", steps: [call({ inputTokens: 0, outputTokens: 0 })] }] },
limits,
),
null,
);
});