From eaa963f98b905f4b0c3175a3f4980b3de8af39c5 Mon Sep 17 00:00:00 2001 From: amosblomqvist Date: Tue, 22 Sep 2026 17:05:44 +0300 Subject: [PATCH] =?UTF-8?q?test(model-catalog):=20machine-aware=20record?= =?UTF-8?q?=20assertions=20=E2=80=94=20the=20suite=20runs=20green=20on=20a?= =?UTF-8?q?ny=20tree?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Master reported the only red on main: 'serves the keyed providers' catalogs' pinned defaults/slots NULL, but catalogDefaults/catalogSlots read the REAL data/settings.json — the owner's tree carries configured slots, so the payload is legitimately non-null there while the platform worktree's scratch record is empty. Same class of fragility in two more tests the scratch run couldn't see: the bare-level validation verdicts (validateChatSettings / validateModelsSettings) fall back to the record's resolved strong tier. data/settings.json belongs to settings-store.test.ts under the one-fs-owner discipline, so the tests READ the resolved truth instead of pinning it — the chat-flow 503-gate pattern, applied to all four: - modelCatalog payload: defaults/slots must carry EXACTLY what the machine's record says (configured slots surface as configured, empty record surfaces as null) — never a phantom id in either direction. - catalogSlots: projects each stored tier's model verbatim; a tier without a model reads null. - bare-level verdicts: expectBareLevelVerdict mirrors the fallback chain — no resolved tier → rides unchecked; a tier whose model left the catalog → rejected honestly; else the supported-level check. Verified green against four record states: empty, configured (glm with unsupported minimal), vanished model, and a level-only stored slot (inert). Gates: typecheck, lint, 825/825, build. --- tests/model-catalog.test.ts | 73 ++++++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 9 deletions(-) diff --git a/tests/model-catalog.test.ts b/tests/model-catalog.test.ts index d900fce..c0262ac 100644 --- a/tests/model-catalog.test.ts +++ b/tests/model-catalog.test.ts @@ -1,11 +1,14 @@ import { describe, it } from "node:test"; import assert from "node:assert/strict"; import { modelCatalog, validateChatSettings, validateModelsSettings, catalogDefaultsFor, catalogProviders, catalogSlots, toModelDto, rolePinsFor, catalogRolePins } from "../src/server/model-catalog.ts"; +import type { SettingsValidation } from "../src/server/model-catalog.ts"; import { clampThinkingLevel, getSupportedThinkingLevels } from "@earendil-works/pi-ai"; import { DEFAULT_THINKING_ORDER } from "../config.ts"; import { piEngineForSpec, piModels } from "../src/agent/models.ts"; import { BACKGROUND_MODEL_ROLES } from "../src/core/settings.ts"; import { resolveSettings } from "../src/core/settings-resolution.ts"; +import { readSettings } from "../src/store/settings.ts"; +import { effectiveSettings } from "../src/server/stores.ts"; // The model-control wave (2026-09-12): pi-ai owns the registry — these pins // hold our projection + validation honest against the shipped catalog, and @@ -26,6 +29,28 @@ function glmModel() { return glm!; } +/** The record's resolved tier as seen by the validation fallbacks — + * machine-aware truth (data/settings.json belongs to settings-store.test.ts + * under the one-fs-owner discipline, so READ it, never pin it): on a + * scratch worktree the record is empty, on the owner's machine slots are + * configured; the expected verdicts below follow whichever holds. */ +/** The machine-aware expected verdict for a bare-level validation: mirrors + * the fallback chain exactly — no resolved tier → rides unchecked; a tier + * whose model left the catalog → rejected; else the supported-level check. */ +function expectBareLevelVerdict(verdict: SettingsValidation, tier: "strong" | "cheap" | "vision") { + const slot = effectiveSettings().models[tier]; + if (!slot) { + assert.deepEqual(verdict, { ok: true }, "no model to check against → accepted (clamps at run)"); + return; + } + const model = piModels().getModel(slot.provider, slot.id); + if (!model) { + assert.equal(verdict.ok, false, "a vanished model is rejected honestly"); + return; + } + assert.equal(verdict.ok, getSupportedThinkingLevels(model).includes("minimal")); +} + describe("the model catalog projection", () => { it("serves the keyed providers' catalogs with the picker's DTO shape", () => { const { models, defaults, providers, slots } = modelCatalog(); @@ -55,11 +80,29 @@ describe("the model catalog projection", () => { for (const id of keyless) { assert.ok(!models.some((m) => m.provider === id), `keyless ${id} must not federate`); } - // BYOK: nothing configured in this environment's record → defaults NULL, - // slots NULL — no phantom ids anywhere in the payload - assert.equal(defaults, null); + // BYOK: the payload carries ONLY what the machine's record actually + // says — never a phantom id. data/settings.json belongs to + // settings-store.test.ts (the one-fs-owner discipline), so READ the + // resolved truth here instead of pinning it: on a scratch worktree + // the record is empty (defaults/slots NULL), on the owner's machine + // configured slots must surface exactly as configured. + const record = readSettings(); + const chat = record?.chat; + const strong = effectiveSettings().models.strong; + if (chat?.model || strong) { + assert.ok(defaults, "a configured chain yields a default"); + assert.deepEqual(defaults!.model, chat?.model ?? { provider: strong!.provider, id: strong!.id }); + } else { + assert.equal(defaults, null); + } for (const name of ["strong", "cheap", "vision"] as const) { - assert.equal(slots[name], null, name); + const stored = record?.models?.[name]; + if (stored?.model) { + assert.ok(slots[name], name); + assert.deepEqual(slots[name]!.model, stored.model); + } else { + assert.equal(slots[name], null, name); + } } }); @@ -129,10 +172,18 @@ describe("the background roles' resolved pins (rolePins)", () => { }); describe("the model slots surface (model slots wave, 2026-09-20)", () => { - it("catalogSlots reports NULL when a tier holds no model", () => { + it("catalogSlots projects the record's tiers — NULL only where a tier holds no model", () => { + // machine-aware (one-fs-owner: READ the record, never pin it) + const record = readSettings(); const slots = catalogSlots(); for (const name of ["strong", "cheap", "vision"] as const) { - assert.equal(slots[name], null, name); + const stored = record?.models?.[name]; + if (stored?.model) { + assert.ok(slots[name], name); + assert.deepEqual(slots[name]!.model, stored.model); + } else { + assert.equal(slots[name], null, name); + } } }); @@ -177,8 +228,10 @@ describe("the models-section validation gate", () => { false, ); // (a level-only slot never passes the WRITE parser — owner, 2026-09-22; - // the validation gate tolerates it here with nothing to check against) - assert.equal(validateModelsSettings({ strong: { thinkingLevel: "minimal" } }).ok, true); + // the validation gate falls back to the tier's RESOLVED model — + // machine-aware: the record's strong tier decides the verdict) + const levelOnly = validateModelsSettings({ strong: { thinkingLevel: "minimal" } }); + expectBareLevelVerdict(levelOnly, "strong"); }); it("accepts empty settings (the clear-slots write)", () => { @@ -199,7 +252,9 @@ describe("the settings validation gate", () => { }); it("a bare thinking level checks against the chat model, else the strong slot, else rides unchecked", () => { - assert.deepEqual(validateChatSettings({ thinkingLevel: "minimal" }), { ok: true }, "no model to check against → accepted (clamps at run)"); + // machine-aware: with a record-carried strong slot the level checks + // against it; with none it rides unchecked (clamps at run). + expectBareLevelVerdict(validateChatSettings({ thinkingLevel: "minimal" }), "strong"); }); it("rejects unknown providers and models", () => {