feat(models): BYOK all the way down — factory pins deleted, per-need refusal gates

Owner decision 2026-09-22: 'it doesn't make sense to have any default
model configured anywhere; the application is bring your own keys.'

config.ts: MODELS deleted; ModelRole becomes an explicit 9-role union;
DEFAULT_THINKING_ORDER (medium, low, high, minimal, xhigh, max) bakes
the unset-level default — first entry the model supports; explicitly
set levels keep pi's native nearest-clamp.

core/settings-resolution: the factory layer dissolves — the chain is
role override → slot → NOTHING. resolveSettings(record) takes no
factory. Empty chains resolve to null/undefined; a thinkingLevel-only
slot is inert (stored, never run). resolveConversationSpec: chat.model
override over the strong slot, null when both empty. resolveRoleSpec
spans all nine roles for the job/extraction engines.

Refusal envelope (core/settings.ts): modelsNotConfigured(missing) —
{error, code: 'models_not_configured', missing: [slot tiers]} +
ModelsNotConfiguredError with .envelope. Per-need gates, each honest:
- chat-handler: universal turn gate (new + continuing turns) returns
  503 before beginTurn — no liveness, no write-ahead persist, no graph
  trigger before the gate fires.
- runner.runJob: unresolvable engine lands the job as error with the
  envelope wording; no phantom loop, no result steer.
- extraction: refuseIfStagesUnconfigured throws before any stage
  (turn-service info-skips so coverage retries next turn; the import
  route surfaces per-file code+missing); labeler-less runs skip
  cluster naming with a console line and still merge.

Write side: slot entries must carry a model — level-only slot writes
are 400s (absent slot key = clear). piEngineForSpec resolves unset
thinking levels via DEFAULT_THINKING_ORDER; piEngineFor now throws
honestly; thinkingLevelFor/modelIdFor deleted; teacher-agent, pi-run,
visual-maker, pdf-reader all read engine.level.

model-catalog: defaults/slots/rolePins null when the chain is empty
(contracts nullable) — no phantom ids in payloads. Stores drop
MODEL_PINS entirely.

Tests: pin-leaners converted to explicit injection (per-chat model
pins on scripted chats — exactly what the setup flow writes; faux
engines for agent-construction tests). New coverage: the 503 gate
envelope, the runner refusal, the extraction refusal + labeler skip.

Gates: typecheck, lint, 815/815, build.
This commit is contained in:
amosblomqvist
2026-09-22 16:59:16 +03:00
parent 8675b19bef
commit bfa906e2f8
26 changed files with 889 additions and 562 deletions
+50 -20
View File
@@ -4,6 +4,7 @@ import type { Usage } from "@earendil-works/pi-ai";
import { z } from "zod";
import { lhTool, toPiTools } from "../lh-tool.ts";
import type { ModelRole } from "../../../config.ts";
import { ModelsNotConfiguredError, ROLE_SLOT, type ModelSlotName } from "../../core/settings.ts";
import type { PiEngine } from "../models.ts";
import { runSubagent } from "./pi-run.ts";
import type {
@@ -562,11 +563,12 @@ export interface ExtractionModels {
auditor?: PiEngine;
/** Test seam for the cluster namer (post-extraction garnish). */
labeler?: PiEngine;
/** Settings-resolved engine per role (model slots wave, 2026-09-20) —
* the same slot treatment the job runner got: the role-override →
* slot → factory-pin chain, read FRESH per stage call (a settings PUT
* between stages lands). Absent → config pins, unchanged. */
engineFor?: (role: ModelRole) => PiEngine;
/** Settings-resolved engine per role — the same slot treatment the
* job runner got, read FRESH per stage call (a settings PUT between
* stages lands). BYOK (owner, 2026-09-22): the chain (role override →
* slot → NOTHING) may end EMPTY → undefined; the front doors refuse
* with ModelsNotConfiguredError before any stage runs. */
engineFor?: (role: ModelRole) => PiEngine | undefined;
}
export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {
@@ -615,13 +617,24 @@ export function callUsage(usage: Usage): CallUsage {
export function createExtractionPipeline(deps: ExtractionPipelineDeps): ExtractionPipeline {
const { chatStore, graphStore, embedder: embed } = deps;
// Stage-engine resolution (model slots wave, 2026-09-20): a direct test
// value wins; otherwise the engineFor chain (role override → slot →
// factory pin, resolved FRESH per stage call server-side) — the job
// runner's exact treatment, closing the flagged gap where extraction
// stayed config-pin-bound.
// Stage-engine resolution: a direct test value wins; otherwise the
// engineFor chain (role override → slot → nothing, resolved FRESH per
// stage call server-side) — the job runner's exact treatment. undefined
// = the BYOK chain came up empty; the front-door guards turn that into
// an honest refusal.
const stageEngine = (role: ModelRole, override?: PiEngine): PiEngine | undefined =>
override ?? deps.engineFor?.(role);
/** The per-need refusal guard: which slot tiers the pipeline's stages
* found unset (test values count as configured — they ARE engines). */
function refuseIfStagesUnconfigured(): void {
const missing = new Set<ModelSlotName>();
if (!stageEngine("extractor", deps.extractor)) missing.add(ROLE_SLOT.extractor);
if (!stageEngine("auditor", deps.auditor)) missing.add(ROLE_SLOT.auditor);
if (missing.size > 0) {
console.info(`[extraction] refused — no model configured for: ${[...missing].join(", ")}`);
throw new ModelsNotConfiguredError([...missing]);
}
}
// THE SHARED REGISTRIES (runs() above — globalThis): every pipeline
// instance (route entries in dev, tests) sees the same runs, the same
// single-flight guard. No module-level state left in this factory.
@@ -743,6 +756,10 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
try {
const record = chatStore.getChat(chatId);
if (!record) throw new Error(`Chat not found: ${chatId}`);
// BYOK per-need gate (owner, 2026-09-22): the write channel needs
// extractor + auditor engines; an empty chain refuses BEFORE any
// stage runs (typed — the trigger skips, routes surface 503).
refuseIfStagesUnconfigured();
const covered = coverageOf(record.messages as UIMessage[], graphStore.readCoverage(), chatId);
if (covered.uncovered.length === 0) {
console.info(`[extraction ${chatId}] nothing uncovered — no-op`);
@@ -811,11 +828,16 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
);
// Post-extraction garnish (DECIDED 2026-09-13): name new/changed
// clusters. Fire-and-forget — naming never fails the run.
refreshClusterLabels({
graphStore,
engine: stageEngine("labeler", deps.labeler),
signal: runningExtractions.get(chatId)?.controller.signal,
}).catch(() => {});
const labelerEngine = stageEngine("labeler", deps.labeler);
if (labelerEngine) {
refreshClusterLabels({
graphStore,
engine: labelerEngine,
signal: runningExtractions.get(chatId)?.controller.signal,
}).catch(() => {});
} else {
console.info(`[extraction ${chatId}] cluster naming skipped — no labeler model configured`);
}
return stats;
} catch (err) {
persistRunUsage(chatId, Math.round(performance.now() - t0), batchesDone > 0 ? total : undefined);
@@ -908,6 +930,9 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
inFlight.add(id);
const t0 = performance.now();
try {
// BYOK per-need gate (owner, 2026-09-22): same refusal as the chat
// front door — no extractor/auditor engines, no run.
refuseIfStagesUnconfigured();
console.info(`[extraction ${id}] notes pipeline start (${input.filePath})`);
setPhase(id, "extracting", "notes", input.filePath);
const findings = await notesExtract(input);
@@ -933,11 +958,16 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
`[extraction ${id}] merged: ${stats.nodes} nodes, ${stats.edges} edges, ${stats.events} events ` +
`(${Math.round(performance.now() - t0)}ms total)`,
);
refreshClusterLabels({
graphStore,
engine: stageEngine("labeler", deps.labeler),
signal: runningExtractions.get(id)?.controller.signal,
}).catch(() => {});
const labelerEngine = stageEngine("labeler", deps.labeler);
if (labelerEngine) {
refreshClusterLabels({
graphStore,
engine: labelerEngine,
signal: runningExtractions.get(id)?.controller.signal,
}).catch(() => {});
} else {
console.info(`[extraction ${id}] cluster naming skipped — no labeler model configured`);
}
return stats;
} catch (err) {
persistRunUsage(id, Math.round(performance.now() - t0), undefined);
+6 -4
View File
@@ -1,7 +1,7 @@
import type { AssistantMessage, Message, ToolResultMessage, Usage } from "@earendil-works/pi-ai";
import { z } from "zod";
import { lhTool, piToolDeclarations, type LhTool } from "../lh-tool.ts";
import { piEngineFor, thinkingLevelFor, type PiEngine } from "../models.ts";
import { piEngineFor, type PiEngine, type ThinkingLevel } from "../models.ts";
import type { PdfResult, PdfToolInput } from "../../core/contracts.ts";
import { getHeadless, type HeadlessApi } from "./headless.ts";
import { pdfExtract, pdfInfo, pdfProbe, pdfSearch, type PdfSearchHit } from "./pdf-ops.ts";
@@ -181,7 +181,9 @@ export async function runPdfReaderAgent(
const tools = createPdfTools(pdfPath);
const toolDeclarations = piToolDeclarations(tools);
const engine = opts.engine ?? piEngineFor("pdfReader");
const reasoning = thinkingLevelFor("pdfReader");
// BYOK (owner, 2026-09-22): the level rides the injected engine — no
// factory pin to consult (engine.level is the resolved/clamped value).
const reasoning = engine.level;
const prompt = [
`PDF path: ${pdfPath}`,
`Question (what the teacher needs from this document): ${input.question || "(no question — give a structural overview)"}`,
@@ -392,11 +394,11 @@ async function streamOnce(
systemPrompt: string,
messages: Message[],
tools: ReturnType<typeof piToolDeclarations>,
reasoning: ReturnType<typeof thinkingLevelFor>,
reasoning: ThinkingLevel | undefined,
abortSignal?: AbortSignal,
): Promise<AssistantMessage> {
const stream = engine.streamFn(engine.model, { systemPrompt, messages, tools }, {
reasoning: reasoning === "off" ? undefined : reasoning,
reasoning: reasoning && reasoning !== "off" ? reasoning : undefined,
apiKey: process.env.OPENROUTER_API_KEY || undefined,
signal: abortSignal,
} as never);
+5 -2
View File
@@ -1,6 +1,6 @@
import { Agent, type AgentEvent, type AgentMessage, type AgentTool } from "@earendil-works/pi-agent-core";
import type { AssistantMessage, ToolResultMessage, Usage } from "@earendil-works/pi-ai";
import { openRouterApiKey, piEngineFor, thinkingLevelFor, type PiEngine } from "../models.ts";
import { openRouterApiKey, piEngineFor, type PiEngine } from "../models.ts";
import type { ModelRole } from "../../../config.ts";
/**
@@ -47,6 +47,9 @@ export interface SubagentRunResult {
}
export function subagentEngine(role: ModelRole, override?: PiEngine): PiEngine {
// BYOK (owner, 2026-09-22): no factory pins — the override is the only
// real path (the runner injects settings-resolved engines); a missed
// injection hits the honest thrower in models.ts instead of a guess.
return override ?? piEngineFor(role);
}
@@ -57,7 +60,7 @@ export async function runSubagent(tools: AgentTool<any, any>[], opts: SubagentRu
initialState: {
systemPrompt: opts.systemPrompt,
model: engine.model,
thinkingLevel: thinkingLevelFor(opts.role),
thinkingLevel: engine.level,
tools,
messages: opts.messages ?? [],
},
+29 -25
View File
@@ -20,7 +20,8 @@ import {
type VisualToolInput,
} from "../../core/contracts.ts";
import { registerJobAbort, unregisterJobAbort } from "./aborts.ts";
import { MODELS, type ModelRole } from "../../../config.ts";
import type { ModelRole } from "../../../config.ts";
import { ROLE_SLOT, modelsNotConfigured } from "../../core/settings.ts";
import type { ChatStore, JobStore } from "../../core/ports.ts";
import { runDemoBuilder } from "./demo-builder.ts";
import { runPdfReaderAgent } from "./pdf-reader.ts";
@@ -76,7 +77,10 @@ export interface JobRunnerDeps {
* old gap where per-role settings overrides never reached job engines.
* Test overrides (below) still win; absent → config pins.
*/
engineFor?: (role: ModelRole) => PiEngine;
/** Settings-resolved engine per role — BYOK (owner, 2026-09-22): the
* chain (role override → slot → NOTHING) may end empty → undefined;
* runJob then refuses the spawn honestly instead of guessing. */
engineFor?: (role: ModelRole) => PiEngine | undefined;
/** The search seam (search port wave, 2026-09-20): resolves the LIVE
* provider fresh per job — server/search.ts composition injected by
* the wiring; null = honestly unavailable (the researcher's web_search
@@ -114,20 +118,6 @@ const KIND_ROLE: Record<Exclude<JobKind, "graph-context">, ModelRole> = {
pdf: "pdfReader",
};
/** The config-pin model stamp for a job kind — JobUsage.model when no
* engine was resolved (engineFor absent AND no test override). Stamps are
* provider-qualified ("provider:id") since the model slots wave, matching
* the price table's uniform keys. Context-injection jobs make NO model
* call ("" — onUsage never fires for them; the kind is deliberately
* absent from the runners below). */
const KIND_MODEL: Record<JobKind, string> = {
research: `${MODELS.researcher.provider}:${MODELS.researcher.model}`,
demo: `${MODELS.demoBuilder.provider}:${MODELS.demoBuilder.model}`,
visual: `${MODELS.visualMaker.provider}:${MODELS.visualMaker.model}`,
pdf: `${MODELS.pdfReader.provider}:${MODELS.pdfReader.model}`,
"graph-context": "",
};
export function createJobRunner(deps: JobRunnerDeps): JobRunner {
const { chatStore, jobStore, deliver } = deps;
@@ -135,10 +125,10 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
* then writes through the job store so the poll sees progress). */
const usageTotals = new Map<string, JobUsage>();
function makeLoopContext(jobId: string, kind: JobKind, engine?: PiEngine): LoopContext {
// Provider-qualified stamp (model slots wave): the engine's actual
// model, else the legacy config-pin id (see KIND_MODEL).
const model = engine ? `${engine.model.provider}:${engine.model.id}` : KIND_MODEL[kind];
function makeLoopContext(jobId: string, engine: PiEngine): LoopContext {
// Provider-qualified stamp: the engine's actual model — the only
// thing that ran (BYOK: no config-pin fallback stamps exist).
const model = `${engine.model.provider}:${engine.model.id}`;
// Real-kill wiring (2026-09 abort semantics): the controller is what
// the abort endpoint fires (agent/subagents/aborts.ts registry); the
// signal rides into every model call of the loop.
@@ -171,8 +161,8 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
// are kind-correct by construction; transcript fallbacks are cast after).
/** The engine one job runs on: test override first (the old seam —
* tests swap scripted engines without touching deps), then the injected
* settings-resolved engineFor (model slots wave, 2026-09-20), else
* undefined → the sub-agent factories fall back to config pins. */
* settings-resolved engineFor. undefined = the BYOK chain came up
* empty (owner, 2026-09-22) — runJob refuses below, honestly. */
function pickEngine(kind: Exclude<JobKind, "graph-context">): PiEngine | undefined {
const testOverride: Record<string, { current?: PiEngine } | undefined> = {
researcher: researchModelOverride,
@@ -185,7 +175,7 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
const KIND_RUNNERS: Record<
Exclude<JobKind, "graph-context">,
(jobId: string, engine: PiEngine | undefined, input: JobToolInput | undefined, loop: LoopContext) => Promise<void>
(jobId: string, engine: PiEngine, input: JobToolInput | undefined, loop: LoopContext) => Promise<void>
> = {
research: (jobId, engine, input, loop) => runResearchJob(jobId, engine, input, loop),
demo: (jobId, engine, input, loop) => runDemoJob(jobId, engine, input, loop),
@@ -212,9 +202,23 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
jobStore.updateJob(jobId, { status: "error", error: `unknown kind ${job.kind}`, activity: "failed" });
return;
}
const engine = pickEngine(job.kind);
if (!engine) {
// BYOK per-need refusal (owner, 2026-09-22): this kind's tier has
// no model — the job record surfaces the honest error (dock +
// teacher read it), and no engine-less loop ever starts.
const tier = ROLE_SLOT[KIND_ROLE[job.kind]];
const envelope = modelsNotConfigured([tier]);
console.info(`[job ${jobId}] refused — ${JSON.stringify(envelope)}`);
jobStore.updateJob(jobId, {
status: "error",
error: `${envelope.error} (missing: ${tier})`,
activity: "failed",
});
return;
}
try {
const engine = pickEngine(job.kind);
await run(jobId, engine, input, makeLoopContext(jobId, job.kind, engine));
await run(jobId, engine, input, makeLoopContext(jobId, engine));
} finally {
unregisterJobAbort(jobId);
// The last usage write may have been mid-flight when the loop died;
+6 -4
View File
@@ -1,7 +1,7 @@
import type { AssistantMessage, Message, ToolResultMessage, Usage } from "@earendil-works/pi-ai";
import { z } from "zod";
import { lhTool, piToolDeclarations, type LhTool } from "../lh-tool.ts";
import { piEngineFor, thinkingLevelFor, type PiEngine } from "../models.ts";
import { piEngineFor, type PiEngine, type ThinkingLevel } from "../models.ts";
import type { VisualResult } from "../../core/contracts.ts";
import { getHeadless, type HeadlessApi } from "./headless.ts";
@@ -132,7 +132,9 @@ export async function runVisualMaker(input: VisualMakerInput, opts: VisualMakerO
const tools = createVisualTools();
const toolDeclarations = piToolDeclarations(tools);
const engine = opts.engine ?? piEngineFor("visualMaker");
const reasoning = thinkingLevelFor("visualMaker");
// BYOK (owner, 2026-09-22): the level rides the injected engine — no
// factory pin to consult (engine.level is the resolved/clamped value).
const reasoning = engine.level;
const prompt = [
"Brief for ONE precise figure (from the teacher, verbatim — it is everything you know):",
brief,
@@ -300,11 +302,11 @@ async function streamOnce(
systemPrompt: string,
messages: Message[],
tools: ReturnType<typeof piToolDeclarations>,
reasoning: ReturnType<typeof thinkingLevelFor>,
reasoning: ThinkingLevel | undefined,
abortSignal?: AbortSignal,
): Promise<AssistantMessage> {
const stream = engine.streamFn(engine.model, { systemPrompt, messages, tools }, {
reasoning: reasoning === "off" ? undefined : reasoning,
reasoning: reasoning && reasoning !== "off" ? reasoning : undefined,
apiKey: process.env.OPENROUTER_API_KEY || undefined,
signal: abortSignal,
} as never);