Key-presence refusal at every gate: a pinned model is not enough
Owner-diagnosed: the turn gate checked only model coordinates, so a pinned slot whose provider key was deleted passed the gate and died mid-stream masked to 'An error occurred.' The key is now gated where the model is — by what would RUN (env-first keyValue truth, so an env-shadowed key passes). - core/settings.ts: the provider_key_missing envelope family (code, provider, slots, human line naming the provider from AUTH_PROVIDER_KEYS labels) + ProviderKeyMissingError + isProviderKeyRefusal matcher. - chat-handler turn gate: refuses 503 before beginTurn when the resolved conversation's provider key is gone (slots: [strong]). - Job runner: same check via an injected dep (agent layer cannot read the key store) — a keyless engine refuses the spawn through the existing failure-shape convention, no phantom run. - Extraction pipeline: refuseIfStagesUnready refuses BEFORE any batch (first offender: extractor before auditor); the trigger skips info- level, /api/graph/import surfaces the 503 envelope per file. - Tests: 8 new pins (turn gate both ways, runner refusal, pipeline first-offender + pass-through, envelope factory, matcher); the three turn-driving suites seed OPENROUTER_API_KEY like any real setup. - Doc fix: stale EXTRACTION.chunkTokens comment in the manual-extract route (left over from eed4b4e). Known residual (pre-existing, flagged for master): the manual extract route logs BOTH config refusals as console.error (fire-and-forget path); the coverage ledger keeps everything uncovered either way.
This commit is contained in:
@@ -3,7 +3,8 @@
|
||||
* (vision-extraction-coverage.md): run the pipeline over ONE session's
|
||||
* uncovered content, whatever its size — the user's answer to an aborted
|
||||
* run, a pipeline error, or a session that went dormant under the chunk
|
||||
* budget. The pipeline itself batches under EXTRACTION.chunkTokens and
|
||||
* budget. The pipeline itself batches under the graph.extractionChunkTokens setting (extraction-UX redesign,
|
||||
* 2026-09-24) and
|
||||
* checkpoints coverage per merged batch, so this is the same run the
|
||||
* trigger fires, just user-initiated. 409 while a run is already in
|
||||
* flight (the dock is the source of truth for that); failures are logged
|
||||
|
||||
@@ -14,7 +14,12 @@
|
||||
* /api/graph/status, as with session runs.
|
||||
*/
|
||||
import { extraction } from "@/src/server";
|
||||
import { ModelsNotConfiguredError, type EmissionStats } from "@/src/core";
|
||||
import {
|
||||
ModelsNotConfiguredError,
|
||||
PROVIDER_KEY_MISSING_CODE,
|
||||
ProviderKeyMissingError,
|
||||
type EmissionStats,
|
||||
} from "@/src/core";
|
||||
|
||||
export const dynamic = "force-dynamic";
|
||||
|
||||
@@ -23,10 +28,14 @@ interface ImportResult {
|
||||
ok: boolean;
|
||||
stats?: EmissionStats;
|
||||
error?: string;
|
||||
/** The machine discriminator when ok:false is the BYOK refusal (owner,
|
||||
* 2026-09-22) — the client can open the setup flow off it. */
|
||||
code?: "models_not_configured";
|
||||
/** The machine discriminator when ok:false is a config refusal — the
|
||||
* BYOK models refusal (owner, 2026-09-22) or the provider-key refusal
|
||||
* (2026-09-25); the client can open the setup/keys flow off it. */
|
||||
code?: "models_not_configured" | (typeof PROVIDER_KEY_MISSING_CODE);
|
||||
missing?: string[];
|
||||
/** The key-refusal specifics (provider_key_missing only). */
|
||||
provider?: string;
|
||||
slots?: string[];
|
||||
}
|
||||
|
||||
export async function POST(request: Request) {
|
||||
@@ -57,6 +66,17 @@ export async function POST(request: Request) {
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (err instanceof ProviderKeyMissingError) {
|
||||
results.push({
|
||||
filePath: file.name,
|
||||
ok: false,
|
||||
error: err.envelope.error,
|
||||
code: PROVIDER_KEY_MISSING_CODE,
|
||||
provider: err.envelope.provider,
|
||||
slots: err.envelope.slots,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
results.push({ filePath: file.name, ok: false, error: err instanceof Error ? err.message : String(err) });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,14 @@ import type { Usage } from "@earendil-works/pi-ai";
|
||||
import { z } from "zod";
|
||||
import { lhTool, toPiTools } from "../lh-tool.ts";
|
||||
import type { ModelRole } from "../../../config.ts";
|
||||
import { FACTORY_SETTINGS, ModelsNotConfiguredError, ROLE_SLOT, type ModelSlotName } from "../../core/settings.ts";
|
||||
import {
|
||||
FACTORY_SETTINGS,
|
||||
ModelsNotConfiguredError,
|
||||
ProviderKeyMissingError,
|
||||
ROLE_SLOT,
|
||||
type ModelSlotName,
|
||||
type ProviderKeyMissing,
|
||||
} from "../../core/settings.ts";
|
||||
import type { PiEngine } from "../models.ts";
|
||||
import { runSubagent } from "./pi-run.ts";
|
||||
import type {
|
||||
@@ -566,6 +573,14 @@ export interface ExtractionModels {
|
||||
* slot → NOTHING) may end EMPTY → undefined; the front doors refuse
|
||||
* with ModelsNotConfiguredError before any stage runs. */
|
||||
engineFor?: (role: ModelRole) => PiEngine | undefined;
|
||||
/** Key-presence gate (provider-key interlude, 2026-09-25): given a
|
||||
* stage engine's provider + the role's slot tier, the injected check
|
||||
* returns the refusal envelope when the provider's key is missing
|
||||
* (null = run). Same injection discipline as the job runner's seam
|
||||
* (the agent layer may not read the key store); absent in tests that
|
||||
* inject engines directly (test engines never owe a key). A keyless
|
||||
* stage refuses with ProviderKeyMissingError BEFORE any batch runs. */
|
||||
providerKeyMissing?: (provider: string, tier: ModelSlotName) => ProviderKeyMissing | null;
|
||||
}
|
||||
|
||||
export interface ExtractionPipelineDeps extends GraphDeps, ExtractionModels {
|
||||
@@ -623,15 +638,30 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
const stageEngine = (role: ModelRole, override?: PiEngine): PiEngine | undefined =>
|
||||
override ?? deps.engineFor?.(role);
|
||||
/** The per-need refusal guard: which slot tiers the pipeline's stages
|
||||
* found unset (test values count as configured — they ARE engines). */
|
||||
function refuseIfStagesUnconfigured(): void {
|
||||
* found unset (test values count as configured — they ARE engines),
|
||||
* and whether a configured stage's provider key is gone (provider-key
|
||||
* interlude, 2026-09-25) — both refuse BEFORE any batch runs. */
|
||||
function refuseIfStagesUnready(): void {
|
||||
const stages: readonly [ModelRole, PiEngine | undefined][] = [
|
||||
["extractor", stageEngine("extractor", deps.extractor)],
|
||||
["auditor", stageEngine("auditor", deps.auditor)],
|
||||
];
|
||||
const missing = new Set<ModelSlotName>();
|
||||
if (!stageEngine("extractor", deps.extractor)) missing.add(ROLE_SLOT.extractor);
|
||||
if (!stageEngine("auditor", deps.auditor)) missing.add(ROLE_SLOT.auditor);
|
||||
for (const [role, engine] of stages) if (!engine) missing.add(ROLE_SLOT[role]);
|
||||
if (missing.size > 0) {
|
||||
console.info(`[extraction] refused — no model configured for: ${[...missing].join(", ")}`);
|
||||
throw new ModelsNotConfiguredError([...missing]);
|
||||
}
|
||||
// Key-presence parity: first offender surfaces (extractor before
|
||||
// auditor); fixing that key re-gates and the next run surfaces the
|
||||
// next offender, if any.
|
||||
for (const [role, engine] of stages) {
|
||||
const refusal = deps.providerKeyMissing?.(engine!.model.provider, ROLE_SLOT[role]);
|
||||
if (refusal) {
|
||||
console.info(`[extraction] refused — ${JSON.stringify(refusal)}`);
|
||||
throw new ProviderKeyMissingError(refusal.provider, refusal.slots);
|
||||
}
|
||||
}
|
||||
}
|
||||
// THE SHARED REGISTRIES (runs() above — globalThis): every pipeline
|
||||
// instance (route entries in dev, tests) sees the same runs, the same
|
||||
@@ -758,10 +788,11 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
try {
|
||||
const record = chatStore.getChat(chatId);
|
||||
if (!record) throw new Error(`Chat not found: ${chatId}`);
|
||||
// BYOK per-need gate (owner, 2026-09-22): the write channel needs
|
||||
// extractor + auditor engines; an empty chain refuses BEFORE any
|
||||
// BYOK per-need gate (owner, 2026-09-22) + key presence
|
||||
// (2026-09-25): the write channel needs extractor + auditor
|
||||
// engines AND their providers' keys; a refusal lands BEFORE any
|
||||
// stage runs (typed — the trigger skips, routes surface 503).
|
||||
refuseIfStagesUnconfigured();
|
||||
refuseIfStagesUnready();
|
||||
const covered = coverageOf(record.messages as UIMessage[], graphStore.readCoverage(), chatId);
|
||||
if (covered.uncovered.length === 0) {
|
||||
console.info(`[extraction ${chatId}] nothing uncovered — no-op`);
|
||||
@@ -921,9 +952,10 @@ export function createExtractionPipeline(deps: ExtractionPipelineDeps): Extracti
|
||||
inFlight.add(id);
|
||||
const t0 = performance.now();
|
||||
try {
|
||||
// BYOK per-need gate (owner, 2026-09-22): same refusal as the chat
|
||||
// front door — no extractor/auditor engines, no run.
|
||||
refuseIfStagesUnconfigured();
|
||||
// BYOK per-need gate (owner, 2026-09-22) + key presence
|
||||
// (2026-09-25): same refusal as the chat front door — no
|
||||
// extractor/auditor engines, or a stage key gone, no run.
|
||||
refuseIfStagesUnready();
|
||||
console.info(`[extraction ${id}] notes pipeline start (${input.filePath})`);
|
||||
setPhase(id, "extracting", "notes", input.filePath);
|
||||
const findings = await notesExtract(input);
|
||||
|
||||
@@ -21,7 +21,7 @@ import {
|
||||
} from "../../core/contracts.ts";
|
||||
import { registerJobAbort, unregisterJobAbort } from "./aborts.ts";
|
||||
import type { ModelRole } from "../../../config.ts";
|
||||
import { ROLE_SLOT, modelsNotConfigured } from "../../core/settings.ts";
|
||||
import { ROLE_SLOT, modelsNotConfigured, type ModelSlotName, type ProviderKeyMissing } from "../../core/settings.ts";
|
||||
import type { ChatStore, JobStore } from "../../core/ports.ts";
|
||||
import { runDemoBuilder } from "./demo-builder.ts";
|
||||
import { runPdfReaderAgent } from "./pdf-reader.ts";
|
||||
@@ -84,6 +84,12 @@ export interface JobRunnerDeps {
|
||||
* drop (jobTools' searchAvailable) uses the same signal — jobs already
|
||||
* in flight when a key disappears still fail honestly, never crash. */
|
||||
searchProvider?: () => SearchProvider | null;
|
||||
/** Key-presence gate (provider-key interlude, 2026-09-25): given the
|
||||
* resolved engine's provider + the kind's slot tier, return the
|
||||
* refusal envelope when the provider's key is missing (null = run).
|
||||
* Injected because the agent layer may not read the key store; absent
|
||||
* in tests that inject engines directly (test engines never owe a key). */
|
||||
providerKeyMissing?: (provider: string, tier: ModelSlotName) => ProviderKeyMissing | null;
|
||||
}
|
||||
|
||||
export interface JobRunner {
|
||||
@@ -189,18 +195,15 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
|
||||
return;
|
||||
}
|
||||
const engine = pickEngine(job.kind);
|
||||
if (!engine) {
|
||||
// BYOK per-need refusal (owner, 2026-09-22): this kind's tier has
|
||||
// no model. Refuse through the SAME convention as every other
|
||||
// failed job — record the honest error (no status flip:
|
||||
// finalizeJob owns the terminal flip after delivery), then steer
|
||||
// the kind's failure shape through finishJob so the wake
|
||||
// pipeline resumes the lesson and the teacher reads the refusal
|
||||
// instead of waiting forever on a job it already dispatched.
|
||||
const tier = ROLE_SLOT[KIND_ROLE[job.kind]];
|
||||
const envelope = modelsNotConfigured([tier]);
|
||||
const reason = `${envelope.error} (missing: ${tier})`;
|
||||
console.info(`[job ${jobId}] refused — ${JSON.stringify(envelope)}`);
|
||||
// The per-need refusal guards (BYOK 2026-09-22 / provider-key
|
||||
// interlude 2026-09-25): no engine for the kind's tier, or the
|
||||
// resolved engine's provider key gone — both refuse through the SAME
|
||||
// convention as every other failed job: record the honest error (no
|
||||
// status flip: finalizeJob owns the terminal flip after delivery),
|
||||
// then steer the kind's failure shape through finishJob so the wake
|
||||
// pipeline resumes the lesson and the teacher reads the refusal
|
||||
// instead of waiting forever on a job it already dispatched.
|
||||
const refuse = async (reason: string): Promise<void> => {
|
||||
if (stillRunning(jobId)) jobStore.updateJob(jobId, { error: reason });
|
||||
const resolved = resolveInput(job, input) as
|
||||
| ResearchToolInput
|
||||
@@ -229,6 +232,23 @@ export function createJobRunner(deps: JobRunnerDeps): JobRunner {
|
||||
pagesConsulted: [],
|
||||
};
|
||||
await finishJob(jobId, failure);
|
||||
};
|
||||
if (!engine) {
|
||||
const tier = ROLE_SLOT[KIND_ROLE[job.kind]];
|
||||
const envelope = modelsNotConfigured([tier]);
|
||||
console.info(`[job ${jobId}] refused — ${JSON.stringify(envelope)}`);
|
||||
await refuse(`${envelope.error} (missing: ${tier})`);
|
||||
return;
|
||||
}
|
||||
// Key presence (2026-09-25): the engine exists, but its provider's
|
||||
// key may be gone — refuse the spawn BEFORE the loop, same
|
||||
// convention (the injected check reads the key store; absent in
|
||||
// engine-injecting tests).
|
||||
const keyRefusal =
|
||||
deps.providerKeyMissing?.(engine.model.provider, ROLE_SLOT[KIND_ROLE[job.kind]]) ?? null;
|
||||
if (keyRefusal) {
|
||||
console.info(`[job ${jobId}] refused — ${JSON.stringify(keyRefusal)}`);
|
||||
await refuse(keyRefusal.error);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
|
||||
@@ -28,6 +28,7 @@
|
||||
*/
|
||||
|
||||
import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
|
||||
import { AUTH_PROVIDER_KEYS } from "./auth-keys.ts";
|
||||
import type { ModelRole } from "../../config.ts";
|
||||
import type { ChatSettings } from "./contracts/chat.ts";
|
||||
import type { SearchEngineId } from "./search.ts";
|
||||
@@ -133,6 +134,56 @@ export class ModelsNotConfiguredError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/** The key-presence refusal (provider-key interlude, owner, 2026-09-25):
|
||||
* a PINNED model is not enough — its provider's key must be present, or
|
||||
* the action dies mid-stream masked to a generic error. Same envelope
|
||||
* family as modelsNotConfigured: HTTP callers get 503 + this JSON;
|
||||
* in-band consumers (the job record's error field) carry the human
|
||||
* `error` line. `provider` names whose key is gone; `slots` names the
|
||||
* slot tier(s) the refused action resolves to that provider. */
|
||||
export const PROVIDER_KEY_MISSING_CODE = "provider_key_missing" as const;
|
||||
export interface ProviderKeyMissing {
|
||||
/** Human-readable — safe to render as-is. */
|
||||
error: string;
|
||||
/** The machine discriminator the client switches on. */
|
||||
code: typeof PROVIDER_KEY_MISSING_CODE;
|
||||
/** The provider whose key is gone, e.g. "openrouter". */
|
||||
provider: string;
|
||||
/** Which slot tiers the refused action resolves to that provider. */
|
||||
slots: ModelSlotName[];
|
||||
}
|
||||
/** The refusal wording's invariant tail — the matcher below keys on it
|
||||
* (the label prefix varies by provider). */
|
||||
const PROVIDER_KEY_MISSING_TAIL = "key is set — add one in Settings → Keys";
|
||||
export function providerKeyMissing(provider: string, slots: ModelSlotName[]): ProviderKeyMissing {
|
||||
const label = AUTH_PROVIDER_KEYS.find((p) => p.id === provider)?.label ?? provider;
|
||||
return {
|
||||
error: `No ${label} ${PROVIDER_KEY_MISSING_TAIL}`,
|
||||
code: PROVIDER_KEY_MISSING_CODE,
|
||||
provider,
|
||||
slots,
|
||||
};
|
||||
}
|
||||
/** The refusal error text (same discipline as isModelsRefusal): UI
|
||||
* consumers detect key-refused work by the wording (info tone, not
|
||||
* scary-error) — the envelope's home module owns the string so the
|
||||
* writer and the matcher can never drift. */
|
||||
export function isProviderKeyRefusal(error: string | undefined): boolean {
|
||||
return !!error && error.endsWith(PROVIDER_KEY_MISSING_TAIL);
|
||||
}
|
||||
/** The typed throw for in-process seams (extraction pipeline) — the
|
||||
* ProviderKeyMissing sibling of ModelsNotConfiguredError: callers catch
|
||||
* it to skip honestly (trigger) or surface the 503 envelope (routes). */
|
||||
export class ProviderKeyMissingError extends Error {
|
||||
readonly envelope: ProviderKeyMissing;
|
||||
constructor(provider: string, slots: ModelSlotName[]) {
|
||||
const envelope = providerKeyMissing(provider, slots);
|
||||
super(envelope.error);
|
||||
this.name = "ProviderKeyMissingError";
|
||||
this.envelope = envelope;
|
||||
}
|
||||
}
|
||||
|
||||
/** One slot's stored state. BYOK (owner, 2026-09-22): the WRITE side
|
||||
* requires model (a level alone can never run — no factory floor); the
|
||||
* sanitizer keeps whatever is on disk, but RESOLUTION treats a slot
|
||||
|
||||
@@ -10,13 +10,14 @@ import { recordToPiMessages } from "./pi/record-adapter.ts";
|
||||
import { createPiTurnResponse } from "./pi/ui-stream.ts";
|
||||
import { effectiveSettings, stores } from "./stores.ts";
|
||||
import { searchAvailable, searchProvider } from "./search.ts";
|
||||
import { providerKeyPresent } from "./keys.ts";
|
||||
import {
|
||||
extractionEnabled,
|
||||
graphToolsEnabled,
|
||||
resolveConversationSpec,
|
||||
resolveRoleSpec,
|
||||
} from "../core/settings-resolution.ts";
|
||||
import { modelsNotConfigured } from "../core/settings.ts";
|
||||
import { modelsNotConfigured, providerKeyMissing as providerKeyMissingEnvelope, type ProviderKeyMissing } from "../core/settings.ts";
|
||||
import { beginTurn, endTurn } from "./turn-liveness.ts";
|
||||
import { persistTurn, triggerExtraction, withMessageIds } from "./turn-service.ts";
|
||||
import type { CallUsage } from "../core/contracts.ts";
|
||||
@@ -50,6 +51,12 @@ const jobRunner = createJobRunner({
|
||||
// undefined and the runner refuses the spawn honestly.
|
||||
engineFor: (role) =>
|
||||
resolveEngine(role),
|
||||
// Key-presence gate (provider-key interlude, 2026-09-25): the runner
|
||||
// reads the resolved engine's provider and asks THIS check (the agent
|
||||
// layer may not read the key store); a missing key refuses the spawn
|
||||
// with the 503-envelope family instead of dying mid-run.
|
||||
providerKeyMissing: (provider, tier) =>
|
||||
providerKeyPresent(provider) ? null : providerKeyMissingEnvelope(provider, [tier]),
|
||||
// Search port wave (2026-09-20): the researcher's web_search runs on the
|
||||
// injected port — resolved FRESH per job (key rotations land immediately).
|
||||
// null = honestly unavailable; the dispatch-level drop (below) uses the
|
||||
@@ -67,6 +74,11 @@ export const extraction = createExtractionPipeline({
|
||||
// with the typed ModelsNotConfiguredError. Direct test values still win.
|
||||
engineFor: (role) =>
|
||||
resolveEngine(role),
|
||||
// Key-presence gate (provider-key interlude, 2026-09-25): same check
|
||||
// the job runner gets — a stage engine whose provider key is gone
|
||||
// refuses BEFORE any batch runs (the typed throw), never mid-run.
|
||||
providerKeyMissing: (provider, tier) =>
|
||||
providerKeyPresent(provider) ? null : providerKeyMissingEnvelope(provider, [tier]),
|
||||
});
|
||||
|
||||
|
||||
@@ -151,6 +163,21 @@ export async function handleChatRequest(
|
||||
return Response.json(envelope, { status: 503 });
|
||||
}
|
||||
|
||||
// THE KEY-PRESENCE GATE (provider-key interlude, owner, 2026-09-25): a
|
||||
// pinned model is not enough — its PROVIDER's key must be present, or
|
||||
// the turn dies mid-stream masked to a generic error. Truth is env-first
|
||||
// (providerKeyPresent reads what pi-ai would read at request time), so
|
||||
// an env-shadowed key passes. Same placement discipline as the models
|
||||
// gate: BEFORE beginTurn (no liveness effects) and the write-ahead
|
||||
// persist (nothing durable changes); wake-pipeline turns ride the same
|
||||
// path. The affected tier is strong — the conversation chain's own tier
|
||||
// (a chat.model override still resolves through it).
|
||||
if (!providerKeyPresent(conversation.provider)) {
|
||||
const envelope = providerKeyMissingEnvelope(conversation.provider, ["strong"]);
|
||||
console.info(`[chat ${chatId}] turn refused — ${JSON.stringify(envelope)}`);
|
||||
return Response.json(envelope, { status: 503 });
|
||||
}
|
||||
|
||||
// One turn per chat, ever. A second request while a turn runs means a
|
||||
// stale/remounted client raced a live one — its message list is outdated,
|
||||
// and letting it through would double-run the teacher and clobber the
|
||||
|
||||
@@ -193,6 +193,17 @@ export function keyValue(id: string): string | undefined {
|
||||
return process.env[id] ?? authStore.authVars()[id];
|
||||
}
|
||||
|
||||
/** The key-presence truth for one model provider (provider-key interlude,
|
||||
* 2026-09-25): would this provider's key be present at request time?
|
||||
* Reads through keyValue — the SAME env-first lookup pi-ai performs
|
||||
* natively at every request — so an env-shadowed key counts as present
|
||||
* (gate by what would run). Providers absent from AUTH_PROVIDER_KEYS take
|
||||
* no key at all: nothing to gate, present by definition. */
|
||||
export function providerKeyPresent(provider: string): boolean {
|
||||
const info = AUTH_PROVIDER_KEYS.find((p) => p.id === provider);
|
||||
return info ? keyValue(info.envVar) !== undefined : true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Save one key (PUT). Shape gate first (owner decision: bad pastes are
|
||||
* 400s, never stored), then the file write, then the env reflection. A
|
||||
|
||||
@@ -20,7 +20,7 @@ import { withPlanBlocks } from "../core/plan-blocks.ts";
|
||||
import { coverageOf } from "../core/coverage.ts";
|
||||
import type { ChatStore, GraphStore } from "../core/ports.ts";
|
||||
import type { ExtractionPipeline } from "../agent/subagents/extraction.ts";
|
||||
import { ModelsNotConfiguredError } from "../core/settings.ts";
|
||||
import { ModelsNotConfiguredError, ProviderKeyMissingError } from "../core/settings.ts";
|
||||
|
||||
/**
|
||||
* The canonical state write: the client owns the message list between turns
|
||||
@@ -101,13 +101,18 @@ export function triggerExtraction(
|
||||
const covered = coverageOf(record.messages as UIMessage[], deps.graphStore.readCoverage(), chatId);
|
||||
if (covered.uncoveredTokens < chunkTokens) return;
|
||||
void extraction.runExtraction(chatId).catch((err) => {
|
||||
// BYOK (owner, 2026-09-22): an unconfigured write channel SKIPS
|
||||
// (the coverage ledger keeps the messages uncovered — the next
|
||||
// trigger retries once slots are set); only real failures log.
|
||||
// BYOK (owner, 2026-09-22) + key presence (provider-key interlude,
|
||||
// 2026-09-25): an unconfigured OR keyless write channel SKIPS (the
|
||||
// coverage ledger keeps the messages uncovered — the next trigger
|
||||
// retries once slots/keys are set); only real failures log.
|
||||
if (err instanceof ModelsNotConfiguredError) {
|
||||
console.info(`[chat ${chatId}] extraction skipped — ${err.envelope.error} (missing: ${err.envelope.missing.join(", ")})`);
|
||||
return;
|
||||
}
|
||||
if (err instanceof ProviderKeyMissingError) {
|
||||
console.info(`[chat ${chatId}] extraction skipped — ${err.envelope.error} (provider: ${err.envelope.provider})`);
|
||||
return;
|
||||
}
|
||||
console.error(`[chat ${chatId}] extraction pipeline failed:`, err);
|
||||
});
|
||||
} catch (err) {
|
||||
|
||||
@@ -2,7 +2,13 @@ import { describe, it } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { parseModelsRefusal } from "../components/chat.tsx";
|
||||
import { prefillFromSlots, prefillLevelFromPin, setupStage } from "../components/setup-dialog.tsx";
|
||||
import { isModelsRefusal, modelsNotConfigured } from "../src/core/settings.ts";
|
||||
import {
|
||||
isModelsRefusal,
|
||||
isProviderKeyRefusal,
|
||||
modelsNotConfigured,
|
||||
providerKeyMissing,
|
||||
PROVIDER_KEY_MISSING_CODE,
|
||||
} from "../src/core/settings.ts";
|
||||
import type { ModelCatalogEntry, ModelSlotName } from "../src/core/index.ts";
|
||||
|
||||
/**
|
||||
@@ -42,6 +48,35 @@ describe("isModelsRefusal — the job-record matcher", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("providerKeyMissing — the envelope factory (provider-key interlude, 2026-09-25)", () => {
|
||||
it("names the provider from the curated label and carries the exact machine code", () => {
|
||||
const envelope = providerKeyMissing("openrouter", ["strong"]);
|
||||
assert.deepEqual(envelope, {
|
||||
error: "No OpenRouter key is set — add one in Settings → Keys",
|
||||
code: PROVIDER_KEY_MISSING_CODE,
|
||||
provider: "openrouter",
|
||||
slots: ["strong"],
|
||||
});
|
||||
assert.equal(envelope.code, "provider_key_missing");
|
||||
});
|
||||
|
||||
it("an unknown provider falls back to its raw id as the label", () => {
|
||||
const envelope = providerKeyMissing("weird", ["cheap"]);
|
||||
assert.equal(envelope.error, "No weird key is set — add one in Settings → Keys");
|
||||
});
|
||||
});
|
||||
|
||||
describe("isProviderKeyRefusal — the job-record matcher (provider-key interlude)", () => {
|
||||
it("matches the envelope wording regardless of provider label, not real failures", () => {
|
||||
assert.equal(isProviderKeyRefusal(providerKeyMissing("together", ["cheap"]).error), true);
|
||||
assert.equal(isProviderKeyRefusal(providerKeyMissing("openai", ["strong"]).error), true);
|
||||
assert.equal(isProviderKeyRefusal(""), false);
|
||||
assert.equal(isProviderKeyRefusal(undefined), false);
|
||||
assert.equal(isProviderKeyRefusal("FAILED: headless crashed"), false);
|
||||
assert.equal(isProviderKeyRefusal(modelsNotConfigured(["strong"]).error), false, "the models refusal is a different failure");
|
||||
});
|
||||
});
|
||||
|
||||
describe("prefillFromSlots — the SetupDialog prefill (BYOK wave)", () => {
|
||||
const entry = (id: string, provider: string): ModelCatalogEntry => ({
|
||||
id,
|
||||
|
||||
@@ -36,6 +36,11 @@ import type { TeachingAgent } from "../src/agent/teacher-agent.ts";
|
||||
import type { UIMessage } from "ai";
|
||||
import { PATHS } from "../config.ts";
|
||||
|
||||
// The key-presence turn gate (provider-key interlude, 2026-09-25): scripted
|
||||
// turns pin openrouter per chat — the gate reads env-first truth, so the
|
||||
// key must be present in the test process like any real setup.
|
||||
process.env.OPENROUTER_API_KEY ??= "test-key-presence-gate";
|
||||
|
||||
const QUIZ_INPUT = {
|
||||
question: "Which planet is closest to the sun?",
|
||||
options: [
|
||||
@@ -612,3 +617,56 @@ describe("the BYOK turn gate (no factory floor — owner, 2026-09-22)", () => {
|
||||
await drain(res);
|
||||
});
|
||||
});
|
||||
|
||||
// ─── the key-presence turn gate (provider-key interlude, owner, 2026-09-25) ─
|
||||
|
||||
describe("the key-presence turn gate (provider-key interlude, 2026-09-25)", () => {
|
||||
// The gate's truth is env-first (keyValue): in tests process.env is the
|
||||
// WHOLE truth (the throwaway data dir has no auth.json), so each test
|
||||
// pins its provider's env var explicitly and restores it after.
|
||||
it("a pinned provider whose key is gone refuses with 503 provider_key_missing — nothing lingers", async () => {
|
||||
const chat = chatStore.createChat("Test: keyless provider");
|
||||
createdChats.push(chat.id);
|
||||
const saved = process.env.TOGETHER_API_KEY;
|
||||
delete process.env.TOGETHER_API_KEY;
|
||||
try {
|
||||
// The chat's own model override pins the provider — the same
|
||||
// chain the strong slot rides. The gate fires BEFORE the engine
|
||||
// build, so the fake model id never hits the catalog.
|
||||
chatStore.saveSettings(chat.id, { model: { provider: "together", id: "no-such-model" } });
|
||||
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "teach me something" }] };
|
||||
const res = await handleChatRequest(post(chat.id, [firstUser]), agentFactory(scriptedTeacher(defaultLesson).engine));
|
||||
|
||||
assert.equal(res.status, 503, "the key-presence gate refuses before beginTurn");
|
||||
const body = (await res.json()) as { error: string; code: string; provider: string; slots: string[] };
|
||||
assert.equal(body.code, "provider_key_missing");
|
||||
assert.equal(body.provider, "together", "the envelope names whose key is gone");
|
||||
assert.deepEqual(body.slots, ["strong"], "the conversation tier is the affected slot");
|
||||
assert.equal(body.error, "No Together key is set — add one in Settings → Keys", "the human line names the provider");
|
||||
assert.equal(isTurnRunning(chat.id), false, "the refusal never registers a live turn");
|
||||
assert.equal(chatStore.getChat(chat.id)!.messages.length, 0, "no write-ahead persist before the gate");
|
||||
} finally {
|
||||
if (saved !== undefined) process.env.TOGETHER_API_KEY = saved;
|
||||
}
|
||||
});
|
||||
|
||||
it("an env-present key passes the gate (gate by what would run)", async () => {
|
||||
const chat = chatStore.createChat("Test: env-keyed provider");
|
||||
createdChats.push(chat.id);
|
||||
const saved = process.env.OPENAI_API_KEY;
|
||||
process.env.OPENAI_API_KEY = "test-key-presence-gate";
|
||||
try {
|
||||
// A REAL catalog model: the engine build AFTER the gate must
|
||||
// resolve — the injected agent replaces the loop, not the build.
|
||||
chatStore.saveSettings(chat.id, { model: { provider: "openai", id: "gpt-4" } });
|
||||
const firstUser = { id: "u1", role: "user", parts: [{ type: "text", text: "teach me something" }] };
|
||||
const res = await handleChatRequest(post(chat.id, [firstUser]), agentFactory(scriptedTeacher(defaultLesson).engine));
|
||||
|
||||
assert.equal(res.status, 200, "the env-shadowed key counts as present — the turn runs");
|
||||
await drain(res);
|
||||
} finally {
|
||||
if (saved !== undefined) process.env.OPENAI_API_KEY = saved;
|
||||
else delete process.env.OPENAI_API_KEY;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
|
||||
/**
|
||||
* Demo + visual artifact tests (W6) — deterministic, zero network, no real
|
||||
* Chrome:
|
||||
@@ -83,6 +84,13 @@ function testRunner() {
|
||||
}
|
||||
import { artifactRenderers } from "../components/artifacts/index.ts";
|
||||
|
||||
// The key-presence turn gate (provider-key interlude, 2026-09-25): scripted
|
||||
// turns pin openrouter per chat — the gate reads env-first truth, so the
|
||||
// key must be present in the test process like any real setup.
|
||||
process.env.OPENROUTER_API_KEY ??= "test-key-presence-gate";
|
||||
|
||||
|
||||
|
||||
const JOBS_DIR = path.join(PATHS.data, "jobs");
|
||||
const createdChats: string[] = [];
|
||||
const createdJobs: string[] = [];
|
||||
|
||||
@@ -39,7 +39,7 @@ import type {
|
||||
} from "../src/core/contracts.ts";
|
||||
import type { GraphStore } from "../src/core/ports.ts";
|
||||
import { abortExtraction, createExtractionPipeline, extractionStatuses, transcriptView } from "../src/agent/subagents/extraction.ts";
|
||||
import { ModelsNotConfiguredError } from "../src/core/settings.ts";
|
||||
import { ModelsNotConfiguredError, ProviderKeyMissingError, providerKeyMissing, type ModelSlotName } from "../src/core/settings.ts";
|
||||
import { batchTag, coverageOf, coverageSlots } from "../src/core/coverage.ts";
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { PATHS } from "../config.ts";
|
||||
@@ -394,6 +394,54 @@ describe("the extraction pipeline (extractor → auditor → merger)", () => {
|
||||
assert.equal(ledgers.readCoverage().length, 0);
|
||||
});
|
||||
|
||||
it("key-presence refusal: a stage engine whose provider key is gone throws before any batch (provider-key interlude, 2026-09-25)", async () => {
|
||||
const { chatId } = fixtureChat();
|
||||
const ledgers = memoryLedgers();
|
||||
const extractor = fauxEngine([]);
|
||||
const auditor = fauxEngine([]);
|
||||
const seen: Array<[string, ModelSlotName]> = [];
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
extractor,
|
||||
auditor,
|
||||
providerKeyMissing: (provider, tier) => {
|
||||
seen.push([provider, tier]);
|
||||
return providerKeyMissing(provider, [tier]);
|
||||
},
|
||||
});
|
||||
await assert.rejects(pipeline.runExtraction(chatId), (err: unknown) => {
|
||||
assert.ok(err instanceof ProviderKeyMissingError);
|
||||
assert.equal(err.envelope.code, "provider_key_missing");
|
||||
assert.equal(err.envelope.provider, extractor.model.provider, "the provider of the keyless engine");
|
||||
assert.deepEqual(err.envelope.slots, ["cheap"], "extractor + auditor are the cheap tier");
|
||||
return true;
|
||||
});
|
||||
// FIRST offender surfaces (extractor before auditor) — and nothing
|
||||
// ran: the refusal lands before any batch, no coverage writes.
|
||||
assert.deepEqual(seen, [[extractor.model.provider, "cheap"]]);
|
||||
assert.equal(ledgers.readCoverage().length, 0);
|
||||
});
|
||||
|
||||
it("key-presence pass-through: null from the check lets a stage through — the NEXT keyless stage surfaces (first-offender discipline)", async () => {
|
||||
const { chatId } = fixtureChat();
|
||||
const ledgers = memoryLedgers();
|
||||
let calls = 0;
|
||||
const pipeline = createExtractionPipeline({
|
||||
chatStore,
|
||||
graphStore: ledgers,
|
||||
extractor: fauxEngine([]),
|
||||
auditor: fauxEngine([]),
|
||||
providerKeyMissing: (provider, tier) => (calls++ === 0 ? null : providerKeyMissing(provider, [tier])),
|
||||
});
|
||||
await assert.rejects(pipeline.runExtraction(chatId), (err: unknown) => {
|
||||
assert.ok(err instanceof ProviderKeyMissingError, "the auditor's refusal surfaces after the extractor passed");
|
||||
return true;
|
||||
});
|
||||
assert.equal(calls, 2, "both stages were checked, in order");
|
||||
assert.equal(ledgers.readCoverage().length, 0);
|
||||
});
|
||||
|
||||
it("the LLM cluster-namer is DEAD (wave 7): a merge mints no naming runs — labels are session names now", async () => {
|
||||
const { chatId } = fixtureChat();
|
||||
const ledgers = memoryLedgers();
|
||||
|
||||
@@ -58,6 +58,7 @@ import { persistTurn, withMessageIds } from "../src/server/turn-service.ts";
|
||||
import { chatStore } from "../src/store/chats.ts";
|
||||
import { jobStore } from "../src/store/jobs.ts";
|
||||
import { createJobRunner, pdfModelOverride, researchModelOverride } from "../src/agent/subagents/runner.ts";
|
||||
import { providerKeyMissing, type ModelSlotName } from "../src/core/settings.ts";
|
||||
import type { PiEngine } from "../src/agent/models.ts";
|
||||
import type { SearchProvider } from "../src/core/search.ts";
|
||||
import { PATHS } from "../config.ts";
|
||||
@@ -65,6 +66,11 @@ import { PATHS } from "../config.ts";
|
||||
const JOBS_DIR = path.join(PATHS.data, "jobs");
|
||||
|
||||
const createdChats: string[] = [];
|
||||
|
||||
// The key-presence turn gate (provider-key interlude, 2026-09-25): scripted
|
||||
// turns pin openrouter per chat — the gate reads env-first truth, so the
|
||||
// key must be present in the test process like any real setup.
|
||||
process.env.OPENROUTER_API_KEY ??= "test-key-presence-gate";
|
||||
const createdJobs: string[] = [];
|
||||
|
||||
function trackChat(id: string) {
|
||||
@@ -1391,3 +1397,56 @@ describe("BYOK job refusal (no factory floor)", () => {
|
||||
assert.equal(result.sources.length, 0, "the failure result matches the research convention");
|
||||
});
|
||||
});
|
||||
|
||||
// ─── provider-key: the key-presence job refusal (owner, 2026-09-25) ─────────
|
||||
|
||||
describe("provider-key job refusal (key-presence interlude)", () => {
|
||||
it("an engine whose provider key is gone refuses the spawn — envelope wording, no phantom run", async () => {
|
||||
const chat = chatStore.createChat("Test: keyless spawn");
|
||||
trackChat(chat.id);
|
||||
chatStore.saveMessages(chat.id, [
|
||||
{ id: "u1", role: "user", parts: [{ type: "text", text: "hi" }] },
|
||||
{
|
||||
id: "a1",
|
||||
role: "assistant",
|
||||
parts: [{ type: "tool-research", toolCallId: "call-km1", state: "output-available", input: RESEARCH_INPUT, output: "Dispatched." }],
|
||||
},
|
||||
] as never);
|
||||
|
||||
const job = jobStore.createJob({ kind: "research", chatId: chat.id, toolCallId: "call-km1" });
|
||||
trackJob(job.id);
|
||||
|
||||
const delivery = fakeDeliver();
|
||||
// The chain RESOLVES (an engine exists) — the KEY is what's missing.
|
||||
// The injected check is the server seam's stand-in: it sees the
|
||||
// engine's provider + the kind's slot tier, exactly as wired.
|
||||
const engine = fauxEngine([]);
|
||||
const seen: Array<[string, ModelSlotName]> = [];
|
||||
const runner = createJobRunner({
|
||||
chatStore,
|
||||
jobStore,
|
||||
deliver: delivery.deliver,
|
||||
engineFor: () => engine,
|
||||
providerKeyMissing: (provider, tier) => {
|
||||
seen.push([provider, tier]);
|
||||
return providerKeyMissing(provider, [tier]);
|
||||
},
|
||||
});
|
||||
await runner.runJob(job.id);
|
||||
|
||||
// The refusal lands through the SAME convention as every failed job:
|
||||
// honest record, kind failure shape steered, terminal flip after.
|
||||
const done = jobStore.getJob(job.id)!;
|
||||
assert.equal(done.status, "error", "the job lands error, never a phantom run");
|
||||
assert.equal(done.error, providerKeyMissing(engine.model.provider, ["cheap"]).error, "the envelope's error line verbatim");
|
||||
assert.deepEqual(seen, [[engine.model.provider, "cheap"]], "the check saw the engine's provider + the researcher's tier");
|
||||
assert.equal(delivery.steers.length, 1, "the refusal STEERED — the teacher is not stranded");
|
||||
const text = delivery.steers[0]!.parts[0];
|
||||
assert.ok(
|
||||
text.type === "text" && text.text.includes(`Research failed: No ${engine.model.provider} key is set`),
|
||||
"the steer carries the kind's failure shape",
|
||||
);
|
||||
const result = done.result as ResearchResult;
|
||||
assert.equal(result.sources.length, 0, "the failure result matches the research convention");
|
||||
});
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user