feat: user pins for local model limits (context, max output, thinking)

A local endpoint that publishes no limits — LM Studio's /models says nothing —
left every model on the 8192/4096 defaults with no way to say "this one has a
131k window". Each discovered model now gets three optional pins in the
endpoint editor: context window, max output tokens, thinking (auto/on/off).

A pin always wins: user pin → what the endpoint published → compiled default.
Blank means no opinion, so discovery keeps ruling wherever you don't pin. Pins
live in customProvider.overrides keyed by model id, stored apart from the
discovery snapshot: a reconnect refreshes the models and prunes the pins of
models that vanished while keeping the rest.

Pins are written by PATCH /api/custom-provider, which never probes — the whole
point is configuring a server that isn't answering /models correctly. Connect
still probes and still fails loudly when the endpoint is down. A pin under 2048
tokens is rejected on the write side and clamped up on the read path, matching
the compaction-floor discipline.
This commit is contained in:
Daniel Buckner
2026-10-27 16:25:03 -07:00
parent 286a95ba04
commit 695948271f
7 changed files with 455 additions and 28 deletions
+41 -1
View File
@@ -18,13 +18,14 @@ import type {
AppearanceSettings,
BackgroundModelRole,
CompactionSettings,
CustomModelOverride,
GeneralSettings,
DictationSettings,
GraphSettings,
ModelsSettings,
RoleModelOverride,
} from "../core/settings.ts";
import { BACKGROUND_MODEL_ROLES, COMPACTION_THRESHOLD_FLOOR, EXTRACTION_CHUNK_TOKENS_FLOOR, MODEL_SLOTS } from "../core/settings.ts";
import { BACKGROUND_MODEL_ROLES, COMPACTION_THRESHOLD_FLOOR, CUSTOM_MODEL_PIN_FLOOR, EXTRACTION_CHUNK_TOKENS_FLOOR, MODEL_SLOTS } from "../core/settings.ts";
import { WRITABLE_TOOL_KEYS } from "../core/pending.ts";
import { isDictationLanguage } from "../core/dictation.ts";
@@ -386,6 +387,45 @@ export function parseDictationSettingsBody(body: unknown): SettingsBodyResult<Di
return { ok: true, settings: { language: raw.language } };
}
/** The custom endpoint's per-model pins (PATCH /api/custom-provider): the
* whole desired map, replace-whole like every section. Strict write side — a
* token count must be a whole number at or above the floor (the read side
* clamps, this rejects — the compaction discipline). `reasoning` is the
* tri-state pin: true forces thinking on, false forces it off. */
export function parseCustomModelOverridesBody(
body: unknown,
): SettingsBodyResult<Record<string, CustomModelOverride>> {
const raw = asRecord(body);
if (!raw) return { ok: false, error: "Invalid JSON body" };
const map = asRecord(raw.overrides);
if (!map) return { ok: false, error: "overrides must be a { modelId: { context?, maxTokens?, reasoning? } } map" };
const out: Record<string, CustomModelOverride> = {};
for (const [key, value] of Object.entries(map)) {
const id = key.trim();
if (!id) return { ok: false, error: "override keys must be non-empty model ids" };
const entry = asRecord(value);
if (!entry) return { ok: false, error: `override for ${id} must be an object` };
const pin: CustomModelOverride = {};
for (const field of ["context", "maxTokens"] as const) {
const v = entry[field];
if (v === undefined) continue;
if (typeof v !== "number" || !Number.isInteger(v) || v < CUSTOM_MODEL_PIN_FLOOR) {
return {
ok: false,
error: `${field} for ${id} must be a whole number of tokens at least ${CUSTOM_MODEL_PIN_FLOOR}`,
};
}
pin[field] = v;
}
if (entry.reasoning !== undefined) {
if (typeof entry.reasoning !== "boolean") return { ok: false, error: `reasoning for ${id} must be true or false` };
pin.reasoning = entry.reasoning;
}
out[id] = pin;
}
return { ok: true, settings: out };
}
/** The section→parser map for PUT /api/settings/[section]. The custom endpoint
* is NOT a section — it connects via POST /api/custom-provider. */
export const SETTINGS_SECTIONS = {