Compare commits

..
2 Commits
30 changed files with 952 additions and 94 deletions
+2 -2
View File
@@ -1,12 +1,12 @@
name: Linux desktop release
# Manual until a native Linux runner is registered. The current 0.5.2
# Manual until a native Linux runner is registered. The current 0.6.0
# packages are built locally using the identical gates documented below.
on:
workflow_dispatch:
inputs:
tag:
description: Existing stable version tag to package (for example v0.5.2)
description: Existing stable version tag to package (for example v0.6.0)
required: true
type: string
+1 -1
View File
@@ -48,7 +48,7 @@ Future teaching sessions can read the graph instead of starting from scratch. Pr
- **Windows (x64):** run the installer. It is currently unsigned; Windows may show an “unknown publisher” or SmartScreen warning.
- **Linux (x64):** make the AppImage executable and open it, or install the Debian/Ubuntu `.deb`.
Add your provider API key and choose your models, then start a session with what you want to understand. LearnHarness supports 30+ provider options, including OpenAI, Anthropic, Google, DeepSeek, and OpenRouter. Optional dictation and web search use separate service keys.
Add your provider API key and choose your models, then start a session with what you want to understand. LearnHarness supports 30+ provider options, including OpenAI, Anthropic, Google, DeepSeek, and OpenRouter. Optional web search uses a separate service key. Dictation supports direct Deepgram Nova-3 streaming or MAI-Transcribe-2 using your OpenRouter API key; choose the speech recognition model in Settings → Speech recognition.
All three platforms support in-app updates.
+12
View File
@@ -0,0 +1,12 @@
import { keyValue, openRouterTranscription } from "@/src/server";
export const runtime = "nodejs";
/** Check availability before recording; the token stays on the server. */
export function GET() {
return Response.json({ configured: Boolean(keyValue("OPENROUTER_API_KEY")) }, { headers: { "Cache-Control": "no-store" } });
}
export async function POST(request: Request) {
return openRouterTranscription(request);
}
+113 -15
View File
@@ -3,14 +3,17 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { Loader2Icon, MicIcon } from "lucide-react";
import { useSettingsDialog } from "@/components/settings/settings-dialog";
import { dictationUrl } from "@/src/core";
import { DEFAULT_STT_MODEL, OPENROUTER_STT_MODEL, STT_MAX_AUDIO_BYTES, WAV_HEADER_BYTES, dictationUrl, dictationWav, type SttModel, type SttLanguage } from "@/src/core";
import { LanguageSelect, useI18n } from "@/components/i18n";
import { Hint } from "@/components/ui/hint";
/**
* Voice dictation — the cloud engine (STT wave 1, 2026-09-17).
* Voice dictation — direct Deepgram streaming or whole-recording OpenRouter STT.
* MAI-Transcribe-2 records locally, drains the final PCM frame on Stop, then
* sends one WAV through /api/stt/transcribe. That route holds the API key.
* Both engines buffer text and append once when dictation finishes.
*
* Architecture (docs/research/stt.md): the renderer captures mic audio in
* Deepgram architecture: the renderer captures mic audio in
* an AudioWorklet (public/worklets/pcm-capture.js → linear16 16kHz mono),
* opens the Deepgram streaming WebSocket DIRECTLY (audio never loops
* through our server) with a short-lived token minted by /api/stt/token
@@ -107,6 +110,7 @@ export const DEEPGRAM_KEY_ID = "DEEPGRAM_API_KEY";
* instead. Exported for pins; the errored idle button's click recognizes
* it and deep-links Settings → Keys → the Deepgram row. */
export const DICTATION_KEYLESS = "Dictation needs a Deepgram key";
export const OPENROUTER_DICTATION_KEYLESS = "Dictation needs an OpenRouter key";
/** Mint a fresh short-lived token (the route holds the real key
* server-side). Module-level: the start path's warm cache and the
@@ -239,7 +243,7 @@ export function useDictationTarget(append: (text: string) => void, claimed: bool
}, [append, claimed]);
}
export type DictationPhase = "idle" | "connecting" | "recording" | "stopping";
export type DictationPhase = "idle" | "connecting" | "recording" | "stopping" | "transcribing";
export type Dictation = {
phase: DictationPhase;
@@ -283,6 +287,14 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
const selectedUrl = useRef("");
selectedUrl.current = dictationUrl(sttLanguage, general.sttPreferredLanguages);
const recordingUrl = useRef(selectedUrl.current);
const selected = useRef({ model: general.sttModel ?? DEFAULT_STT_MODEL, language: sttLanguage });
selected.current = { model: general.sttModel ?? DEFAULT_STT_MODEL, language: sttLanguage };
const recordingConfig = useRef<{ model: SttModel; language: SttLanguage }>(selected.current);
const audioFrames = useRef<Uint8Array[]>([]);
const audioBytes = useRef(0);
const upload = useRef<AbortController | null>(null);
const drain = useRef<(() => void) | null>(null);
const stopping = useRef(false);
const [phase, setPhase] = useState<DictationPhase>("idle");
const [error, setError] = useState<string | null>(null);
/** Latest RMS from the worklet — the meter samples this (dictate pattern). */
@@ -303,7 +315,7 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
/** Frames captured before the socket is OPEN (mitigation A): flushed in
* order on open; capped at 256 frames (≈32s) with the oldest dropped;
* cleared at teardown so a dead run's audio can't leak into the next. */
const pendingRef = useRef<ArrayBuffer[]>([]);
const pendingRef = useRef<Array<ArrayBuffer | Int16Array>>([]);
const genRef = useRef(0); // stale-handler guard (dictate pattern)
const safetyRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const reconnectsRef = useRef(0);
@@ -327,6 +339,13 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
/** Full teardown of one run's machinery. Idempotent; generation-guarded by callers. */
const cleanup = useCallback(() => {
genRef.current++;
upload.current?.abort();
upload.current = null;
drain.current?.();
drain.current = null;
audioFrames.current = [];
audioBytes.current = 0;
stopping.current = false;
dictationActiveFlag = false;
setDictating(false);
flush(); // the deferred transcript appears when the run ends (owner round 2)
@@ -356,7 +375,7 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
for (const w of waitersRef.current.splice(0)) w();
// Pre-mint for the next run (mitigation D) — best-effort; a keyless
// page just doesn't warm.
warmTokens.refresh();
if (selected.current.model === DEFAULT_STT_MODEL) warmTokens.refresh();
}, []);
/** Open the Deepgram socket and wire its messages. `gen` guards staleness. */
@@ -427,6 +446,11 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
const start = useCallback(async () => {
// Pin the complete configuration once; reconnects must retain both language and hints.
recordingUrl.current = selectedUrl.current;
recordingConfig.current = { ...selected.current };
const batch = recordingConfig.current.model === OPENROUTER_STT_MODEL;
audioFrames.current = [];
audioBytes.current = 0;
stopping.current = false;
setDictating(true);
dictationActiveFlag = true;
setError(null);
@@ -442,7 +466,17 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
// the token runs concurrently with it. 127.0.0.1 is a secure
// context, so getUserMedia works without TLS; macOS TCC prompts on
// first use.
const tokenPromise = warmTokens.mint();
const controller = new AbortController();
upload.current = controller;
const tokenPromise = batch ? fetch("/api/stt/transcribe", { signal: controller.signal }).then(async (response) => {
if (!response.ok) throw new Error("OpenRouter transcription is unavailable");
const status = await response.json() as { configured?: boolean };
if (!status.configured) throw new Error(OPENROUTER_DICTATION_KEYLESS);
return "";
}).catch((error: unknown) => {
if (error instanceof Error && error.message === OPENROUTER_DICTATION_KEYLESS) throw error;
throw new Error("OpenRouter transcription is unavailable");
}) : warmTokens.mint();
tokenPromise.catch(() => {}); // surfaced below; no unhandled rejection
const micPromise = navigator.mediaDevices.getUserMedia({
audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true },
@@ -466,10 +500,26 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
const worklet = new AudioWorkletNode(ctx, "pcm-capture");
workletRef.current = worklet;
worklet.port.onmessage = (e) => {
const msg = e.data as { type: string; rms?: number; buffer?: ArrayBuffer };
if (gen !== genRef.current) return;
const msg = e.data as { type: string; rms?: number; buffer?: ArrayBuffer | Int16Array };
if (msg.type === "level") {
levelRef.current = msg.rms ?? 0; // never reset to zero (dictate pattern)
} else if (msg.type === "flushed") {
drain.current?.();
drain.current = null;
} else if (msg.type === "audio") {
if (batch) {
const raw = msg.buffer!;
const bytes = raw instanceof ArrayBuffer ? new Uint8Array(raw) : new Uint8Array(raw.buffer, raw.byteOffset, raw.byteLength);
if (audioBytes.current + bytes.byteLength + WAV_HEADER_BYTES > STT_MAX_AUDIO_BYTES) {
setError("Recording exceeds OpenRouter's 25 MB audio limit");
cleanup();
return;
}
audioFrames.current.push(bytes);
audioBytes.current += bytes.byteLength;
return;
}
const ws = wsRef.current;
if (ws && ws.readyState === WebSocket.OPEN) ws.send(msg.buffer!);
// Mitigation A: not OPEN yet (handshake or reconnect gap) —
@@ -511,12 +561,15 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
if (gen !== genRef.current) return;
// Mitigation B: the handshake fires the moment the graph is live,
// overlapping any remaining token wait — mic-gated, never before it.
openSocket(gen, token);
if (batch) { if (!stopping.current) setPhase("recording"); }
else openSocket(gen, token);
}, [openSocket, cleanup]);
const stop = useCallback(() => {
const gen = genRef.current;
if (phase !== "recording" && phase !== "connecting") return;
if (stopping.current) return;
stopping.current = true;
setPhase("stopping");
// Mic dies FIRST — no more audio enqueues (dictate pattern). The
@@ -525,6 +578,43 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
streamRef.current = null;
sourceRef.current?.disconnect();
sourceRef.current = null;
if (recordingConfig.current.model === OPENROUTER_STT_MODEL) {
const worklet = workletRef.current;
if (!worklet) { cleanup(); return; }
void (async () => {
try {
// Source is disconnected; drain its last partial PCM frame before wrapping the WAV.
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => { drain.current = null; reject(new Error("Could not finish microphone capture")); }, 2000);
drain.current = () => { clearTimeout(timer); resolve(); };
worklet.port.postMessage({ type: "flush" });
});
if (gen !== genRef.current) return;
worklet.disconnect();
workletRef.current = null;
if (audioBytes.current === 0) { cleanup(); return; }
setPhase("transcribing");
const wav = dictationWav(audioFrames.current);
audioFrames.current = [];
const params = new URLSearchParams(recordingConfig.current);
const response = await fetch(`/api/stt/transcribe?${params}`, {
method: "POST", headers: { "Content-Type": "audio/wav" }, body: wav,
signal: AbortSignal.any([upload.current!.signal, AbortSignal.timeout(75_000)]),
});
const body = await response.json() as { text?: string; message?: string };
if (gen !== genRef.current) return;
if (!response.ok) throw new Error(body.message ?? "OpenRouter transcription is unavailable");
if (typeof body.text !== "string") throw new Error("OpenRouter returned an invalid transcript");
if (body.text.trim()) bufferRef.current.push(body.text.trim());
cleanup();
} catch (error) {
if (gen !== genRef.current) return;
setError(error instanceof Error && error.name === "TimeoutError" ? "OpenRouter transcription timed out" : error instanceof Error && error.name === "Error" ? error.message : "OpenRouter transcription is unavailable");
cleanup();
}
})();
return;
}
workletRef.current?.disconnect();
workletRef.current = null;
@@ -587,8 +677,13 @@ export function useDictation({ append }: { append: (text: string) => void }): Di
// page owns ONE hook instance, and only the mounted page's listener is
// attached.
useEffect(() => {
warmTokens.refresh(); // pre-mint at mount (mitigation D) — silent if keyless
}, []);
if ((general.sttModel ?? DEFAULT_STT_MODEL) === DEFAULT_STT_MODEL) warmTokens.refresh();
}, [general.sttModel]);
useEffect(() => () => {
// Navigation cancels a pending upload and releases the microphone without inserting text.
bufferRef.current = [];
cleanup();
}, [cleanup]);
useEffect(() => {
window.addEventListener("lh:kbd-toggle-dictation", toggle);
return () => window.removeEventListener("lh:kbd-toggle-dictation", toggle);
@@ -631,9 +726,8 @@ export function DictationBar({ dictation }: { dictation: Dictation }) {
* idle button deep-links Settings → Keys → the Deepgram row and clears
* the error — the surface is clean when the user returns. Any other
* error keeps the old behavior (click = retry, which clears). */
const keyless = error === DICTATION_KEYLESS;
const keyless = error === DICTATION_KEYLESS || error === OPENROUTER_DICTATION_KEYLESS;
const [levels, setLevels] = useState<number[]>(() => new Array(METER_CELLS).fill(0));
const recording = phase === "recording" || phase === "stopping";
/** Adaptive cell count (owner round 2): derived from the meter's
* measured width — narrow rows show fewer bars instead of
* overflowing under neighbors. */
@@ -691,12 +785,12 @@ export function DictationBar({ dictation }: { dictation: Dictation }) {
onClick={() => {
if (keyless) {
clearError();
openSettings("keys", DEEPGRAM_KEY_ID);
openSettings("keys", error === OPENROUTER_DICTATION_KEYLESS ? "OPENROUTER_API_KEY" : DEEPGRAM_KEY_ID);
} else {
toggle();
}
}}
aria-label={error ? t("Dictation failed: {v0}", { v0: error }) : t("Start dictation")}
aria-label={error ? t("Dictation failed: {v0}", { v0: t(error) }) : t("Start dictation")}
data-testid="dictation-toggle"
// Same blurring as the floating chrome (owner, fix wave round 3),
// but lower opacity — the mic stays quieter than the model pills.
@@ -710,6 +804,10 @@ export function DictationBar({ dictation }: { dictation: Dictation }) {
</div>
);
}
if (phase === "transcribing") return <div className="flex h-7 w-full min-w-0 items-center gap-2 text-xs text-muted-foreground" role="status" data-testid="dictation-transcribing">
<Loader2Icon className="size-3.5 shrink-0 animate-spin" /><span className="min-w-0 flex-1 truncate">{t("Transcribing…")}</span>
<button type="button" onClick={dictation.cancel} className="shrink-0 rounded-sm border border-border px-1.5 py-1 hover:bg-muted">{t("Cancel")}</button>
</div>;
return (
<div className="flex w-full min-w-0 items-center gap-2" data-testid="dictation-bar">
+60 -27
View File
@@ -1,10 +1,11 @@
"use client";
import { createContext, useCallback, useContext, useEffect, useId, useMemo, useRef, useState, type ReactNode } from "react";
import { resolveLanguage, resolveSttLanguage, translator, type GeneralSettings, type Language, type SttLanguage, type Translator } from "@/src/core";
import { DEFAULT_STT_MODEL, OPENROUTER_STT_MODEL, STT_MODELS, resolveLanguage, resolveSttLanguage, translator, type GeneralSettings, type Language, type SttLanguage, type SttModel, type Translator } from "@/src/core";
import { notifySettingsSaved, useSettingsSaved } from "./settings-signal";
import { MenuShell } from "./menu-shell";
import { MENU_CLOSE_EVENT, MENU_OPEN_EVENT, MenuShell } from "./menu-shell";
import { CheckIcon } from "lucide-react";
import { DropdownMenu } from "radix-ui";
type LanguageField = "language" | "sttLanguage";
type I18nValue = {
@@ -58,7 +59,7 @@ export function I18nProvider({ children, initialGeneral = {}, system = "en" }: {
if (!current.ok) throw new Error("Could not load language settings");
const body = await current.json() as { settings?: { general?: GeneralSettings } };
const next = { ...(body.settings?.general ?? latest.current) };
for (const field of ["language", "sttLanguage", "sttPreferredLanguages", "autoUpdate"] as const) {
for (const field of ["language", "sttLanguage", "sttModel", "sttPreferredLanguages", "autoUpdate"] as const) {
if (!Object.hasOwn(patch, field)) continue;
if (patch[field] === undefined) delete next[field];
else Object.assign(next, { [field]: patch[field] });
@@ -77,21 +78,27 @@ export function I18nProvider({ children, initialGeneral = {}, system = "en" }: {
return <I18nContext.Provider value={value}>{children}</I18nContext.Provider>;
}
export function LanguageSelect({ field, disabled = false, compact = false }: { field: "language" | "sttLanguage"; disabled?: boolean; compact?: boolean }) {
const { general, t, saveLanguage, sttLanguage, dictating } = useI18n();
export function LanguageSelect(props: { field: LanguageField; disabled?: boolean; compact?: boolean }) {
return <GeneralSelect {...props} />;
}
export function SttModelSelect() { return <GeneralSelect field="sttModel" />; }
function GeneralSelect({ field, disabled = false, compact = false }: { field: LanguageField | "sttModel"; disabled?: boolean; compact?: boolean }) {
const { general, t, saveGeneral, sttLanguage, dictating } = useI18n();
const [saving, setSaving] = useState(false);
const [error, setError] = useState<string | null>(null);
const [open, setOpen] = useState(false);
const trigger = useRef<HTMLButtonElement>(null);
const menuId = useId();
const locked = disabled || saving || (field === "sttLanguage" && dictating);
const locked = disabled || saving || (field !== "language" && dictating);
const menuOpen = open && !locked;
const compactStt = compact && field === "sttLanguage";
const saved = general[field] ?? "auto";
const saved = general[field] ?? (field === "sttModel" ? DEFAULT_STT_MODEL : "auto");
const selected = compactStt && saved === "auto" ? sttLanguage : saved;
const label = t(field === "language" ? "Interface language" : "Speech recognition language");
const testid = `language-select-${field}${compact ? "-compact" : ""}`;
const options = [
const label = t(field === "language" ? "Interface language" : field === "sttModel" ? "Speech recognition model" : "Speech recognition language");
const testid = field === "sttModel" ? "stt-model-select" : `language-select-${field}${compact ? "-compact" : ""}`;
const options: readonly { value: string; label: string }[] = field === "sttModel" ? STT_MODELS.map((model) => ({ value: model.id, label: model.name })) : [
...(!compactStt ? [{ value: "auto", label: t("System default") } as const] : []),
{ value: "en", label: "English" },
{ value: "ru", label: "Русский" },
@@ -119,17 +126,18 @@ export function LanguageSelect({ field, disabled = false, compact = false }: { f
document.removeEventListener("keydown", escape, true);
};
}, [menuOpen, menuId]);
const pick = (value: "auto" | SttLanguage) => {
const pick = (value: string) => {
if (locked) return;
setOpen(false);
trigger.current?.focus();
if (value === saved) return;
setSaving(true); setError(null);
void saveLanguage(field, value === "auto" ? undefined : value).catch((e: unknown) => setError(e instanceof Error ? e.message : "Could not save language settings")).finally(() => setSaving(false));
const patch = field === "sttModel" ? { sttModel: value as SttModel } : { [field]: value === "auto" ? undefined : value };
void saveGeneral(patch).catch((e: unknown) => setError(e instanceof Error ? e.message : "Could not save language settings")).finally(() => setSaving(false));
};
return <span className={compact ? "flex shrink-0 items-center gap-1" : "flex flex-col items-end gap-1"}>
return <span className={compact ? "flex shrink-0 items-center gap-1" : "flex max-w-full shrink-0 flex-col items-end gap-1"}>
<button ref={trigger} type="button" aria-label={label} title={label} aria-haspopup="menu" aria-expanded={menuOpen} aria-controls={menuOpen ? menuId : undefined} disabled={locked} data-testid={testid}
className={`flex h-7 max-w-full items-center gap-1 rounded-sm border border-border bg-background px-1.5 font-ui text-xs text-foreground transition-colors hover:border-muted-foreground/50 disabled:cursor-not-allowed disabled:opacity-50 ${compact ? "w-24" : "w-36"}`}
className={`flex h-7 max-w-full items-center gap-1 rounded-sm border border-border bg-background px-1.5 font-ui text-xs text-foreground transition-colors hover:border-muted-foreground/50 disabled:cursor-not-allowed disabled:opacity-50 ${compact ? "w-24" : field === "sttModel" ? "w-64" : "w-36"}`}
onClick={() => setOpen(!menuOpen)}
onKeyDown={(event) => {
if (event.key !== "ArrowDown" && event.key !== "ArrowUp") return;
@@ -138,7 +146,7 @@ export function LanguageSelect({ field, disabled = false, compact = false }: { f
<span className="min-w-0 flex-1 truncate text-left">{compact && selected === "multi" ? t("Multi") : options.find((option) => option.value === selected)?.label}</span>
<span aria-hidden className="text-muted-foreground">▾</span>
</button>
{menuOpen && <MenuShell anchorRef={trigger} testid={`${testid}-menu`} preferredSide={compact ? "up" : "down"} menuClass="flex w-44 flex-col gap-0.5 rounded-xl border border-border bg-card p-2 text-xs shadow-xl">
{menuOpen && <MenuShell anchorRef={trigger} testid={`${testid}-menu`} preferredSide={compact ? "up" : "down"} menuClass={`flex ${field === "sttModel" ? "w-72" : "w-44"} flex-col gap-0.5 rounded-xl border border-border bg-card p-2 text-xs shadow-xl`}>
<div id={menuId} role="menu" aria-label={label} className="flex flex-col gap-0.5">
{options.map((option) => <button key={option.value} type="button" role="menuitemradio" aria-checked={selected === option.value} onClick={() => pick(option.value)}
className={`flex items-center justify-between rounded-lg px-2 py-1.5 text-left transition-colors hover:bg-muted ${selected === option.value ? "bg-muted/60 text-card-foreground" : "text-card-foreground"}`}>
@@ -155,25 +163,50 @@ export function SttPreferredLanguages() {
const { general, sttLanguage, dictating, saveGeneral, t } = useI18n();
const [saving, setSaving] = useState(false);
const [error, setError] = useState<string | null>(null);
const [open, setOpen] = useState(false);
const busy = useRef(false);
const preferred = general.sttPreferredLanguages ?? [];
const locked = sttLanguage !== "multi" || dictating || saving;
const unavailable = sttLanguage !== "multi" || general.sttModel === OPENROUTER_STT_MODEL || dictating;
const menuOpen = open && !unavailable;
const summary = (["en", "ru"] as const).filter((language) => preferred.includes(language)).map((language) => language === "en" ? "English" : "Русский").join(", ") || t("No preference");
useEffect(() => { if (unavailable) setOpen(false); }, [unavailable]);
useEffect(() => {
if (!menuOpen) return;
window.dispatchEvent(new CustomEvent(MENU_OPEN_EVENT));
return () => { window.dispatchEvent(new CustomEvent(MENU_CLOSE_EVENT)); };
}, [menuOpen]);
const toggle = (language: Language) => {
if (locked) return;
if (unavailable || busy.current) return;
const next = (["en", "ru"] as const).filter((code) => code === language ? !preferred.includes(code) : preferred.includes(code));
busy.current = true;
setSaving(true); setError(null);
void saveGeneral({ sttPreferredLanguages: next })
.catch(() => setError("Could not save preferred languages"))
.finally(() => setSaving(false));
.finally(() => { busy.current = false; setSaving(false); });
};
return <span className="flex max-w-full flex-col items-start gap-1">
<span role="group" aria-label={t("Preferred languages")} className="flex max-w-full flex-wrap gap-2">
{(["en", "ru"] as const).map((language) => <button key={language} type="button" role="checkbox" aria-checked={preferred.includes(language)}
data-testid={`stt-preferred-${language}`} disabled={locked} onClick={() => toggle(language)}
className={`flex h-7 items-center gap-1.5 rounded-sm border border-border px-1.5 font-ui text-xs transition-colors disabled:cursor-not-allowed disabled:opacity-50 ${preferred.includes(language) ? "bg-muted text-foreground" : "bg-background text-muted-foreground hover:border-muted-foreground/50"}`}>
<span aria-hidden className="flex size-3.5 shrink-0 items-center justify-center rounded-xs border border-border">{preferred.includes(language) && <CheckIcon className="size-3" />}</span>
{language === "en" ? "English" : "Русский"}
</button>)}
</span>
return <span className="flex max-w-full shrink-0 flex-col items-end gap-1">
<DropdownMenu.Root open={menuOpen} onOpenChange={setOpen} modal={false}>
<DropdownMenu.Trigger asChild>
<button type="button" data-testid="stt-preferred-select" aria-label={t("Preferred languages")} title={summary} disabled={unavailable || saving}
className={`flex h-7 w-44 max-w-full items-center gap-1 rounded-sm border border-border bg-background px-1.5 font-ui text-xs text-foreground transition-colors hover:border-muted-foreground/50 ${unavailable ? "disabled:cursor-not-allowed disabled:opacity-50" : "disabled:cursor-default"}`}
onKeyDown={(event) => { if (event.key === "ArrowUp" && !unavailable && !saving) { event.preventDefault(); event.stopPropagation(); setOpen(true); } }}>
<span className="min-w-0 flex-1 truncate text-left">{summary}</span><span aria-hidden className="text-muted-foreground">▾</span>
</button>
</DropdownMenu.Trigger>
<DropdownMenu.Portal>
<DropdownMenu.Content data-testid="stt-preferred-menu" data-lh-menu="" aria-label={t("Preferred languages")} side="bottom" align="end" sideOffset={4} collisionPadding={8} loop
className="z-60 flex w-44 max-h-[min(60vh,var(--radix-dropdown-menu-content-available-height))] flex-col gap-0.5 overflow-y-auto rounded-xl border border-border bg-card p-2 text-xs shadow-xl outline-none"
onKeyDown={(event) => event.stopPropagation()}
onEscapeKeyDown={(event) => event.stopPropagation()}>
{(["en", "ru"] as const).map((language) => <DropdownMenu.CheckboxItem key={language} data-testid={`stt-preferred-${language}`} checked={preferred.includes(language)} disabled={saving}
onCheckedChange={() => toggle(language)} onSelect={(event) => event.preventDefault()}
className={`flex cursor-default select-none items-center gap-2 rounded-lg px-2 py-1.5 text-left text-card-foreground outline-none transition-colors data-highlighted:bg-muted ${preferred.includes(language) ? "bg-muted/60" : ""}`}>
<span aria-hidden className="flex size-3.5 shrink-0 items-center justify-center rounded-xs border border-border"><DropdownMenu.ItemIndicator><CheckIcon className="size-3" /></DropdownMenu.ItemIndicator></span>
{language === "en" ? "English" : "Русский"}
</DropdownMenu.CheckboxItem>)}
</DropdownMenu.Content>
</DropdownMenu.Portal>
</DropdownMenu.Root>
{error && <span role="alert" className="max-w-56 text-xs text-destructive">{t(error)}</span>}
</span>;
}
+2 -1
View File
@@ -703,7 +703,8 @@ export function AgentSection({
<button type="button" data-testid={`slot-fast-${slot}`} aria-label={t("{v0} fast mode", { v0: t(SLOT_META[slot].label) })}
aria-pressed={provider === "chatgpt" && override?.fastMode === true} disabled={provider !== "chatgpt" || !chosen || savingModels}
onClick={() => { setOpenMenu(null); void updateModels(composeSlotFast(models, slot, !override?.fastMode)); }}
className={cn("flex h-7 items-center gap-1 rounded-sm border border-border px-1.5 font-ui text-xs transition-colors disabled:cursor-not-allowed disabled:opacity-40",
className={cn("flex h-7 items-center gap-1 rounded-sm border border-border px-1.5 font-ui text-xs transition-colors disabled:opacity-40",
provider === "chatgpt" && chosen ? "disabled:cursor-default" : "disabled:cursor-not-allowed",
provider === "chatgpt" && override?.fastMode ? "bg-muted text-foreground" : "bg-background text-muted-foreground hover:border-muted-foreground/50")}>
<ZapIcon className="size-3.5" aria-hidden /><span>{t("Fast")}</span>
</button>
+11
View File
@@ -335,6 +335,7 @@ export function KeysSection({
/** The paste editor for a freshly chosen Add-provider target. */
const [adding, setAdding] = useState<ProviderKeyInfo | null>(null);
const [addError, setAddError] = useState<string | null>(null);
const followedProvider = useRef<string | null>(null);
// Escape folds any open editor (unfolded row or the Add-provider paste)
// WITHOUT closing the dialog. Claimed at document CAPTURE phase — the
@@ -372,7 +373,17 @@ export function KeysSection({
// named row; a fresh mount seeds it via the useState initializer.
useEffect(() => {
if (initialRow !== undefined) setOpenRow(initialRow);
followedProvider.current = null;
}, [initialRow]);
// A missing model-provider key has no table row yet. Deep-link into the
// existing Add-provider editor instead (for example, OpenRouter STT setup).
useEffect(() => {
if (!initialRow || !statuses || !providers || followedProvider.current === initialRow) return;
followedProvider.current = initialRow;
if (statuses.some((status) => status.id === initialRow)) return;
const provider = providers.find((provider) => provider.envVar === initialRow);
if (provider) { setAdding(provider); setAddError(null); setOpenRow(null); }
}, [initialRow, statuses, providers]);
async function saveKey(id: string, value: string): Promise<string | null> {
const res = await fetch(`/api/keys/${id}`, {
+11 -5
View File
@@ -39,7 +39,7 @@ import {
type GraphSettings,
type UserSettings,
} from "@/src/core";
import { LanguageSelect, SttPreferredLanguages, useI18n } from "@/components/i18n";
import { LanguageSelect, SttModelSelect, SttPreferredLanguages, useI18n } from "@/components/i18n";
import { useAppearance } from "./appearance.tsx";
import { SettingsSection, SettingsRow, Toggle } from "./controls.tsx";
import { AgentSection } from "./agent-section.tsx";
@@ -62,6 +62,7 @@ const SECTIONS = [
{ id: "appearance", label: "Appearance" },
{ id: "graph", label: "Graph" },
{ id: "keyboard", label: "Keyboard", hint: "Stored on this device" },
{ id: "stt", label: "Speech recognition" },
{ id: "general", label: "General" },
] as const;
type SectionId = (typeof SECTIONS)[number]["id"];
@@ -367,10 +368,6 @@ export function SettingsDialogBody({
testid="section-general"
>
<SettingsRow label={t("Interface language")} testid="row-language"><LanguageSelect field="language" /></SettingsRow>
<SettingsRow label={t("Speech recognition language")} testid="row-stt-language"><LanguageSelect field="sttLanguage" /></SettingsRow>
<SettingsRow label={t("Preferred languages")} hint={t("Used with Multilingual. Favors selected languages; other languages may still be recognized.")} testid="row-stt-preferred-languages" className="flex-col items-start gap-2">
<SttPreferredLanguages />
</SettingsRow>
<SettingsRow
label={t("Auto-update")}
hint={AUTO_UPDATE_HINT}
@@ -386,6 +383,15 @@ export function SettingsDialogBody({
</SettingsRow>
</SettingsSection>
)}
{active === "stt" && (
<SettingsSection title={t("Speech recognition")} testid="section-stt">
<SettingsRow label={t("Speech recognition model")} testid="row-stt-model" className="flex-wrap items-start gap-y-2 [&>span:first-child]:flex-[1_1_12rem]"><SttModelSelect /></SettingsRow>
<SettingsRow label={t("Speech recognition language")} testid="row-stt-language"><LanguageSelect field="sttLanguage" /></SettingsRow>
<SettingsRow label={t("Preferred languages")} testid="row-stt-preferred-languages">
<SttPreferredLanguages />
</SettingsRow>
</SettingsSection>
)}
{active === "keys" && (
<SettingsSection title={t("Providers")} testid="section-keys">
<KeysSection initialRow={initialRow ?? undefined} />
+1 -1
View File
@@ -1,6 +1,6 @@
# Desktop builds and releases
For this fork’s Linux and Windows 0.5.2 release on Gitea, see [Gitea releases](gitea-release.md).
For this fork’s Linux and Windows 0.6.0 release on Gitea, see [Gitea releases](gitea-release.md).
The three-platform GitHub process below documents the upstream reference.
## Platforms
+16 -16
View File
@@ -2,7 +2,7 @@
The release destination is
https://server2.sorokin-oleg-edu.xyz/replicated/learnharness.
Version **0.5.2 targets Linux x64 and Windows x64**, with each platform built
Version **0.6.0 targets Linux x64 and Windows x64**, with each platform built
and verified on its native operating system. The upstream GitHub workflow and three-platform packaging
configuration remain available as reference in `.github/workflows/release.yml`
and `electron-builder.yml`.
@@ -48,18 +48,18 @@ node scripts/smoke-provider-runtime.mjs --standalone dist/linux-unpacked/resourc
xvfb-run -a node scripts/smoke-demo-runtime.mjs --standalone dist/linux-unpacked/resources/server/standalone --runtime dist/linux-unpacked/learnharness --browser
```
The Linux portion of 0.5.2 has these attachments:
The Linux portion of 0.6.0 has these attachments:
- `LearnHarness-0.5.2-x86_64.AppImage`
- `LearnHarness-0.5.2-amd64.deb`
- `LearnHarness-0.6.0-x86_64.AppImage`
- `LearnHarness-0.6.0-amd64.deb`
- `latest-linux.yml`
- `SHA256SUMS`
## Preparing Windows packages
Use a clean checkout of the same `v0.5.2` tag on Windows x64 with Node 26 and Git
Use a clean checkout of the same `v0.6.0` tag on Windows x64 with Node 26 and Git
for Windows. Keep build tools and logs outside the checkout. The Windows
release checkout is `~/Documents/Gitea/learnharness-release-0.5.2`, linked to the
release checkout is `~/Documents/Gitea/learnharness-release-0.6.0`, linked to the
existing repository in `~/Documents/Gitea/learnharness`.
In PowerShell, stop after any failed command:
@@ -85,13 +85,13 @@ node scripts/smoke-demo-runtime.mjs --standalone dist/win-unpacked/resources/ser
Windows contributes three attachments:
- `LearnHarness-0.5.2-x64-setup.exe`
- `LearnHarness-0.5.2-x64-setup.exe.blockmap`
- `LearnHarness-0.6.0-x64-setup.exe`
- `LearnHarness-0.6.0-x64-setup.exe.blockmap`
- `latest.yml`
The combined release has **seven attachments**, including one `SHA256SUMS`
covering all six other files. Verify that `latest.yml` names the Windows
installer exactly, reports version `0.5.2`, and matches the installer's actual
installer exactly, reports version `0.6.0`, and matches the installer's actual
size and SHA-512. Its gzip blockmap must cover the installer and match every
block checksum. Confirm the packaged `resources/app-update.yml` uses the Gitea
generic provider. Windows builds follow upstream's unsigned, per-user installer
@@ -102,8 +102,8 @@ configuration.
Commit and push the release source, then push its exact annotated tag:
```sh
git tag -a v0.5.2 -m 'LearnHarness 0.5.2'
git push gitea main v0.5.2
git tag -a v0.6.0 -m 'LearnHarness 0.6.0'
git push gitea main v0.6.0
```
Supply `GITEA_TOKEN` only to the upload command. A token file may remain outside
@@ -117,14 +117,14 @@ The tool requires the tag to exist in Gitea, creates or reuses one draft at that
tag, uploads missing assets, and downloads every attachment to verify its actual
SHA-256 against the local verified build. Existing mismatched attachments and
published releases are refused. It never publishes or changes repository
visibility. Release notes come from `docs/release-0.5.2.md`.
visibility. Release notes come from `docs/release-0.6.0.md`.
This uploader and its remote verifier enforce a **Linux-only** inventory. They
will refuse a combined Linux/Windows draft. For the combined 0.5.2 release, add
will refuse a combined Linux/Windows draft. For the combined 0.6.0 release, add
the verified Windows attachments to the existing draft and replace `SHA256SUMS`
with the combined manifest. Keep the Linux installers and feed unchanged. Check
all seven attachment names and sizes against the verified local builds. For
the owner-approved 0.5.2 release, local installer/feed/blockmap validation and
the owner-approved 0.6.0 release, local installer/feed/blockmap validation and
the combined checksum manifest are completed before upload. Confirm successful
upload responses and publish immediately afterward; do not download attachments
again or run post-upload hash checks. Do not publish until both platforms'
@@ -145,12 +145,12 @@ public `latest` feed and in-app automatic updates expose this version.
workflow with an existing release tag as input. It requires a native Linux x64
runner labeled `ubuntu-24.04`, the label used by upstream's Linux release job,
plus the native Electron dependencies and `xvfb-run`. There were no runners
registered when 0.5.2 was prepared, so its packages are built locally. A tag push
registered when 0.6.0 was prepared, so its packages are built locally. A tag push
does not queue a job that cannot run.
The workflow uses the repository's ephemeral `GITEA_TOKEN` with release-write
permission only in the final upload step. It checks out the requested tag without
persisting credentials and leaves the result as a verified draft. Public pull
requests cannot trigger this publishing workflow. This workflow remains
Linux-only; the combined 0.5.2 draft was assembled from local native builds.
Linux-only; the combined 0.6.0 draft was assembled from local native builds.
Windows has its own standalone payload and smoke checks, as described above.
+63
View File
@@ -0,0 +1,63 @@
# LearnHarness 0.6.0
## English
This release adds OpenRouter speech recognition and a dedicated speech-recognition
settings page, with cleaner controls in English and Russian.
- **OpenRouter STT:** choose **MAI-Transcribe-2 · OpenRouter** and add your
OpenRouter API key under **Settings → Providers**. The app records locally,
sends the whole recording after **Stop**, and shows **Transcribing…** until
the complete transcript is ready. Audio is not split into speech segments.
Cancelling aborts the request; submitting an answer or message waits for
transcription. The permanent API key stays on the server and audio is not
saved to disk. Recordings are limited to 25 MB, about 13 minutes.
- **Settings → Speech recognition** now contains the model, recognition language,
and preferred languages. The two model choices use the same format:
**Nova-3 · Deepgram** and **MAI-Transcribe-2 · OpenRouter**. General settings
contains only interface language and automatic updates.
- **Preferred languages** uses a styled checkbox dropdown aligned on the right.
It stays open while selecting English and Russian, shows **No preference**
when neither is selected, and no longer dims or flashes while saving.
These hints remain available for Deepgram Multilingual. MAI uses automatic
language detection for Multilingual; its API does not expose a RU + EN hint
list. Saved hints are retained when switching models.
- On Linux, clicking an available **Fast** button keeps the normal cursor while
the setting is saved.
Existing settings remain compatible and direct Deepgram Nova-3 remains the
default. Linux x64 packages include AppImage and Debian; Windows x64 has an
installer. Both platforms use this fork's public Gitea releases for automatic
updates. The Windows installer remains unsigned, as in previous releases.
## Русский
В этом выпуске добавлено распознавание речи через OpenRouter и отдельный раздел
его настроек. Элементы управления обновлены на русском и английском языках.
- **Распознавание через OpenRouter:** выберите **MAI-Transcribe-2 · OpenRouter**
и добавьте API-ключ OpenRouter в разделе **Настройки → Провайдеры**. Приложение
записывает речь локально, отправляет всю запись после остановки и показывает
**Распознавание…** до получения полного текста. Речь не разбивается на
отдельные фрагменты. Отмена прерывает запрос, а отправка ответа или сообщения
дожидается завершения распознавания. Постоянный API-ключ остаётся на сервере,
аудио не сохраняется на диск. Максимальный размер записи — 25 МБ, примерно
13 минут.
- Новый раздел **Настройки → Распознавание речи** содержит выбор модели, языка
распознавания и предпочитаемых языков. Названия моделей оформлены одинаково:
**Nova-3 · Deepgram** и **MAI-Transcribe-2 · OpenRouter**. В разделе «Общие»
остались только язык интерфейса и автоматические обновления.
- **Предпочитаемые языки** выбираются в оформленном в стиле приложения
выпадающем меню с флажками справа. Меню остаётся открытым при выборе русского
и английского. Если оба флажка сняты, отображается **Без предпочтений**.
Во время сохранения меню больше не тускнеет и не мигает. Подсказки работают
в многоязычном режиме Deepgram. MAI определяет языки автоматически: его API
не предоставляет списка подсказок RU + EN. Сохранённые предпочтения остаются
при переключении моделей.
- В Linux нажатие доступной кнопки **Быстро** больше не меняет курсор на время
сохранения настройки.
Прежние настройки совместимы; по умолчанию по-прежнему используется Deepgram
Nova-3 напрямую. Для Linux x64 доступны AppImage и Debian, для Windows x64 —
установщик. Обе версии получают автоматические обновления из публичных выпусков
этого форка в Gitea. Установщик Windows не подписан, как и в предыдущих выпусках.
+18
View File
@@ -0,0 +1,18 @@
# Speech recognition
Settings → Speech recognition contains the model, recognition language and preferred-language controls. The model picker offers two choices:
- **Nova-3 · Deepgram** (the default): streams microphone audio directly to Deepgram using a short-lived token. Add a Deepgram API key under Providers.
- **MAI-Transcribe-2 · OpenRouter**: records locally in memory and sends the complete recording in one request after Stop. Add an OpenRouter API key under Providers; the existing key also works if OpenRouter is already configured for chat. The permanent key stays on the server.
Both insert the transcript after recording finishes. OpenRouter displays **Transcribing…** while waiting. Sending a chat message or answer during recording waits for transcription before continuing. Cancel or Escape discards dictation; cancelling during transcription aborts the request and ignores late results. Navigating away releases the microphone and cancels pending transcription.
The interface language, recognition language, and STT model are saved independently. Model and language changes apply to the next recording, and their controls lock during recording/transcription. Existing settings continue to use Deepgram until another model is selected.
English and Russian select a single recognition language. **Multilingual** uses automatic language detection in MAI, including mixed-language speech. Its documented API accepts only one forced language and has no preferred-language list; saved RU/EN hints remain available for Deepgram and are disabled when MAI is selected. They are not sent as keyword hints or an ignored prompt.
OpenRouter audio is a lossless WAV container around the same 16 kHz mono PCM capture used for Deepgram. Stop drains the microphone's final partial frame before sending it. There is no speech splitting, silence removal, automatic provider fallback, or automatic transcription retry. The inline audio limit is 25 MB (about 13 minutes at this format); exceeding it stops recording with an error. Provider processing can time out for long recordings even within the size limit. Audio is not saved to disk.
The renderer sends WAV bytes to `POST /api/stt/transcribe?model=microsoft/mai-transcribe-2&language=en|ru|multi`. The server validates the model, language, WAV format and body size, reads `OPENROUTER_API_KEY` through the normal key store, and posts base64 audio to OpenRouter's dedicated transcription endpoint. Provider diagnostics are replaced with localized app errors, without returning credentials or audio. Offline tests mock all provider calls.
References: [OpenRouter transcription API](https://openrouter.ai/docs/guides/overview/multimodal/stt), [MAI-Transcribe-2](https://openrouter.ai/microsoft/mai-transcribe-2), [Microsoft language and model options](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/mai-transcribe).
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "learnharness",
"version": "0.5.2",
"version": "0.6.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "learnharness",
"version": "0.5.2",
"version": "0.6.0",
"license": "AGPL-3.0-only",
"dependencies": {
"@earendil-works/pi-agent-core": "1.4.0",
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "learnharness",
"version": "0.5.2",
"version": "0.6.0",
"license": "AGPL-3.0-only",
"author": {
"name": "Amos Blomqvist",
+10
View File
@@ -27,6 +27,16 @@ class PcmCaptureProcessor extends AudioWorkletProcessor {
super();
this.frame = new Int16Array(2048);
this.count = 0;
// Whole-recording STT must include the final, partially filled frame.
this.port.onmessage = (event) => {
if (event.data?.type !== "flush") return;
if (this.count > 0) {
const tail = this.frame.slice(0, this.count);
this.port.postMessage({ type: "audio", buffer: tail }, [tail.buffer]);
this.count = 0;
}
this.port.postMessage({ type: "flushed" });
};
}
process(inputs) {
+1
View File
@@ -81,3 +81,4 @@ export * from "./coverage.ts";
export * from "./session-markdown.ts";
export * from "./i18n.ts";
export * from "./stt.ts";
+20 -2
View File
@@ -252,7 +252,6 @@
"Multilingual": "Multilingual",
"Multi": "Multi",
"Preferred languages": "Preferred languages",
"Used with Multilingual. Favors selected languages; other languages may still be recognized.": "Used with Multilingual. Favors selected languages; other languages may still be recognized.",
"Could not save preferred languages": "Could not save preferred languages",
"sttPreferredLanguages must be an array of \"en\" and \"ru\"": "sttPreferredLanguages must be an array of \"en\" and \"ru\"",
"Global": "Global",
@@ -848,5 +847,24 @@
"Fast mode uses more ChatGPT allowance.": "Fast mode uses more ChatGPT allowance.",
"Fast mode is available only for ChatGPT models.": "Fast mode is available only for ChatGPT models.",
"Could not save model settings": "Could not save model settings",
"Fast mode is unavailable for this ChatGPT model or account. Turn off Fast mode or choose another model.": "Fast mode is unavailable for this ChatGPT model or account. Turn off Fast mode or choose another model."
"Fast mode is unavailable for this ChatGPT model or account. Turn off Fast mode or choose another model.": "Fast mode is unavailable for this ChatGPT model or account. Turn off Fast mode or choose another model.",
"Speech recognition model": "Speech recognition model",
"Unknown speech recognition model": "Unknown speech recognition model",
"Invalid speech recognition language": "Invalid speech recognition language",
"Invalid dictation audio": "Invalid dictation audio",
"Recording exceeds OpenRouter's 25 MB audio limit": "Recording exceeds OpenRouter's 25 MB audio limit",
"Could not read dictation audio": "Could not read dictation audio",
"Transcription cancelled": "Transcription cancelled",
"Dictation needs an OpenRouter key": "Dictation needs an OpenRouter key",
"OpenRouter rejected the API key": "OpenRouter rejected the API key",
"OpenRouter credits are insufficient": "OpenRouter credits are insufficient",
"OpenRouter rate limit reached; try again later": "OpenRouter rate limit reached; try again later",
"OpenRouter transcription failed ({v0})": "OpenRouter transcription failed ({v0})",
"OpenRouter returned an invalid transcript": "OpenRouter returned an invalid transcript",
"OpenRouter transcription timed out": "OpenRouter transcription timed out",
"OpenRouter transcription is unavailable": "OpenRouter transcription is unavailable",
"Could not finish microphone capture": "Could not finish microphone capture",
"Transcribing…": "Transcribing…",
"Speech recognition": "Speech recognition",
"No preference": "No preference"
}
+20 -2
View File
@@ -252,7 +252,6 @@
"Multilingual": "Несколько языков",
"Multi": "Мульти",
"Preferred languages": "Предпочитаемые языки",
"Used with Multilingual. Favors selected languages; other languages may still be recognized.": "Используется в режиме «Несколько языков». Выбранные языки получают приоритет, но могут распознаваться и другие.",
"Could not save preferred languages": "Не удалось сохранить предпочитаемые языки",
"sttPreferredLanguages must be an array of \"en\" and \"ru\"": "Предпочитаемые языки должны быть списком кодов «en» и «ru»",
"Global": "Общие",
@@ -884,5 +883,24 @@
"Fast mode uses more ChatGPT allowance.": "Быстрый режим быстрее расходует лимит ChatGPT.",
"Fast mode is available only for ChatGPT models.": "Быстрый режим доступен только для моделей ChatGPT.",
"Could not save model settings": "Не удалось сохранить настройки моделей",
"Fast mode is unavailable for this ChatGPT model or account. Turn off Fast mode or choose another model.": "Быстрый режим недоступен для этой модели или учётной записи ChatGPT. Отключите быстрый режим или выберите другую модель."
"Fast mode is unavailable for this ChatGPT model or account. Turn off Fast mode or choose another model.": "Быстрый режим недоступен для этой модели или учётной записи ChatGPT. Отключите быстрый режим или выберите другую модель.",
"Speech recognition model": "Модель распознавания речи",
"Unknown speech recognition model": "Неизвестная модель распознавания речи",
"Invalid speech recognition language": "Недопустимый язык распознавания речи",
"Invalid dictation audio": "Некорректная аудиозапись",
"Recording exceeds OpenRouter's 25 MB audio limit": "Запись превышает ограничение OpenRouter в 25 МБ",
"Could not read dictation audio": "Не удалось прочитать аудиозапись",
"Transcription cancelled": "Распознавание отменено",
"Dictation needs an OpenRouter key": "Для диктовки нужен ключ OpenRouter",
"OpenRouter rejected the API key": "OpenRouter отклонил API-ключ",
"OpenRouter credits are insufficient": "Недостаточно средств на балансе OpenRouter",
"OpenRouter rate limit reached; try again later": "Превышен лимит запросов OpenRouter. Попробуйте позже",
"OpenRouter transcription failed ({v0})": "Ошибка распознавания OpenRouter ({v0})",
"OpenRouter returned an invalid transcript": "OpenRouter вернул некорректный результат распознавания",
"OpenRouter transcription timed out": "Время ожидания распознавания OpenRouter истекло",
"OpenRouter transcription is unavailable": "Распознавание через OpenRouter недоступно",
"Could not finish microphone capture": "Не удалось завершить запись с микрофона",
"Transcribing…": "Распознавание…",
"Speech recognition": "Распознавание речи",
"No preference": "Без предпочтений"
}
+1
View File
@@ -184,6 +184,7 @@ export function resolveSettings(record: UserSettings | null): EffectiveSettings
autoUpdate: record?.general?.autoUpdate ?? FACTORY_SETTINGS.general.autoUpdate,
...(record?.general?.language ? { language: record.general.language } : {}),
...(record?.general?.sttLanguage ? { sttLanguage: record.general.sttLanguage } : {}),
...(record?.general?.sttModel ? { sttModel: record.general.sttModel } : {}),
...(record?.general?.sttPreferredLanguages ? { sttPreferredLanguages: [...record.general.sttPreferredLanguages] } : {}),
},
agent: {
+4
View File
@@ -33,6 +33,7 @@ import type { ModelRole } from "../../config.ts";
import type { ChatSettings } from "./contracts/chat.ts";
import type { SearchEngineId } from "./search.ts";
import { isLanguage, isSttLanguage, type Language, type SttLanguage } from "./i18n.ts";
import { isSttModel, type SttModel } from "./stt.ts";
/** The schema version this code understands. Bump ONLY for breaking shape
* changes (a field renamed/meaning changed); pure additions never bump. */
@@ -240,6 +241,8 @@ export interface GeneralSettings {
language?: Language;
/** Independent dictation language; multi = multilingual, absent = browser/system language. */
sttLanguage?: SttLanguage;
/** Absent preserves direct Deepgram Nova-3 streaming. */
sttModel?: SttModel;
/** Optional recognition hints for multilingual dictation; absent/empty = automatic detection. */
sttPreferredLanguages?: Language[];
}
@@ -452,6 +455,7 @@ function sanitizeGeneral(raw: Record<string, unknown>): GeneralSettings {
if (autoUpdate !== undefined) out.autoUpdate = autoUpdate;
if (isLanguage(raw.language)) out.language = raw.language;
if (isSttLanguage(raw.sttLanguage)) out.sttLanguage = raw.sttLanguage;
if (isSttModel(raw.sttModel)) out.sttModel = raw.sttModel;
if (Array.isArray(raw.sttPreferredLanguages)) out.sttPreferredLanguages = [...new Set(raw.sttPreferredLanguages.filter(isLanguage))];
return out;
}
+36
View File
@@ -0,0 +1,36 @@
/** Dictation model choices are independent of the conversational model tiers. */
export const STT_MODELS = [
{ id: "deepgram/nova-3", name: "Nova-3 · Deepgram" },
{ id: "microsoft/mai-transcribe-2", name: "MAI-Transcribe-2 · OpenRouter" },
] as const;
export type SttModel = (typeof STT_MODELS)[number]["id"];
export const DEFAULT_STT_MODEL: SttModel = "deepgram/nova-3";
export const OPENROUTER_STT_MODEL: SttModel = "microsoft/mai-transcribe-2";
export function isSttModel(value: unknown): value is SttModel {
return STT_MODELS.some((model) => model.id === value);
}
/** OpenRouter's inline audio limit. WAV uses the existing 16 kHz mono PCM capture. */
export const STT_MAX_AUDIO_BYTES = 25 * 1024 * 1024;
export const STT_SAMPLE_RATE = 16000;
export const WAV_HEADER_BYTES = 44;
/** Wrap captured PCM in a WAV container without resampling or encoding loss. */
export function dictationWav(frames: readonly Uint8Array[]): Uint8Array<ArrayBuffer> {
const size = frames.reduce((total, frame) => total + frame.byteLength, 0);
if (size % 2 !== 0) throw new Error("Invalid dictation audio");
if (size + WAV_HEADER_BYTES > STT_MAX_AUDIO_BYTES) throw new Error("Recording exceeds OpenRouter's 25 MB audio limit");
const wav = new Uint8Array(WAV_HEADER_BYTES + size);
const view = new DataView(wav.buffer);
const text = (offset: number, value: string) => {
for (let i = 0; i < value.length; i++) view.setUint8(offset + i, value.charCodeAt(i));
};
text(0, "RIFF"); view.setUint32(4, 36 + size, true); text(8, "WAVE");
text(12, "fmt "); view.setUint32(16, 16, true); view.setUint16(20, 1, true);
view.setUint16(22, 1, true); view.setUint32(24, STT_SAMPLE_RATE, true);
view.setUint32(28, STT_SAMPLE_RATE * 2, true); view.setUint16(32, 2, true); view.setUint16(34, 16, true);
text(36, "data"); view.setUint32(40, size, true);
let offset = WAV_HEADER_BYTES;
for (const frame of frames) { wav.set(frame, offset); offset += frame.byteLength; }
return wav;
}
+1
View File
@@ -46,3 +46,4 @@ export * from "./model-catalog.ts";
export { sessionCost } from "./pricing.ts";
export { applicationLanguage, chatLanguage } from "./language.ts";
export * from "./stt.ts";
+5 -1
View File
@@ -12,7 +12,7 @@
*/
import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
import { isHexColor, isLanguage, isSttLanguage, type ChatSettings } from "../core";
import { isHexColor, isLanguage, isSttLanguage, isSttModel, type ChatSettings } from "../core";
import type {
AgentSettings,
AppearanceSettings,
@@ -98,6 +98,10 @@ export function parseGeneralSettingsBody(body: unknown): SettingsBodyResult<Gene
if (!Array.isArray(raw.sttPreferredLanguages) || !raw.sttPreferredLanguages.every(isLanguage)) return { ok: false, error: 'sttPreferredLanguages must be an array of "en" and "ru"' };
settings.sttPreferredLanguages = [...new Set(raw.sttPreferredLanguages)];
}
if (raw.sttModel !== undefined) {
if (!isSttModel(raw.sttModel)) return { ok: false, error: "Unknown speech recognition model" };
settings.sttModel = raw.sttModel;
}
return { ok: true, settings };
}
+75
View File
@@ -0,0 +1,75 @@
import { OPENROUTER_STT_MODEL, STT_MAX_AUDIO_BYTES, STT_SAMPLE_RATE, WAV_HEADER_BYTES } from "../core/stt.ts";
import { isSttLanguage } from "../core/i18n.ts";
import { keyValue } from "./keys.ts";
const fail = (status: number, message: string) => Response.json({ message }, { status });
/** One complete recording per request. Credentials and audio never go to disk. */
export async function openRouterTranscription(request: Request, key = keyValue("OPENROUTER_API_KEY")): Promise<Response> {
if (!key) return fail(503, "Dictation needs an OpenRouter key");
const params = new URL(request.url).searchParams;
if (params.get("model") !== OPENROUTER_STT_MODEL) return fail(400, "Unknown speech recognition model");
const language = params.get("language");
if (!isSttLanguage(language)) return fail(400, "Invalid speech recognition language");
if (request.headers.get("content-type") !== "audio/wav") return fail(415, "Invalid dictation audio");
if (Number(request.headers.get("content-length")) > STT_MAX_AUDIO_BYTES) return fail(413, "Recording exceeds OpenRouter's 25 MB audio limit");
if (!request.body) return fail(400, "Invalid dictation audio");
const reader = request.body.getReader();
const frames: Uint8Array[] = [];
let size = 0;
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
size += value.byteLength;
if (size > STT_MAX_AUDIO_BYTES) {
await reader.cancel();
return fail(413, "Recording exceeds OpenRouter's 25 MB audio limit");
}
frames.push(value);
}
} catch {
return fail(request.signal.aborted ? 499 : 400, "Could not read dictation audio");
} finally { reader.releaseLock(); }
if (request.signal.aborted) return fail(499, "Transcription cancelled");
const audio = Buffer.concat(frames, size);
// Accept exactly the lossless PCM container produced by our capture graph.
if (size <= WAV_HEADER_BYTES || size % 2 !== 0 || audio.toString("ascii", 0, 4) !== "RIFF"
|| audio.toString("ascii", 8, 12) !== "WAVE" || audio.toString("ascii", 12, 16) !== "fmt "
|| audio.readUInt32LE(4) !== size - 8 || audio.readUInt32LE(16) !== 16 || audio.readUInt16LE(20) !== 1
|| audio.readUInt16LE(22) !== 1 || audio.readUInt32LE(24) !== STT_SAMPLE_RATE
|| audio.readUInt32LE(28) !== STT_SAMPLE_RATE * 2 || audio.readUInt16LE(32) !== 2 || audio.readUInt16LE(34) !== 16
|| audio.toString("ascii", 36, 40) !== "data" || audio.readUInt32LE(40) !== size - WAV_HEADER_BYTES) {
return fail(400, "Invalid dictation audio");
}
const signal = AbortSignal.any([request.signal, AbortSignal.timeout(65_000)]);
try {
const response = await fetch("https://openrouter.ai/api/v1/audio/transcriptions", {
method: "POST",
headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" },
body: JSON.stringify({
model: OPENROUTER_STT_MODEL,
input_audio: { data: audio.toString("base64"), format: "wav" },
// MAI allows one forced language; automatic detection supports mixed speech.
...(language === "multi" ? {} : { language }),
response_format: "json",
}),
signal,
});
if (!response.ok) {
// Provider diagnostics can contain credentials or echoed audio. Return only app-authored errors.
await response.body?.cancel();
if (response.status === 401 || response.status === 403) return fail(502, "OpenRouter rejected the API key");
if (response.status === 402) return fail(502, "OpenRouter credits are insufficient");
if (response.status === 429) return fail(429, "OpenRouter rate limit reached; try again later");
return fail(502, `OpenRouter transcription failed (${response.status})`);
}
const result = await response.json() as { text?: unknown };
if (typeof result.text !== "string") return fail(502, "OpenRouter returned an invalid transcript");
return Response.json({ text: result.text });
} catch (error) {
if (request.signal.aborted) return fail(499, "Transcription cancelled");
if (signal.aborted || (error instanceof Error && error.name === "TimeoutError")) return fail(504, "OpenRouter transcription timed out");
return fail(502, "OpenRouter transcription is unavailable");
}
}
+1 -1
View File
@@ -71,7 +71,7 @@ afterEach(async () => { await act(async () => root?.unmount()); host?.remove();
test("Providers is the first/default page; General is last and other pages keep relative order", () => {
const html = renderToStaticMarkup(<SettingsDialogBody onClose={() => {}} />);
const ids = [...html.matchAll(/data-testid="settings-nav-([^"]+)"/g)].map((m) => m[1]);
assert.deepEqual(ids, ["keys", "agent", "appearance", "graph", "keyboard", "general"]);
assert.deepEqual(ids, ["keys", "agent", "appearance", "graph", "keyboard", "stt", "general"]);
assert.match(html, /settings-nav-keys[^>]*aria-current="true"/); assert.match(html, />Providers<\/button>/);
assert.doesNotMatch(html, />Model providers<\/button>/);
});
+256
View File
@@ -0,0 +1,256 @@
import React, { act } from "react";
(globalThis as Record<string, unknown>).React = React;
import assert from "node:assert/strict";
import { afterEach, it } from "node:test";
import { createRoot, type Root } from "react-dom/client";
import { parseHTML } from "linkedom";
import { I18nProvider, useI18n } from "../components/i18n.tsx";
import { KeysSection } from "../components/settings/keys-section.tsx";
import { DictationBar, finishDictation, isDictationActive, useDictation, useDictationTarget, type Dictation } from "../components/dictation.tsx";
import type { GeneralSettings } from "../src/core/settings.ts";
import { STT_MAX_AUDIO_BYTES } from "../src/core/stt.ts";
const dom = parseHTML("<html><body></body></html>");
Object.assign(globalThis, {
window: dom.window, document: dom.document, HTMLElement: dom.window.HTMLElement, Element: dom.window.Element,
CustomEvent: dom.window.CustomEvent, IS_REACT_ACT_ENVIRONMENT: true,
ResizeObserver: class { observe() {} disconnect() {} },
});
const originalFetch = globalThis.fetch;
let root: Root | null = null;
let host: HTMLDivElement;
let dictation: Dictation;
let setGeneral: (patch: Partial<GeneralSettings>) => Promise<void>;
let saved: GeneralSettings;
let transcripts: string[];
let questionTranscripts: string[];
let uploads: { url: URL; init: RequestInit; audio: Buffer }[];
let heldResponse: ((response: Response) => void) | undefined;
let trackStops = 0;
let configured = true;
let micGate: Promise<void> | undefined;
class MockWorklet {
static latest: MockWorklet;
disconnected = false;
port = {
onmessage: (_event: { data: unknown }) => {},
postMessage: (_message: unknown) => {
queueMicrotask(() => {
this.audio([3, 0]); // final partial frame, available only at Stop
this.port.onmessage({ data: { type: "flushed" } });
});
},
};
constructor() { MockWorklet.latest = this; }
connect() {}
disconnect() { this.disconnected = true; }
audio(samples: number[]) { this.port.onmessage({ data: { type: "audio", buffer: new Uint8Array(samples).buffer } }); }
}
class MockSocket {
static OPEN = 1; static CONNECTING = 0;
static all: MockSocket[] = [];
readyState = 0;
sent: unknown[] = [];
handlers = new Map<string, ((event: any) => void)[]>();
constructor(public url: string) {
MockSocket.all.push(this);
queueMicrotask(() => { this.readyState = 1; this.emit("open", {}); });
}
addEventListener(type: string, callback: (event: any) => void) { this.handlers.set(type, [...(this.handlers.get(type) ?? []), callback]); }
emit(type: string, event: unknown) { for (const handler of this.handlers.get(type) ?? []) handler(event); }
send(value: unknown) {
this.sent.push(value);
if (value === '{"type":"CloseStream"}') queueMicrotask(() => {
this.emit("message", { data: JSON.stringify({ type: "Results", is_final: true, channel: { alternatives: [{ transcript: "Deepgram final" }] } }) });
this.readyState = 3; this.emit("close", { code: 1000 });
});
}
close() { this.readyState = 3; this.emit("close", { code: 1000 }); }
}
Object.assign(globalThis, {
AudioWorkletNode: MockWorklet,
WebSocket: MockSocket,
AudioContext: class {
state = "running";
audioWorklet = { addModule: async () => {} };
destination = {};
createMediaStreamSource() { return { connect() {}, disconnect() {} }; }
},
});
Object.defineProperty(globalThis, "navigator", { configurable: true, value: {
languages: ["ru-RU"],
mediaDevices: { getUserMedia: async () => {
await micGate;
return { getTracks: () => [{ stop: () => { trackStops++; } }] };
} },
} });
async function mount(general: GeneralSettings = { language: "ru", sttModel: "microsoft/mai-transcribe-2", sttLanguage: "multi", sttPreferredLanguages: ["ru", "en"] }, question = false) {
saved = general; transcripts = []; questionTranscripts = []; uploads = []; heldResponse = undefined;
trackStops = 0; configured = true; micGate = undefined; MockSocket.all = [];
globalThis.fetch = async (input, init) => {
const url = String(input);
if (url === "/api/settings") return Response.json({ settings: { general: saved } });
if (url === "/api/settings/general") { saved = JSON.parse(String(init?.body)); return Response.json({ settings: { general: saved } }); }
if (url === "/api/stt/transcribe") return Response.json({ configured });
if (url === "/api/stt/token") return Response.json({ token: "test-deepgram-token" });
assert.equal(init?.method, "POST");
const parsed = new URL(url, "http://localhost");
assert.equal(parsed.pathname, "/api/stt/transcribe");
uploads.push({ url: parsed, init, audio: Buffer.from(init.body as Uint8Array) });
return new Promise<Response>((resolve, reject) => {
heldResponse = resolve;
init.signal?.addEventListener("abort", () => reject(new DOMException("cancelled", "AbortError")), { once: true });
});
};
host = document.createElement("div"); document.body.appendChild(host); root = createRoot(host);
function Harness() {
setGeneral = useI18n().saveGeneral;
dictation = useDictation({ append: (text) => transcripts.push(text) });
useDictationTarget((text) => questionTranscripts.push(text), question);
return <DictationBar dictation={dictation} />;
}
await act(async () => root!.render(<I18nProvider initialGeneral={general}><Harness /></I18nProvider>));
}
async function start() { await act(async () => dictation.toggle()); assert.equal(dictation.phase, "recording"); }
async function audio(bytes = [1, 0, 2, 0]) { await act(async () => MockWorklet.latest.audio(bytes)); }
async function stop() { await act(async () => dictation.stop()); }
async function respond(response = Response.json({ text: "Привет, hello!" })) { await act(async () => heldResponse!(response)); }
afterEach(async () => {
await act(async () => root?.unmount()); root = null; host?.remove();
globalThis.fetch = originalFetch;
});
it("uploads exactly one entire recording after Stop, includes its tail and inserts text only after completion", async () => {
await mount(); await start(); await audio(); await audio([4, 0]);
assert.equal(uploads.length, 0); assert.deepEqual(transcripts, []);
assert.equal(MockSocket.all.length, 0, "OpenRouter never opens a Deepgram socket");
await stop();
assert.equal(dictation.phase, "transcribing");
assert.equal(trackStops, 1, "microphone is off before uploading");
assert.equal(uploads.length, 1);
assert.deepEqual([...uploads[0]!.audio.subarray(44)], [1, 0, 2, 0, 4, 0, 3, 0]);
assert.equal(uploads[0]!.url.searchParams.get("model"), "microsoft/mai-transcribe-2");
assert.equal(uploads[0]!.url.searchParams.get("language"), "multi");
assert.equal(uploads[0]!.init.headers && new Headers(uploads[0]!.init.headers).get("Authorization"), null);
assert(document.querySelector('[data-testid="dictation-transcribing"]')?.textContent?.includes("Распознавание…"));
assert.deepEqual(transcripts, []);
await respond();
assert.equal(dictation.phase, "idle"); assert.deepEqual(transcripts, ["Привет, hello!"]);
assert.equal(isDictationActive(), false);
});
it("pins model and language at start and uses changed settings only on the next recording", async () => {
await mount(); await start(); await audio();
await act(async () => setGeneral({ sttLanguage: "en", sttModel: "deepgram/nova-3" }));
await stop();
assert.equal(uploads[0]!.url.searchParams.get("language"), "multi");
assert.equal(uploads[0]!.url.searchParams.get("model"), "microsoft/mai-transcribe-2");
await respond(); await start();
assert.equal(MockSocket.all.length, 1);
assert.equal(new URL(MockSocket.all[0]!.url).searchParams.get("language"), "en");
});
it("waits for the complete transcript before finishing a submit and targets a pending question", async () => {
await mount(undefined, true); await start(); await audio();
let finished = false;
let completion: Promise<void>;
await act(async () => { completion = finishDictation().then(() => { finished = true; }); });
assert.equal(finished, false); assert.deepEqual(questionTranscripts, []);
await respond(); await completion!;
assert.equal(finished, true); assert.deepEqual(transcripts, []);
assert.deepEqual(questionTranscripts, ["Привет, hello!"]);
});
it("cancel during recording discards audio without sending it", async () => {
await mount(); await start(); await audio();
await act(async () => dictation.cancel());
assert.equal(trackStops, 1); assert.equal(uploads.length, 0);
assert.equal(dictation.phase, "idle"); assert.deepEqual(transcripts, []);
});
it("cancel during transcription aborts the upload and ignores a late result", async () => {
await mount(); await start(); await audio(); await stop();
const signal = uploads[0]!.init.signal!;
await act(async () => dictation.cancel());
assert.equal(signal.aborted, true); await respond();
assert.deepEqual(transcripts, []); assert.equal(dictation.phase, "idle");
});
it("navigation during transcription aborts the upload without inserting text", async () => {
await mount(); await start(); await audio(); await stop();
await act(async () => root!.unmount()); root = null;
assert.equal(uploads[0]!.init.signal!.aborted, true);
assert.deepEqual(transcripts, []); assert.equal(isDictationActive(), false);
});
it("shows provider failures without inserting a false transcript, then allows another recording", async () => {
await mount(); await start(); await audio(); await stop();
await respond(Response.json({ message: "OpenRouter credits are insufficient" }, { status: 502 }));
assert.equal(dictation.error, "OpenRouter credits are insufficient");
assert(document.querySelector('[data-testid="dictation-toggle"]')?.textContent?.includes("Недостаточно средств"));
assert.deepEqual(transcripts, []);
await start(); assert.equal(dictation.error, null);
});
it("reports a missing OpenRouter key and shuts the microphone down", async () => {
await mount(); configured = false;
await act(async () => dictation.toggle());
assert.equal(dictation.phase, "idle");
assert.equal(dictation.error, "Dictation needs an OpenRouter key");
assert.equal(trackStops, 1); assert.equal(uploads.length, 0);
});
it("opens the existing Add-provider editor for a missing OpenRouter key and respects Cancel", async () => {
await mount();
const appFetch = globalThis.fetch;
globalThis.fetch = async (input, init) => String(input) === "/api/keys" ? Response.json({ keys: [], providers: [{ id: "openrouter", envVar: "OPENROUTER_API_KEY", label: "OpenRouter" }] }) : appFetch(input, init);
const render = async (initialRow?: string) => act(async () => root!.render(<I18nProvider initialGeneral={saved}><KeysSection initialRow={initialRow} /></I18nProvider>));
await render("OPENROUTER_API_KEY");
assert(document.querySelector('[data-testid="key-openrouter-input"]'));
await act(async () => document.querySelector<HTMLElement>('[data-testid="key-openrouter-cancel"]')!.click());
assert.equal(document.querySelector('[data-testid="key-openrouter-input"]'), null);
await render("OPENROUTER_API_KEY");
assert.equal(document.querySelector('[data-testid="key-openrouter-input"]'), null, "ordinary renders must not reopen a cancelled editor");
await render(undefined); await render("OPENROUTER_API_KEY");
assert(document.querySelector('[data-testid="key-openrouter-input"]'), "a new deep-link opens the editor again");
});
it("does not duplicate uploads when Stop is pressed twice before rendering", async () => {
await mount(); await start(); await audio();
await act(async () => { dictation.stop(); dictation.stop(); });
assert.equal(uploads.length, 1); await respond();
assert.deepEqual(transcripts, ["Привет, hello!"]);
});
it("stops oversized capture with an explicit error and never uploads a partial recording", async () => {
await mount(); await start();
await act(async () => MockWorklet.latest.port.onmessage({ data: { type: "audio", buffer: new Uint8Array(STT_MAX_AUDIO_BYTES).buffer } }));
assert.equal(dictation.phase, "idle");
assert.match(dictation.error!, /25 MB/);
assert.equal(trackStops, 1); assert.equal(uploads.length, 0);
});
it("cancels safely while permission is pending and stops a late microphone stream", async () => {
await mount(); let allowMic: () => void;
micGate = new Promise<void>((resolve) => { allowMic = resolve; });
await act(async () => dictation.toggle());
assert.equal(dictation.phase, "connecting");
await stop(); await act(async () => allowMic!());
assert.equal(trackStops, 1); assert.equal(uploads.length, 0); assert.equal(dictation.phase, "idle");
});
it("keeps Deepgram streaming audio during recording and appending its finals after Stop", async () => {
await mount({ sttModel: "deepgram/nova-3", sttLanguage: "multi", sttPreferredLanguages: ["ru", "en"] });
await start(); await audio();
const socket = MockSocket.all[0]!;
assert.equal(socket.sent.length, 1);
assert.deepEqual(new URL(socket.url).searchParams.getAll("language_hint"), ["ru", "en"]);
assert.equal(uploads.length, 0); assert.deepEqual(transcripts, []);
await stop(); assert.deepEqual(transcripts, ["Deepgram final"]);
assert.equal(dictation.phase, "idle");
});
+73 -13
View File
@@ -4,21 +4,26 @@ import assert from "node:assert/strict";
import { afterEach, it } from "node:test";
import { createRoot, type Root } from "react-dom/client";
import { parseHTML } from "linkedom";
import { I18nProvider, LanguageSelect, SttPreferredLanguages, useI18n } from "../components/i18n.tsx";
import { MENU_CLOSE_EVENT, MENU_OPEN_EVENT } from "../components/menu-shell.tsx";
import { dictationUrl, translate, type GeneralSettings, type SttLanguage } from "../src/core/index.ts";
import { useEnterSubmits } from "../components/enter-submits.ts";
const dom = parseHTML("<html><body></body></html>");
Object.assign(globalThis, {
window: dom.window, document: dom.document, Element: dom.window.Element,
HTMLElement: dom.window.HTMLElement, HTMLInputElement: dom.window.HTMLInputElement,
HTMLTextAreaElement: dom.window.HTMLTextAreaElement, CustomEvent: dom.window.CustomEvent,
Node: dom.window.Node, MutationObserver: dom.window.MutationObserver,
NodeFilter: { SHOW_ELEMENT: 1, FILTER_ACCEPT: 1, FILTER_SKIP: 3 },
ResizeObserver: class { observe() {} unobserve() {} disconnect() {} },
getComputedStyle: () => ({ display: "block", visibility: "visible", direction: "ltr", getPropertyValue: () => "" }),
IS_REACT_ACT_ENVIRONMENT: true,
requestAnimationFrame: (callback: FrameRequestCallback) => setTimeout(() => callback(performance.now()), 0),
cancelAnimationFrame: (id: number) => clearTimeout(id),
});
Object.defineProperty(globalThis, "navigator", { configurable: true, value: { languages: ["en-US"] } });
Object.assign(dom.window, { innerWidth: 1280, innerHeight: 800 });
const { I18nProvider, LanguageSelect, SttModelSelect, SttPreferredLanguages, useI18n } = await import("../components/i18n.tsx");
const originalFetch = globalThis.fetch;
let root: Root | null = null;
let host: HTMLDivElement;
@@ -28,6 +33,7 @@ let failSave = false;
let holdSave: (() => Promise<void>) | undefined;
let recording: (active: boolean) => void;
let resolvedSttLanguage: SttLanguage;
let submits = 0;
function element(selector: string): HTMLElement {
const found = document.querySelector<HTMLElement>(selector);
@@ -37,7 +43,7 @@ function element(selector: string): HTMLElement {
function trigger(field: string, compact = false) { return element(`[data-testid="language-select-${field}${compact ? "-compact" : ""}"]`); }
async function click(target: HTMLElement) {
await act(async () => {
target.dispatchEvent(new dom.window.Event("pointerdown", { bubbles: true }));
target.dispatchEvent(Object.assign(new dom.window.Event("pointerdown", { bubbles: true }), { button: 0, ctrlKey: false, pointerType: "mouse", isPrimary: true }));
target.dispatchEvent(new dom.window.Event("click", { bubbles: true }));
});
}
@@ -48,7 +54,7 @@ async function pick(text: string) {
}
async function mount(initialGeneral: GeneralSettings = { language: "ru", sttLanguage: "en", autoUpdate: false }) {
general = initialGeneral;
writes = []; failSave = false; holdSave = undefined;
writes = []; failSave = false; holdSave = undefined; submits = 0;
globalThis.fetch = async (input, init) => {
const url = String(input);
if ((init?.method ?? "GET") === "GET") {
@@ -65,9 +71,10 @@ async function mount(initialGeneral: GeneralSettings = { language: "ru", sttLang
};
host = document.createElement("div"); document.body.appendChild(host); root = createRoot(host);
function Controls() {
useEnterSubmits(() => { submits++; });
recording = useI18n().setDictating;
resolvedSttLanguage = useI18n().sttLanguage;
return <><LanguageSelect field="language" /><LanguageSelect field="sttLanguage" /><LanguageSelect field="sttLanguage" compact /><SttPreferredLanguages /></>;
return <><LanguageSelect field="language" /><SttModelSelect /><LanguageSelect field="sttLanguage" /><LanguageSelect field="sttLanguage" compact /><SttPreferredLanguages /></>;
}
await act(async () => { root!.render(<I18nProvider initialGeneral={general}><Controls /></I18nProvider>); });
}
@@ -75,6 +82,40 @@ afterEach(async () => {
await act(async () => { root?.unmount(); }); root = null; host?.remove(); globalThis.fetch = originalFetch;
});
it("selects only MAI or direct Deepgram, saves independently, and restores hints when returning to Deepgram", async () => {
await mount({ language: "ru", sttLanguage: "multi", sttPreferredLanguages: ["ru", "en"], autoUpdate: false });
const model = () => element('[data-testid="stt-model-select"]');
assert(model().textContent?.includes("Nova-3 · Deepgram"));
await click(model());
assert.equal(document.querySelectorAll('[role="menuitemradio"]').length, 2);
await pick("MAI-Transcribe-2");
assert.deepEqual(general, { language: "ru", sttLanguage: "multi", sttPreferredLanguages: ["ru", "en"], autoUpdate: false, sttModel: "microsoft/mai-transcribe-2" });
assert.equal(preferredTrigger().textContent, "English, Русский▾");
assert(preferredTrigger().disabled, "unsupported hints cannot be changed for MAI");
await click(trigger("language")); await pick("English");
assert.equal(general.sttModel, "microsoft/mai-transcribe-2");
assert.deepEqual(general.sttPreferredLanguages, ["ru", "en"]);
await click(model()); await pick("Nova-3 · Deepgram");
assert.equal(general.sttModel, "deepgram/nova-3");
assert(!preferredTrigger().disabled);
assert.deepEqual(general.sttPreferredLanguages, ["ru", "en"]);
});
it("locks model changes during recording and leaves the selected model unchanged if saving fails", async () => {
await mount();
const model = () => element('[data-testid="stt-model-select"]');
await click(model());
await act(async () => recording(true));
assert(model().hasAttribute("disabled"));
assert.equal(document.querySelector('[role="menu"]'), null);
await act(async () => recording(false));
failSave = true;
await click(model()); await pick("MAI-Transcribe-2");
assert(model().textContent?.includes("Nova-3 · Deepgram"));
assert.equal(general.sttModel, undefined);
assert(document.querySelector('[role="alert"]'));
});
it("picks languages through the shared menu and preserves the other language and update preference", async () => {
await mount();
assert.equal(document.querySelector("select"), null);
@@ -202,34 +243,42 @@ it("serializes rapid picks in independent selectors without losing either prefer
assert.equal(writes.length, 2);
});
function preferred(language: "en" | "ru") { return element(`[data-testid="stt-preferred-${language}"]`) as HTMLButtonElement; }
function preferred(language: "en" | "ru") { return element(`[data-testid="stt-preferred-${language}"]`); }
function preferredTrigger() { return element('[data-testid="stt-preferred-select"]') as HTMLButtonElement; }
async function openPreferred() { await click(preferredTrigger()); assert(document.querySelector('[data-testid="stt-preferred-menu"]')); }
async function closePreferred() { await click(preferredTrigger()); assert.equal(document.querySelector('[data-testid="stt-preferred-menu"]'), null); }
it("saves RU + EN hints, restores them after reload, and retains the choices in single-language mode", async () => {
await mount();
assert(preferred("en").disabled); assert(preferred("ru").disabled);
assert(preferredTrigger().disabled);
assert.equal(writes.length, 0);
await click(trigger("sttLanguage")); await pick("Несколько языков");
assert(!preferred("en").disabled);
assert(!preferredTrigger().disabled); await openPreferred();
await click(preferred("en")); await click(preferred("ru"));
assert(document.querySelector('[data-testid="stt-preferred-menu"]'), "checkbox picks keep the dropdown open");
assert.deepEqual(general, { language: "ru", sttLanguage: "multi", sttPreferredLanguages: ["en", "ru"], autoUpdate: false });
assert.deepEqual(new URL(dictationUrl(resolvedSttLanguage, general.sttPreferredLanguages)).searchParams.getAll("language_hint"), ["en", "ru"]);
const saved = general;
await act(async () => root!.unmount()); root = null; host.remove();
await mount(saved);
assert(preferredTrigger().textContent?.includes("English, Русский")); await openPreferred();
assert.equal(preferred("en").getAttribute("aria-checked"), "true");
assert.equal(preferred("ru").getAttribute("aria-checked"), "true");
await click(trigger("sttLanguage", true)); await pick("English");
assert(preferred("en").disabled); assert(preferred("ru").disabled);
await closePreferred(); await click(trigger("sttLanguage", true)); await pick("English");
assert(preferredTrigger().disabled);
assert.deepEqual(general.sttPreferredLanguages, ["en", "ru"]);
assert.deepEqual(new URL(dictationUrl(resolvedSttLanguage, general.sttPreferredLanguages)).searchParams.getAll("language_hint"), []);
await click(trigger("sttLanguage")); await pick("Несколько языков");
await openPreferred();
assert.equal(preferred("en").getAttribute("aria-checked"), "true");
await click(preferred("en")); await click(preferred("ru"));
assert.deepEqual(general.sttPreferredLanguages, []);
assert(preferredTrigger().textContent?.includes("Без предпочтений"));
});
it("locks preference edits while saving and recording, and reports a failed save without changing the choice", async () => {
await mount({ language: "ru", sttLanguage: "multi", autoUpdate: false });
await openPreferred();
failSave = true;
await click(preferred("en"));
assert.equal(preferred("en").getAttribute("aria-checked"), "false");
@@ -239,22 +288,23 @@ it("locks preference edits while saving and recording, and reports a failed save
let release!: () => void;
const pending = new Promise<void>((resolve) => { release = resolve; }); holdSave = () => pending;
await click(preferred("en"));
assert(preferred("en").disabled); assert(preferred("ru").disabled);
assert.equal(preferred("en").getAttribute("aria-disabled"), "true"); assert.equal(preferred("ru").getAttribute("aria-disabled"), "true");
await click(preferred("ru"));
assert.equal(writes.length, 2);
await act(async () => release()); holdSave = undefined;
assert.equal(element('[data-testid="stt-preferred-en"]').getAttribute("aria-checked"), "true");
assert.equal(document.querySelector('[role="alert"]'), null);
await act(async () => recording(true));
assert(preferred("en").disabled); assert(preferred("ru").disabled);
await click(preferred("en"));
assert(preferredTrigger().disabled); assert.equal(document.querySelector('[data-testid="stt-preferred-menu"]'), null);
await click(preferredTrigger());
assert.deepEqual(general.sttPreferredLanguages, ["en"]);
await act(async () => recording(false));
assert(!preferred("en").disabled);
assert(!preferredTrigger().disabled);
});
it("serializes hint and interface-language saves without losing either preference", async () => {
await mount({ language: "ru", sttLanguage: "multi", autoUpdate: false });
await openPreferred();
let release!: () => void;
const pending = new Promise<void>((resolve) => { release = resolve; }); holdSave = () => pending;
await click(preferred("ru"));
@@ -262,3 +312,13 @@ it("serializes hint and interface-language saves without losing either preferenc
await act(async () => release());
assert.deepEqual(general, { language: "en", sttLanguage: "multi", sttPreferredLanguages: ["ru"], autoUpdate: false });
});
it("keeps keyboard checkbox selection inside the menu without submitting the underlying composer", async () => {
await mount({ language: "en", sttLanguage: "multi" });
assert(preferredTrigger().textContent?.includes("No preference"));
await openPreferred();
await act(async () => preferred("en").dispatchEvent(Object.assign(new dom.window.Event("keydown", { bubbles: true, cancelable: true }), { key: "Enter" })));
assert.deepEqual(general.sttPreferredLanguages, ["en"]);
assert.equal(submits, 0);
assert(document.querySelector('[data-testid="stt-preferred-menu"]'));
});
+3 -3
View File
@@ -41,12 +41,12 @@ describe("SettingsDialog (the designed surface, SSR)", () => {
assert.ok(!html.includes("settings-dialog"));
});
it("the dialog frame: role=dialog + the ✕ + the six-section nav, Providers first and General last", () => {
it("the dialog frame: role=dialog + the ✕ + the seven-section nav, Providers first and General last", () => {
const html = renderToStaticMarkup(<SettingsDialogBody onClose={() => {}} />);
assert.ok(html.includes('role="dialog"'));
assert.ok(html.includes('aria-modal="true"'));
assert.ok(html.includes('aria-label="Close settings"'));
for (const id of ["general", "keys", "agent", "appearance", "graph", "keyboard"]) {
for (const id of ["general", "keys", "agent", "appearance", "graph", "keyboard", "stt"]) {
assert.ok(html.includes(`data-testid="settings-nav-${id}"`), id);
}
// Providers is the first landing section; internal keys deep links survive.
@@ -281,4 +281,4 @@ describe("KeyboardSection (thin SSR)", () => {
assert.ok(!KEYBOARD_ACTIONS.some((a) => a.id as string === "model-menu-walk"));
assert.ok(!KEYBOARD_ACTIONS.some((a) => a.id as string === "model-menu-pick"));
});
});
});
+1 -1
View File
@@ -49,7 +49,7 @@ describe("the settings store", () => {
assert.deepEqual(out.graph, { enabled: false });
});
it("persists multilingual STT independently of the interface language and update preference", () => {
const general = { language: "ru" as const, sttLanguage: "multi" as const, sttPreferredLanguages: ["en" as const, "ru" as const], autoUpdate: false };
const general = { language: "ru" as const, sttLanguage: "multi" as const, sttModel: "microsoft/mai-transcribe-2" as const, sttPreferredLanguages: ["en" as const, "ru" as const], autoUpdate: false };
writeSettings({ version: 1, general });
assert.deepEqual(JSON.parse(fs.readFileSync(SETTINGS_FILE, "utf-8")).general, general);
assert.deepEqual(readSettings()?.general, general);
+132
View File
@@ -0,0 +1,132 @@
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { afterEach, describe, it } from "node:test";
import { runInNewContext } from "node:vm";
import { DEFAULT_STT_MODEL, OPENROUTER_STT_MODEL, STT_MAX_AUDIO_BYTES, dictationWav } from "../src/core/stt.ts";
import { sanitizeUserSettings } from "../src/core/settings.ts";
import { resolveSettings } from "../src/core/settings-resolution.ts";
import { parseGeneralSettingsBody } from "../src/server/settings-body.ts";
import { openRouterTranscription } from "../src/server/stt.ts";
const originalFetch = globalThis.fetch;
afterEach(() => { globalThis.fetch = originalFetch; });
const frames = [new Uint8Array([0, 128, 255, 127]), new Uint8Array([1, 0, 255, 255])];
function request(language = "multi", extra: RequestInit = {}) {
return new Request(`http://localhost/api/stt/transcribe?model=${OPENROUTER_STT_MODEL}&language=${language}`, {
method: "POST", headers: { "Content-Type": "audio/wav" }, body: dictationWav(frames), ...extra,
});
}
describe("STT configuration and lossless capture", () => {
it("round-trips both STT models without changing languages, hints or update preferences", () => {
for (const sttModel of [DEFAULT_STT_MODEL, OPENROUTER_STT_MODEL]) {
const general = { sttModel, sttLanguage: "multi" as const, sttPreferredLanguages: ["en" as const, "ru" as const], language: "ru" as const, autoUpdate: false };
assert.deepEqual(parseGeneralSettingsBody(general), { ok: true, settings: general });
const saved = sanitizeUserSettings({ version: 1, general });
assert.deepEqual(saved?.general, general);
assert.deepEqual(resolveSettings(saved).general, general);
}
assert.deepEqual(sanitizeUserSettings({ version: 1, general: { sttModel: "other" } })?.general, {});
assert.equal(parseGeneralSettingsBody({ sttModel: "other" }).ok, false);
assert.equal(resolveSettings(null).general.sttModel, undefined, "old records retain the Deepgram default");
});
it("keeps every PCM sample in order, including short final frames", () => {
const wav = Buffer.from(dictationWav(frames));
assert.equal(wav.toString("ascii", 0, 4), "RIFF");
assert.equal(wav.readUInt32LE(4), wav.length - 8);
assert.equal(wav.readUInt32LE(24), 16000);
assert.equal(wav.readUInt16LE(22), 1);
assert.equal(wav.readUInt16LE(34), 16);
assert.equal(wav.readUInt32LE(40), 8);
assert.deepEqual([...wav.subarray(44)], frames.flatMap((frame) => [...frame]));
assert.throws(() => dictationWav([new Uint8Array(1)]), /Invalid dictation audio/);
assert.throws(() => dictationWav([new Uint8Array(STT_MAX_AUDIO_BYTES)]), /25 MB/);
});
it("drains the worklet's incomplete last frame exactly once", () => {
let Processor: new () => { process: (inputs: Float32Array[][]) => void; port: { onmessage: (event: unknown) => void } };
const messages: { type: string; buffer?: Int16Array }[] = [];
runInNewContext(readFileSync(new URL("../public/worklets/pcm-capture.js", import.meta.url), "utf8"), {
AudioWorkletProcessor: class { port = { postMessage: (message: { type: string; buffer?: Int16Array }) => messages.push(message) }; },
registerProcessor: (_name: string, processor: typeof Processor) => { Processor = processor; },
});
const processor = new Processor!();
processor.process([[new Float32Array([0, 1, -1])]]);
assert.equal(messages.filter((message) => message.type === "audio").length, 0);
processor.port.onmessage({ data: { type: "flush" } });
assert.deepEqual(Array.from(messages.find((message) => message.type === "audio")!.buffer!), [0, 32767, -32768]);
assert.equal(messages.at(-1)?.type, "flushed");
processor.port.onmessage({ data: { type: "flush" } });
assert.equal(messages.filter((message) => message.type === "audio").length, 1);
});
});
describe("OpenRouter whole-recording transcription", () => {
for (const language of ["en", "ru", "multi"]) it(`sends one complete WAV using server auth and the correct language (${language})`, async () => {
let calls = 0;
globalThis.fetch = async (input, init) => {
calls++;
assert.equal(String(input), "https://openrouter.ai/api/v1/audio/transcriptions");
assert.equal(new Headers(init?.headers).get("Authorization"), "Bearer test-only-key");
const body = JSON.parse(String(init?.body));
assert.equal(body.model, OPENROUTER_STT_MODEL);
assert.equal(body.response_format, "json");
assert.equal(body.input_audio.format, "wav");
assert.deepEqual(Buffer.from(body.input_audio.data, "base64"), Buffer.from(dictationWav(frames)));
assert.deepEqual(Object.keys(body).sort(), (language === "multi" ? ["model", "input_audio", "response_format"] : ["model", "input_audio", "response_format", "language"]).sort());
assert.equal(body.language, language === "multi" ? undefined : language);
return Response.json({ text: "Привет, hello!", usage: { cost: 0.01 } });
};
const response = await openRouterTranscription(request(language), "test-only-key");
assert.equal(response.status, 200);
assert.deepEqual(await response.json(), { text: "Привет, hello!" });
assert.equal(calls, 1);
});
it("refuses missing keys, unknown models and invalid languages without any provider request", async () => {
globalThis.fetch = async () => { throw new Error("must not reach provider"); };
assert.equal((await openRouterTranscription(request(), "")).status, 503);
assert.equal((await openRouterTranscription(new Request("http://localhost?model=other&language=en", { method: "POST" }), "test")).status, 400);
assert.equal((await openRouterTranscription(request("en,ru"), "test")).status, 400);
});
it("rejects invalid audio and enforces the upload limit before contacting OpenRouter", async () => {
globalThis.fetch = async () => { throw new Error("must not reach provider"); };
assert.equal((await openRouterTranscription(request("multi", { body: "not wav" }), "test")).status, 400);
assert.equal((await openRouterTranscription(request("multi", { headers: { "Content-Type": "audio/wav", "Content-Length": String(STT_MAX_AUDIO_BYTES + 1) } }), "test")).status, 413);
const damaged = dictationWav(frames); damaged[24] = 0;
assert.equal((await openRouterTranscription(request("multi", { body: damaged }), "test")).status, 400);
const stream = new ReadableStream({ start(controller) { controller.enqueue(new Uint8Array(STT_MAX_AUDIO_BYTES + 1)); controller.close(); } });
const streamingRequest = request("multi", { body: stream, duplex: "half" } as RequestInit);
assert.equal((await openRouterTranscription(streamingRequest, "test")).status, 413);
});
for (const status of [401, 403, 402, 429, 500]) it(`handles provider status ${status} without leaking diagnostics or retrying`, async () => {
let calls = 0;
globalThis.fetch = async () => { calls++; return new Response("test-only-key PRIVATE_AUDIO", { status }); };
const response = await openRouterTranscription(request(), "test-only-key");
assert.equal(response.status, status === 429 ? 429 : 502);
assert.doesNotMatch(await response.text(), /test-only-key|PRIVATE_AUDIO/);
assert.equal(calls, 1);
});
it("accepts an empty transcript for silence, but rejects malformed provider output", async () => {
globalThis.fetch = async () => Response.json({ text: "" });
assert.deepEqual(await (await openRouterTranscription(request(), "test")).json(), { text: "" });
globalThis.fetch = async () => Response.json({ text: null });
assert.equal((await openRouterTranscription(request(), "test")).status, 502);
});
it("reports timeouts and respects cancellation", async () => {
globalThis.fetch = async () => { throw new DOMException("timeout", "TimeoutError"); };
assert.equal((await openRouterTranscription(request(), "test")).status, 504);
const controller = new AbortController(); controller.abort();
assert.equal((await openRouterTranscription(request("multi", { signal: controller.signal }), "test")).status, 499);
});
it("propagates cancellation to an in-flight provider request", async () => {
let started: () => void;
const ready = new Promise<void>((resolve) => { started = resolve; });
globalThis.fetch = async (_input, init) => new Promise<Response>((_resolve, reject) => {
init!.signal!.addEventListener("abort", () => reject(new DOMException("cancelled", "AbortError")), { once: true });
started();
});
const controller = new AbortController();
const pending = openRouterTranscription(request("multi", { signal: controller.signal }), "test");
await ready; controller.abort();
assert.equal((await pending).status, 499);
});
});