diff --git a/README.md b/README.md index a9ff694..10c5aa7 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Future teaching sessions can read the graph instead of starting from scratch. Pr - **Windows (x64):** run the installer. It is currently unsigned; Windows may show an “unknown publisher” or SmartScreen warning. - **Linux (x64):** make the AppImage executable and open it, or install the Debian/Ubuntu `.deb`. -Add your provider API key and choose your models, then start a session with what you want to understand. LearnHarness supports 30+ provider options, including OpenAI, Anthropic, Google, DeepSeek, and OpenRouter. Optional dictation and web search use separate service keys. +Add your provider API key and choose your models, then start a session with what you want to understand. LearnHarness supports 30+ provider options, including OpenAI, Anthropic, Google, DeepSeek, and OpenRouter. Optional web search uses a separate service key. Dictation supports direct Deepgram Nova-3 streaming or MAI-Transcribe-2 using your OpenRouter API key; choose the speech recognition model in Settings → Speech recognition. All three platforms support in-app updates. diff --git a/app/api/stt/transcribe/route.ts b/app/api/stt/transcribe/route.ts new file mode 100644 index 0000000..8a71a0f --- /dev/null +++ b/app/api/stt/transcribe/route.ts @@ -0,0 +1,12 @@ +import { keyValue, openRouterTranscription } from "@/src/server"; + +export const runtime = "nodejs"; + +/** Check availability before recording; the token stays on the server. */ +export function GET() { + return Response.json({ configured: Boolean(keyValue("OPENROUTER_API_KEY")) }, { headers: { "Cache-Control": "no-store" } }); +} + +export async function POST(request: Request) { + return openRouterTranscription(request); +} diff --git a/components/dictation.tsx b/components/dictation.tsx index e1da5cc..bdcda42 100644 --- a/components/dictation.tsx +++ b/components/dictation.tsx @@ -3,14 +3,17 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { Loader2Icon, MicIcon } from "lucide-react"; import { useSettingsDialog } from "@/components/settings/settings-dialog"; -import { dictationUrl } from "@/src/core"; +import { DEFAULT_STT_MODEL, OPENROUTER_STT_MODEL, STT_MAX_AUDIO_BYTES, WAV_HEADER_BYTES, dictationUrl, dictationWav, type SttModel, type SttLanguage } from "@/src/core"; import { LanguageSelect, useI18n } from "@/components/i18n"; import { Hint } from "@/components/ui/hint"; /** - * Voice dictation — the cloud engine (STT wave 1, 2026-09-17). + * Voice dictation — direct Deepgram streaming or whole-recording OpenRouter STT. + * MAI-Transcribe-2 records locally, drains the final PCM frame on Stop, then + * sends one WAV through /api/stt/transcribe. That route holds the API key. + * Both engines buffer text and append once when dictation finishes. * - * Architecture (docs/research/stt.md): the renderer captures mic audio in + * Deepgram architecture: the renderer captures mic audio in * an AudioWorklet (public/worklets/pcm-capture.js → linear16 16kHz mono), * opens the Deepgram streaming WebSocket DIRECTLY (audio never loops * through our server) with a short-lived token minted by /api/stt/token @@ -107,6 +110,7 @@ export const DEEPGRAM_KEY_ID = "DEEPGRAM_API_KEY"; * instead. Exported for pins; the errored idle button's click recognizes * it and deep-links Settings → Keys → the Deepgram row. */ export const DICTATION_KEYLESS = "Dictation needs a Deepgram key"; +export const OPENROUTER_DICTATION_KEYLESS = "Dictation needs an OpenRouter key"; /** Mint a fresh short-lived token (the route holds the real key * server-side). Module-level: the start path's warm cache and the @@ -239,7 +243,7 @@ export function useDictationTarget(append: (text: string) => void, claimed: bool }, [append, claimed]); } -export type DictationPhase = "idle" | "connecting" | "recording" | "stopping"; +export type DictationPhase = "idle" | "connecting" | "recording" | "stopping" | "transcribing"; export type Dictation = { phase: DictationPhase; @@ -283,6 +287,14 @@ export function useDictation({ append }: { append: (text: string) => void }): Di const selectedUrl = useRef(""); selectedUrl.current = dictationUrl(sttLanguage, general.sttPreferredLanguages); const recordingUrl = useRef(selectedUrl.current); + const selected = useRef({ model: general.sttModel ?? DEFAULT_STT_MODEL, language: sttLanguage }); + selected.current = { model: general.sttModel ?? DEFAULT_STT_MODEL, language: sttLanguage }; + const recordingConfig = useRef<{ model: SttModel; language: SttLanguage }>(selected.current); + const audioFrames = useRef([]); + const audioBytes = useRef(0); + const upload = useRef(null); + const drain = useRef<(() => void) | null>(null); + const stopping = useRef(false); const [phase, setPhase] = useState("idle"); const [error, setError] = useState(null); /** Latest RMS from the worklet — the meter samples this (dictate pattern). */ @@ -303,7 +315,7 @@ export function useDictation({ append }: { append: (text: string) => void }): Di /** Frames captured before the socket is OPEN (mitigation A): flushed in * order on open; capped at 256 frames (≈32s) with the oldest dropped; * cleared at teardown so a dead run's audio can't leak into the next. */ - const pendingRef = useRef([]); + const pendingRef = useRef>([]); const genRef = useRef(0); // stale-handler guard (dictate pattern) const safetyRef = useRef | null>(null); const reconnectsRef = useRef(0); @@ -327,6 +339,13 @@ export function useDictation({ append }: { append: (text: string) => void }): Di /** Full teardown of one run's machinery. Idempotent; generation-guarded by callers. */ const cleanup = useCallback(() => { genRef.current++; + upload.current?.abort(); + upload.current = null; + drain.current?.(); + drain.current = null; + audioFrames.current = []; + audioBytes.current = 0; + stopping.current = false; dictationActiveFlag = false; setDictating(false); flush(); // the deferred transcript appears when the run ends (owner round 2) @@ -356,7 +375,7 @@ export function useDictation({ append }: { append: (text: string) => void }): Di for (const w of waitersRef.current.splice(0)) w(); // Pre-mint for the next run (mitigation D) — best-effort; a keyless // page just doesn't warm. - warmTokens.refresh(); + if (selected.current.model === DEFAULT_STT_MODEL) warmTokens.refresh(); }, []); /** Open the Deepgram socket and wire its messages. `gen` guards staleness. */ @@ -427,6 +446,11 @@ export function useDictation({ append }: { append: (text: string) => void }): Di const start = useCallback(async () => { // Pin the complete configuration once; reconnects must retain both language and hints. recordingUrl.current = selectedUrl.current; + recordingConfig.current = { ...selected.current }; + const batch = recordingConfig.current.model === OPENROUTER_STT_MODEL; + audioFrames.current = []; + audioBytes.current = 0; + stopping.current = false; setDictating(true); dictationActiveFlag = true; setError(null); @@ -442,7 +466,17 @@ export function useDictation({ append }: { append: (text: string) => void }): Di // the token runs concurrently with it. 127.0.0.1 is a secure // context, so getUserMedia works without TLS; macOS TCC prompts on // first use. - const tokenPromise = warmTokens.mint(); + const controller = new AbortController(); + upload.current = controller; + const tokenPromise = batch ? fetch("/api/stt/transcribe", { signal: controller.signal }).then(async (response) => { + if (!response.ok) throw new Error("OpenRouter transcription is unavailable"); + const status = await response.json() as { configured?: boolean }; + if (!status.configured) throw new Error(OPENROUTER_DICTATION_KEYLESS); + return ""; + }).catch((error: unknown) => { + if (error instanceof Error && error.message === OPENROUTER_DICTATION_KEYLESS) throw error; + throw new Error("OpenRouter transcription is unavailable"); + }) : warmTokens.mint(); tokenPromise.catch(() => {}); // surfaced below; no unhandled rejection const micPromise = navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true }, @@ -466,10 +500,26 @@ export function useDictation({ append }: { append: (text: string) => void }): Di const worklet = new AudioWorkletNode(ctx, "pcm-capture"); workletRef.current = worklet; worklet.port.onmessage = (e) => { - const msg = e.data as { type: string; rms?: number; buffer?: ArrayBuffer }; + if (gen !== genRef.current) return; + const msg = e.data as { type: string; rms?: number; buffer?: ArrayBuffer | Int16Array }; if (msg.type === "level") { levelRef.current = msg.rms ?? 0; // never reset to zero (dictate pattern) + } else if (msg.type === "flushed") { + drain.current?.(); + drain.current = null; } else if (msg.type === "audio") { + if (batch) { + const raw = msg.buffer!; + const bytes = raw instanceof ArrayBuffer ? new Uint8Array(raw) : new Uint8Array(raw.buffer, raw.byteOffset, raw.byteLength); + if (audioBytes.current + bytes.byteLength + WAV_HEADER_BYTES > STT_MAX_AUDIO_BYTES) { + setError("Recording exceeds OpenRouter's 25 MB audio limit"); + cleanup(); + return; + } + audioFrames.current.push(bytes); + audioBytes.current += bytes.byteLength; + return; + } const ws = wsRef.current; if (ws && ws.readyState === WebSocket.OPEN) ws.send(msg.buffer!); // Mitigation A: not OPEN yet (handshake or reconnect gap) — @@ -511,12 +561,15 @@ export function useDictation({ append }: { append: (text: string) => void }): Di if (gen !== genRef.current) return; // Mitigation B: the handshake fires the moment the graph is live, // overlapping any remaining token wait — mic-gated, never before it. - openSocket(gen, token); + if (batch) { if (!stopping.current) setPhase("recording"); } + else openSocket(gen, token); }, [openSocket, cleanup]); const stop = useCallback(() => { const gen = genRef.current; if (phase !== "recording" && phase !== "connecting") return; + if (stopping.current) return; + stopping.current = true; setPhase("stopping"); // Mic dies FIRST — no more audio enqueues (dictate pattern). The @@ -525,6 +578,43 @@ export function useDictation({ append }: { append: (text: string) => void }): Di streamRef.current = null; sourceRef.current?.disconnect(); sourceRef.current = null; + if (recordingConfig.current.model === OPENROUTER_STT_MODEL) { + const worklet = workletRef.current; + if (!worklet) { cleanup(); return; } + void (async () => { + try { + // Source is disconnected; drain its last partial PCM frame before wrapping the WAV. + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { drain.current = null; reject(new Error("Could not finish microphone capture")); }, 2000); + drain.current = () => { clearTimeout(timer); resolve(); }; + worklet.port.postMessage({ type: "flush" }); + }); + if (gen !== genRef.current) return; + worklet.disconnect(); + workletRef.current = null; + if (audioBytes.current === 0) { cleanup(); return; } + setPhase("transcribing"); + const wav = dictationWav(audioFrames.current); + audioFrames.current = []; + const params = new URLSearchParams(recordingConfig.current); + const response = await fetch(`/api/stt/transcribe?${params}`, { + method: "POST", headers: { "Content-Type": "audio/wav" }, body: wav, + signal: AbortSignal.any([upload.current!.signal, AbortSignal.timeout(75_000)]), + }); + const body = await response.json() as { text?: string; message?: string }; + if (gen !== genRef.current) return; + if (!response.ok) throw new Error(body.message ?? "OpenRouter transcription is unavailable"); + if (typeof body.text !== "string") throw new Error("OpenRouter returned an invalid transcript"); + if (body.text.trim()) bufferRef.current.push(body.text.trim()); + cleanup(); + } catch (error) { + if (gen !== genRef.current) return; + setError(error instanceof Error && error.name === "TimeoutError" ? "OpenRouter transcription timed out" : error instanceof Error && error.name === "Error" ? error.message : "OpenRouter transcription is unavailable"); + cleanup(); + } + })(); + return; + } workletRef.current?.disconnect(); workletRef.current = null; @@ -587,8 +677,13 @@ export function useDictation({ append }: { append: (text: string) => void }): Di // page owns ONE hook instance, and only the mounted page's listener is // attached. useEffect(() => { - warmTokens.refresh(); // pre-mint at mount (mitigation D) — silent if keyless - }, []); + if ((general.sttModel ?? DEFAULT_STT_MODEL) === DEFAULT_STT_MODEL) warmTokens.refresh(); + }, [general.sttModel]); + useEffect(() => () => { + // Navigation cancels a pending upload and releases the microphone without inserting text. + bufferRef.current = []; + cleanup(); + }, [cleanup]); useEffect(() => { window.addEventListener("lh:kbd-toggle-dictation", toggle); return () => window.removeEventListener("lh:kbd-toggle-dictation", toggle); @@ -631,9 +726,8 @@ export function DictationBar({ dictation }: { dictation: Dictation }) { * idle button deep-links Settings → Keys → the Deepgram row and clears * the error — the surface is clean when the user returns. Any other * error keeps the old behavior (click = retry, which clears). */ - const keyless = error === DICTATION_KEYLESS; + const keyless = error === DICTATION_KEYLESS || error === OPENROUTER_DICTATION_KEYLESS; const [levels, setLevels] = useState(() => new Array(METER_CELLS).fill(0)); - const recording = phase === "recording" || phase === "stopping"; /** Adaptive cell count (owner round 2): derived from the meter's * measured width — narrow rows show fewer bars instead of * overflowing under neighbors. */ @@ -691,12 +785,12 @@ export function DictationBar({ dictation }: { dictation: Dictation }) { onClick={() => { if (keyless) { clearError(); - openSettings("keys", DEEPGRAM_KEY_ID); + openSettings("keys", error === OPENROUTER_DICTATION_KEYLESS ? "OPENROUTER_API_KEY" : DEEPGRAM_KEY_ID); } else { toggle(); } }} - aria-label={error ? t("Dictation failed: {v0}", { v0: error }) : t("Start dictation")} + aria-label={error ? t("Dictation failed: {v0}", { v0: t(error) }) : t("Start dictation")} data-testid="dictation-toggle" // Same blurring as the floating chrome (owner, fix wave round 3), // but lower opacity — the mic stays quieter than the model pills. @@ -710,6 +804,10 @@ export function DictationBar({ dictation }: { dictation: Dictation }) { ); } + if (phase === "transcribing") return
+ {t("Transcribing…")} + +
; return (
diff --git a/components/i18n.tsx b/components/i18n.tsx index b9df373..ec1dd22 100644 --- a/components/i18n.tsx +++ b/components/i18n.tsx @@ -1,10 +1,11 @@ "use client"; import { createContext, useCallback, useContext, useEffect, useId, useMemo, useRef, useState, type ReactNode } from "react"; -import { resolveLanguage, resolveSttLanguage, translator, type GeneralSettings, type Language, type SttLanguage, type Translator } from "@/src/core"; +import { DEFAULT_STT_MODEL, OPENROUTER_STT_MODEL, STT_MODELS, resolveLanguage, resolveSttLanguage, translator, type GeneralSettings, type Language, type SttLanguage, type SttModel, type Translator } from "@/src/core"; import { notifySettingsSaved, useSettingsSaved } from "./settings-signal"; -import { MenuShell } from "./menu-shell"; +import { MENU_CLOSE_EVENT, MENU_OPEN_EVENT, MenuShell } from "./menu-shell"; import { CheckIcon } from "lucide-react"; +import { DropdownMenu } from "radix-ui"; type LanguageField = "language" | "sttLanguage"; type I18nValue = { @@ -58,7 +59,7 @@ export function I18nProvider({ children, initialGeneral = {}, system = "en" }: { if (!current.ok) throw new Error("Could not load language settings"); const body = await current.json() as { settings?: { general?: GeneralSettings } }; const next = { ...(body.settings?.general ?? latest.current) }; - for (const field of ["language", "sttLanguage", "sttPreferredLanguages", "autoUpdate"] as const) { + for (const field of ["language", "sttLanguage", "sttModel", "sttPreferredLanguages", "autoUpdate"] as const) { if (!Object.hasOwn(patch, field)) continue; if (patch[field] === undefined) delete next[field]; else Object.assign(next, { [field]: patch[field] }); @@ -77,21 +78,27 @@ export function I18nProvider({ children, initialGeneral = {}, system = "en" }: { return {children}; } -export function LanguageSelect({ field, disabled = false, compact = false }: { field: "language" | "sttLanguage"; disabled?: boolean; compact?: boolean }) { - const { general, t, saveLanguage, sttLanguage, dictating } = useI18n(); +export function LanguageSelect(props: { field: LanguageField; disabled?: boolean; compact?: boolean }) { + return ; +} + +export function SttModelSelect() { return ; } + +function GeneralSelect({ field, disabled = false, compact = false }: { field: LanguageField | "sttModel"; disabled?: boolean; compact?: boolean }) { + const { general, t, saveGeneral, sttLanguage, dictating } = useI18n(); const [saving, setSaving] = useState(false); const [error, setError] = useState(null); const [open, setOpen] = useState(false); const trigger = useRef(null); const menuId = useId(); - const locked = disabled || saving || (field === "sttLanguage" && dictating); + const locked = disabled || saving || (field !== "language" && dictating); const menuOpen = open && !locked; const compactStt = compact && field === "sttLanguage"; - const saved = general[field] ?? "auto"; + const saved = general[field] ?? (field === "sttModel" ? DEFAULT_STT_MODEL : "auto"); const selected = compactStt && saved === "auto" ? sttLanguage : saved; - const label = t(field === "language" ? "Interface language" : "Speech recognition language"); - const testid = `language-select-${field}${compact ? "-compact" : ""}`; - const options = [ + const label = t(field === "language" ? "Interface language" : field === "sttModel" ? "Speech recognition model" : "Speech recognition language"); + const testid = field === "sttModel" ? "stt-model-select" : `language-select-${field}${compact ? "-compact" : ""}`; + const options: readonly { value: string; label: string }[] = field === "sttModel" ? STT_MODELS.map((model) => ({ value: model.id, label: model.name })) : [ ...(!compactStt ? [{ value: "auto", label: t("System default") } as const] : []), { value: "en", label: "English" }, { value: "ru", label: "Русский" }, @@ -119,17 +126,18 @@ export function LanguageSelect({ field, disabled = false, compact = false }: { f document.removeEventListener("keydown", escape, true); }; }, [menuOpen, menuId]); - const pick = (value: "auto" | SttLanguage) => { + const pick = (value: string) => { if (locked) return; setOpen(false); trigger.current?.focus(); if (value === saved) return; setSaving(true); setError(null); - void saveLanguage(field, value === "auto" ? undefined : value).catch((e: unknown) => setError(e instanceof Error ? e.message : "Could not save language settings")).finally(() => setSaving(false)); + const patch = field === "sttModel" ? { sttModel: value as SttModel } : { [field]: value === "auto" ? undefined : value }; + void saveGeneral(patch).catch((e: unknown) => setError(e instanceof Error ? e.message : "Could not save language settings")).finally(() => setSaving(false)); }; - return + return - {menuOpen && + {menuOpen &&