diff --git a/app/api/stt/token/route.ts b/app/api/stt/token/route.ts new file mode 100644 index 0000000..3ff213f --- /dev/null +++ b/app/api/stt/token/route.ts @@ -0,0 +1,46 @@ +/** + * POST-less BYOK mint: the renderer never holds the real Deepgram key. + * + * The key lives server-side only (auth.json → process.env in the desktop + * app; .env.local in dev — same seam as OPENROUTER_API_KEY). A dictation + * run starts here: mint a short-lived access token (TTL 300s — a run can + * outlast the 30s default), the renderer opens its WebSocket to Deepgram + * with it, and audio flows renderer → Deepgram directly, never through + * this server. The token is only checked at the WebSocket handshake, so + * long dictation sessions survive the TTL fine (Deepgram docs). + * + * NOTE: the key needs ≥ Member permission in the Deepgram console to call + * /auth/grant (their docs, token-based auth FAQ) — plain read/transcribe + * keys return FORBIDDEN here. + */ +export async function GET() { + const key = process.env.DEEPGRAM_API_KEY; + if (!key) { + return Response.json( + { message: "No DEEPGRAM_API_KEY — add it to auth.json (desktop) or .env.local (dev)." }, + { status: 503 }, + ); + } + + try { + const res = await fetch("https://api.deepgram.com/v1/auth/grant", { + method: "POST", + headers: { Authorization: `Token ${key}`, "Content-Type": "application/json" }, + body: JSON.stringify({ ttl_seconds: 300 }), + }); + if (!res.ok) { + const detail = await res.text(); + return Response.json( + { message: `Deepgram token grant failed (${res.status}): ${detail.slice(0, 200)}` }, + { status: 502 }, + ); + } + const data = (await res.json()) as { token?: string }; + if (!data.token) { + return Response.json({ message: "Deepgram grant returned no token" }, { status: 502 }); + } + return Response.json({ token: data.token }); + } catch { + return Response.json({ message: "Deepgram unreachable" }, { status: 502 }); + } +} diff --git a/components/chat.tsx b/components/chat.tsx index 1a110e6..d3e0a7b 100644 --- a/components/chat.tsx +++ b/components/chat.tsx @@ -19,6 +19,7 @@ import { AnsweredCard } from "@/components/answered-card"; import { artifactRenderers } from "@/components/artifacts"; import { SubagentSteerCard } from "@/components/artifacts/steer-card"; import { Composer } from "@/components/composer"; +import { DictationBar, useDictation } from "@/components/dictation"; import { JobsContext } from "@/components/jobs-context"; import { StatsBar } from "@/components/stats-bar"; import { GraphContextMarker } from "@/components/graph-context-marker"; @@ -246,6 +247,12 @@ export function Chat({ (messages.at(-1)?.parts.some((p) => isToolUIPart(p) && p.state === "input-streaming") ?? false); const [text, setText] = useState(""); + // Voice dictation (STT wave, 2026-09-17): finals append at the end of + // the current text (the page owns the separator, dictate-flush style); + // interims ghost in the stats bar's middle slot next to the meter. + const dictation = useDictation({ + append: (t) => setText((v) => (v ? v.replace(/\s+$/, "") + " " + t : t)), + }); // Jobs never gate the composer (contracts.ts "THE ASYNC PROTOCOL" §4): job // tools resolve with an ack mid-turn, so the learner can interject while // sub-agents run — a feature, not a race: the wake pipeline queues steers @@ -864,6 +871,7 @@ export function Chat({ phase={phases.at(-1) ?? "probe"} turns={turns} onReserve={setDockReserve} + dictationSlot={} composer={ stop()} canSend={canSend} + onEscape={dictation.active ? dictation.stop : undefined} verdict={ planVerdict ? { @@ -940,6 +949,7 @@ function DockOverlay({ onReserve, plan, onOpenPlan, + dictationSlot, }: { pending: ReturnType; /** Teach-phase pending question: arrives collapsed behind the reveal button (W5). */ @@ -962,6 +972,9 @@ function DockOverlay({ plan?: React.ReactNode; /** Opens the full-DAG popup (the marginalia + chip share it). */ onOpenPlan: () => void; + /** The dictation strip (STT wave): rendered in the stats bar's middle + * slot — the mic button + live meter (components/dictation.tsx). */ + dictationSlot?: React.ReactNode; }) { const { isAtBottom, scrollToBottom, scrollRef } = useStickToBottomContext(); // Teach-phase mount (session reload with a pending check): the dock starts @@ -1226,6 +1239,7 @@ function DockOverlay({ chatId={chatId} settings={settings} onSettingsSaved={onSettingsSaved} + dictationSlot={dictationSlot} /> {/* The plan marginalia (owner, plan-placement round): the bare diff --git a/components/composer.tsx b/components/composer.tsx index ea1dffa..ed21548 100644 --- a/components/composer.tsx +++ b/components/composer.tsx @@ -20,6 +20,7 @@ export function Composer({ canSend, autoFocus, verdict, + onEscape, }: { busy: boolean; value: string; @@ -32,6 +33,9 @@ export function Composer({ * word, the send square becomes two stacked buttons — green accept, * red decline. Typed text rides the verdict as his note. */ verdict?: { onAccept: () => void; onDecline: () => void }; + /** Escape while a dictation runs stops it (STT wave, 2026-09-17). No-op + * when absent — Esc otherwise means nothing in the composer. */ + onEscape?: () => void; }) { return (
onChange(e.target.value)} onKeyDown={(e) => { + // Esc ends dictation before anything else Esc might someday mean + // here (the dictate decisions: click-again or Esc to stop). + if (e.key === "Escape" && onEscape) { + e.preventDefault(); + onEscape(); + return; + } // Chat convention: Enter sends, Shift+Enter is a newline. // ⌘/Ctrl+Enter also sends — the universal submit shortcut // across every input in the app (thinking field, answers). diff --git a/components/dictation.tsx b/components/dictation.tsx new file mode 100644 index 0000000..a7ec9bc --- /dev/null +++ b/components/dictation.tsx @@ -0,0 +1,373 @@ +"use client"; + +import { useCallback, useEffect, useRef, useState } from "react"; +import { Loader2Icon, MicIcon } from "lucide-react"; + +/** + * Voice dictation — the cloud engine (STT wave 1, 2026-09-17). + * + * Architecture (docs/research/stt.md): the renderer captures mic audio in + * an AudioWorklet (public/worklets/pcm-capture.js → linear16 16kHz mono), + * opens the Deepgram streaming WebSocket DIRECTLY (audio never loops + * through our server) with a short-lived token minted by /api/stt/token + * (the real key stays server-side), and streams frames while it listens. + * Finals append into the composer; interims ghost beside the meter. + * + * Ported patterns from the owner's pi `dictate` extension (his words: + * "very well tuned") — see ~/.pi/agent/extensions/dictate: + * - generation counter: every start/stop increments it; all async + * handlers no-op when stale, so a dying session's socket can never + * tear down the current one (a real bug he hit in the terminal). + * - meter: latest-RMS-never-resets — the level ref keeps the last + * observed value, so a tick with no fresh audio re-renders the last + * level instead of dropping to silence (60ms tick samples the ref). + * - dB mapping floor −50 / ceiling −10: the sweet spot for showing + * conversational speech as mid-height bars without pegging. + * - stop: mic dies FIRST (no more audio enqueues), then CloseStream, + * then a 3s safety net in case Deepgram never closes the socket. + * + * Meter rendering lives in (own 60ms timer over the level + * ref) so the 16-renders/sec never touch the page component — only the + * tiny bar strip repaints. + */ + +/* + * Deepgram streaming params (tuned in the owner's real-use dictate + * extension + the research pass): nova-3, his endpointing=300 (faster + * finals after a pause), utterance_end_ms=1000 as the noise-robust + * backstop, interim_results for the live ghost. Silence streams too + * (the worklet never pauses), so the 10s idle timeout can't bite — no + * KeepAlive needed. + */ +const DG_URL = + "wss://api.deepgram.com/v1/listen" + + "?model=nova-3" + + "&encoding=linear16" + + "&sample_rate=16000" + + "&channels=1" + + "&interim_results=true" + + "&smart_format=true" + + "&punctuate=true" + + "&endpointing=300" + + "&utterance_end_ms=1000"; + +/** Meter tuning — the owner's tested values, see the file header. */ +const METER_FLOOR_DB = -50; +const METER_CEILING_DB = -10; +export const METER_TICK_MS = 60; +/** Rolling history: 48 cells × 60ms ≈ 2.9s of recent loudness. */ +export const METER_CELLS = 48; + +/** RMS (0..1) → normalized 0..1 across the audible dB window. */ +export function rmsToMeter(rms: number): number { + if (rms <= 0) return 0; + const db = 20 * Math.log10(rms); + const t = (db - METER_FLOOR_DB) / (METER_CEILING_DB - METER_FLOOR_DB); + return Math.max(0, Math.min(1, t)); +} + +export type DictationPhase = "idle" | "connecting" | "recording" | "stopping"; + +export type Dictation = { + phase: DictationPhase; + /** Terminal-state explanation surfaced next to the mic (cleared on the next start). */ + error: string | null; + /** Live in-flight partial (the ghost); cleared when the final commits. */ + interim: string; + /** Latest observed RMS, 0..1 — sampled by the meter's own tick, never reset. */ + levelRef: { current: number }; + /** Recording or finalizing — true from start to stop completion. */ + active: boolean; + toggle: () => void; + stop: () => void; +}; + +/** + * One dictation instance per composer-bearing page. `append` receives each + * committed utterance (final) — the page joins it into its text state + * (separator handling is the page's: it owns the current text). + */ +export function useDictation({ append }: { append: (text: string) => void }): Dictation { + const [phase, setPhase] = useState("idle"); + const [error, setError] = useState(null); + const [interim, setInterim] = useState(""); + /** Latest RMS from the worklet — the meter samples this (dictate pattern). */ + const levelRef = useRef(0); + + // Live machinery (non-render values). AudioContext/MediaStream are + // closed on stop; the socket is recreated per run. + const wsRef = useRef(null); + const ctxRef = useRef(null); + const streamRef = useRef(null); + const genRef = useRef(0); // stale-handler guard (dictate pattern) + const safetyRef = useRef | null>(null); + const reconnectsRef = useRef(0); + + /** Full teardown of one run's machinery. Idempotent; generation-guarded by callers. */ + const cleanup = useCallback(() => { + genRef.current++; + if (safetyRef.current) { + clearTimeout(safetyRef.current); + safetyRef.current = null; + } + for (const track of streamRef.current?.getTracks() ?? []) track.stop(); + streamRef.current = null; + void ctxRef.current?.close().catch(() => {}); + ctxRef.current = null; + const ws = wsRef.current; + wsRef.current = null; + if (ws && (ws.readyState === WebSocket.OPEN || ws.readyState === WebSocket.CONNECTING)) { + try { + ws.close(); + } catch {} + } + setInterim(""); + setPhase("idle"); + }, []); + + /** Mint a short-lived token — the run's first step, BEFORE the mic: a + * keyless app fails fast without ever triggering the macOS TCC prompt. */ + const mintToken = useCallback(async () => { + const res = await fetch("/api/stt/token"); + const data = (await res.json()) as { token?: string; message?: string }; + if (!res.ok || !data.token) throw new Error(data.message ?? `token route ${res.status}`); + return data.token; + }, []); + + /** Open the Deepgram socket and wire its messages. `gen` guards staleness. */ + const openSocket = useCallback( + (gen: number, token: string) => { + const ws = new WebSocket(DG_URL, ["token", token]); + wsRef.current = ws; + + ws.addEventListener("open", () => { + if (gen !== genRef.current) return; + setPhase("recording"); + }); + ws.addEventListener("message", (ev) => { + if (gen !== genRef.current) return; + let msg: any; + try { + msg = JSON.parse(ev.data as string); + } catch { + return; + } + if (msg.type === "Results") { + const text: string | undefined = msg.channel?.alternatives?.[0]?.transcript; + if (msg.is_final) { + // The committed utterance: hand to the page, clear the ghost. + if (text) append(text); + setInterim(""); + } else if (text) { + setInterim(text); + } + } else if (msg.type === "UtteranceEnd") { + // Backstop: the endpointing VAD missed, the word-gap one didn't. + setInterim(""); + } + }); + ws.addEventListener("close", (ev) => { + if (gen !== genRef.current) return; + if (ev.code === 1000) { + // Our own CloseStream landing: finalize now — the stop path's + // safety net stays as the no-close backstop only. + cleanup(); + return; + } + // Abnormal close mid-run: ONE reconnect with a fresh token (the + // research pass: resume is our job, no SDK to lean on). + if (reconnectsRef.current < 1) { + reconnectsRef.current++; + mintToken() + .then((t) => { + if (gen === genRef.current) openSocket(gen, t); + }) + .catch(() => { + if (gen === genRef.current) { + setError("Deepgram reconnect failed"); + cleanup(); + } + }); + } else { + setError(`Deepgram connection lost (${ev.code})`); + cleanup(); + } + }); + }, + [append, cleanup, mintToken], + ); + + const start = useCallback(async () => { + setError(null); + setPhase("connecting"); + reconnectsRef.current = 0; + const gen = ++genRef.current; + + // 1. Token first — fail fast, no TCC prompt for a keyless app. + try { + const token = await mintToken(); + if (gen !== genRef.current) return; + openSocket(gen, token); + } catch (e) { + if (gen !== genRef.current) return; + setError(e instanceof Error ? e.message : "token fetch failed"); + setPhase("idle"); + return; + } + + try { + // 2. Mic + capture graph. 127.0.0.1 is a secure context — + // getUserMedia works without TLS (research); macOS TCC prompts + // on first use. + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true, autoGainControl: true }, + }); + if (gen !== genRef.current) { + for (const track of stream.getTracks()) track.stop(); + return; + } + streamRef.current = stream; + + const ctx = new AudioContext({ sampleRate: 16000 }); + ctxRef.current = ctx; + await ctx.audioWorklet.addModule("/worklets/pcm-capture.js"); + + const worklet = new AudioWorkletNode(ctx, "pcm-capture"); + worklet.port.onmessage = (e) => { + const msg = e.data as { type: string; rms?: number; buffer?: ArrayBuffer }; + if (msg.type === "level") { + levelRef.current = msg.rms ?? 0; // never reset to zero (dictate pattern) + } else if (msg.type === "audio") { + const ws = wsRef.current; + if (ws && ws.readyState === WebSocket.OPEN) ws.send(msg.buffer!); + } + }; + ctx.createMediaStreamSource(stream).connect(worklet); + // Pull the graph with silence out (the worklet writes no output) — + // an unconnected worklet node is never clocked. + worklet.connect(ctx.destination); + } catch (e) { + if (gen !== genRef.current) return; + const msg = + e instanceof DOMException && (e.name === "NotAllowedError" || e.name === "SecurityError") + ? "Microphone permission denied" + : e instanceof Error + ? e.message + : "mic unavailable"; + setError(msg); + cleanup(); + } + }, [openSocket, cleanup, mintToken]); + + const stop = useCallback(() => { + const gen = genRef.current; + if (phase !== "recording" && phase !== "connecting") return; + setPhase("stopping"); + + // Mic dies FIRST — no more audio enqueues (dictate pattern). + for (const track of streamRef.current?.getTracks() ?? []) track.stop(); + streamRef.current = null; + void ctxRef.current?.close().catch(() => {}); + ctxRef.current = null; + + const ws = wsRef.current; + if (ws && ws.readyState === WebSocket.OPEN) { + try { + ws.send(JSON.stringify({ type: "CloseStream" })); + } catch { + cleanup(); + return; + } + // Deepgram flushes remaining finals then closes; if it never + // does, the safety net ends the run anyway (dictate pattern). + safetyRef.current = setTimeout(() => { + if (gen === genRef.current) cleanup(); + }, 3000); + } else { + cleanup(); + } + }, [phase, cleanup]); + + const toggle = useCallback(() => { + if (phase === "idle") void start(); + else if (phase === "recording" || phase === "connecting") stop(); + // "stopping" presses are ignored — the flush is in flight. + }, [phase, start, stop]); + + return { phase, error, interim, levelRef, active: phase !== "idle", toggle, stop }; +} + +/** + * The stats-bar dictation slot (owner placement, 2026-09-17): the mic + * button rides the middle of the stats row — idle it is just the quiet mic + * affordance; recording, the meter unfurls to fill the free middle space + * (width follows what the row has), with the live interim ghost at its + * flank. Red dot + red mic while live; spinner while Deepgram flushes. + */ +export function DictationBar({ dictation }: { dictation: Dictation }) { + const { phase, error, interim, levelRef, toggle, active } = dictation; + const [levels, setLevels] = useState(() => new Array(METER_CELLS).fill(0)); + const recording = phase === "recording" || phase === "stopping"; + + // The meter's own heartbeat — 60ms samples of the level ref into a + // rolling ring. Only THIS strip re-renders; the page never does. + useEffect(() => { + if (!recording) return; + setLevels(new Array(METER_CELLS).fill(0)); + const id = setInterval(() => { + setLevels((ring) => { + const next = ring.slice(1); + next.push(rmsToMeter(levelRef.current)); + return next; + }); + }, METER_TICK_MS); + return () => clearInterval(id); + }, [recording, levelRef]); + + return ( +
+ + {error && ( + + {error} + + )} + {!error && recording && ( + <> + + {/* Bars flex to the row's free width — the owner's "fill up + the whole middle section" ask: pitch is whatever space + exists between session data and the model control. */} +
+ {levels.map((v, i) => ( + + ))} +
+ {interim && ( + + {interim} + + )} + + )} +
+ ); +} diff --git a/components/home.tsx b/components/home.tsx index e7a5c1e..9005c75 100644 --- a/components/home.tsx +++ b/components/home.tsx @@ -5,6 +5,7 @@ import { useRouter } from "next/navigation"; import { CornerDownLeftIcon } from "lucide-react"; import { GraphView } from "@/components/graph-view"; import { GraphContextControl, effectiveContextTokens } from "@/components/graph-context-control"; +import { DictationBar, useDictation } from "@/components/dictation"; import { SidebarToggle } from "@/components/sidebar"; import { StatsBar } from "@/components/stats-bar"; import type { ChatSettings, TurnUsage } from "@/src/core"; @@ -18,6 +19,11 @@ import type { ChatSettings, TurnUsage } from "@/src/core"; export function Home() { const router = useRouter(); const [topic, setTopic] = useState(""); + // Voice dictation (STT wave, 2026-09-17): finals append at the end of the + // topic draft; the strip lives in the stats bar's middle slot. + const dictation = useDictation({ + append: (t) => setTopic((v) => (v ? v.replace(/\s+$/, "") + " " + t : t)), + }); const [busy, setBusy] = useState(false); // The model+thinking selection made pre-chat — lifted from the StatsBar's // ModelControl, carried to the chat POST then PUT-on-create (the model @@ -97,6 +103,12 @@ export function Home() { value={topic} onChange={(e) => setTopic(e.target.value)} onKeyDown={(e) => { + // Esc ends dictation (the dictate decisions: click-again or Esc). + if (e.key === "Escape" && dictation.active) { + e.preventDefault(); + dictation.stop(); + return; + } if (e.key === "Enter" && !e.shiftKey) { e.preventDefault(); void start(); @@ -121,6 +133,7 @@ export function Home() { budget (session-data slot), while the RIGHT keeps the model + thinking control (deferred save). */} } turns={[] as TurnUsage[]} chatId="" settings={settings} diff --git a/components/new-session.tsx b/components/new-session.tsx index 3c66e76..d06e24f 100644 --- a/components/new-session.tsx +++ b/components/new-session.tsx @@ -4,6 +4,7 @@ import { useCallback, useState } from "react"; import { useRouter } from "next/navigation"; import { Conversation, ConversationContent, ConversationEmptyState } from "@/components/ai-elements/conversation"; import { Composer } from "@/components/composer"; +import { DictationBar, useDictation } from "@/components/dictation"; import { GraphContextControl, effectiveContextTokens } from "@/components/graph-context-control"; import { PhaseChip } from "@/components/phase-divider"; import { SidebarToggle } from "@/components/sidebar"; @@ -74,6 +75,11 @@ export function DraftShell({ onSettingsSaved: (s: ChatSettings) => void; }) { const canSend = text.trim().length > 0; + // Voice dictation (STT wave, 2026-09-17): finals append at the end of the + // draft text; the strip lives in the stats bar's middle slot. + const dictation = useDictation({ + append: (t) => onChange(text ? text.replace(/\s+$/, "") + " " + t : t), + }); return (
@@ -110,9 +116,11 @@ export function DraftShell({ onStop={() => {}} canSend={canSend} autoFocus + onEscape={dictation.active ? dictation.stop : undefined} />
} turns={[]} chatId="" settings={settings} diff --git a/components/stats-bar.tsx b/components/stats-bar.tsx index 2520325..8fc5f55 100644 --- a/components/stats-bar.tsx +++ b/components/stats-bar.tsx @@ -33,6 +33,7 @@ export function StatsBar({ settings, onSettingsSaved, leftSlot, + dictationSlot, }: { turns: TurnUsage[]; chatId: string; @@ -41,6 +42,11 @@ export function StatsBar({ /** Pre-chat only: replaces the (empty) session-data summary — home and * the /chat/new draft mount the knowledge-context slider here. */ leftSlot?: ReactNode; + /** The dictation strip (STT wave, 2026-09-17): the row's middle is dead + * space between the session data and the model control — the mic + * button lives there, and while recording the meter unfurls to fill + * whatever width the row has (owner placement). */ + dictationSlot?: ReactNode; }) { const [open, setOpen] = useState(false); // Same dropdown discipline as the model control (owner, df4433f treatment, @@ -155,6 +161,12 @@ export function StatsBar({ )}
)} + {/* The middle slot: dictation (owner, 2026-09-17). flex-1 lets the + meter expand into the row's free width; justify-center keeps the + lone idle mic button visually centered. */} + {dictationSlot && ( +
{dictationSlot}
+ )} {/* The model + thinking control — one controller for the conversation agents (prober/teacher); the record's `turns` above still folds the ACTUAL models billed, regardless of what the picker says. */} diff --git a/public/worklets/pcm-capture.js b/public/worklets/pcm-capture.js new file mode 100644 index 0000000..2b4ec72 --- /dev/null +++ b/public/worklets/pcm-capture.js @@ -0,0 +1,55 @@ +/* + * pcm-capture — the dictation audio path (STT wave, 2026-09-17). + * + * Runs OFF the main thread (AudioWorklet): converts the mic's Float32 + * samples to 16-bit little-endian PCM (linear16) and hands buffers to the + * main thread in two streams: + * + * { type: "level", rms } — every render quantum (128 samples ≈ 8ms at + * 16kHz): the quantum's RMS level, 0..1. This + * is the meter's heartbeat; posting per quantum + * is what makes the meter feel instant (the + * dominant latency in the owner's terminal + * version was a 256ms IO buffer — see + * ~/.pi/agent/extensions/dictate). + * { type: "audio", buffer } — Int16Array frames of 2048 samples (128ms), + * zero-copy (transferred). Deepgram recommends + * ~100–200ms chunks; silence is streamed too, + * so the socket never idles (no KeepAlive + * needed — the 10s NET-0001 timeout only bites + * a stream that stops SENDING). + * + * Ships as a static file under public/ so it rides the standalone bundle. + * Plain JS by necessity (worklets run outside the module graph). + */ +class PcmCaptureProcessor extends AudioWorkletProcessor { + constructor() { + super(); + this.frame = new Int16Array(2048); + this.count = 0; + } + + process(inputs) { + const channel = inputs[0] && inputs[0][0]; + if (!channel) return true; // no input yet — stay alive + + // RMS for the meter (float domain, before conversion). + let sumSquares = 0; + for (let i = 0; i < channel.length; i++) sumSquares += channel[i] * channel[i]; + this.port.postMessage({ type: "level", rms: Math.sqrt(sumSquares / channel.length) }); + + // Float32 → Int16 (clamp first: a hot mic can exceed ±1.0). + for (let i = 0; i < channel.length; i++) { + const s = Math.max(-1, Math.min(1, channel[i])); + this.frame[this.count++] = s < 0 ? s * 0x8000 : s * 0x7fff; + if (this.count === this.frame.length) { + this.port.postMessage({ type: "audio", buffer: this.frame }, [this.frame.buffer]); + this.frame = new Int16Array(2048); + this.count = 0; + } + } + return true; + } +} + +registerProcessor("pcm-capture", PcmCaptureProcessor);