sorokin.oleg.edu/learnharness
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 6 Wiki Activity

feat: add OpenRouter dictation and speech recognition settings

Browse Source
This commit is contained in:
sorokin.oleg.edu
2026-10-26 23:42:44 +03:00
parent 168cad7470
commit ecbd2b0bb9
24 changed files with 867 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/core/index.ts
+1
View File
@@ -81,3 +81,4 @@ export * from "./coverage.ts";
export * from "./session-markdown.ts";
export * from "./i18n.ts";
export * from "./stt.ts";
src/core/locales/en.json
+20 -2
View File
@@ -252,7 +252,6 @@
"Multilingual": "Multilingual",
"Multi": "Multi",
"Preferred languages": "Preferred languages",
"Used with Multilingual. Favors selected languages; other languages may still be recognized.": "Used with Multilingual. Favors selected languages; other languages may still be recognized.",
"Could not save preferred languages": "Could not save preferred languages",
"sttPreferredLanguages must be an array of \"en\" and \"ru\"": "sttPreferredLanguages must be an array of \"en\" and \"ru\"",
"Global": "Global",
@@ -848,5 +847,24 @@
"Fast mode uses more ChatGPT allowance.": "Fast mode uses more ChatGPT allowance.",
"Fast mode is available only for ChatGPT models.": "Fast mode is available only for ChatGPT models.",
"Could not save model settings": "Could not save model settings",
"Fast mode is unavailable for this ChatGPT model or account. Turn off Fast mode or choose another model.": "Fast mode is unavailable for this ChatGPT model or account. Turn off Fast mode or choose another model."
"Fast mode is unavailable for this ChatGPT model or account. Turn off Fast mode or choose another model.": "Fast mode is unavailable for this ChatGPT model or account. Turn off Fast mode or choose another model.",
"Speech recognition model": "Speech recognition model",
"Unknown speech recognition model": "Unknown speech recognition model",
"Invalid speech recognition language": "Invalid speech recognition language",
"Invalid dictation audio": "Invalid dictation audio",
"Recording exceeds OpenRouter's 25 MB audio limit": "Recording exceeds OpenRouter's 25 MB audio limit",
"Could not read dictation audio": "Could not read dictation audio",
"Transcription cancelled": "Transcription cancelled",
"Dictation needs an OpenRouter key": "Dictation needs an OpenRouter key",
"OpenRouter rejected the API key": "OpenRouter rejected the API key",
"OpenRouter credits are insufficient": "OpenRouter credits are insufficient",
"OpenRouter rate limit reached; try again later": "OpenRouter rate limit reached; try again later",
"OpenRouter transcription failed ({v0})": "OpenRouter transcription failed ({v0})",
"OpenRouter returned an invalid transcript": "OpenRouter returned an invalid transcript",
"OpenRouter transcription timed out": "OpenRouter transcription timed out",
"OpenRouter transcription is unavailable": "OpenRouter transcription is unavailable",
"Could not finish microphone capture": "Could not finish microphone capture",
"Transcribing…": "Transcribing…",
"Speech recognition": "Speech recognition",
"No preference": "No preference"
}
src/core/locales/ru.json
+20 -2
View File
@@ -252,7 +252,6 @@
"Multilingual": "Несколько языков",
"Multi": "Мульти",
"Preferred languages": "Предпочитаемые языки",
"Used with Multilingual. Favors selected languages; other languages may still be recognized.": "Используется в режиме «Несколько языков». Выбранные языки получают приоритет, но могут распознаваться и другие.",
"Could not save preferred languages": "Не удалось сохранить предпочитаемые языки",
"sttPreferredLanguages must be an array of \"en\" and \"ru\"": "Предпочитаемые языки должны быть списком кодов «en» и «ru»",
"Global": "Общие",
@@ -884,5 +883,24 @@
"Fast mode uses more ChatGPT allowance.": "Быстрый режим быстрее расходует лимит ChatGPT.",
"Fast mode is available only for ChatGPT models.": "Быстрый режим доступен только для моделей ChatGPT.",
"Could not save model settings": "Не удалось сохранить настройки моделей",
"Fast mode is unavailable for this ChatGPT model or account. Turn off Fast mode or choose another model.": "Быстрый режим недоступен для этой модели или учётной записи ChatGPT. Отключите быстрый режим или выберите другую модель."
"Fast mode is unavailable for this ChatGPT model or account. Turn off Fast mode or choose another model.": "Быстрый режим недоступен для этой модели или учётной записи ChatGPT. Отключите быстрый режим или выберите другую модель.",
"Speech recognition model": "Модель распознавания речи",
"Unknown speech recognition model": "Неизвестная модель распознавания речи",
"Invalid speech recognition language": "Недопустимый язык распознавания речи",
"Invalid dictation audio": "Некорректная аудиозапись",
"Recording exceeds OpenRouter's 25 MB audio limit": "Запись превышает ограничение OpenRouter в 25 МБ",
"Could not read dictation audio": "Не удалось прочитать аудиозапись",
"Transcription cancelled": "Распознавание отменено",
"Dictation needs an OpenRouter key": "Для диктовки нужен ключ OpenRouter",
"OpenRouter rejected the API key": "OpenRouter отклонил API-ключ",
"OpenRouter credits are insufficient": "Недостаточно средств на балансе OpenRouter",
"OpenRouter rate limit reached; try again later": "Превышен лимит запросов OpenRouter. Попробуйте позже",
"OpenRouter transcription failed ({v0})": "Ошибка распознавания OpenRouter ({v0})",
"OpenRouter returned an invalid transcript": "OpenRouter вернул некорректный результат распознавания",
"OpenRouter transcription timed out": "Время ожидания распознавания OpenRouter истекло",
"OpenRouter transcription is unavailable": "Распознавание через OpenRouter недоступно",
"Could not finish microphone capture": "Не удалось завершить запись с микрофона",
"Transcribing…": "Распознавание…",
"Speech recognition": "Распознавание речи",
"No preference": "Без предпочтений"
}
src/core/settings-resolution.ts
+1
View File
@@ -184,6 +184,7 @@ export function resolveSettings(record: UserSettings | null): EffectiveSettings
autoUpdate: record?.general?.autoUpdate ?? FACTORY_SETTINGS.general.autoUpdate,
...(record?.general?.language ? { language: record.general.language } : {}),
...(record?.general?.sttLanguage ? { sttLanguage: record.general.sttLanguage } : {}),
...(record?.general?.sttModel ? { sttModel: record.general.sttModel } : {}),
...(record?.general?.sttPreferredLanguages ? { sttPreferredLanguages: [...record.general.sttPreferredLanguages] } : {}),
},
agent: {
src/core/settings.ts
+4
View File
@@ -33,6 +33,7 @@ import type { ModelRole } from "../../config.ts";
import type { ChatSettings } from "./contracts/chat.ts";
import type { SearchEngineId } from "./search.ts";
import { isLanguage, isSttLanguage, type Language, type SttLanguage } from "./i18n.ts";
import { isSttModel, type SttModel } from "./stt.ts";
/** The schema version this code understands. Bump ONLY for breaking shape
* changes (a field renamed/meaning changed); pure additions never bump. */
@@ -240,6 +241,8 @@ export interface GeneralSettings {
language?: Language;
/** Independent dictation language; multi = multilingual, absent = browser/system language. */
sttLanguage?: SttLanguage;
/** Absent preserves direct Deepgram Nova-3 streaming. */
sttModel?: SttModel;
/** Optional recognition hints for multilingual dictation; absent/empty = automatic detection. */
sttPreferredLanguages?: Language[];
}
@@ -452,6 +455,7 @@ function sanitizeGeneral(raw: Record<string, unknown>): GeneralSettings {
if (autoUpdate !== undefined) out.autoUpdate = autoUpdate;
if (isLanguage(raw.language)) out.language = raw.language;
if (isSttLanguage(raw.sttLanguage)) out.sttLanguage = raw.sttLanguage;
if (isSttModel(raw.sttModel)) out.sttModel = raw.sttModel;
if (Array.isArray(raw.sttPreferredLanguages)) out.sttPreferredLanguages = [...new Set(raw.sttPreferredLanguages.filter(isLanguage))];
return out;
}
src/core/stt.ts
+36
View File
@@ -0,0 +1,36 @@
/** Dictation model choices are independent of the conversational model tiers. */
export const STT_MODELS = [
{ id: "deepgram/nova-3", name: "Nova-3 · Deepgram" },
{ id: "microsoft/mai-transcribe-2", name: "MAI-Transcribe-2 · OpenRouter" },
] as const;
export type SttModel = (typeof STT_MODELS)[number]["id"];
export const DEFAULT_STT_MODEL: SttModel = "deepgram/nova-3";
export const OPENROUTER_STT_MODEL: SttModel = "microsoft/mai-transcribe-2";
export function isSttModel(value: unknown): value is SttModel {
return STT_MODELS.some((model) => model.id === value);
}
/** OpenRouter's inline audio limit. WAV uses the existing 16 kHz mono PCM capture. */
export const STT_MAX_AUDIO_BYTES = 25 * 1024 * 1024;
export const STT_SAMPLE_RATE = 16000;
export const WAV_HEADER_BYTES = 44;
/** Wrap captured PCM in a WAV container without resampling or encoding loss. */
export function dictationWav(frames: readonly Uint8Array[]): Uint8Array<ArrayBuffer> {
const size = frames.reduce((total, frame) => total + frame.byteLength, 0);
if (size % 2 !== 0) throw new Error("Invalid dictation audio");
if (size + WAV_HEADER_BYTES > STT_MAX_AUDIO_BYTES) throw new Error("Recording exceeds OpenRouter's 25 MB audio limit");
const wav = new Uint8Array(WAV_HEADER_BYTES + size);
const view = new DataView(wav.buffer);
const text = (offset: number, value: string) => {
for (let i = 0; i < value.length; i++) view.setUint8(offset + i, value.charCodeAt(i));
};
text(0, "RIFF"); view.setUint32(4, 36 + size, true); text(8, "WAVE");
text(12, "fmt "); view.setUint32(16, 16, true); view.setUint16(20, 1, true);
view.setUint16(22, 1, true); view.setUint32(24, STT_SAMPLE_RATE, true);
view.setUint32(28, STT_SAMPLE_RATE * 2, true); view.setUint16(32, 2, true); view.setUint16(34, 16, true);
text(36, "data"); view.setUint32(40, size, true);
let offset = WAV_HEADER_BYTES;
for (const frame of frames) { wav.set(frame, offset); offset += frame.byteLength; }
return wav;
}
src/server/index.ts
+1
View File
@@ -46,3 +46,4 @@ export * from "./model-catalog.ts";
export { sessionCost } from "./pricing.ts";
export { applicationLanguage, chatLanguage } from "./language.ts";
export * from "./stt.ts";
src/server/settings-body.ts
+5 -1
View File
@@ -12,7 +12,7 @@
*/
import type { ThinkingLevel } from "@earendil-works/pi-agent-core";
import { isHexColor, isLanguage, isSttLanguage, type ChatSettings } from "../core";
import { isHexColor, isLanguage, isSttLanguage, isSttModel, type ChatSettings } from "../core";
import type {
AgentSettings,
AppearanceSettings,
@@ -98,6 +98,10 @@ export function parseGeneralSettingsBody(body: unknown): SettingsBodyResult<Gene
if (!Array.isArray(raw.sttPreferredLanguages) || !raw.sttPreferredLanguages.every(isLanguage)) return { ok: false, error: 'sttPreferredLanguages must be an array of "en" and "ru"' };
settings.sttPreferredLanguages = [...new Set(raw.sttPreferredLanguages)];
}
if (raw.sttModel !== undefined) {
if (!isSttModel(raw.sttModel)) return { ok: false, error: "Unknown speech recognition model" };
settings.sttModel = raw.sttModel;
}
return { ok: true, settings };
}
src/server/stt.ts
+75
View File
@@ -0,0 +1,75 @@
import { OPENROUTER_STT_MODEL, STT_MAX_AUDIO_BYTES, STT_SAMPLE_RATE, WAV_HEADER_BYTES } from "../core/stt.ts";
import { isSttLanguage } from "../core/i18n.ts";
import { keyValue } from "./keys.ts";
const fail = (status: number, message: string) => Response.json({ message }, { status });
/** One complete recording per request. Credentials and audio never go to disk. */
export async function openRouterTranscription(request: Request, key = keyValue("OPENROUTER_API_KEY")): Promise<Response> {
if (!key) return fail(503, "Dictation needs an OpenRouter key");
const params = new URL(request.url).searchParams;
if (params.get("model") !== OPENROUTER_STT_MODEL) return fail(400, "Unknown speech recognition model");
const language = params.get("language");
if (!isSttLanguage(language)) return fail(400, "Invalid speech recognition language");
if (request.headers.get("content-type") !== "audio/wav") return fail(415, "Invalid dictation audio");
if (Number(request.headers.get("content-length")) > STT_MAX_AUDIO_BYTES) return fail(413, "Recording exceeds OpenRouter's 25 MB audio limit");
if (!request.body) return fail(400, "Invalid dictation audio");
const reader = request.body.getReader();
const frames: Uint8Array[] = [];
let size = 0;
try {
while (true) {
const { done, value } = await reader.read();
if (done) break;
size += value.byteLength;
if (size > STT_MAX_AUDIO_BYTES) {
await reader.cancel();
return fail(413, "Recording exceeds OpenRouter's 25 MB audio limit");
}
frames.push(value);
}
} catch {
return fail(request.signal.aborted ? 499 : 400, "Could not read dictation audio");
} finally { reader.releaseLock(); }
if (request.signal.aborted) return fail(499, "Transcription cancelled");
const audio = Buffer.concat(frames, size);
// Accept exactly the lossless PCM container produced by our capture graph.
if (size <= WAV_HEADER_BYTES || size % 2 !== 0 || audio.toString("ascii", 0, 4) !== "RIFF"
|| audio.toString("ascii", 8, 12) !== "WAVE" || audio.toString("ascii", 12, 16) !== "fmt "
|| audio.readUInt32LE(4) !== size - 8 || audio.readUInt32LE(16) !== 16 || audio.readUInt16LE(20) !== 1
|| audio.readUInt16LE(22) !== 1 || audio.readUInt32LE(24) !== STT_SAMPLE_RATE
|| audio.readUInt32LE(28) !== STT_SAMPLE_RATE * 2 || audio.readUInt16LE(32) !== 2 || audio.readUInt16LE(34) !== 16
|| audio.toString("ascii", 36, 40) !== "data" || audio.readUInt32LE(40) !== size - WAV_HEADER_BYTES) {
return fail(400, "Invalid dictation audio");
}
const signal = AbortSignal.any([request.signal, AbortSignal.timeout(65_000)]);
try {
const response = await fetch("https://openrouter.ai/api/v1/audio/transcriptions", {
method: "POST",
headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" },
body: JSON.stringify({
model: OPENROUTER_STT_MODEL,
input_audio: { data: audio.toString("base64"), format: "wav" },
// MAI allows one forced language; automatic detection supports mixed speech.
...(language === "multi" ? {} : { language }),
response_format: "json",
}),
signal,
});
if (!response.ok) {
// Provider diagnostics can contain credentials or echoed audio. Return only app-authored errors.
await response.body?.cancel();
if (response.status === 401 || response.status === 403) return fail(502, "OpenRouter rejected the API key");
if (response.status === 402) return fail(502, "OpenRouter credits are insufficient");
if (response.status === 429) return fail(429, "OpenRouter rate limit reached; try again later");
return fail(502, `OpenRouter transcription failed (${response.status})`);
}
const result = await response.json() as { text?: unknown };
if (typeof result.text !== "string") return fail(502, "OpenRouter returned an invalid transcript");
return Response.json({ text: result.text });
} catch (error) {
if (request.signal.aborted) return fail(499, "Transcription cancelled");
if (signal.aborted || (error instanceof Error && error.name === "TimeoutError")) return fail(504, "OpenRouter transcription timed out");
return fail(502, "OpenRouter transcription is unavailable");
}
}