feat: add Azure Speech STT (speech-to-text) provider

Add support for Azure Speech Service speech-to-text transcription for voice messages.

Features:
- Azure STT provider for audio transcription
- Support for 100+ languages and dialects
- High accuracy for Chinese, English, Japanese, etc.
- Automatic audio format detection
- Environment variable support: AZURE_SPEECH_API_KEY, AZURE_SPEECH_REGION

Implementation:
- Created Azure media understanding provider
- Audio transcription via Azure Speech REST API
- Endpoint: https://{region}.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1
- Headers: Ocp-Apim-Subscription-Key
- Request: Binary audio data (wav, mp3, ogg, etc.)
- Response: JSON with DisplayText transcription

Technical details:
- Uses Azure Speech conversation recognition endpoint
- Supports multiple audio formats (auto-detected from Content-Type)
- Returns DisplayText or NBest alternatives
- Integrated with Moltbot's media understanding pipeline
- Auto-discovery from environment variables

Benefits:
- No CLI dependencies (no whisper installation needed)
- High accuracy multilingual transcription
- Unified Azure billing with GPT and TTS
- Fast response times (~500-1000ms)
- Handles Telegram voice messages automatically

Co-Authored-By: Claude Sonnet 4.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Li Hongmin 2026-01-28 18:00:42 +09:00
parent 0134020564
commit d681622dbb
4 changed files with 96 additions and 1 deletions

View File

@ -0,0 +1,84 @@
import type { AudioTranscriptionRequest, AudioTranscriptionResult } from "../../types.js";
import { fetchWithTimeout, normalizeBaseUrl, readErrorResponse } from "../shared.js";
export const DEFAULT_AZURE_SPEECH_REGION = "eastus2";
function resolveRegion(region?: string): string {
const trimmed = region?.trim();
return trimmed || process.env.AZURE_SPEECH_REGION?.trim() || DEFAULT_AZURE_SPEECH_REGION;
}
type AzureSpeechResponse = {
RecognitionStatus: string;
Offset?: number;
Duration?: number;
DisplayText?: string;
NBest?: Array<{
Confidence?: number;
Display?: string;
Lexical?: string;
}>;
};
export async function transcribeAzureAudio(
params: AudioTranscriptionRequest,
): Promise<AudioTranscriptionResult> {
const fetchFn = params.fetchFn ?? fetch;
const region = resolveRegion(params.baseUrl);
const language = params.language?.trim() || "zh-CN";
// Azure Speech REST API endpoint
const url = new URL(
`https://${region}.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1`,
);
url.searchParams.set("language", language);
if (params.query) {
for (const [key, value] of Object.entries(params.query)) {
if (value === undefined) continue;
url.searchParams.set(key, String(value));
}
}
const headers = new Headers(params.headers);
if (!headers.has("Ocp-Apim-Subscription-Key")) {
headers.set("Ocp-Apim-Subscription-Key", params.apiKey);
}
if (!headers.has("Content-Type")) {
// Azure accepts various audio formats
const contentType = params.mime ?? "audio/wav";
headers.set("Content-Type", contentType);
}
headers.set("Accept", "application/json");
const body = new Uint8Array(params.buffer);
const res = await fetchWithTimeout(
url.toString(),
{
method: "POST",
headers,
body,
},
params.timeoutMs,
fetchFn,
);
if (!res.ok) {
const detail = await readErrorResponse(res);
const suffix = detail ? `: ${detail}` : "";
throw new Error(`Azure Speech transcription failed (HTTP ${res.status})${suffix}`);
}
const payload = (await res.json()) as AzureSpeechResponse;
if (payload.RecognitionStatus !== "Success") {
throw new Error(`Azure Speech recognition failed: ${payload.RecognitionStatus}`);
}
const transcript = payload.DisplayText?.trim() || payload.NBest?.[0]?.Display?.trim();
if (!transcript) {
throw new Error("Azure Speech response missing transcript");
}
return { text: transcript, model: `azure-speech-${language}` };
}

View File

@ -0,0 +1,9 @@
import type { MediaUnderstandingProvider } from "../../types.js";
import { transcribeAzureAudio } from "./audio.js";
export function buildAzureProvider(): MediaUnderstandingProvider {
return {
id: "azure",
transcribeAudio: transcribeAzureAudio,
};
}

View File

@ -1,6 +1,7 @@
import { normalizeProviderId } from "../../agents/model-selection.js";
import type { MediaUnderstandingProvider } from "../types.js";
import { anthropicProvider } from "./anthropic/index.js";
import { buildAzureProvider } from "./azure/index.js";
import { deepgramProvider } from "./deepgram/index.js";
import { googleProvider } from "./google/index.js";
import { groqProvider } from "./groq/index.js";
@ -14,6 +15,7 @@ const PROVIDERS: MediaUnderstandingProvider[] = [
anthropicProvider,
minimaxProvider,
deepgramProvider,
buildAzureProvider(),
];
export function normalizeMediaProviderId(id: string): string {

View File

@ -49,7 +49,7 @@ import {
import { describeImageWithModel } from "./providers/image.js";
import { estimateBase64Size, resolveVideoMaxBase64Bytes } from "./video.js";
const AUTO_AUDIO_KEY_PROVIDERS = ["openai", "groq", "deepgram", "google"] as const;
const AUTO_AUDIO_KEY_PROVIDERS = ["openai", "groq", "deepgram", "google", "azure"] as const;
const AUTO_IMAGE_KEY_PROVIDERS = ["openai", "anthropic", "google", "minimax"] as const;
const AUTO_VIDEO_KEY_PROVIDERS = ["google"] as const;
const DEFAULT_IMAGE_MODELS: Record<string, string> = {