diff --git a/src/media-understanding/providers/azure/audio.ts b/src/media-understanding/providers/azure/audio.ts new file mode 100644 index 000000000..9325dacbc --- /dev/null +++ b/src/media-understanding/providers/azure/audio.ts @@ -0,0 +1,84 @@ +import type { AudioTranscriptionRequest, AudioTranscriptionResult } from "../../types.js"; +import { fetchWithTimeout, normalizeBaseUrl, readErrorResponse } from "../shared.js"; + +export const DEFAULT_AZURE_SPEECH_REGION = "eastus2"; + +function resolveRegion(region?: string): string { + const trimmed = region?.trim(); + return trimmed || process.env.AZURE_SPEECH_REGION?.trim() || DEFAULT_AZURE_SPEECH_REGION; +} + +type AzureSpeechResponse = { + RecognitionStatus: string; + Offset?: number; + Duration?: number; + DisplayText?: string; + NBest?: Array<{ + Confidence?: number; + Display?: string; + Lexical?: string; + }>; +}; + +export async function transcribeAzureAudio( + params: AudioTranscriptionRequest, +): Promise { + const fetchFn = params.fetchFn ?? fetch; + const region = resolveRegion(params.baseUrl); + const language = params.language?.trim() || "zh-CN"; + + // Azure Speech REST API endpoint + const url = new URL( + `https://${region}.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1`, + ); + url.searchParams.set("language", language); + + if (params.query) { + for (const [key, value] of Object.entries(params.query)) { + if (value === undefined) continue; + url.searchParams.set(key, String(value)); + } + } + + const headers = new Headers(params.headers); + if (!headers.has("Ocp-Apim-Subscription-Key")) { + headers.set("Ocp-Apim-Subscription-Key", params.apiKey); + } + if (!headers.has("Content-Type")) { + // Azure accepts various audio formats + const contentType = params.mime ?? "audio/wav"; + headers.set("Content-Type", contentType); + } + headers.set("Accept", "application/json"); + + const body = new Uint8Array(params.buffer); + const res = await fetchWithTimeout( + url.toString(), + { + method: "POST", + headers, + body, + }, + params.timeoutMs, + fetchFn, + ); + + if (!res.ok) { + const detail = await readErrorResponse(res); + const suffix = detail ? `: ${detail}` : ""; + throw new Error(`Azure Speech transcription failed (HTTP ${res.status})${suffix}`); + } + + const payload = (await res.json()) as AzureSpeechResponse; + + if (payload.RecognitionStatus !== "Success") { + throw new Error(`Azure Speech recognition failed: ${payload.RecognitionStatus}`); + } + + const transcript = payload.DisplayText?.trim() || payload.NBest?.[0]?.Display?.trim(); + if (!transcript) { + throw new Error("Azure Speech response missing transcript"); + } + + return { text: transcript, model: `azure-speech-${language}` }; +} diff --git a/src/media-understanding/providers/azure/index.ts b/src/media-understanding/providers/azure/index.ts new file mode 100644 index 000000000..88e61d0fd --- /dev/null +++ b/src/media-understanding/providers/azure/index.ts @@ -0,0 +1,9 @@ +import type { MediaUnderstandingProvider } from "../../types.js"; +import { transcribeAzureAudio } from "./audio.js"; + +export function buildAzureProvider(): MediaUnderstandingProvider { + return { + id: "azure", + transcribeAudio: transcribeAzureAudio, + }; +} diff --git a/src/media-understanding/providers/index.ts b/src/media-understanding/providers/index.ts index a20ba92fb..5ffe5fc27 100644 --- a/src/media-understanding/providers/index.ts +++ b/src/media-understanding/providers/index.ts @@ -1,6 +1,7 @@ import { normalizeProviderId } from "../../agents/model-selection.js"; import type { MediaUnderstandingProvider } from "../types.js"; import { anthropicProvider } from "./anthropic/index.js"; +import { buildAzureProvider } from "./azure/index.js"; import { deepgramProvider } from "./deepgram/index.js"; import { googleProvider } from "./google/index.js"; import { groqProvider } from "./groq/index.js"; @@ -14,6 +15,7 @@ const PROVIDERS: MediaUnderstandingProvider[] = [ anthropicProvider, minimaxProvider, deepgramProvider, + buildAzureProvider(), ]; export function normalizeMediaProviderId(id: string): string { diff --git a/src/media-understanding/runner.ts b/src/media-understanding/runner.ts index ffc6e4d64..432086f25 100644 --- a/src/media-understanding/runner.ts +++ b/src/media-understanding/runner.ts @@ -49,7 +49,7 @@ import { import { describeImageWithModel } from "./providers/image.js"; import { estimateBase64Size, resolveVideoMaxBase64Bytes } from "./video.js"; -const AUTO_AUDIO_KEY_PROVIDERS = ["openai", "groq", "deepgram", "google"] as const; +const AUTO_AUDIO_KEY_PROVIDERS = ["openai", "groq", "deepgram", "google", "azure"] as const; const AUTO_IMAGE_KEY_PROVIDERS = ["openai", "anthropic", "google", "minimax"] as const; const AUTO_VIDEO_KEY_PROVIDERS = ["google"] as const; const DEFAULT_IMAGE_MODELS: Record = {