diff --git a/docs/providers/azure.md b/docs/providers/azure.md index b70de09eb..2df8987c0 100644 --- a/docs/providers/azure.md +++ b/docs/providers/azure.md @@ -222,9 +222,76 @@ Check [Azure OpenAI API documentation](https://learn.microsoft.com/azure/ai-serv | Data residency | Regional | Global | | Enterprise features | Azure integration | OpenAI org settings | +## Azure Speech Service (TTS) + +Azure Speech Service provides high-quality text-to-speech with neural voices in 100+ languages. + +### Configuration + +Add Azure TTS to your `moltbot.json`: + +```json +{ + "messages": { + "tts": { + "provider": "azure", + "azure": { + "apiKey": "your-speech-api-key", + "region": "eastus2", + "voice": "zh-CN-XiaoxiaoNeural", + "lang": "zh-CN", + "outputFormat": "audio-24khz-48kbitrate-mono-mp3" + } + } + } +} +``` + +### Environment Variables + +```bash +export AZURE_SPEECH_API_KEY="your-speech-api-key" +export AZURE_SPEECH_REGION="eastus2" +``` + +### Popular Voices + +**Chinese**: +- `zh-CN-XiaoxiaoNeural` - Female, natural and friendly +- `zh-CN-YunxiNeural` - Male, warm and steady +- `zh-CN-YunyangNeural` - Male, professional news anchor + +**English**: +- `en-US-JennyNeural` - Female, friendly assistant +- `en-US-GuyNeural` - Male, professional +- `en-US-AriaNeural` - Female, conversational + +**Japanese**: +- `ja-JP-NanamiNeural` - Female, polite and clear +- `ja-JP-KeitaNeural` - Male, friendly + +See [full voice list](https://learn.microsoft.com/azure/ai-services/speech-service/language-support?tabs=tts). + +### Output Formats + +Common formats for Moltbot: +- `audio-24khz-48kbitrate-mono-mp3` - Default, good quality +- `audio-48khz-96kbitrate-mono-mp3` - High quality +- `webm-24khz-16bit-mono-opus` - WebM for web +- `ogg-24khz-16bit-mono-opus` - OGG for compatibility + +### Advantages over Edge TTS + +- **Higher quality**: Neural voices with better prosody +- **More voices**: 400+ neural voices vs Edge's limited set +- **Custom voices**: Create custom neural voices (preview) +- **SSML control**: Fine-grained control over speech +- **Unified billing**: Same Azure subscription as GPT models + ## Additional Resources - [Azure AI Services Documentation](https://learn.microsoft.com/azure/ai-services/) - [Azure OpenAI Quickstart](https://learn.microsoft.com/azure/ai-services/openai/quickstart) +- [Azure Speech Service](https://learn.microsoft.com/azure/ai-services/speech-service/) +- [Speech TTS API Reference](https://learn.microsoft.com/azure/ai-services/speech-service/rest-text-to-speech) - [Model Deployments](https://learn.microsoft.com/azure/ai-services/openai/how-to/create-resource) -- [API Reference](https://learn.microsoft.com/azure/ai-services/openai/reference) diff --git a/src/config/types.tts.ts b/src/config/types.tts.ts index 4eb4989b9..03739f0ef 100644 --- a/src/config/types.tts.ts +++ b/src/config/types.tts.ts @@ -1,4 +1,4 @@ -export type TtsProvider = "elevenlabs" | "openai" | "edge"; +export type TtsProvider = "elevenlabs" | "openai" | "edge" | "azure"; export type TtsMode = "final" | "all"; @@ -73,6 +73,14 @@ export type TtsConfig = { proxy?: string; timeoutMs?: number; }; + /** Azure Speech Service configuration. */ + azure?: { + apiKey?: string; + region?: string; + voice?: string; + outputFormat?: string; + lang?: string; + }; /** Optional path for local TTS user preferences JSON. */ prefsPath?: string; /** Hard cap for text sent to TTS (chars). */ diff --git a/src/config/zod-schema.core.ts b/src/config/zod-schema.core.ts index 4a8c80bcc..2ad2ce34a 100644 --- a/src/config/zod-schema.core.ts +++ b/src/config/zod-schema.core.ts @@ -156,7 +156,7 @@ export const MarkdownConfigSchema = z .strict() .optional(); -export const TtsProviderSchema = z.enum(["elevenlabs", "openai", "edge"]); +export const TtsProviderSchema = z.enum(["elevenlabs", "openai", "edge", "azure"]); export const TtsModeSchema = z.enum(["final", "all"]); export const TtsAutoSchema = z.enum(["off", "always", "inbound", "tagged"]); export const TtsConfigSchema = z @@ -224,6 +224,16 @@ export const TtsConfigSchema = z }) .strict() .optional(), + azure: z + .object({ + apiKey: z.string().optional(), + region: z.string().optional(), + voice: z.string().optional(), + outputFormat: z.string().optional(), + lang: z.string().optional(), + }) + .strict() + .optional(), prefsPath: z.string().optional(), maxTextLength: z.number().int().min(1).optional(), timeoutMs: z.number().int().min(1000).max(120000).optional(), diff --git a/src/tts/tts.ts b/src/tts/tts.ts index af3d7fda5..f71b291cd 100644 --- a/src/tts/tts.ts +++ b/src/tts/tts.ts @@ -51,6 +51,10 @@ const DEFAULT_OPENAI_VOICE = "alloy"; const DEFAULT_EDGE_VOICE = "en-US-MichelleNeural"; const DEFAULT_EDGE_LANG = "en-US"; const DEFAULT_EDGE_OUTPUT_FORMAT = "audio-24khz-48kbitrate-mono-mp3"; +const DEFAULT_AZURE_VOICE = "en-US-JennyNeural"; +const DEFAULT_AZURE_LANG = "en-US"; +const DEFAULT_AZURE_OUTPUT_FORMAT = "audio-24khz-48kbitrate-mono-mp3"; +const DEFAULT_AZURE_REGION = "eastus2"; const DEFAULT_ELEVENLABS_VOICE_SETTINGS = { stability: 0.5, @@ -124,6 +128,13 @@ export type ResolvedTtsConfig = { proxy?: string; timeoutMs?: number; }; + azure: { + apiKey?: string; + region: string; + voice: string; + outputFormat: string; + lang: string; + }; prefsPath?: string; maxTextLength: number; timeoutMs: number; @@ -296,6 +307,16 @@ export function resolveTtsConfig(cfg: MoltbotConfig): ResolvedTtsConfig { proxy: raw.edge?.proxy?.trim() || undefined, timeoutMs: raw.edge?.timeoutMs, }, + azure: { + apiKey: raw.azure?.apiKey?.trim() || undefined, + region: + raw.azure?.region?.trim() || + process.env.AZURE_SPEECH_REGION?.trim() || + DEFAULT_AZURE_REGION, + voice: raw.azure?.voice?.trim() || DEFAULT_AZURE_VOICE, + outputFormat: raw.azure?.outputFormat?.trim() || DEFAULT_AZURE_OUTPUT_FORMAT, + lang: raw.azure?.lang?.trim() || DEFAULT_AZURE_LANG, + }, prefsPath: raw.prefsPath, maxTextLength: raw.maxTextLength ?? DEFAULT_MAX_TEXT_LENGTH, timeoutMs: raw.timeoutMs ?? DEFAULT_TIMEOUT_MS, @@ -474,10 +495,13 @@ export function resolveTtsApiKey( if (provider === "openai") { return config.openai.apiKey || process.env.OPENAI_API_KEY; } + if (provider === "azure") { + return config.azure.apiKey || process.env.AZURE_SPEECH_API_KEY || process.env.AZURE_API_KEY; + } return undefined; } -export const TTS_PROVIDERS = ["openai", "elevenlabs", "edge"] as const; +export const TTS_PROVIDERS = ["openai", "elevenlabs", "edge", "azure"] as const; export function resolveTtsProviderOrder(primary: TtsProvider): TtsProvider[] { return [primary, ...TTS_PROVIDERS.filter((provider) => provider !== primary)]; @@ -485,6 +509,7 @@ export function resolveTtsProviderOrder(primary: TtsProvider): TtsProvider[] { export function isTtsProviderConfigured(config: ResolvedTtsConfig, provider: TtsProvider): boolean { if (provider === "edge") return config.edge.enabled; + if (provider === "azure") return Boolean(resolveTtsApiKey(config, provider)); return Boolean(resolveTtsApiKey(config, provider)); } @@ -1047,6 +1072,54 @@ function inferEdgeExtension(outputFormat: string): string { return ".mp3"; } +async function azureTTS(params: { + text: string; + apiKey: string; + region: string; + voice: string; + outputFormat: string; + lang: string; + timeoutMs: number; +}): Promise { + const { text, apiKey, region, voice, outputFormat, lang, timeoutMs } = params; + + // Construct SSML (Speech Synthesis Markup Language) request + const ssml = ` + + ${text.replace(/&/g, "&").replace(//g, ">")} + +`; + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + + try { + const response = await fetch( + `https://${region}.tts.speech.microsoft.com/cognitiveservices/v1`, + { + method: "POST", + headers: { + "Ocp-Apim-Subscription-Key": apiKey, + "Content-Type": "application/ssml+xml", + "X-Microsoft-OutputFormat": outputFormat, + "User-Agent": "Moltbot", + }, + body: ssml, + signal: controller.signal, + }, + ); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Azure Speech TTS error (${response.status}): ${errorText}`); + } + + return Buffer.from(await response.arrayBuffer()); + } finally { + clearTimeout(timeout); + } +} + async function edgeTTS(params: { text: string; outputPath: string; @@ -1172,7 +1245,17 @@ export async function textToSpeech(params: { } let audioBuffer: Buffer; - if (provider === "elevenlabs") { + if (provider === "azure") { + audioBuffer = await azureTTS({ + text: params.text, + apiKey, + region: config.azure.region, + voice: config.azure.voice, + outputFormat: config.azure.outputFormat, + lang: config.azure.lang, + timeoutMs: config.timeoutMs, + }); + } else if (provider === "elevenlabs") { const voiceIdOverride = params.overrides?.elevenlabs?.voiceId; const modelIdOverride = params.overrides?.elevenlabs?.modelId; const voiceSettings = {