diff --git a/docs/providers/pocket-tts.md b/docs/providers/pocket-tts.md new file mode 100644 index 000000000..96530f4d7 --- /dev/null +++ b/docs/providers/pocket-tts.md @@ -0,0 +1,262 @@ +--- +summary: "Local CPU-based TTS with Pocket TTS" +read_when: + - You want local text-to-speech without cloud APIs + - You need free, offline TTS + - You want voice cloning capabilities +--- + +# Pocket TTS + +[Pocket TTS](https://github.com/kyutai-labs/pocket-tts) is a local, CPU-only text-to-speech engine from Kyutai Labs. It runs entirely on your machine with no API keys required. + +**Highlights:** +- ~100M parameter model, runs on CPU +- ~200ms latency to first audio chunk +- 6× realtime on MacBook Air M4 +- Voice cloning via reference audio file +- 8 built-in voices +- Fully offline after initial model download (~400MB) + +## Quick start + +1. Install Pocket TTS: + +```bash +# Using pip +pip install pocket-tts + +# Or using uv (faster) +uv pip install pocket-tts +``` + +2. Start the server: + +```bash +pocket-tts serve --voice alba +``` + +3. Configure Clawdbot: + +```json5 +{ + messages: { + tts: { + provider: "pocket", + pocket: { + baseUrl: "http://localhost:8000", // default + voice: "alba" // default + } + } + } +} +``` + +4. Enable TTS: + +``` +/tts always +``` + +## Built-in voices + +Pocket TTS includes 8 built-in voices (Les Misérables characters): + +| Voice | Description | +|-------|-------------| +| `alba` | Default voice | +| `marius` | Male voice | +| `javert` | Male voice | +| `jean` | Male voice | +| `fantine` | Female voice | +| `cosette` | Female voice | +| `eponine` | Female voice | +| `azelma` | Female voice | + +## Voice cloning + +Clone any voice by providing a reference audio file: + +```bash +# Start server with custom voice +pocket-tts serve --voice /path/to/your-voice.wav +``` + +Or use a HuggingFace voice: + +```json5 +{ + messages: { + tts: { + pocket: { + voice: "hf://kyutai/tts-voices/custom-voice.wav" + } + } + } +} +``` + +## Configuration + +### Full config options + +```json5 +{ + messages: { + tts: { + provider: "pocket", // Use pocket as primary provider + pocket: { + enabled: true, // Enable/disable pocket (default: true) + baseUrl: "http://localhost:8000", // Server URL (also used for auto-start binding) + voice: "alba", // Voice name or URL + autoStart: false // Auto-start server if not running (default: false) + } + } + } +} +``` + +### Auto-start mode + +Clawdbot can automatically start `pocket-tts serve` when it's not running: + +```json5 +{ + messages: { + tts: { + provider: "pocket", + pocket: { + baseUrl: "http://localhost:8000", // Host/port derived from this URL + autoStart: true, // Spawn server automatically + voice: "alba" // Voice to use when starting + } + } + } +} +``` + +**How it works:** +1. Clawdbot checks `/health` endpoint +2. If server is down and `autoStart: true`, spawns `pocket-tts serve` +3. Host and port are derived from `baseUrl` (e.g., `http://localhost:9000` → `--host localhost --port 9000`) +4. Waits up to 30s for server to become healthy (model loading) +5. Server is stopped automatically when Clawdbot exits + +**Note:** First request may be slow (~10-30s) while the model loads. Subsequent requests are fast (~200ms). + +### Environment variables + +Pocket TTS doesn't require API keys: + +```bash +# Manual start (recommended for production) +pocket-tts serve --voice alba + +# Or let Clawdbot auto-start via config +``` + +## Provider fallback + +Clawdbot tries providers in order: **OpenAI → ElevenLabs → Edge → Pocket** + +When Pocket TTS is configured but the server isn't running: +1. If `autoStart: true`, Clawdbot tries to start the server +2. If that fails (or `autoStart: false`), falls back to next provider + +To check if the server is running: + +```bash +curl http://localhost:8000/health +# Returns: {"status": "healthy"} +``` + +### Using Pocket as primary + +To use Pocket first (before cloud providers): + +```json5 +{ + messages: { + tts: { + provider: "pocket" // Try pocket first, fall back to others + } + } +} +``` + +## Comparison with other providers + +| Provider | API Key | Latency | Cost | Offline | Output | +|----------|---------|---------|------|---------|--------| +| Pocket TTS | No | ~200ms | Free | Yes | WAV | +| Edge TTS | No | ~500ms | Free | No | MP3 | +| OpenAI | Yes | ~300ms | $0.015/1K chars | No | MP3/WAV | +| ElevenLabs | Yes | ~400ms | $0.30/1K chars | No | MP3 | + +**Note:** Pocket TTS outputs WAV format (uncompressed). File sizes are ~5-10x larger than MP3 (~100KB vs ~15KB for short phrases). Most messaging platforms handle this fine. + +**First run:** Downloads ~400MB model from HuggingFace on first use. After that, it's fully offline. + +## Troubleshooting + +### Server not running + +If you see "pocket: server not running", start the server: + +```bash +pocket-tts serve --voice alba +``` + +Or enable auto-start in config: + +```json5 +{ messages: { tts: { pocket: { autoStart: true } } } } +``` + +### pocket-tts not installed + +If auto-start fails with "ENOENT" or "command not found": + +```bash +# Install pocket-tts +pip install pocket-tts + +# Verify installation +pocket-tts --help +``` + +### Wrong Python version + +Pocket TTS requires Python 3.10+: + +```bash +python3 --version +# Should be 3.10 or higher +``` + +### Model loading slow + +First request takes 10-30s to load the model. This is normal. +Subsequent requests are fast (~200ms). + +### Invalid voice error + +Valid voices are: +- Built-in: `alba`, `marius`, `javert`, `jean`, `fantine`, `cosette`, `eponine`, `azelma` +- HuggingFace: `hf://kyutai/tts-voices/...` +- HTTP/HTTPS URLs + +Local file paths are **not** supported via the API. Use `pocket-tts serve --voice /path/to/file.wav` instead. + +### Missing dependencies + +```bash +# Reinstall with all dependencies +pip install pocket-tts[audio] +``` + +## Links + +- [Pocket TTS GitHub](https://github.com/kyutai-labs/pocket-tts) +- [Kyutai Labs](https://kyutai.org) +- [HuggingFace Model](https://huggingface.co/kyutai/pocket-tts-v1) diff --git a/docs/tts.md b/docs/tts.md index 22dacd611..5d756c333 100644 --- a/docs/tts.md +++ b/docs/tts.md @@ -8,7 +8,7 @@ read_when: # Text-to-speech (TTS) -Clawdbot can convert outbound replies into audio using ElevenLabs, OpenAI, or Edge TTS. +Clawdbot can convert outbound replies into audio using ElevenLabs, OpenAI, Edge TTS, or Pocket TTS. It works anywhere Clawdbot can send audio; Telegram gets a round voice-note bubble. ## Supported services @@ -16,6 +16,7 @@ It works anywhere Clawdbot can send audio; Telegram gets a round voice-note bubb - **ElevenLabs** (primary or fallback provider) - **OpenAI** (primary or fallback provider; also used for summaries) - **Edge TTS** (primary or fallback provider; uses `node-edge-tts`, default when no API keys) +- **Pocket TTS** (local CPU-based TTS; see [Pocket TTS docs](providers/pocket-tts.md)) ### Edge TTS notes @@ -50,6 +51,7 @@ so that provider must also be authenticated if you enable summaries. - [ElevenLabs Authentication](https://elevenlabs.io/docs/api-reference/authentication) - [node-edge-tts](https://github.com/SchneeHertz/node-edge-tts) - [Microsoft Speech output formats](https://learn.microsoft.com/azure/ai-services/speech-service/rest-text-to-speech#audio-outputs) +- [Pocket TTS](https://github.com/kyutai-labs/pocket-tts) (local CPU-based TTS) ## Is it enabled by default? diff --git a/src/commands/doctor-tts.ts b/src/commands/doctor-tts.ts new file mode 100644 index 000000000..ea9d33e82 --- /dev/null +++ b/src/commands/doctor-tts.ts @@ -0,0 +1,90 @@ +import type { ClawdbotConfig } from "../config/config.js"; +import { runExec } from "../process/exec.js"; +import { note } from "../terminal/note.js"; + +/** + * Check if pocket-tts CLI is installed and accessible. + */ +async function isPocketTtsInstalled(): Promise { + try { + await runExec("pocket-tts", ["--version"], { timeoutMs: 5_000 }); + return true; + } catch { + return false; + } +} + +/** + * Check if pocket-tts server is running at the given URL. + */ +async function isPocketServerHealthy(baseUrl: string): Promise { + try { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 3000); + try { + const url = baseUrl.replace(/\/+$/, "") + "/health"; + const response = await fetch(url, { + method: "GET", + signal: controller.signal, + }); + return response.ok; + } finally { + clearTimeout(timeout); + } + } catch { + return false; + } +} + +/** + * Note TTS configuration warnings. + * Called from doctor.ts. + */ +export async function noteTtsConfigWarnings(cfg: ClawdbotConfig): Promise { + const ttsConfig = cfg.messages?.tts; + if (!ttsConfig) return; + + const pocket = ttsConfig.pocket; + const isPocketProvider = ttsConfig.provider === "pocket"; + const isPocketEnabled = pocket?.enabled !== false; // defaults to true + + // Only check pocket if it's relevant + if (!isPocketProvider && !isPocketEnabled) return; + + const warnings: string[] = []; + + // Check if pocket-tts CLI is installed + const installed = await isPocketTtsInstalled(); + if (!installed) { + if (isPocketProvider) { + warnings.push( + "pocket-tts is not installed but configured as the TTS provider.", + " Install: pip install pocket-tts", + " Or change provider: clawdbot config set messages.tts.provider openai", + ); + } else if (pocket?.autoStart) { + warnings.push( + "pocket.autoStart is enabled but pocket-tts is not installed.", + " Install: pip install pocket-tts", + " Or disable: clawdbot config set messages.tts.pocket.autoStart false", + ); + } + } + + // Check if server is running (only if autoStart is off) + if (installed && isPocketProvider && !pocket?.autoStart) { + const baseUrl = pocket?.baseUrl || "http://localhost:8000"; + const healthy = await isPocketServerHealthy(baseUrl); + if (!healthy) { + warnings.push( + `pocket-tts server is not running at ${baseUrl}.`, + " Start manually: pocket-tts serve --voice alba", + " Or enable auto-start: clawdbot config set messages.tts.pocket.autoStart true", + ); + } + } + + if (warnings.length > 0) { + note(warnings.join("\n"), "TTS"); + } +} diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index aa4f4d7a3..957e54523 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -48,6 +48,7 @@ import { maybeRepairUiProtocolFreshness } from "./doctor-ui.js"; import { maybeOfferUpdateBeforeDoctor } from "./doctor-update.js"; import { MEMORY_SYSTEM_PROMPT, shouldSuggestMemorySystem } from "./doctor-workspace.js"; import { noteWorkspaceStatus } from "./doctor-workspace-status.js"; +import { noteTtsConfigWarnings } from "./doctor-tts.js"; import { applyWizardMetadata, printWizardHeader, randomToken } from "./onboard-helpers.js"; import { ensureSystemdUserLingerInteractive } from "./systemd-linger.js"; @@ -252,6 +253,7 @@ export async function doctorCommand( } noteWorkspaceStatus(cfg); + await noteTtsConfigWarnings(cfg); const { healthOk } = await checkGatewayHealth({ runtime, diff --git a/src/config/types.tts.ts b/src/config/types.tts.ts index 4eb4989b9..a0d472f58 100644 --- a/src/config/types.tts.ts +++ b/src/config/types.tts.ts @@ -1,4 +1,4 @@ -export type TtsProvider = "elevenlabs" | "openai" | "edge"; +export type TtsProvider = "elevenlabs" | "openai" | "edge" | "pocket"; export type TtsMode = "final" | "all"; @@ -73,6 +73,17 @@ export type TtsConfig = { proxy?: string; timeoutMs?: number; }; + /** Pocket TTS (local CPU-based TTS) configuration. */ + pocket?: { + /** Explicitly allow Pocket TTS usage (no API key required, but requires pocket-tts serve running). */ + enabled?: boolean; + /** Base URL for pocket-tts serve (default: http://localhost:8000). Used for both API calls and auto-start binding. */ + baseUrl?: string; + /** Voice name or URL (built-in: alba, marius, javert, jean, fantine, cosette, eponine, azelma; or hf:// URL). */ + voice?: string; + /** Auto-start pocket-tts serve if not running (default: false). Requires pocket-tts to be installed. */ + autoStart?: boolean; + }; /** Optional path for local TTS user preferences JSON. */ prefsPath?: string; /** Hard cap for text sent to TTS (chars). */ diff --git a/src/tts/tts.test.ts b/src/tts/tts.test.ts index 8462cba01..2f3bc03b4 100644 --- a/src/tts/tts.test.ts +++ b/src/tts/tts.test.ts @@ -560,4 +560,207 @@ describe("tts", () => { process.env.CLAWDBOT_TTS_PREFS = prevPrefs; }); }); + + describe("pocket TTS", () => { + const { + POCKET_VOICES, + checkPocketHealth, + normalizePocketBaseUrl, + parsePocketBaseUrl, + isValidPocketVoice, + } = _test; + + describe("POCKET_VOICES", () => { + it("contains all built-in pocket-tts voices", () => { + expect(POCKET_VOICES).toContain("alba"); + expect(POCKET_VOICES).toContain("marius"); + expect(POCKET_VOICES).toContain("javert"); + expect(POCKET_VOICES).toContain("jean"); + expect(POCKET_VOICES).toContain("fantine"); + expect(POCKET_VOICES).toContain("cosette"); + expect(POCKET_VOICES).toContain("eponine"); + expect(POCKET_VOICES).toContain("azelma"); + expect(POCKET_VOICES.length).toBe(8); + }); + }); + + describe("normalizePocketBaseUrl", () => { + it("returns default URL for empty input", () => { + expect(normalizePocketBaseUrl("")).toBe("http://localhost:8000"); + expect(normalizePocketBaseUrl(" ")).toBe("http://localhost:8000"); + }); + + it("strips trailing slashes", () => { + expect(normalizePocketBaseUrl("http://localhost:8000/")).toBe("http://localhost:8000"); + expect(normalizePocketBaseUrl("http://localhost:8000///")).toBe("http://localhost:8000"); + }); + + it("preserves valid URLs without trailing slash", () => { + expect(normalizePocketBaseUrl("http://localhost:8000")).toBe("http://localhost:8000"); + expect(normalizePocketBaseUrl("http://192.168.1.100:9000")).toBe( + "http://192.168.1.100:9000", + ); + }); + }); + + describe("parsePocketBaseUrl", () => { + it("extracts host and port from URL", () => { + expect(parsePocketBaseUrl("http://localhost:8000")).toEqual({ + host: "localhost", + port: 8000, + }); + expect(parsePocketBaseUrl("http://192.168.1.100:9000")).toEqual({ + host: "192.168.1.100", + port: 9000, + }); + expect(parsePocketBaseUrl("http://myserver:3000/")).toEqual({ + host: "myserver", + port: 3000, + }); + }); + + it("uses defaults for missing port", () => { + expect(parsePocketBaseUrl("http://localhost")).toEqual({ host: "localhost", port: 8000 }); + }); + + it("handles invalid URLs gracefully", () => { + expect(parsePocketBaseUrl("not-a-url")).toEqual({ host: "localhost", port: 8000 }); + expect(parsePocketBaseUrl("")).toEqual({ host: "localhost", port: 8000 }); + }); + }); + + describe("isValidPocketVoice", () => { + it("accepts built-in voices", () => { + expect(isValidPocketVoice("alba")).toBe(true); + expect(isValidPocketVoice("marius")).toBe(true); + expect(isValidPocketVoice("cosette")).toBe(true); + }); + + it("accepts URL-based voices", () => { + expect(isValidPocketVoice("http://example.com/voice.wav")).toBe(true); + expect(isValidPocketVoice("https://example.com/voice.wav")).toBe(true); + expect(isValidPocketVoice("hf://kyutai/tts-voices/custom.wav")).toBe(true); + }); + + it("rejects unknown voice names", () => { + expect(isValidPocketVoice("unknown_voice")).toBe(false); + expect(isValidPocketVoice("my_custom_voice")).toBe(false); + expect(isValidPocketVoice("/path/to/local/file.wav")).toBe(false); + }); + }); + + describe("checkPocketHealth", () => { + it("returns false when server is not reachable", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = vi.fn(async () => { + throw new Error("Connection refused"); + }) as unknown as typeof fetch; + + const result = await checkPocketHealth("http://localhost:9999", 1000); + expect(result).toBe(false); + + globalThis.fetch = originalFetch; + }); + + it("returns true when server responds with ok", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = vi.fn(async () => ({ + ok: true, + json: async () => ({ status: "healthy" }), + })) as unknown as typeof fetch; + + const result = await checkPocketHealth("http://localhost:8000", 1000); + expect(result).toBe(true); + + globalThis.fetch = originalFetch; + }); + + it("returns false when server responds with error status", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = vi.fn(async () => ({ + ok: false, + status: 500, + })) as unknown as typeof fetch; + + const result = await checkPocketHealth("http://localhost:8000", 1000); + expect(result).toBe(false); + + globalThis.fetch = originalFetch; + }); + }); + + describe("resolveTtsConfig pocket defaults", () => { + it("resolves pocket config with defaults", () => { + const config = resolveTtsConfig({}); + expect(config.pocket.enabled).toBe(true); + expect(config.pocket.baseUrl).toBe("http://localhost:8000"); + expect(config.pocket.voice).toBe("alba"); + expect(config.pocket.autoStart).toBe(false); + }); + + it("respects custom pocket config", () => { + const config = resolveTtsConfig({ + messages: { + tts: { + pocket: { + enabled: false, + baseUrl: "http://myserver:9000", + voice: "marius", + autoStart: true, + }, + }, + }, + }); + expect(config.pocket.enabled).toBe(false); + expect(config.pocket.baseUrl).toBe("http://myserver:9000"); + expect(config.pocket.voice).toBe("marius"); + expect(config.pocket.autoStart).toBe(true); + }); + }); + + describe("auto-start configuration", () => { + it("autoStart defaults to false for safety", () => { + const config = resolveTtsConfig({ + messages: { + tts: { + provider: "pocket", + }, + }, + }); + expect(config.pocket.autoStart).toBe(false); + }); + + it("can enable autoStart via config", () => { + const config = resolveTtsConfig({ + messages: { + tts: { + pocket: { + autoStart: true, + }, + }, + }, + }); + expect(config.pocket.autoStart).toBe(true); + }); + + it("derives host/port from baseUrl for auto-start", () => { + // This tests that baseUrl is the single source of truth + const config = resolveTtsConfig({ + messages: { + tts: { + pocket: { + baseUrl: "http://192.168.1.50:9999", + autoStart: true, + }, + }, + }, + }); + expect(config.pocket.baseUrl).toBe("http://192.168.1.50:9999"); + // parsePocketBaseUrl will extract host/port from baseUrl + const { host, port } = parsePocketBaseUrl(config.pocket.baseUrl); + expect(host).toBe("192.168.1.50"); + expect(port).toBe(9999); + }); + }); + }); }); diff --git a/src/tts/tts.ts b/src/tts/tts.ts index 847876d04..d92c900ef 100644 --- a/src/tts/tts.ts +++ b/src/tts/tts.ts @@ -52,6 +52,19 @@ const DEFAULT_EDGE_VOICE = "en-US-MichelleNeural"; const DEFAULT_EDGE_LANG = "en-US"; const DEFAULT_EDGE_OUTPUT_FORMAT = "audio-24khz-48kbitrate-mono-mp3"; +const DEFAULT_POCKET_BASE_URL = "http://localhost:8000"; +const DEFAULT_POCKET_VOICE = "alba"; +const POCKET_VOICES = [ + "alba", + "marius", + "javert", + "jean", + "fantine", + "cosette", + "eponine", + "azelma", +] as const; + const DEFAULT_ELEVENLABS_VOICE_SETTINGS = { stability: 0.5, similarityBoost: 0.75, @@ -124,6 +137,12 @@ export type ResolvedTtsConfig = { proxy?: string; timeoutMs?: number; }; + pocket: { + enabled: boolean; + baseUrl: string; + voice: string; + autoStart: boolean; + }; prefsPath?: string; maxTextLength: number; timeoutMs: number; @@ -165,6 +184,9 @@ type TtsDirectiveOverrides = { languageCode?: string; voiceSettings?: Partial; }; + pocket?: { + voice?: string; + }; }; type TtsDirectiveParseResult = { @@ -296,6 +318,12 @@ export function resolveTtsConfig(cfg: ClawdbotConfig): ResolvedTtsConfig { proxy: raw.edge?.proxy?.trim() || undefined, timeoutMs: raw.edge?.timeoutMs, }, + pocket: { + enabled: raw.pocket?.enabled ?? true, + baseUrl: raw.pocket?.baseUrl?.trim() || DEFAULT_POCKET_BASE_URL, + voice: raw.pocket?.voice?.trim() || DEFAULT_POCKET_VOICE, + autoStart: raw.pocket?.autoStart ?? false, + }, prefsPath: raw.prefsPath, maxTextLength: raw.maxTextLength ?? DEFAULT_MAX_TEXT_LENGTH, timeoutMs: raw.timeoutMs ?? DEFAULT_TIMEOUT_MS, @@ -477,7 +505,7 @@ export function resolveTtsApiKey( return undefined; } -export const TTS_PROVIDERS = ["openai", "elevenlabs", "edge"] as const; +export const TTS_PROVIDERS = ["openai", "elevenlabs", "edge", "pocket"] as const; export function resolveTtsProviderOrder(primary: TtsProvider): TtsProvider[] { return [primary, ...TTS_PROVIDERS.filter((provider) => provider !== primary)]; @@ -485,6 +513,7 @@ export function resolveTtsProviderOrder(primary: TtsProvider): TtsProvider[] { export function isTtsProviderConfigured(config: ResolvedTtsConfig, provider: TtsProvider): boolean { if (provider === "edge") return config.edge.enabled; + if (provider === "pocket") return config.pocket.enabled; return Boolean(resolveTtsApiKey(config, provider)); } @@ -1068,6 +1097,237 @@ async function edgeTTS(params: { await tts.ttsPromise(text, outputPath); } +function normalizePocketBaseUrl(baseUrl: string): string { + const trimmed = baseUrl.trim(); + if (!trimmed) return DEFAULT_POCKET_BASE_URL; + return trimmed.replace(/\/+$/, ""); +} + +function parsePocketBaseUrl(baseUrl: string): { host: string; port: number } { + try { + const url = new URL(normalizePocketBaseUrl(baseUrl)); + return { + host: url.hostname || "localhost", + port: url.port ? parseInt(url.port, 10) : 8000, + }; + } catch { + return { host: "localhost", port: 8000 }; + } +} + +function isValidPocketVoice(voice: string): boolean { + if (!voice) return false; + // Built-in voices + if (POCKET_VOICES.includes(voice as (typeof POCKET_VOICES)[number])) return true; + // URL-based voices (http://, https://, hf://) + if (voice.startsWith("http://") || voice.startsWith("https://") || voice.startsWith("hf://")) { + return true; + } + return false; +} + +function formatPocketVoiceWarning(voice: string): string { + return `Pocket TTS voice "${voice}" may not be valid. Valid options: ${POCKET_VOICES.join(", ")}, or hf:// / http:// URLs`; +} + +async function pocketTTS(params: { + text: string; + baseUrl: string; + voice: string; + timeoutMs: number; +}): Promise { + const { text, baseUrl, voice, timeoutMs } = params; + + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + + try { + const url = new URL(`${normalizePocketBaseUrl(baseUrl)}/tts`); + + // Build form data + const formData = new FormData(); + formData.append("text", text); + + // Always send voice_url - let server validate/reject invalid voices + // This ensures explicit errors rather than silent fallback to server default + if (voice) { + formData.append("voice_url", voice); + } + + const response = await fetch(url.toString(), { + method: "POST", + body: formData, + signal: controller.signal, + }); + + if (!response.ok) { + let errorDetail = ""; + try { + const errorText = await response.text(); + // Try to parse JSON error response from FastAPI + const parsed = JSON.parse(errorText); + errorDetail = parsed.detail || errorText; + } catch { + // If not JSON, use raw text + } + throw new Error( + `Pocket TTS API error (${response.status})${errorDetail ? `: ${errorDetail}` : ""}`, + ); + } + + return Buffer.from(await response.arrayBuffer()); + } finally { + clearTimeout(timeout); + } +} + +async function checkPocketHealth(baseUrl: string, timeoutMs: number): Promise { + const url = `${normalizePocketBaseUrl(baseUrl)}/health`; + try { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), Math.min(timeoutMs, 5000)); + try { + const response = await fetch(url, { + method: "GET", + signal: controller.signal, + }); + return response.ok; + } finally { + clearTimeout(timeout); + } + } catch (err) { + // Only log on explicit check, not during polling (too noisy) + // logVerbose(`TTS: pocket health check failed: ${err instanceof Error ? err.message : err}`); + return false; + } +} + +// Pocket TTS subprocess management +let pocketProcess: import("node:child_process").ChildProcess | null = null; +let pocketStartupPromise: Promise | null = null; +let pocketLastFailure: number = 0; +const POCKET_STARTUP_COOLDOWN_MS = 60000; // 1 minute cooldown after failure + +async function startPocketServer(config: ResolvedTtsConfig["pocket"]): Promise { + // If already running, return true + if (pocketProcess) { + return true; + } + + // If startup is in progress, wait for it (fixes race condition) + if (pocketStartupPromise) { + return pocketStartupPromise; + } + + // Cooldown after failure to prevent repeated spawn attempts + if (pocketLastFailure && Date.now() - pocketLastFailure < POCKET_STARTUP_COOLDOWN_MS) { + const remainingSec = Math.ceil( + (POCKET_STARTUP_COOLDOWN_MS - (Date.now() - pocketLastFailure)) / 1000, + ); + logVerbose(`TTS: pocket-tts startup on cooldown (${remainingSec}s remaining)`); + return false; + } + + // Start new startup process + pocketStartupPromise = (async () => { + try { + const { spawn } = await import("node:child_process"); + const { host, port } = parsePocketBaseUrl(config.baseUrl); + + const args = ["serve", "--host", host, "--port", String(port)]; + + // Add voice if specified + if (config.voice) { + args.push("--voice", config.voice); + } + + logVerbose(`TTS: Starting pocket-tts server: pocket-tts ${args.join(" ")}`); + + pocketProcess = spawn("pocket-tts", args, { + stdio: ["ignore", "ignore", "pipe"], // stdin: ignore, stdout: ignore (prevent buffer hang), stderr: pipe + detached: false, + }); + + let spawnError: Error | null = null; + + pocketProcess.on("error", (err) => { + spawnError = err; + pocketProcess = null; + }); + + pocketProcess.on("exit", (code, signal) => { + logVerbose(`TTS: pocket-tts exited (code=${code}, signal=${signal})`); + pocketProcess = null; + }); + + // Capture stderr for debugging + pocketProcess.stderr?.on("data", (data: Buffer) => { + const msg = data.toString().trim(); + if (msg) logVerbose(`TTS: pocket-tts: ${msg}`); + }); + + // Wait for server to become healthy (up to 30s for model loading) + const maxWait = 30000; + const pollInterval = 500; + const startTime = Date.now(); + + while (Date.now() - startTime < maxWait) { + await new Promise((r) => setTimeout(r, pollInterval)); + + // Check for spawn error (e.g., pocket-tts not installed) + if (spawnError) { + const err = spawnError as Error; // Type narrowing doesn't work across async callbacks + const hint = err.message.includes("ENOENT") + ? " (is pocket-tts installed? Run: pip install pocket-tts)" + : ""; + logVerbose(`TTS: pocket-tts spawn failed: ${err.message}${hint}`); + pocketLastFailure = Date.now(); + return false; + } + + if (!pocketProcess) { + // Process died during startup + pocketLastFailure = Date.now(); + return false; + } + + const healthy = await checkPocketHealth(config.baseUrl, 2000); + if (healthy) { + logVerbose(`TTS: pocket-tts server ready after ${Date.now() - startTime}ms`); + pocketLastFailure = 0; // Reset on success + return true; + } + } + + logVerbose("TTS: pocket-tts server failed to become healthy within 30s"); + stopPocketServer(); + pocketLastFailure = Date.now(); + return false; + } catch (err) { + logVerbose(`TTS: Failed to start pocket-tts: ${err instanceof Error ? err.message : err}`); + pocketLastFailure = Date.now(); + return false; + } finally { + pocketStartupPromise = null; + } + })(); + + return pocketStartupPromise; +} + +function stopPocketServer(): void { + if (pocketProcess) { + logVerbose("TTS: Stopping pocket-tts server"); + pocketProcess.kill("SIGTERM"); + pocketProcess = null; + } +} + +// Clean up on process exit (don't call process.exit - let Clawdbot handle graceful shutdown) +process.on("exit", stopPocketServer); +process.on("SIGINT", stopPocketServer); +process.on("SIGTERM", stopPocketServer); + export async function textToSpeech(params: { text: string; cfg: ClawdbotConfig; @@ -1165,6 +1425,67 @@ export async function textToSpeech(params: { }; } + if (provider === "pocket") { + if (!config.pocket.enabled) { + lastError = "pocket: disabled"; + continue; + } + + // Check if pocket-tts server is running + let isHealthy = await checkPocketHealth(config.pocket.baseUrl, config.timeoutMs); + + // Try auto-starting if configured and not healthy + if (!isHealthy && config.pocket.autoStart) { + logVerbose("TTS: pocket-tts not running, attempting auto-start..."); + const started = await startPocketServer(config.pocket); + if (started) { + isHealthy = true; + } + } + + if (!isHealthy) { + const hint = config.pocket.autoStart + ? "auto-start failed" + : "run `pocket-tts serve` or set pocket.autoStart=true"; + lastError = `pocket: server not running (${hint})`; + continue; + } + + // Support voice override from model directives + const pocketVoice = params.overrides?.pocket?.voice ?? config.pocket.voice; + + // Warn if voice looks invalid (but still try - server will give authoritative error) + if (pocketVoice && !isValidPocketVoice(pocketVoice)) { + logVerbose(`TTS: ${formatPocketVoiceWarning(pocketVoice)}`); + } + + const audioBuffer = await pocketTTS({ + text: params.text, + baseUrl: config.pocket.baseUrl, + voice: pocketVoice, + timeoutMs: config.timeoutMs, + }); + + const latencyMs = Date.now() - providerStart; + + // Pocket TTS returns WAV format + const tempDir = mkdtempSync(path.join(tmpdir(), "tts-")); + const audioPath = path.join(tempDir, `voice-${Date.now()}.wav`); + writeFileSync(audioPath, audioBuffer); + scheduleCleanup(tempDir); + + const voiceCompatible = isVoiceCompatibleAudio({ fileName: audioPath }); + + return { + success: true, + audioPath, + latencyMs, + provider, + outputFormat: "wav", + voiceCompatible, + }; + } + const apiKey = resolveTtsApiKey(config, provider); if (!apiKey) { lastError = `No API key for ${provider}`; @@ -1463,9 +1784,17 @@ export const _test = { isValidOpenAIModel, OPENAI_TTS_MODELS, OPENAI_TTS_VOICES, + POCKET_VOICES, parseTtsDirectives, resolveModelOverridePolicy, summarizeText, resolveOutputFormat, resolveEdgeOutputFormat, + checkPocketHealth, + normalizePocketBaseUrl, + parsePocketBaseUrl, + isValidPocketVoice, + formatPocketVoiceWarning, + startPocketServer, + stopPocketServer, };