import type { Api, AssistantMessage, Context, Model } from "@mariozechner/pi-ai"; import { complete } from "@mariozechner/pi-ai"; import { discoverAuthStorage, discoverModels } from "@mariozechner/pi-coding-agent"; import { getApiKeyForModel, requireApiKey } from "../../agents/model-auth.js"; import { setupLlmProxy } from "../../agents/llm-proxy.js"; import { ensureClawdbotModelsJson } from "../../agents/models-config.js"; import { minimaxUnderstandImage } from "../../agents/minimax-vlm.js"; import { coerceImageAssistantText } from "../../agents/tools/image-tool.helpers.js"; import type { ImageDescriptionRequest, ImageDescriptionResult } from "../types.js"; export async function describeImageWithModel( params: ImageDescriptionRequest, ): Promise { // Set up LLM proxy if configured (must be done before any fetch calls) setupLlmProxy(params.cfg); await ensureClawdbotModelsJson(params.cfg, params.agentDir); const authStorage = discoverAuthStorage(params.agentDir); const modelRegistry = discoverModels(authStorage, params.agentDir); const model = modelRegistry.find(params.provider, params.model) as Model | null; if (!model) { throw new Error(`Unknown model: ${params.provider}/${params.model}`); } if (!model.input?.includes("image")) { throw new Error(`Model does not support images: ${params.provider}/${params.model}`); } const apiKeyInfo = await getApiKeyForModel({ model, cfg: params.cfg, agentDir: params.agentDir, profileId: params.profile, preferredProfile: params.preferredProfile, }); const apiKey = requireApiKey(apiKeyInfo, model.provider); authStorage.setRuntimeApiKey(model.provider, apiKey); const base64 = params.buffer.toString("base64"); if (model.provider === "minimax") { const text = await minimaxUnderstandImage({ apiKey, prompt: params.prompt ?? "Describe the image.", imageDataUrl: `data:${params.mime ?? "image/jpeg"};base64,${base64}`, modelBaseUrl: model.baseUrl, }); return { text, model: model.id }; } const context: Context = { messages: [ { role: "user", content: [ { type: "text", text: params.prompt ?? "Describe the image." }, { type: "image", data: base64, mimeType: params.mime ?? "image/jpeg" }, ], timestamp: Date.now(), }, ], }; const message = (await complete(model, context, { apiKey, maxTokens: params.maxTokens ?? 512, })) as AssistantMessage; const text = coerceImageAssistantText({ message, provider: model.provider, model: model.id, }); return { text, model: model.id }; }