fix: packaging and OpenAI vision format conversion
- Fix missing bin field in package.json to resolve Windows installer ENOENT error - Add image format conversion in OpenAI-compatible API to support vision models
This commit is contained in:
parent
c20035094d
commit
48094ffbf1
@ -11,8 +11,8 @@
|
|||||||
"./cli-entry": "./moltbot.mjs"
|
"./cli-entry": "./moltbot.mjs"
|
||||||
},
|
},
|
||||||
"bin": {
|
"bin": {
|
||||||
"moltbot": "./moltbot.mjs",
|
"moltbot": "moltbot.mjs",
|
||||||
"clawdbot": "./moltbot.mjs"
|
"clawdbot": "moltbot.mjs"
|
||||||
},
|
},
|
||||||
"files": [
|
"files": [
|
||||||
"dist/acp/**",
|
"dist/acp/**",
|
||||||
|
|||||||
@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto";
|
|||||||
import type { IncomingMessage, ServerResponse } from "node:http";
|
import type { IncomingMessage, ServerResponse } from "node:http";
|
||||||
|
|
||||||
import { buildHistoryContextFromEntries, type HistoryEntry } from "../auto-reply/reply/history.js";
|
import { buildHistoryContextFromEntries, type HistoryEntry } from "../auto-reply/reply/history.js";
|
||||||
|
import type { ImageContent } from "../commands/agent/types.js";
|
||||||
import { createDefaultDeps } from "../cli/deps.js";
|
import { createDefaultDeps } from "../cli/deps.js";
|
||||||
import { agentCommand } from "../commands/agent.js";
|
import { agentCommand } from "../commands/agent.js";
|
||||||
import { emitAgentEvent, onAgentEvent } from "../infra/agent-events.js";
|
import { emitAgentEvent, onAgentEvent } from "../infra/agent-events.js";
|
||||||
@ -64,23 +65,49 @@ function extractTextContent(content: unknown): string {
|
|||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function extractImages(content: unknown): ImageContent[] {
|
||||||
|
if (!Array.isArray(content)) return [];
|
||||||
|
const images: ImageContent[] = [];
|
||||||
|
for (const part of content) {
|
||||||
|
if (!part || typeof part !== "object") continue;
|
||||||
|
const p = part as { type?: string; image_url?: { url?: string } };
|
||||||
|
if (p.type === "image_url" && typeof p.image_url?.url === "string") {
|
||||||
|
const url = p.image_url.url;
|
||||||
|
if (url.startsWith("data:")) {
|
||||||
|
const match = url.match(/^data:([^;]+);base64,(.+)$/);
|
||||||
|
if (match) {
|
||||||
|
images.push({
|
||||||
|
type: "image",
|
||||||
|
mimeType: match[1]!,
|
||||||
|
data: match[2]!,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return images;
|
||||||
|
}
|
||||||
|
|
||||||
function buildAgentPrompt(messagesUnknown: unknown): {
|
function buildAgentPrompt(messagesUnknown: unknown): {
|
||||||
message: string;
|
message: string;
|
||||||
extraSystemPrompt?: string;
|
extraSystemPrompt?: string;
|
||||||
|
images?: ImageContent[];
|
||||||
} {
|
} {
|
||||||
const messages = asMessages(messagesUnknown);
|
const messages = asMessages(messagesUnknown);
|
||||||
|
|
||||||
const systemParts: string[] = [];
|
const systemParts: string[] = [];
|
||||||
const conversationEntries: Array<{ role: "user" | "assistant" | "tool"; entry: HistoryEntry }> =
|
const conversationEntries: Array<{ role: "user" | "assistant" | "tool"; entry: HistoryEntry }> =
|
||||||
[];
|
[];
|
||||||
|
let lastUserImages: ImageContent[] | undefined;
|
||||||
|
|
||||||
for (const msg of messages) {
|
for (const msg of messages) {
|
||||||
if (!msg || typeof msg !== "object") continue;
|
if (!msg || typeof msg !== "object") continue;
|
||||||
const role = typeof msg.role === "string" ? msg.role.trim() : "";
|
const role = typeof msg.role === "string" ? msg.role.trim() : "";
|
||||||
const content = extractTextContent(msg.content).trim();
|
const content = extractTextContent(msg.content).trim();
|
||||||
if (!role || !content) continue;
|
if (!role && !Array.isArray(msg.content)) continue;
|
||||||
|
|
||||||
if (role === "system" || role === "developer") {
|
if (role === "system" || role === "developer") {
|
||||||
systemParts.push(content);
|
if (content) systemParts.push(content);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -89,6 +116,10 @@ function buildAgentPrompt(messagesUnknown: unknown): {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (normalizedRole === "user") {
|
||||||
|
lastUserImages = extractImages(msg.content);
|
||||||
|
}
|
||||||
|
|
||||||
const name = typeof msg.name === "string" ? msg.name.trim() : "";
|
const name = typeof msg.name === "string" ? msg.name.trim() : "";
|
||||||
const sender =
|
const sender =
|
||||||
normalizedRole === "assistant"
|
normalizedRole === "assistant"
|
||||||
@ -135,6 +166,7 @@ function buildAgentPrompt(messagesUnknown: unknown): {
|
|||||||
return {
|
return {
|
||||||
message,
|
message,
|
||||||
extraSystemPrompt: systemParts.length > 0 ? systemParts.join("\n\n") : undefined,
|
extraSystemPrompt: systemParts.length > 0 ? systemParts.join("\n\n") : undefined,
|
||||||
|
images: lastUserImages,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -206,6 +238,7 @@ export async function handleOpenAiHttpRequest(
|
|||||||
{
|
{
|
||||||
message: prompt.message,
|
message: prompt.message,
|
||||||
extraSystemPrompt: prompt.extraSystemPrompt,
|
extraSystemPrompt: prompt.extraSystemPrompt,
|
||||||
|
images: prompt.images,
|
||||||
sessionKey,
|
sessionKey,
|
||||||
runId,
|
runId,
|
||||||
deliver: false,
|
deliver: false,
|
||||||
@ -313,6 +346,7 @@ export async function handleOpenAiHttpRequest(
|
|||||||
{
|
{
|
||||||
message: prompt.message,
|
message: prompt.message,
|
||||||
extraSystemPrompt: prompt.extraSystemPrompt,
|
extraSystemPrompt: prompt.extraSystemPrompt,
|
||||||
|
images: prompt.images,
|
||||||
sessionKey,
|
sessionKey,
|
||||||
runId,
|
runId,
|
||||||
deliver: false,
|
deliver: false,
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user