romtuck/openclaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add file attachment extraction tests

Browse Source
This commit is contained in:
frankekn 2026-01-29 07:24:20 +08:00
parent 9bfaf34518
commit 06a7721ac5
2 changed files with 97 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
src/media-understanding/apply.test.ts
View File

@ -41,7 +41,7 @@ describe("applyMediaUnderstanding", () => {
mockedResolveApiKey.mockClear();
mockedFetchRemoteMedia.mockReset();
mockedFetchRemoteMedia.mockResolvedValue({
buffer: Buffer.from("audio-bytes"),
buffer: Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]),
contentType: "audio/ogg",
fileName: "note.ogg",
});
@ -51,7 +51,7 @@ describe("applyMediaUnderstanding", () => {
const { applyMediaUnderstanding } = await loadApply();
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
const audioPath = path.join(dir, "note.ogg");
await fs.writeFile(audioPath, "hello");
await fs.writeFile(audioPath, Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8]));
const ctx: MsgContext = {
Body: "<media:audio>",
@ -94,7 +94,7 @@ describe("applyMediaUnderstanding", () => {
const { applyMediaUnderstanding } = await loadApply();
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
const audioPath = path.join(dir, "note.ogg");
await fs.writeFile(audioPath, "hello");
await fs.writeFile(audioPath, Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8]));
const ctx: MsgContext = {
Body: "<media:audio> /capture status",
@ -176,7 +176,7 @@ describe("applyMediaUnderstanding", () => {
const { applyMediaUnderstanding } = await loadApply();
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
const audioPath = path.join(dir, "large.wav");
await fs.writeFile(audioPath, "0123456789");
await fs.writeFile(audioPath, Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]));
const ctx: MsgContext = {
Body: "<media:audio>",
@ -211,7 +211,7 @@ describe("applyMediaUnderstanding", () => {
const { applyMediaUnderstanding } = await loadApply();
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
const audioPath = path.join(dir, "note.ogg");
await fs.writeFile(audioPath, "hello");
await fs.writeFile(audioPath, Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8]));
const ctx: MsgContext = {
Body: "<media:audio>",
@ -352,7 +352,7 @@ describe("applyMediaUnderstanding", () => {
const { applyMediaUnderstanding } = await loadApply();
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
const audioPath = path.join(dir, "fallback.ogg");
await fs.writeFile(audioPath, "hello");
await fs.writeFile(audioPath, Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6]));
const ctx: MsgContext = {
Body: "<media:audio>",
@ -390,8 +390,8 @@ describe("applyMediaUnderstanding", () => {
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
const audioPathA = path.join(dir, "note-a.ogg");
const audioPathB = path.join(dir, "note-b.ogg");
await fs.writeFile(audioPathA, "hello");
await fs.writeFile(audioPathB, "world");
await fs.writeFile(audioPathA, Buffer.from([200, 201, 202, 203, 204, 205, 206, 207, 208]));
await fs.writeFile(audioPathB, Buffer.from([200, 201, 202, 203, 204, 205, 206, 207, 208]));
const ctx: MsgContext = {
Body: "<media:audio>",
@ -435,7 +435,7 @@ describe("applyMediaUnderstanding", () => {
const audioPath = path.join(dir, "note.ogg");
const videoPath = path.join(dir, "clip.mp4");
await fs.writeFile(imagePath, "image-bytes");
await fs.writeFile(audioPath, "audio-bytes");
await fs.writeFile(audioPath, Buffer.from([200, 201, 202, 203, 204, 205, 206, 207, 208]));
await fs.writeFile(videoPath, "video-bytes");
const ctx: MsgContext = {
@ -487,4 +487,63 @@ describe("applyMediaUnderstanding", () => {
expect(ctx.CommandBody).toBe("audio ok");
expect(ctx.BodyForCommands).toBe("audio ok");
});
it("treats text-like audio attachments as CSV (comma wins over tabs)", async () => {
const { applyMediaUnderstanding } = await loadApply();
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
const csvPath = path.join(dir, "data.mp3");
const csvText = '"a","b"\t"c"\n"1","2"\t"3"';
const csvBuffer = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(csvText, "utf16le")]);
await fs.writeFile(csvPath, csvBuffer);
const ctx: MsgContext = {
Body: "<media:audio>",
MediaPath: csvPath,
MediaType: "audio/mpeg",
};
const cfg: MoltbotConfig = {
tools: {
media: {
audio: { enabled: false },
image: { enabled: false },
video: { enabled: false },
},
},
};
const result = await applyMediaUnderstanding({ ctx, cfg });
expect(result.appliedFile).toBe(true);
expect(ctx.Body).toContain('<file name="data.mp3" mime="text/csv">');
expect(ctx.Body).toContain('"a","b"\t"c"');
});
it("infers TSV when tabs are present without commas", async () => {
const { applyMediaUnderstanding } = await loadApply();
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
const tsvPath = path.join(dir, "report.mp3");
const tsvText = "a\tb\tc\n1\t2\t3";
await fs.writeFile(tsvPath, tsvText);
const ctx: MsgContext = {
Body: "<media:audio>",
MediaPath: tsvPath,
MediaType: "audio/mpeg",
};
const cfg: MoltbotConfig = {
tools: {
media: {
audio: { enabled: false },
image: { enabled: false },
video: { enabled: false },
},
},
};
const result = await applyMediaUnderstanding({ ctx, cfg });
expect(result.appliedFile).toBe(true);
expect(ctx.Body).toContain('<file name="report.mp3" mime="text/tab-separated-values">');
expect(ctx.Body).toContain("a\tb\tc");
});
});

47
src/media-understanding/apply.ts
View File

@ -104,19 +104,25 @@ function appendFileBlocks(body: string | undefined, blocks: string[]): string {
return `${base}\n\n${suffix}`.trim();
}
function isLikelyUtf16(buffer?: Buffer): boolean {
if (!buffer || buffer.length < 2) return false;
function resolveUtf16Charset(buffer?: Buffer): "utf-16le" | "utf-16be" | undefined {
if (!buffer || buffer.length < 2) return undefined;
const b0 = buffer[0];
const b1 = buffer[1];
if ((b0 === 0xff && b1 === 0xfe) || (b0 === 0xfe && b1 === 0xff)) {
return true;
if (b0 === 0xff && b1 === 0xfe) {
return "utf-16le";
}
if (b0 === 0xfe && b1 === 0xff) {
return "utf-16be";
}
const sampleLen = Math.min(buffer.length, 2048);
let zeroCount = 0;
for (let i = 0; i < sampleLen; i += 1) {
if (buffer[i] === 0) zeroCount += 1;
}
return zeroCount / sampleLen > 0.2;
if (zeroCount / sampleLen > 0.2) {
return "utf-16le";
}
return undefined;
}
function looksLikeUtf8Text(buffer?: Buffer): boolean {
@ -144,16 +150,16 @@ function looksLikeUtf8Text(buffer?: Buffer): boolean {
function decodeTextSample(buffer?: Buffer): string {
if (!buffer || buffer.length === 0) return "";
const sample = buffer.subarray(0, Math.min(buffer.length, 8192));
if (isLikelyUtf16(sample)) {
const isBigEndian = sample[0] === 0xfe && sample[1] === 0xff;
if (isBigEndian) {
const swapped = Buffer.alloc(sample.length);
for (let i = 0; i + 1 < sample.length; i += 2) {
swapped[i] = sample[i + 1];
swapped[i + 1] = sample[i];
}
return new TextDecoder("utf-16le").decode(swapped);
const utf16Charset = resolveUtf16Charset(sample);
if (utf16Charset === "utf-16be") {
const swapped = Buffer.alloc(sample.length);
for (let i = 0; i + 1 < sample.length; i += 2) {
swapped[i] = sample[i + 1];
swapped[i + 1] = sample[i];
}
return new TextDecoder("utf-16le").decode(swapped);
}
if (utf16Charset === "utf-16le") {
return new TextDecoder("utf-16le").decode(sample);
}
return new TextDecoder("utf-8").decode(sample);
@ -219,13 +225,15 @@ async function extractFileBlocks(params: {
}
const nameHint = bufferResult?.fileName ?? attachment.path ?? attachment.url;
const forcedTextMimeResolved = forcedTextMime ?? resolveTextMimeFromName(nameHint ?? "");
const utf16Charset = resolveUtf16Charset(bufferResult?.buffer);
const textSample = decodeTextSample(bufferResult?.buffer);
const textLike = isLikelyUtf16(bufferResult?.buffer) || looksLikeUtf8Text(bufferResult?.buffer);
const textLike = Boolean(utf16Charset) || looksLikeUtf8Text(bufferResult?.buffer);
if (!forcedTextMimeResolved && kind === "audio" && !textLike) {
continue;
}
const guessedDelimited = textLike ? guessDelimitedMime(textSample) : undefined;
const textHint = forcedTextMimeResolved ?? guessedDelimited ?? (textLike ? "text/plain" : undefined);
const textHint =
forcedTextMimeResolved ?? guessedDelimited ?? (textLike ? "text/plain" : undefined);
const rawMime = bufferResult?.mime ?? attachment.mime;
const mimeType = textHint ?? normalizeMimeType(rawMime);
if (!mimeType) {
@ -243,17 +251,20 @@ async function extractFileBlocks(params: {
}
if (!allowedMimes.has(mimeType)) {
if (shouldLogVerbose()) {
logVerbose(`media: file attachment skipped (unsupported mime ${mimeType}) index=${attachment.index}`);
logVerbose(
`media: file attachment skipped (unsupported mime ${mimeType}) index=${attachment.index}`,
);
}
continue;
}
let extracted: Awaited<ReturnType<typeof extractFileContentFromSource>>;
try {
const mediaType = utf16Charset ? `${mimeType}; charset=${utf16Charset}` : mimeType;
extracted = await extractFileContentFromSource({
source: {
type: "base64",
data: bufferResult.buffer.toString("base64"),
mediaType: mimeType,
mediaType,
filename: bufferResult.fileName,
},
limits: {