diff --git a/src/media-understanding/apply.test.ts b/src/media-understanding/apply.test.ts index 8bebb8e20..1fb8ae240 100644 --- a/src/media-understanding/apply.test.ts +++ b/src/media-understanding/apply.test.ts @@ -41,7 +41,7 @@ describe("applyMediaUnderstanding", () => { mockedResolveApiKey.mockClear(); mockedFetchRemoteMedia.mockReset(); mockedFetchRemoteMedia.mockResolvedValue({ - buffer: Buffer.from("audio-bytes"), + buffer: Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]), contentType: "audio/ogg", fileName: "note.ogg", }); @@ -51,7 +51,7 @@ describe("applyMediaUnderstanding", () => { const { applyMediaUnderstanding } = await loadApply(); const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-")); const audioPath = path.join(dir, "note.ogg"); - await fs.writeFile(audioPath, "hello"); + await fs.writeFile(audioPath, Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8])); const ctx: MsgContext = { Body: "", @@ -94,7 +94,7 @@ describe("applyMediaUnderstanding", () => { const { applyMediaUnderstanding } = await loadApply(); const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-")); const audioPath = path.join(dir, "note.ogg"); - await fs.writeFile(audioPath, "hello"); + await fs.writeFile(audioPath, Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8])); const ctx: MsgContext = { Body: " /capture status", @@ -176,7 +176,7 @@ describe("applyMediaUnderstanding", () => { const { applyMediaUnderstanding } = await loadApply(); const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-")); const audioPath = path.join(dir, "large.wav"); - await fs.writeFile(audioPath, "0123456789"); + await fs.writeFile(audioPath, Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])); const ctx: MsgContext = { Body: "", @@ -211,7 +211,7 @@ describe("applyMediaUnderstanding", () => { const { applyMediaUnderstanding } = await loadApply(); const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-")); const audioPath = path.join(dir, "note.ogg"); - await fs.writeFile(audioPath, "hello"); + await fs.writeFile(audioPath, Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8])); const ctx: MsgContext = { Body: "", @@ -352,7 +352,7 @@ describe("applyMediaUnderstanding", () => { const { applyMediaUnderstanding } = await loadApply(); const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-")); const audioPath = path.join(dir, "fallback.ogg"); - await fs.writeFile(audioPath, "hello"); + await fs.writeFile(audioPath, Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6])); const ctx: MsgContext = { Body: "", @@ -390,8 +390,8 @@ describe("applyMediaUnderstanding", () => { const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-")); const audioPathA = path.join(dir, "note-a.ogg"); const audioPathB = path.join(dir, "note-b.ogg"); - await fs.writeFile(audioPathA, "hello"); - await fs.writeFile(audioPathB, "world"); + await fs.writeFile(audioPathA, Buffer.from([200, 201, 202, 203, 204, 205, 206, 207, 208])); + await fs.writeFile(audioPathB, Buffer.from([200, 201, 202, 203, 204, 205, 206, 207, 208])); const ctx: MsgContext = { Body: "", @@ -435,7 +435,7 @@ describe("applyMediaUnderstanding", () => { const audioPath = path.join(dir, "note.ogg"); const videoPath = path.join(dir, "clip.mp4"); await fs.writeFile(imagePath, "image-bytes"); - await fs.writeFile(audioPath, "audio-bytes"); + await fs.writeFile(audioPath, Buffer.from([200, 201, 202, 203, 204, 205, 206, 207, 208])); await fs.writeFile(videoPath, "video-bytes"); const ctx: MsgContext = { @@ -487,4 +487,63 @@ describe("applyMediaUnderstanding", () => { expect(ctx.CommandBody).toBe("audio ok"); expect(ctx.BodyForCommands).toBe("audio ok"); }); + + it("treats text-like audio attachments as CSV (comma wins over tabs)", async () => { + const { applyMediaUnderstanding } = await loadApply(); + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-")); + const csvPath = path.join(dir, "data.mp3"); + const csvText = '"a","b"\t"c"\n"1","2"\t"3"'; + const csvBuffer = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(csvText, "utf16le")]); + await fs.writeFile(csvPath, csvBuffer); + + const ctx: MsgContext = { + Body: "", + MediaPath: csvPath, + MediaType: "audio/mpeg", + }; + const cfg: MoltbotConfig = { + tools: { + media: { + audio: { enabled: false }, + image: { enabled: false }, + video: { enabled: false }, + }, + }, + }; + + const result = await applyMediaUnderstanding({ ctx, cfg }); + + expect(result.appliedFile).toBe(true); + expect(ctx.Body).toContain(''); + expect(ctx.Body).toContain('"a","b"\t"c"'); + }); + + it("infers TSV when tabs are present without commas", async () => { + const { applyMediaUnderstanding } = await loadApply(); + const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-")); + const tsvPath = path.join(dir, "report.mp3"); + const tsvText = "a\tb\tc\n1\t2\t3"; + await fs.writeFile(tsvPath, tsvText); + + const ctx: MsgContext = { + Body: "", + MediaPath: tsvPath, + MediaType: "audio/mpeg", + }; + const cfg: MoltbotConfig = { + tools: { + media: { + audio: { enabled: false }, + image: { enabled: false }, + video: { enabled: false }, + }, + }, + }; + + const result = await applyMediaUnderstanding({ ctx, cfg }); + + expect(result.appliedFile).toBe(true); + expect(ctx.Body).toContain(''); + expect(ctx.Body).toContain("a\tb\tc"); + }); }); diff --git a/src/media-understanding/apply.ts b/src/media-understanding/apply.ts index 398ac6913..e95d27241 100644 --- a/src/media-understanding/apply.ts +++ b/src/media-understanding/apply.ts @@ -104,19 +104,25 @@ function appendFileBlocks(body: string | undefined, blocks: string[]): string { return `${base}\n\n${suffix}`.trim(); } -function isLikelyUtf16(buffer?: Buffer): boolean { - if (!buffer || buffer.length < 2) return false; +function resolveUtf16Charset(buffer?: Buffer): "utf-16le" | "utf-16be" | undefined { + if (!buffer || buffer.length < 2) return undefined; const b0 = buffer[0]; const b1 = buffer[1]; - if ((b0 === 0xff && b1 === 0xfe) || (b0 === 0xfe && b1 === 0xff)) { - return true; + if (b0 === 0xff && b1 === 0xfe) { + return "utf-16le"; + } + if (b0 === 0xfe && b1 === 0xff) { + return "utf-16be"; } const sampleLen = Math.min(buffer.length, 2048); let zeroCount = 0; for (let i = 0; i < sampleLen; i += 1) { if (buffer[i] === 0) zeroCount += 1; } - return zeroCount / sampleLen > 0.2; + if (zeroCount / sampleLen > 0.2) { + return "utf-16le"; + } + return undefined; } function looksLikeUtf8Text(buffer?: Buffer): boolean { @@ -144,16 +150,16 @@ function looksLikeUtf8Text(buffer?: Buffer): boolean { function decodeTextSample(buffer?: Buffer): string { if (!buffer || buffer.length === 0) return ""; const sample = buffer.subarray(0, Math.min(buffer.length, 8192)); - if (isLikelyUtf16(sample)) { - const isBigEndian = sample[0] === 0xfe && sample[1] === 0xff; - if (isBigEndian) { - const swapped = Buffer.alloc(sample.length); - for (let i = 0; i + 1 < sample.length; i += 2) { - swapped[i] = sample[i + 1]; - swapped[i + 1] = sample[i]; - } - return new TextDecoder("utf-16le").decode(swapped); + const utf16Charset = resolveUtf16Charset(sample); + if (utf16Charset === "utf-16be") { + const swapped = Buffer.alloc(sample.length); + for (let i = 0; i + 1 < sample.length; i += 2) { + swapped[i] = sample[i + 1]; + swapped[i + 1] = sample[i]; } + return new TextDecoder("utf-16le").decode(swapped); + } + if (utf16Charset === "utf-16le") { return new TextDecoder("utf-16le").decode(sample); } return new TextDecoder("utf-8").decode(sample); @@ -219,13 +225,15 @@ async function extractFileBlocks(params: { } const nameHint = bufferResult?.fileName ?? attachment.path ?? attachment.url; const forcedTextMimeResolved = forcedTextMime ?? resolveTextMimeFromName(nameHint ?? ""); + const utf16Charset = resolveUtf16Charset(bufferResult?.buffer); const textSample = decodeTextSample(bufferResult?.buffer); - const textLike = isLikelyUtf16(bufferResult?.buffer) || looksLikeUtf8Text(bufferResult?.buffer); + const textLike = Boolean(utf16Charset) || looksLikeUtf8Text(bufferResult?.buffer); if (!forcedTextMimeResolved && kind === "audio" && !textLike) { continue; } const guessedDelimited = textLike ? guessDelimitedMime(textSample) : undefined; - const textHint = forcedTextMimeResolved ?? guessedDelimited ?? (textLike ? "text/plain" : undefined); + const textHint = + forcedTextMimeResolved ?? guessedDelimited ?? (textLike ? "text/plain" : undefined); const rawMime = bufferResult?.mime ?? attachment.mime; const mimeType = textHint ?? normalizeMimeType(rawMime); if (!mimeType) { @@ -243,17 +251,20 @@ async function extractFileBlocks(params: { } if (!allowedMimes.has(mimeType)) { if (shouldLogVerbose()) { - logVerbose(`media: file attachment skipped (unsupported mime ${mimeType}) index=${attachment.index}`); + logVerbose( + `media: file attachment skipped (unsupported mime ${mimeType}) index=${attachment.index}`, + ); } continue; } let extracted: Awaited>; try { + const mediaType = utf16Charset ? `${mimeType}; charset=${utf16Charset}` : mimeType; extracted = await extractFileContentFromSource({ source: { type: "base64", data: bufferResult.buffer.toString("base64"), - mediaType: mimeType, + mediaType, filename: bufferResult.fileName, }, limits: {