diff --git a/ui/src/styles/chat/layout.css b/ui/src/styles/chat/layout.css index e137cb8c8..63e972dfb 100644 --- a/ui/src/styles/chat/layout.css +++ b/ui/src/styles/chat/layout.css @@ -103,8 +103,8 @@ bottom: 0; flex-shrink: 0; display: flex; - align-items: stretch; - gap: 12px; + flex-direction: column; + gap: 8px; margin-top: auto; /* Push to bottom of flex container */ padding: 12px 4px 4px; background: linear-gradient(to bottom, transparent, var(--bg) 20%); @@ -115,6 +115,86 @@ background: linear-gradient(to bottom, transparent, var(--bg-content) 20%); } +/* Attachment preview row */ +.chat-attachments { + display: flex; + gap: 8px; + flex-wrap: wrap; + padding: 4px 0; +} + +.chat-attachment { + position: relative; + width: 64px; + height: 64px; + border-radius: 8px; + overflow: hidden; + border: 1px solid var(--border); + background: var(--panel); + flex-shrink: 0; +} + +.chat-attachment__img { + width: 100%; + height: 100%; + object-fit: cover; + display: block; +} + +.chat-attachment__remove { + position: absolute; + top: 2px; + right: 2px; + width: 20px; + height: 20px; + border-radius: 50%; + border: none; + background: rgba(0, 0, 0, 0.6); + color: #fff; + font-size: 12px; + line-height: 1; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + padding: 0; +} + +.chat-attachment__remove svg { + width: 12px; + height: 12px; + stroke: currentColor; + fill: none; + stroke-width: 2px; +} + +.chat-attachment__remove:hover { + background: rgba(220, 38, 38, 0.8); +} + +/* Compose row (textarea + actions) */ +.chat-compose__row { + display: flex; + align-items: stretch; + gap: 12px; +} + +/* Chat images in message bubbles */ +.chat-images { + display: flex; + gap: 8px; + flex-wrap: wrap; + margin-bottom: 8px; +} + +.chat-image { + max-width: 300px; + max-height: 200px; + border-radius: 8px; + object-fit: contain; + border: 1px solid var(--border); +} + .chat-compose__field { flex: 1 1 auto; min-width: 0; diff --git a/ui/src/ui/app-chat.ts b/ui/src/ui/app-chat.ts index 81aae3c88..e4b61c468 100644 --- a/ui/src/ui/app-chat.ts +++ b/ui/src/ui/app-chat.ts @@ -1,4 +1,4 @@ -import { abortChatRun, loadChatHistory, sendChatMessage } from "./controllers/chat"; +import { abortChatRun, loadChatHistory, sendChatMessage, type ChatAttachment } from "./controllers/chat"; import { loadSessions } from "./controllers/sessions"; import { generateUUID } from "./uuid"; import { resetToolStream } from "./app-tool-stream"; @@ -12,6 +12,7 @@ import type { ClawdbotApp } from "./app"; type ChatHost = { connected: boolean; chatMessage: string; + chatAttachments: ChatAttachment[]; chatQueue: Array<{ id: string; text: string; createdAt: number }>; chatRunId: string | null; chatSending: boolean; @@ -61,10 +62,10 @@ function enqueueChatMessage(host: ChatHost, text: string) { async function sendChatMessageNow( host: ChatHost, message: string, - opts?: { previousDraft?: string; restoreDraft?: boolean }, + opts?: { previousDraft?: string; restoreDraft?: boolean; attachments?: ChatAttachment[] }, ) { resetToolStream(host as unknown as Parameters[0]); - const ok = await sendChatMessage(host as unknown as ClawdbotApp, message); + const ok = await sendChatMessage(host as unknown as ClawdbotApp, message, opts?.attachments); if (!ok && opts?.previousDraft != null) { host.chatMessage = opts.previousDraft; } @@ -104,7 +105,8 @@ export async function handleSendChat( if (!host.connected) return; const previousDraft = host.chatMessage; const message = (messageOverride ?? host.chatMessage).trim(); - if (!message) return; + const attachments = host.chatAttachments.length > 0 ? [...host.chatAttachments] : undefined; + if (!message && !attachments) return; if (isChatStopCommand(message)) { await handleAbortChat(host); @@ -114,6 +116,8 @@ export async function handleSendChat( if (messageOverride == null) { host.chatMessage = ""; } + // Clear attachments after capturing them + host.chatAttachments = []; if (isChatBusy(host)) { enqueueChatMessage(host, message); @@ -123,6 +127,7 @@ export async function handleSendChat( await sendChatMessageNow(host, message, { previousDraft: messageOverride == null ? previousDraft : undefined, restoreDraft: Boolean(messageOverride && opts?.restoreDraft), + attachments, }); } diff --git a/ui/src/ui/app-render.ts b/ui/src/ui/app-render.ts index db29bd7ec..40f991259 100644 --- a/ui/src/ui/app-render.ts +++ b/ui/src/ui/app-render.ts @@ -457,6 +457,7 @@ export function renderApp(state: AppViewState) { stream: state.chatStream, streamStartedAt: state.chatStreamStartedAt, draft: state.chatMessage, + attachments: state.chatAttachments, queue: state.chatQueue, connected: state.connected, canSend: state.connected, @@ -477,6 +478,7 @@ export function renderApp(state: AppViewState) { }, onChatScroll: (event) => state.handleChatScroll(event), onDraftChange: (next) => (state.chatMessage = next), + onAttachmentsChange: (next) => (state.chatAttachments = next), onSend: () => state.handleSendChat(), canAbort: Boolean(state.chatRunId), onAbort: () => void state.handleAbortChat(), diff --git a/ui/src/ui/app-view-state.ts b/ui/src/ui/app-view-state.ts index f589c760c..47d28c34a 100644 --- a/ui/src/ui/app-view-state.ts +++ b/ui/src/ui/app-view-state.ts @@ -22,6 +22,7 @@ import type { import type { ChatQueueItem, CronFormState } from "./ui-types"; import type { EventLogEntry } from "./app-events"; import type { SkillMessage } from "./controllers/skills"; +import type { ChatAttachment } from "./controllers/chat"; import type { ExecApprovalsFile, ExecApprovalsSnapshot, @@ -56,6 +57,7 @@ export type AppViewState = { chatAvatarUrl: string | null; chatThinkingLevel: string | null; chatQueue: ChatQueueItem[]; + chatAttachments: ChatAttachment[]; nodesLoading: boolean; nodes: Array>; devicesLoading: boolean; diff --git a/ui/src/ui/app.ts b/ui/src/ui/app.ts index 0e21d283a..c1def802a 100644 --- a/ui/src/ui/app.ts +++ b/ui/src/ui/app.ts @@ -129,6 +129,7 @@ export class ClawdbotApp extends LitElement { @state() chatAvatarUrl: string | null = null; @state() chatThinkingLevel: string | null = null; @state() chatQueue: ChatQueueItem[] = []; + @state() chatAttachments: import("./controllers/chat").ChatAttachment[] = []; // Sidebar state for tool output viewing @state() sidebarOpen = false; @state() sidebarContent: string | null = null; diff --git a/ui/src/ui/chat/grouped-render.ts b/ui/src/ui/chat/grouped-render.ts index ea1c7ffda..9334ee2ad 100644 --- a/ui/src/ui/chat/grouped-render.ts +++ b/ui/src/ui/chat/grouped-render.ts @@ -9,6 +9,7 @@ import { isToolResultMessage, normalizeRoleForGrouping } from "./message-normali import { extractTextCached, extractThinkingCached, + extractImagesCached, formatReasoningMarkdown, } from "./message-extract"; import { extractToolCards, renderToolCardSidebar } from "./tool-cards"; @@ -181,6 +182,8 @@ function renderGroupedMessage( const hasToolCards = toolCards.length > 0; const extractedText = extractTextCached(message); + const extractedImages = extractImagesCached(message); + const hasImages = extractedImages.length > 0; const extractedThinking = opts.showReasoning && role === "assistant" ? extractThinkingCached(message) @@ -207,7 +210,7 @@ function renderGroupedMessage( )}`; } - if (!markdown && !hasToolCards) return nothing; + if (!markdown && !hasToolCards && !hasImages) return nothing; return html`
@@ -217,6 +220,11 @@ function renderGroupedMessage( toSanitizedMarkdownHtml(reasoningMarkdown), )}
` : nothing} + ${hasImages + ? html`
${extractedImages.map( + (img) => html`Attached image`, + )}
` + : nothing} ${markdown ? html`
${unsafeHTML(toSanitizedMarkdownHtml(markdown))}
` : nothing} diff --git a/ui/src/ui/chat/message-extract.ts b/ui/src/ui/chat/message-extract.ts index 76dcfa591..d1d435d42 100644 --- a/ui/src/ui/chat/message-extract.ts +++ b/ui/src/ui/chat/message-extract.ts @@ -127,6 +127,39 @@ export function extractRawText(message: unknown): string | null { return null; } +export type ImageBlock = { + dataUrl: string; + mimeType: string; +}; + +const imageCache = new WeakMap(); + +export function extractImages(message: unknown): ImageBlock[] { + const m = message as Record; + const content = m.content; + if (!Array.isArray(content)) return []; + const images: ImageBlock[] = []; + for (const p of content) { + const item = p as Record; + if (item.type === "image" && typeof item.dataUrl === "string") { + images.push({ + dataUrl: item.dataUrl as string, + mimeType: (item.mimeType as string) ?? "image/png", + }); + } + } + return images; +} + +export function extractImagesCached(message: unknown): ImageBlock[] { + if (!message || typeof message !== "object") return extractImages(message); + const obj = message as object; + if (imageCache.has(obj)) return imageCache.get(obj)!; + const value = extractImages(message); + imageCache.set(obj, value); + return value; +} + export function formatReasoningMarkdown(text: string): string { const trimmed = text.trim(); if (!trimmed) return ""; diff --git a/ui/src/ui/controllers/chat.ts b/ui/src/ui/controllers/chat.ts index 53027c6ea..b6859fd79 100644 --- a/ui/src/ui/controllers/chat.ts +++ b/ui/src/ui/controllers/chat.ts @@ -2,6 +2,14 @@ import type { GatewayBrowserClient } from "../gateway"; import { extractText } from "../chat/message-extract"; import { generateUUID } from "../uuid"; +export type ChatAttachment = { + id: string; + dataUrl: string; + mimeType: string; + fileName: string; + size: number; +}; + export type ChatState = { client: GatewayBrowserClient | null; connected: boolean; @@ -43,21 +51,46 @@ export async function loadChatHistory(state: ChatState) { } } -export async function sendChatMessage(state: ChatState, message: string): Promise { +export async function sendChatMessage( + state: ChatState, + message: string, + attachments?: ChatAttachment[], +): Promise { if (!state.client || !state.connected) return false; const msg = message.trim(); - if (!msg) return false; + const hasAttachments = attachments && attachments.length > 0; + if (!msg && !hasAttachments) return false; + + // Build local content blocks for chat history + const contentBlocks: unknown[] = []; + if (msg) contentBlocks.push({ type: "text", text: msg }); + if (hasAttachments) { + for (const att of attachments) { + contentBlocks.push({ type: "image", dataUrl: att.dataUrl, mimeType: att.mimeType }); + } + } const now = Date.now(); state.chatMessages = [ ...state.chatMessages, { role: "user", - content: [{ type: "text", text: msg }], + content: contentBlocks, timestamp: now, }, ]; + // Backend requires NonEmptyString for message; use space fallback for image-only sends + const wireMessage = msg || " "; + + // Convert attachments to gateway format (base64 content without data URL prefix) + const wireAttachments = hasAttachments + ? attachments.map((att) => { + const base64 = att.dataUrl.replace(/^data:[^;]+;base64,/, ""); + return { type: "image", mimeType: att.mimeType, fileName: att.fileName, content: base64 }; + }) + : undefined; + state.chatSending = true; state.lastError = null; const runId = generateUUID(); @@ -67,9 +100,10 @@ export async function sendChatMessage(state: ChatState, message: string): Promis try { await state.client.request("chat.send", { sessionKey: state.sessionKey, - message: msg, + message: wireMessage, deliver: false, idempotencyKey: runId, + ...(wireAttachments ? { attachments: wireAttachments } : {}), }); return true; } catch (err) { diff --git a/ui/src/ui/views/chat.ts b/ui/src/ui/views/chat.ts index dd61ca0ec..571420ea2 100644 --- a/ui/src/ui/views/chat.ts +++ b/ui/src/ui/views/chat.ts @@ -3,7 +3,9 @@ import { repeat } from "lit/directives/repeat.js"; import type { SessionsListResult } from "../types"; import type { ChatQueueItem } from "../ui-types"; import type { ChatItem, MessageGroup } from "../types/chat-types"; +import type { ChatAttachment } from "../controllers/chat"; import { icons } from "../icons"; +import { generateUUID } from "../uuid"; import { normalizeMessage, normalizeRoleForGrouping, @@ -37,6 +39,7 @@ export type ChatProps = { streamStartedAt: number | null; assistantAvatarUrl?: string | null; draft: string; + attachments: ChatAttachment[]; queue: ChatQueueItem[]; connected: boolean; canSend: boolean; @@ -56,6 +59,7 @@ export type ChatProps = { onRefresh: () => void; onToggleFocusMode: () => void; onDraftChange: (next: string) => void; + onAttachmentsChange: (next: ChatAttachment[]) => void; onSend: () => void; onAbort?: () => void; onQueueRemove: (id: string) => void; @@ -66,6 +70,45 @@ export type ChatProps = { onChatScroll?: (event: Event) => void; }; +const MAX_ATTACHMENT_BYTES = 5_000_000; // 5 MB + +function handlePaste(e: ClipboardEvent, props: ChatProps) { + const items = e.clipboardData?.items; + if (!items) return; + const imageFiles: File[] = []; + for (const item of items) { + if (item.kind === "file" && item.type.startsWith("image/")) { + const file = item.getAsFile(); + if (file) imageFiles.push(file); + } + } + if (imageFiles.length === 0) return; + e.preventDefault(); + for (const file of imageFiles) { + if (file.size > MAX_ATTACHMENT_BYTES) { + console.warn(`Skipped image "${file.name}": exceeds 5 MB limit (${file.size} bytes)`); + continue; + } + const reader = new FileReader(); + reader.onload = () => { + const dataUrl = reader.result as string; + const attachment: ChatAttachment = { + id: generateUUID(), + dataUrl, + mimeType: file.type, + fileName: file.name || "pasted-image", + size: file.size, + }; + props.onAttachmentsChange([...props.attachments, attachment]); + }; + reader.readAsDataURL(file); + } +} + +function removeAttachment(props: ChatProps, id: string) { + props.onAttachmentsChange(props.attachments.filter((a) => a.id !== id)); +} + const COMPACTION_TOAST_DURATION_MS = 5000; function renderCompactionIndicator(status: CompactionIndicatorStatus | null | undefined) { @@ -235,39 +278,63 @@ export function renderChat(props: ChatProps) { : nothing}
- -
- - + ${props.attachments.length > 0 + ? html` +
+ ${props.attachments.map( + (att) => html` +
+ ${att.fileName} + +
+ `, + )} +
+ ` + : nothing} +
+ +
+ + +