feat(webchat): add image paste support with preview, validation, and rendering

Add clipboard image paste to WebChat compose area:
- Paste handler reads image files from clipboard, converts to data URLs
- Attachment preview thumbnails with remove button in compose area
- 5 MB frontend size validation (matches backend limit)
- Image content blocks rendered in chat history bubbles
- Backend NonEmptyString bypass: image-only sends use space fallback
- Attachments converted to base64 gateway format on send

Fixes #1681, supersedes #1900
This commit is contained in:
Joey Rodriguez 2026-01-30 02:05:56 -05:00
parent c8063bdcd8
commit 363c08db3d
9 changed files with 276 additions and 44 deletions

View File

@ -103,8 +103,8 @@
bottom: 0; bottom: 0;
flex-shrink: 0; flex-shrink: 0;
display: flex; display: flex;
align-items: stretch; flex-direction: column;
gap: 12px; gap: 8px;
margin-top: auto; /* Push to bottom of flex container */ margin-top: auto; /* Push to bottom of flex container */
padding: 12px 4px 4px; padding: 12px 4px 4px;
background: linear-gradient(to bottom, transparent, var(--bg) 20%); background: linear-gradient(to bottom, transparent, var(--bg) 20%);
@ -115,6 +115,86 @@
background: linear-gradient(to bottom, transparent, var(--bg-content) 20%); background: linear-gradient(to bottom, transparent, var(--bg-content) 20%);
} }
/* Attachment preview row */
.chat-attachments {
display: flex;
gap: 8px;
flex-wrap: wrap;
padding: 4px 0;
}
.chat-attachment {
position: relative;
width: 64px;
height: 64px;
border-radius: 8px;
overflow: hidden;
border: 1px solid var(--border);
background: var(--panel);
flex-shrink: 0;
}
.chat-attachment__img {
width: 100%;
height: 100%;
object-fit: cover;
display: block;
}
.chat-attachment__remove {
position: absolute;
top: 2px;
right: 2px;
width: 20px;
height: 20px;
border-radius: 50%;
border: none;
background: rgba(0, 0, 0, 0.6);
color: #fff;
font-size: 12px;
line-height: 1;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
padding: 0;
}
.chat-attachment__remove svg {
width: 12px;
height: 12px;
stroke: currentColor;
fill: none;
stroke-width: 2px;
}
.chat-attachment__remove:hover {
background: rgba(220, 38, 38, 0.8);
}
/* Compose row (textarea + actions) */
.chat-compose__row {
display: flex;
align-items: stretch;
gap: 12px;
}
/* Chat images in message bubbles */
.chat-images {
display: flex;
gap: 8px;
flex-wrap: wrap;
margin-bottom: 8px;
}
.chat-image {
max-width: 300px;
max-height: 200px;
border-radius: 8px;
object-fit: contain;
border: 1px solid var(--border);
}
.chat-compose__field { .chat-compose__field {
flex: 1 1 auto; flex: 1 1 auto;
min-width: 0; min-width: 0;

View File

@ -1,4 +1,4 @@
import { abortChatRun, loadChatHistory, sendChatMessage } from "./controllers/chat"; import { abortChatRun, loadChatHistory, sendChatMessage, type ChatAttachment } from "./controllers/chat";
import { loadSessions } from "./controllers/sessions"; import { loadSessions } from "./controllers/sessions";
import { generateUUID } from "./uuid"; import { generateUUID } from "./uuid";
import { resetToolStream } from "./app-tool-stream"; import { resetToolStream } from "./app-tool-stream";
@ -12,6 +12,7 @@ import type { ClawdbotApp } from "./app";
type ChatHost = { type ChatHost = {
connected: boolean; connected: boolean;
chatMessage: string; chatMessage: string;
chatAttachments: ChatAttachment[];
chatQueue: Array<{ id: string; text: string; createdAt: number }>; chatQueue: Array<{ id: string; text: string; createdAt: number }>;
chatRunId: string | null; chatRunId: string | null;
chatSending: boolean; chatSending: boolean;
@ -61,10 +62,10 @@ function enqueueChatMessage(host: ChatHost, text: string) {
async function sendChatMessageNow( async function sendChatMessageNow(
host: ChatHost, host: ChatHost,
message: string, message: string,
opts?: { previousDraft?: string; restoreDraft?: boolean }, opts?: { previousDraft?: string; restoreDraft?: boolean; attachments?: ChatAttachment[] },
) { ) {
resetToolStream(host as unknown as Parameters<typeof resetToolStream>[0]); resetToolStream(host as unknown as Parameters<typeof resetToolStream>[0]);
const ok = await sendChatMessage(host as unknown as ClawdbotApp, message); const ok = await sendChatMessage(host as unknown as ClawdbotApp, message, opts?.attachments);
if (!ok && opts?.previousDraft != null) { if (!ok && opts?.previousDraft != null) {
host.chatMessage = opts.previousDraft; host.chatMessage = opts.previousDraft;
} }
@ -104,7 +105,8 @@ export async function handleSendChat(
if (!host.connected) return; if (!host.connected) return;
const previousDraft = host.chatMessage; const previousDraft = host.chatMessage;
const message = (messageOverride ?? host.chatMessage).trim(); const message = (messageOverride ?? host.chatMessage).trim();
if (!message) return; const attachments = host.chatAttachments.length > 0 ? [...host.chatAttachments] : undefined;
if (!message && !attachments) return;
if (isChatStopCommand(message)) { if (isChatStopCommand(message)) {
await handleAbortChat(host); await handleAbortChat(host);
@ -114,6 +116,8 @@ export async function handleSendChat(
if (messageOverride == null) { if (messageOverride == null) {
host.chatMessage = ""; host.chatMessage = "";
} }
// Clear attachments after capturing them
host.chatAttachments = [];
if (isChatBusy(host)) { if (isChatBusy(host)) {
enqueueChatMessage(host, message); enqueueChatMessage(host, message);
@ -123,6 +127,7 @@ export async function handleSendChat(
await sendChatMessageNow(host, message, { await sendChatMessageNow(host, message, {
previousDraft: messageOverride == null ? previousDraft : undefined, previousDraft: messageOverride == null ? previousDraft : undefined,
restoreDraft: Boolean(messageOverride && opts?.restoreDraft), restoreDraft: Boolean(messageOverride && opts?.restoreDraft),
attachments,
}); });
} }

View File

@ -457,6 +457,7 @@ export function renderApp(state: AppViewState) {
stream: state.chatStream, stream: state.chatStream,
streamStartedAt: state.chatStreamStartedAt, streamStartedAt: state.chatStreamStartedAt,
draft: state.chatMessage, draft: state.chatMessage,
attachments: state.chatAttachments,
queue: state.chatQueue, queue: state.chatQueue,
connected: state.connected, connected: state.connected,
canSend: state.connected, canSend: state.connected,
@ -477,6 +478,7 @@ export function renderApp(state: AppViewState) {
}, },
onChatScroll: (event) => state.handleChatScroll(event), onChatScroll: (event) => state.handleChatScroll(event),
onDraftChange: (next) => (state.chatMessage = next), onDraftChange: (next) => (state.chatMessage = next),
onAttachmentsChange: (next) => (state.chatAttachments = next),
onSend: () => state.handleSendChat(), onSend: () => state.handleSendChat(),
canAbort: Boolean(state.chatRunId), canAbort: Boolean(state.chatRunId),
onAbort: () => void state.handleAbortChat(), onAbort: () => void state.handleAbortChat(),

View File

@ -22,6 +22,7 @@ import type {
import type { ChatQueueItem, CronFormState } from "./ui-types"; import type { ChatQueueItem, CronFormState } from "./ui-types";
import type { EventLogEntry } from "./app-events"; import type { EventLogEntry } from "./app-events";
import type { SkillMessage } from "./controllers/skills"; import type { SkillMessage } from "./controllers/skills";
import type { ChatAttachment } from "./controllers/chat";
import type { import type {
ExecApprovalsFile, ExecApprovalsFile,
ExecApprovalsSnapshot, ExecApprovalsSnapshot,
@ -56,6 +57,7 @@ export type AppViewState = {
chatAvatarUrl: string | null; chatAvatarUrl: string | null;
chatThinkingLevel: string | null; chatThinkingLevel: string | null;
chatQueue: ChatQueueItem[]; chatQueue: ChatQueueItem[];
chatAttachments: ChatAttachment[];
nodesLoading: boolean; nodesLoading: boolean;
nodes: Array<Record<string, unknown>>; nodes: Array<Record<string, unknown>>;
devicesLoading: boolean; devicesLoading: boolean;

View File

@ -129,6 +129,7 @@ export class ClawdbotApp extends LitElement {
@state() chatAvatarUrl: string | null = null; @state() chatAvatarUrl: string | null = null;
@state() chatThinkingLevel: string | null = null; @state() chatThinkingLevel: string | null = null;
@state() chatQueue: ChatQueueItem[] = []; @state() chatQueue: ChatQueueItem[] = [];
@state() chatAttachments: import("./controllers/chat").ChatAttachment[] = [];
// Sidebar state for tool output viewing // Sidebar state for tool output viewing
@state() sidebarOpen = false; @state() sidebarOpen = false;
@state() sidebarContent: string | null = null; @state() sidebarContent: string | null = null;

View File

@ -9,6 +9,7 @@ import { isToolResultMessage, normalizeRoleForGrouping } from "./message-normali
import { import {
extractTextCached, extractTextCached,
extractThinkingCached, extractThinkingCached,
extractImagesCached,
formatReasoningMarkdown, formatReasoningMarkdown,
} from "./message-extract"; } from "./message-extract";
import { extractToolCards, renderToolCardSidebar } from "./tool-cards"; import { extractToolCards, renderToolCardSidebar } from "./tool-cards";
@ -181,6 +182,8 @@ function renderGroupedMessage(
const hasToolCards = toolCards.length > 0; const hasToolCards = toolCards.length > 0;
const extractedText = extractTextCached(message); const extractedText = extractTextCached(message);
const extractedImages = extractImagesCached(message);
const hasImages = extractedImages.length > 0;
const extractedThinking = const extractedThinking =
opts.showReasoning && role === "assistant" opts.showReasoning && role === "assistant"
? extractThinkingCached(message) ? extractThinkingCached(message)
@ -207,7 +210,7 @@ function renderGroupedMessage(
)}`; )}`;
} }
if (!markdown && !hasToolCards) return nothing; if (!markdown && !hasToolCards && !hasImages) return nothing;
return html` return html`
<div class="${bubbleClasses}"> <div class="${bubbleClasses}">
@ -217,6 +220,11 @@ function renderGroupedMessage(
toSanitizedMarkdownHtml(reasoningMarkdown), toSanitizedMarkdownHtml(reasoningMarkdown),
)}</div>` )}</div>`
: nothing} : nothing}
${hasImages
? html`<div class="chat-images">${extractedImages.map(
(img) => html`<img class="chat-image" src=${img.dataUrl} alt="Attached image" />`,
)}</div>`
: nothing}
${markdown ${markdown
? html`<div class="chat-text">${unsafeHTML(toSanitizedMarkdownHtml(markdown))}</div>` ? html`<div class="chat-text">${unsafeHTML(toSanitizedMarkdownHtml(markdown))}</div>`
: nothing} : nothing}

View File

@ -127,6 +127,39 @@ export function extractRawText(message: unknown): string | null {
return null; return null;
} }
export type ImageBlock = {
dataUrl: string;
mimeType: string;
};
const imageCache = new WeakMap<object, ImageBlock[]>();
export function extractImages(message: unknown): ImageBlock[] {
const m = message as Record<string, unknown>;
const content = m.content;
if (!Array.isArray(content)) return [];
const images: ImageBlock[] = [];
for (const p of content) {
const item = p as Record<string, unknown>;
if (item.type === "image" && typeof item.dataUrl === "string") {
images.push({
dataUrl: item.dataUrl as string,
mimeType: (item.mimeType as string) ?? "image/png",
});
}
}
return images;
}
export function extractImagesCached(message: unknown): ImageBlock[] {
if (!message || typeof message !== "object") return extractImages(message);
const obj = message as object;
if (imageCache.has(obj)) return imageCache.get(obj)!;
const value = extractImages(message);
imageCache.set(obj, value);
return value;
}
export function formatReasoningMarkdown(text: string): string { export function formatReasoningMarkdown(text: string): string {
const trimmed = text.trim(); const trimmed = text.trim();
if (!trimmed) return ""; if (!trimmed) return "";

View File

@ -2,6 +2,14 @@ import type { GatewayBrowserClient } from "../gateway";
import { extractText } from "../chat/message-extract"; import { extractText } from "../chat/message-extract";
import { generateUUID } from "../uuid"; import { generateUUID } from "../uuid";
export type ChatAttachment = {
id: string;
dataUrl: string;
mimeType: string;
fileName: string;
size: number;
};
export type ChatState = { export type ChatState = {
client: GatewayBrowserClient | null; client: GatewayBrowserClient | null;
connected: boolean; connected: boolean;
@ -43,21 +51,46 @@ export async function loadChatHistory(state: ChatState) {
} }
} }
export async function sendChatMessage(state: ChatState, message: string): Promise<boolean> { export async function sendChatMessage(
state: ChatState,
message: string,
attachments?: ChatAttachment[],
): Promise<boolean> {
if (!state.client || !state.connected) return false; if (!state.client || !state.connected) return false;
const msg = message.trim(); const msg = message.trim();
if (!msg) return false; const hasAttachments = attachments && attachments.length > 0;
if (!msg && !hasAttachments) return false;
// Build local content blocks for chat history
const contentBlocks: unknown[] = [];
if (msg) contentBlocks.push({ type: "text", text: msg });
if (hasAttachments) {
for (const att of attachments) {
contentBlocks.push({ type: "image", dataUrl: att.dataUrl, mimeType: att.mimeType });
}
}
const now = Date.now(); const now = Date.now();
state.chatMessages = [ state.chatMessages = [
...state.chatMessages, ...state.chatMessages,
{ {
role: "user", role: "user",
content: [{ type: "text", text: msg }], content: contentBlocks,
timestamp: now, timestamp: now,
}, },
]; ];
// Backend requires NonEmptyString for message; use space fallback for image-only sends
const wireMessage = msg || " ";
// Convert attachments to gateway format (base64 content without data URL prefix)
const wireAttachments = hasAttachments
? attachments.map((att) => {
const base64 = att.dataUrl.replace(/^data:[^;]+;base64,/, "");
return { type: "image", mimeType: att.mimeType, fileName: att.fileName, content: base64 };
})
: undefined;
state.chatSending = true; state.chatSending = true;
state.lastError = null; state.lastError = null;
const runId = generateUUID(); const runId = generateUUID();
@ -67,9 +100,10 @@ export async function sendChatMessage(state: ChatState, message: string): Promis
try { try {
await state.client.request("chat.send", { await state.client.request("chat.send", {
sessionKey: state.sessionKey, sessionKey: state.sessionKey,
message: msg, message: wireMessage,
deliver: false, deliver: false,
idempotencyKey: runId, idempotencyKey: runId,
...(wireAttachments ? { attachments: wireAttachments } : {}),
}); });
return true; return true;
} catch (err) { } catch (err) {

View File

@ -3,7 +3,9 @@ import { repeat } from "lit/directives/repeat.js";
import type { SessionsListResult } from "../types"; import type { SessionsListResult } from "../types";
import type { ChatQueueItem } from "../ui-types"; import type { ChatQueueItem } from "../ui-types";
import type { ChatItem, MessageGroup } from "../types/chat-types"; import type { ChatItem, MessageGroup } from "../types/chat-types";
import type { ChatAttachment } from "../controllers/chat";
import { icons } from "../icons"; import { icons } from "../icons";
import { generateUUID } from "../uuid";
import { import {
normalizeMessage, normalizeMessage,
normalizeRoleForGrouping, normalizeRoleForGrouping,
@ -37,6 +39,7 @@ export type ChatProps = {
streamStartedAt: number | null; streamStartedAt: number | null;
assistantAvatarUrl?: string | null; assistantAvatarUrl?: string | null;
draft: string; draft: string;
attachments: ChatAttachment[];
queue: ChatQueueItem[]; queue: ChatQueueItem[];
connected: boolean; connected: boolean;
canSend: boolean; canSend: boolean;
@ -56,6 +59,7 @@ export type ChatProps = {
onRefresh: () => void; onRefresh: () => void;
onToggleFocusMode: () => void; onToggleFocusMode: () => void;
onDraftChange: (next: string) => void; onDraftChange: (next: string) => void;
onAttachmentsChange: (next: ChatAttachment[]) => void;
onSend: () => void; onSend: () => void;
onAbort?: () => void; onAbort?: () => void;
onQueueRemove: (id: string) => void; onQueueRemove: (id: string) => void;
@ -66,6 +70,45 @@ export type ChatProps = {
onChatScroll?: (event: Event) => void; onChatScroll?: (event: Event) => void;
}; };
const MAX_ATTACHMENT_BYTES = 5_000_000; // 5 MB
function handlePaste(e: ClipboardEvent, props: ChatProps) {
const items = e.clipboardData?.items;
if (!items) return;
const imageFiles: File[] = [];
for (const item of items) {
if (item.kind === "file" && item.type.startsWith("image/")) {
const file = item.getAsFile();
if (file) imageFiles.push(file);
}
}
if (imageFiles.length === 0) return;
e.preventDefault();
for (const file of imageFiles) {
if (file.size > MAX_ATTACHMENT_BYTES) {
console.warn(`Skipped image "${file.name}": exceeds 5 MB limit (${file.size} bytes)`);
continue;
}
const reader = new FileReader();
reader.onload = () => {
const dataUrl = reader.result as string;
const attachment: ChatAttachment = {
id: generateUUID(),
dataUrl,
mimeType: file.type,
fileName: file.name || "pasted-image",
size: file.size,
};
props.onAttachmentsChange([...props.attachments, attachment]);
};
reader.readAsDataURL(file);
}
}
function removeAttachment(props: ChatProps, id: string) {
props.onAttachmentsChange(props.attachments.filter((a) => a.id !== id));
}
const COMPACTION_TOAST_DURATION_MS = 5000; const COMPACTION_TOAST_DURATION_MS = 5000;
function renderCompactionIndicator(status: CompactionIndicatorStatus | null | undefined) { function renderCompactionIndicator(status: CompactionIndicatorStatus | null | undefined) {
@ -235,39 +278,63 @@ export function renderChat(props: ChatProps) {
: nothing} : nothing}
<div class="chat-compose"> <div class="chat-compose">
<label class="field chat-compose__field"> ${props.attachments.length > 0
<span>Message</span> ? html`
<textarea <div class="chat-attachments">
.value=${props.draft} ${props.attachments.map(
?disabled=${!props.connected} (att) => html`
@keydown=${(e: KeyboardEvent) => { <div class="chat-attachment">
if (e.key !== "Enter") return; <img class="chat-attachment__img" src=${att.dataUrl} alt=${att.fileName} />
if (e.isComposing || e.keyCode === 229) return; <button
if (e.shiftKey) return; // Allow Shift+Enter for line breaks class="chat-attachment__remove"
if (!props.connected) return; type="button"
e.preventDefault(); aria-label="Remove attachment"
if (canCompose) props.onSend(); @click=${() => removeAttachment(props, att.id)}
}} >
@input=${(e: Event) => ${icons.x}
props.onDraftChange((e.target as HTMLTextAreaElement).value)} </button>
placeholder=${composePlaceholder} </div>
></textarea> `,
</label> )}
<div class="chat-compose__actions"> </div>
<button `
class="btn" : nothing}
?disabled=${!props.connected || (!canAbort && props.sending)} <div class="chat-compose__row">
@click=${canAbort ? props.onAbort : props.onNewSession} <label class="field chat-compose__field">
> <span>Message</span>
${canAbort ? "Stop" : "New session"} <textarea
</button> .value=${props.draft}
<button ?disabled=${!props.connected}
class="btn primary" @keydown=${(e: KeyboardEvent) => {
?disabled=${!props.connected} if (e.key !== "Enter") return;
@click=${props.onSend} if (e.isComposing || e.keyCode === 229) return;
> if (e.shiftKey) return; // Allow Shift+Enter for line breaks
${isBusy ? "Queue" : "Send"}<kbd class="btn-kbd"></kbd> if (!props.connected) return;
</button> e.preventDefault();
if (canCompose) props.onSend();
}}
@input=${(e: Event) =>
props.onDraftChange((e.target as HTMLTextAreaElement).value)}
@paste=${(e: ClipboardEvent) => handlePaste(e, props)}
placeholder=${composePlaceholder}
></textarea>
</label>
<div class="chat-compose__actions">
<button
class="btn"
?disabled=${!props.connected || (!canAbort && props.sending)}
@click=${canAbort ? props.onAbort : props.onNewSession}
>
${canAbort ? "Stop" : "New session"}
</button>
<button
class="btn primary"
?disabled=${!props.connected}
@click=${props.onSend}
>
${isBusy ? "Queue" : "Send"}<kbd class="btn-kbd"></kbd>
</button>
</div>
</div> </div>
</div> </div>
</section> </section>