feat(webchat): add image paste support with preview, validation, and rendering

Add clipboard image paste to WebChat compose area:
- Paste handler reads image files from clipboard, converts to data URLs
- Attachment preview thumbnails with remove button in compose area
- 5 MB frontend size validation (matches backend limit)
- Image content blocks rendered in chat history bubbles
- Backend NonEmptyString bypass: image-only sends use space fallback
- Attachments converted to base64 gateway format on send

Fixes #1681, supersedes #1900
This commit is contained in:
Joey Rodriguez 2026-01-30 02:05:56 -05:00
parent c8063bdcd8
commit 363c08db3d
9 changed files with 276 additions and 44 deletions

View File

@ -103,8 +103,8 @@
bottom: 0;
flex-shrink: 0;
display: flex;
align-items: stretch;
gap: 12px;
flex-direction: column;
gap: 8px;
margin-top: auto; /* Push to bottom of flex container */
padding: 12px 4px 4px;
background: linear-gradient(to bottom, transparent, var(--bg) 20%);
@ -115,6 +115,86 @@
background: linear-gradient(to bottom, transparent, var(--bg-content) 20%);
}
/* Attachment preview row */
.chat-attachments {
display: flex;
gap: 8px;
flex-wrap: wrap;
padding: 4px 0;
}
.chat-attachment {
position: relative;
width: 64px;
height: 64px;
border-radius: 8px;
overflow: hidden;
border: 1px solid var(--border);
background: var(--panel);
flex-shrink: 0;
}
.chat-attachment__img {
width: 100%;
height: 100%;
object-fit: cover;
display: block;
}
.chat-attachment__remove {
position: absolute;
top: 2px;
right: 2px;
width: 20px;
height: 20px;
border-radius: 50%;
border: none;
background: rgba(0, 0, 0, 0.6);
color: #fff;
font-size: 12px;
line-height: 1;
cursor: pointer;
display: flex;
align-items: center;
justify-content: center;
padding: 0;
}
.chat-attachment__remove svg {
width: 12px;
height: 12px;
stroke: currentColor;
fill: none;
stroke-width: 2px;
}
.chat-attachment__remove:hover {
background: rgba(220, 38, 38, 0.8);
}
/* Compose row (textarea + actions) */
.chat-compose__row {
display: flex;
align-items: stretch;
gap: 12px;
}
/* Chat images in message bubbles */
.chat-images {
display: flex;
gap: 8px;
flex-wrap: wrap;
margin-bottom: 8px;
}
.chat-image {
max-width: 300px;
max-height: 200px;
border-radius: 8px;
object-fit: contain;
border: 1px solid var(--border);
}
.chat-compose__field {
flex: 1 1 auto;
min-width: 0;

View File

@ -1,4 +1,4 @@
import { abortChatRun, loadChatHistory, sendChatMessage } from "./controllers/chat";
import { abortChatRun, loadChatHistory, sendChatMessage, type ChatAttachment } from "./controllers/chat";
import { loadSessions } from "./controllers/sessions";
import { generateUUID } from "./uuid";
import { resetToolStream } from "./app-tool-stream";
@ -12,6 +12,7 @@ import type { ClawdbotApp } from "./app";
type ChatHost = {
connected: boolean;
chatMessage: string;
chatAttachments: ChatAttachment[];
chatQueue: Array<{ id: string; text: string; createdAt: number }>;
chatRunId: string | null;
chatSending: boolean;
@ -61,10 +62,10 @@ function enqueueChatMessage(host: ChatHost, text: string) {
async function sendChatMessageNow(
host: ChatHost,
message: string,
opts?: { previousDraft?: string; restoreDraft?: boolean },
opts?: { previousDraft?: string; restoreDraft?: boolean; attachments?: ChatAttachment[] },
) {
resetToolStream(host as unknown as Parameters<typeof resetToolStream>[0]);
const ok = await sendChatMessage(host as unknown as ClawdbotApp, message);
const ok = await sendChatMessage(host as unknown as ClawdbotApp, message, opts?.attachments);
if (!ok && opts?.previousDraft != null) {
host.chatMessage = opts.previousDraft;
}
@ -104,7 +105,8 @@ export async function handleSendChat(
if (!host.connected) return;
const previousDraft = host.chatMessage;
const message = (messageOverride ?? host.chatMessage).trim();
if (!message) return;
const attachments = host.chatAttachments.length > 0 ? [...host.chatAttachments] : undefined;
if (!message && !attachments) return;
if (isChatStopCommand(message)) {
await handleAbortChat(host);
@ -114,6 +116,8 @@ export async function handleSendChat(
if (messageOverride == null) {
host.chatMessage = "";
}
// Clear attachments after capturing them
host.chatAttachments = [];
if (isChatBusy(host)) {
enqueueChatMessage(host, message);
@ -123,6 +127,7 @@ export async function handleSendChat(
await sendChatMessageNow(host, message, {
previousDraft: messageOverride == null ? previousDraft : undefined,
restoreDraft: Boolean(messageOverride && opts?.restoreDraft),
attachments,
});
}

View File

@ -457,6 +457,7 @@ export function renderApp(state: AppViewState) {
stream: state.chatStream,
streamStartedAt: state.chatStreamStartedAt,
draft: state.chatMessage,
attachments: state.chatAttachments,
queue: state.chatQueue,
connected: state.connected,
canSend: state.connected,
@ -477,6 +478,7 @@ export function renderApp(state: AppViewState) {
},
onChatScroll: (event) => state.handleChatScroll(event),
onDraftChange: (next) => (state.chatMessage = next),
onAttachmentsChange: (next) => (state.chatAttachments = next),
onSend: () => state.handleSendChat(),
canAbort: Boolean(state.chatRunId),
onAbort: () => void state.handleAbortChat(),

View File

@ -22,6 +22,7 @@ import type {
import type { ChatQueueItem, CronFormState } from "./ui-types";
import type { EventLogEntry } from "./app-events";
import type { SkillMessage } from "./controllers/skills";
import type { ChatAttachment } from "./controllers/chat";
import type {
ExecApprovalsFile,
ExecApprovalsSnapshot,
@ -56,6 +57,7 @@ export type AppViewState = {
chatAvatarUrl: string | null;
chatThinkingLevel: string | null;
chatQueue: ChatQueueItem[];
chatAttachments: ChatAttachment[];
nodesLoading: boolean;
nodes: Array<Record<string, unknown>>;
devicesLoading: boolean;

View File

@ -129,6 +129,7 @@ export class ClawdbotApp extends LitElement {
@state() chatAvatarUrl: string | null = null;
@state() chatThinkingLevel: string | null = null;
@state() chatQueue: ChatQueueItem[] = [];
@state() chatAttachments: import("./controllers/chat").ChatAttachment[] = [];
// Sidebar state for tool output viewing
@state() sidebarOpen = false;
@state() sidebarContent: string | null = null;

View File

@ -9,6 +9,7 @@ import { isToolResultMessage, normalizeRoleForGrouping } from "./message-normali
import {
extractTextCached,
extractThinkingCached,
extractImagesCached,
formatReasoningMarkdown,
} from "./message-extract";
import { extractToolCards, renderToolCardSidebar } from "./tool-cards";
@ -181,6 +182,8 @@ function renderGroupedMessage(
const hasToolCards = toolCards.length > 0;
const extractedText = extractTextCached(message);
const extractedImages = extractImagesCached(message);
const hasImages = extractedImages.length > 0;
const extractedThinking =
opts.showReasoning && role === "assistant"
? extractThinkingCached(message)
@ -207,7 +210,7 @@ function renderGroupedMessage(
)}`;
}
if (!markdown && !hasToolCards) return nothing;
if (!markdown && !hasToolCards && !hasImages) return nothing;
return html`
<div class="${bubbleClasses}">
@ -217,6 +220,11 @@ function renderGroupedMessage(
toSanitizedMarkdownHtml(reasoningMarkdown),
)}</div>`
: nothing}
${hasImages
? html`<div class="chat-images">${extractedImages.map(
(img) => html`<img class="chat-image" src=${img.dataUrl} alt="Attached image" />`,
)}</div>`
: nothing}
${markdown
? html`<div class="chat-text">${unsafeHTML(toSanitizedMarkdownHtml(markdown))}</div>`
: nothing}

View File

@ -127,6 +127,39 @@ export function extractRawText(message: unknown): string | null {
return null;
}
export type ImageBlock = {
dataUrl: string;
mimeType: string;
};
const imageCache = new WeakMap<object, ImageBlock[]>();
export function extractImages(message: unknown): ImageBlock[] {
const m = message as Record<string, unknown>;
const content = m.content;
if (!Array.isArray(content)) return [];
const images: ImageBlock[] = [];
for (const p of content) {
const item = p as Record<string, unknown>;
if (item.type === "image" && typeof item.dataUrl === "string") {
images.push({
dataUrl: item.dataUrl as string,
mimeType: (item.mimeType as string) ?? "image/png",
});
}
}
return images;
}
export function extractImagesCached(message: unknown): ImageBlock[] {
if (!message || typeof message !== "object") return extractImages(message);
const obj = message as object;
if (imageCache.has(obj)) return imageCache.get(obj)!;
const value = extractImages(message);
imageCache.set(obj, value);
return value;
}
export function formatReasoningMarkdown(text: string): string {
const trimmed = text.trim();
if (!trimmed) return "";

View File

@ -2,6 +2,14 @@ import type { GatewayBrowserClient } from "../gateway";
import { extractText } from "../chat/message-extract";
import { generateUUID } from "../uuid";
export type ChatAttachment = {
id: string;
dataUrl: string;
mimeType: string;
fileName: string;
size: number;
};
export type ChatState = {
client: GatewayBrowserClient | null;
connected: boolean;
@ -43,21 +51,46 @@ export async function loadChatHistory(state: ChatState) {
}
}
export async function sendChatMessage(state: ChatState, message: string): Promise<boolean> {
export async function sendChatMessage(
state: ChatState,
message: string,
attachments?: ChatAttachment[],
): Promise<boolean> {
if (!state.client || !state.connected) return false;
const msg = message.trim();
if (!msg) return false;
const hasAttachments = attachments && attachments.length > 0;
if (!msg && !hasAttachments) return false;
// Build local content blocks for chat history
const contentBlocks: unknown[] = [];
if (msg) contentBlocks.push({ type: "text", text: msg });
if (hasAttachments) {
for (const att of attachments) {
contentBlocks.push({ type: "image", dataUrl: att.dataUrl, mimeType: att.mimeType });
}
}
const now = Date.now();
state.chatMessages = [
...state.chatMessages,
{
role: "user",
content: [{ type: "text", text: msg }],
content: contentBlocks,
timestamp: now,
},
];
// Backend requires NonEmptyString for message; use space fallback for image-only sends
const wireMessage = msg || " ";
// Convert attachments to gateway format (base64 content without data URL prefix)
const wireAttachments = hasAttachments
? attachments.map((att) => {
const base64 = att.dataUrl.replace(/^data:[^;]+;base64,/, "");
return { type: "image", mimeType: att.mimeType, fileName: att.fileName, content: base64 };
})
: undefined;
state.chatSending = true;
state.lastError = null;
const runId = generateUUID();
@ -67,9 +100,10 @@ export async function sendChatMessage(state: ChatState, message: string): Promis
try {
await state.client.request("chat.send", {
sessionKey: state.sessionKey,
message: msg,
message: wireMessage,
deliver: false,
idempotencyKey: runId,
...(wireAttachments ? { attachments: wireAttachments } : {}),
});
return true;
} catch (err) {

View File

@ -3,7 +3,9 @@ import { repeat } from "lit/directives/repeat.js";
import type { SessionsListResult } from "../types";
import type { ChatQueueItem } from "../ui-types";
import type { ChatItem, MessageGroup } from "../types/chat-types";
import type { ChatAttachment } from "../controllers/chat";
import { icons } from "../icons";
import { generateUUID } from "../uuid";
import {
normalizeMessage,
normalizeRoleForGrouping,
@ -37,6 +39,7 @@ export type ChatProps = {
streamStartedAt: number | null;
assistantAvatarUrl?: string | null;
draft: string;
attachments: ChatAttachment[];
queue: ChatQueueItem[];
connected: boolean;
canSend: boolean;
@ -56,6 +59,7 @@ export type ChatProps = {
onRefresh: () => void;
onToggleFocusMode: () => void;
onDraftChange: (next: string) => void;
onAttachmentsChange: (next: ChatAttachment[]) => void;
onSend: () => void;
onAbort?: () => void;
onQueueRemove: (id: string) => void;
@ -66,6 +70,45 @@ export type ChatProps = {
onChatScroll?: (event: Event) => void;
};
const MAX_ATTACHMENT_BYTES = 5_000_000; // 5 MB
function handlePaste(e: ClipboardEvent, props: ChatProps) {
const items = e.clipboardData?.items;
if (!items) return;
const imageFiles: File[] = [];
for (const item of items) {
if (item.kind === "file" && item.type.startsWith("image/")) {
const file = item.getAsFile();
if (file) imageFiles.push(file);
}
}
if (imageFiles.length === 0) return;
e.preventDefault();
for (const file of imageFiles) {
if (file.size > MAX_ATTACHMENT_BYTES) {
console.warn(`Skipped image "${file.name}": exceeds 5 MB limit (${file.size} bytes)`);
continue;
}
const reader = new FileReader();
reader.onload = () => {
const dataUrl = reader.result as string;
const attachment: ChatAttachment = {
id: generateUUID(),
dataUrl,
mimeType: file.type,
fileName: file.name || "pasted-image",
size: file.size,
};
props.onAttachmentsChange([...props.attachments, attachment]);
};
reader.readAsDataURL(file);
}
}
function removeAttachment(props: ChatProps, id: string) {
props.onAttachmentsChange(props.attachments.filter((a) => a.id !== id));
}
const COMPACTION_TOAST_DURATION_MS = 5000;
function renderCompactionIndicator(status: CompactionIndicatorStatus | null | undefined) {
@ -235,39 +278,63 @@ export function renderChat(props: ChatProps) {
: nothing}
<div class="chat-compose">
<label class="field chat-compose__field">
<span>Message</span>
<textarea
.value=${props.draft}
?disabled=${!props.connected}
@keydown=${(e: KeyboardEvent) => {
if (e.key !== "Enter") return;
if (e.isComposing || e.keyCode === 229) return;
if (e.shiftKey) return; // Allow Shift+Enter for line breaks
if (!props.connected) return;
e.preventDefault();
if (canCompose) props.onSend();
}}
@input=${(e: Event) =>
props.onDraftChange((e.target as HTMLTextAreaElement).value)}
placeholder=${composePlaceholder}
></textarea>
</label>
<div class="chat-compose__actions">
<button
class="btn"
?disabled=${!props.connected || (!canAbort && props.sending)}
@click=${canAbort ? props.onAbort : props.onNewSession}
>
${canAbort ? "Stop" : "New session"}
</button>
<button
class="btn primary"
?disabled=${!props.connected}
@click=${props.onSend}
>
${isBusy ? "Queue" : "Send"}<kbd class="btn-kbd"></kbd>
</button>
${props.attachments.length > 0
? html`
<div class="chat-attachments">
${props.attachments.map(
(att) => html`
<div class="chat-attachment">
<img class="chat-attachment__img" src=${att.dataUrl} alt=${att.fileName} />
<button
class="chat-attachment__remove"
type="button"
aria-label="Remove attachment"
@click=${() => removeAttachment(props, att.id)}
>
${icons.x}
</button>
</div>
`,
)}
</div>
`
: nothing}
<div class="chat-compose__row">
<label class="field chat-compose__field">
<span>Message</span>
<textarea
.value=${props.draft}
?disabled=${!props.connected}
@keydown=${(e: KeyboardEvent) => {
if (e.key !== "Enter") return;
if (e.isComposing || e.keyCode === 229) return;
if (e.shiftKey) return; // Allow Shift+Enter for line breaks
if (!props.connected) return;
e.preventDefault();
if (canCompose) props.onSend();
}}
@input=${(e: Event) =>
props.onDraftChange((e.target as HTMLTextAreaElement).value)}
@paste=${(e: ClipboardEvent) => handlePaste(e, props)}
placeholder=${composePlaceholder}
></textarea>
</label>
<div class="chat-compose__actions">
<button
class="btn"
?disabled=${!props.connected || (!canAbort && props.sending)}
@click=${canAbort ? props.onAbort : props.onNewSession}
>
${canAbort ? "Stop" : "New session"}
</button>
<button
class="btn primary"
?disabled=${!props.connected}
@click=${props.onSend}
>
${isBusy ? "Queue" : "Send"}<kbd class="btn-kbd"></kbd>
</button>
</div>
</div>
</div>
</section>