feat(webchat): add image paste support with preview, validation, and rendering
Add clipboard image paste to WebChat compose area: - Paste handler reads image files from clipboard, converts to data URLs - Attachment preview thumbnails with remove button in compose area - 5 MB frontend size validation (matches backend limit) - Image content blocks rendered in chat history bubbles - Backend NonEmptyString bypass: image-only sends use space fallback - Attachments converted to base64 gateway format on send Fixes #1681, supersedes #1900
This commit is contained in:
parent
c8063bdcd8
commit
363c08db3d
@ -103,8 +103,8 @@
|
||||
bottom: 0;
|
||||
flex-shrink: 0;
|
||||
display: flex;
|
||||
align-items: stretch;
|
||||
gap: 12px;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
margin-top: auto; /* Push to bottom of flex container */
|
||||
padding: 12px 4px 4px;
|
||||
background: linear-gradient(to bottom, transparent, var(--bg) 20%);
|
||||
@ -115,6 +115,86 @@
|
||||
background: linear-gradient(to bottom, transparent, var(--bg-content) 20%);
|
||||
}
|
||||
|
||||
/* Attachment preview row */
|
||||
.chat-attachments {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
flex-wrap: wrap;
|
||||
padding: 4px 0;
|
||||
}
|
||||
|
||||
.chat-attachment {
|
||||
position: relative;
|
||||
width: 64px;
|
||||
height: 64px;
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
border: 1px solid var(--border);
|
||||
background: var(--panel);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.chat-attachment__img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.chat-attachment__remove {
|
||||
position: absolute;
|
||||
top: 2px;
|
||||
right: 2px;
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
border-radius: 50%;
|
||||
border: none;
|
||||
background: rgba(0, 0, 0, 0.6);
|
||||
color: #fff;
|
||||
font-size: 12px;
|
||||
line-height: 1;
|
||||
cursor: pointer;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.chat-attachment__remove svg {
|
||||
width: 12px;
|
||||
height: 12px;
|
||||
stroke: currentColor;
|
||||
fill: none;
|
||||
stroke-width: 2px;
|
||||
}
|
||||
|
||||
.chat-attachment__remove:hover {
|
||||
background: rgba(220, 38, 38, 0.8);
|
||||
}
|
||||
|
||||
/* Compose row (textarea + actions) */
|
||||
.chat-compose__row {
|
||||
display: flex;
|
||||
align-items: stretch;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
/* Chat images in message bubbles */
|
||||
.chat-images {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
flex-wrap: wrap;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.chat-image {
|
||||
max-width: 300px;
|
||||
max-height: 200px;
|
||||
border-radius: 8px;
|
||||
object-fit: contain;
|
||||
border: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.chat-compose__field {
|
||||
flex: 1 1 auto;
|
||||
min-width: 0;
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { abortChatRun, loadChatHistory, sendChatMessage } from "./controllers/chat";
|
||||
import { abortChatRun, loadChatHistory, sendChatMessage, type ChatAttachment } from "./controllers/chat";
|
||||
import { loadSessions } from "./controllers/sessions";
|
||||
import { generateUUID } from "./uuid";
|
||||
import { resetToolStream } from "./app-tool-stream";
|
||||
@ -12,6 +12,7 @@ import type { ClawdbotApp } from "./app";
|
||||
type ChatHost = {
|
||||
connected: boolean;
|
||||
chatMessage: string;
|
||||
chatAttachments: ChatAttachment[];
|
||||
chatQueue: Array<{ id: string; text: string; createdAt: number }>;
|
||||
chatRunId: string | null;
|
||||
chatSending: boolean;
|
||||
@ -61,10 +62,10 @@ function enqueueChatMessage(host: ChatHost, text: string) {
|
||||
async function sendChatMessageNow(
|
||||
host: ChatHost,
|
||||
message: string,
|
||||
opts?: { previousDraft?: string; restoreDraft?: boolean },
|
||||
opts?: { previousDraft?: string; restoreDraft?: boolean; attachments?: ChatAttachment[] },
|
||||
) {
|
||||
resetToolStream(host as unknown as Parameters<typeof resetToolStream>[0]);
|
||||
const ok = await sendChatMessage(host as unknown as ClawdbotApp, message);
|
||||
const ok = await sendChatMessage(host as unknown as ClawdbotApp, message, opts?.attachments);
|
||||
if (!ok && opts?.previousDraft != null) {
|
||||
host.chatMessage = opts.previousDraft;
|
||||
}
|
||||
@ -104,7 +105,8 @@ export async function handleSendChat(
|
||||
if (!host.connected) return;
|
||||
const previousDraft = host.chatMessage;
|
||||
const message = (messageOverride ?? host.chatMessage).trim();
|
||||
if (!message) return;
|
||||
const attachments = host.chatAttachments.length > 0 ? [...host.chatAttachments] : undefined;
|
||||
if (!message && !attachments) return;
|
||||
|
||||
if (isChatStopCommand(message)) {
|
||||
await handleAbortChat(host);
|
||||
@ -114,6 +116,8 @@ export async function handleSendChat(
|
||||
if (messageOverride == null) {
|
||||
host.chatMessage = "";
|
||||
}
|
||||
// Clear attachments after capturing them
|
||||
host.chatAttachments = [];
|
||||
|
||||
if (isChatBusy(host)) {
|
||||
enqueueChatMessage(host, message);
|
||||
@ -123,6 +127,7 @@ export async function handleSendChat(
|
||||
await sendChatMessageNow(host, message, {
|
||||
previousDraft: messageOverride == null ? previousDraft : undefined,
|
||||
restoreDraft: Boolean(messageOverride && opts?.restoreDraft),
|
||||
attachments,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -457,6 +457,7 @@ export function renderApp(state: AppViewState) {
|
||||
stream: state.chatStream,
|
||||
streamStartedAt: state.chatStreamStartedAt,
|
||||
draft: state.chatMessage,
|
||||
attachments: state.chatAttachments,
|
||||
queue: state.chatQueue,
|
||||
connected: state.connected,
|
||||
canSend: state.connected,
|
||||
@ -477,6 +478,7 @@ export function renderApp(state: AppViewState) {
|
||||
},
|
||||
onChatScroll: (event) => state.handleChatScroll(event),
|
||||
onDraftChange: (next) => (state.chatMessage = next),
|
||||
onAttachmentsChange: (next) => (state.chatAttachments = next),
|
||||
onSend: () => state.handleSendChat(),
|
||||
canAbort: Boolean(state.chatRunId),
|
||||
onAbort: () => void state.handleAbortChat(),
|
||||
|
||||
@ -22,6 +22,7 @@ import type {
|
||||
import type { ChatQueueItem, CronFormState } from "./ui-types";
|
||||
import type { EventLogEntry } from "./app-events";
|
||||
import type { SkillMessage } from "./controllers/skills";
|
||||
import type { ChatAttachment } from "./controllers/chat";
|
||||
import type {
|
||||
ExecApprovalsFile,
|
||||
ExecApprovalsSnapshot,
|
||||
@ -56,6 +57,7 @@ export type AppViewState = {
|
||||
chatAvatarUrl: string | null;
|
||||
chatThinkingLevel: string | null;
|
||||
chatQueue: ChatQueueItem[];
|
||||
chatAttachments: ChatAttachment[];
|
||||
nodesLoading: boolean;
|
||||
nodes: Array<Record<string, unknown>>;
|
||||
devicesLoading: boolean;
|
||||
|
||||
@ -129,6 +129,7 @@ export class ClawdbotApp extends LitElement {
|
||||
@state() chatAvatarUrl: string | null = null;
|
||||
@state() chatThinkingLevel: string | null = null;
|
||||
@state() chatQueue: ChatQueueItem[] = [];
|
||||
@state() chatAttachments: import("./controllers/chat").ChatAttachment[] = [];
|
||||
// Sidebar state for tool output viewing
|
||||
@state() sidebarOpen = false;
|
||||
@state() sidebarContent: string | null = null;
|
||||
|
||||
@ -9,6 +9,7 @@ import { isToolResultMessage, normalizeRoleForGrouping } from "./message-normali
|
||||
import {
|
||||
extractTextCached,
|
||||
extractThinkingCached,
|
||||
extractImagesCached,
|
||||
formatReasoningMarkdown,
|
||||
} from "./message-extract";
|
||||
import { extractToolCards, renderToolCardSidebar } from "./tool-cards";
|
||||
@ -181,6 +182,8 @@ function renderGroupedMessage(
|
||||
const hasToolCards = toolCards.length > 0;
|
||||
|
||||
const extractedText = extractTextCached(message);
|
||||
const extractedImages = extractImagesCached(message);
|
||||
const hasImages = extractedImages.length > 0;
|
||||
const extractedThinking =
|
||||
opts.showReasoning && role === "assistant"
|
||||
? extractThinkingCached(message)
|
||||
@ -207,7 +210,7 @@ function renderGroupedMessage(
|
||||
)}`;
|
||||
}
|
||||
|
||||
if (!markdown && !hasToolCards) return nothing;
|
||||
if (!markdown && !hasToolCards && !hasImages) return nothing;
|
||||
|
||||
return html`
|
||||
<div class="${bubbleClasses}">
|
||||
@ -217,6 +220,11 @@ function renderGroupedMessage(
|
||||
toSanitizedMarkdownHtml(reasoningMarkdown),
|
||||
)}</div>`
|
||||
: nothing}
|
||||
${hasImages
|
||||
? html`<div class="chat-images">${extractedImages.map(
|
||||
(img) => html`<img class="chat-image" src=${img.dataUrl} alt="Attached image" />`,
|
||||
)}</div>`
|
||||
: nothing}
|
||||
${markdown
|
||||
? html`<div class="chat-text">${unsafeHTML(toSanitizedMarkdownHtml(markdown))}</div>`
|
||||
: nothing}
|
||||
|
||||
@ -127,6 +127,39 @@ export function extractRawText(message: unknown): string | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
export type ImageBlock = {
|
||||
dataUrl: string;
|
||||
mimeType: string;
|
||||
};
|
||||
|
||||
const imageCache = new WeakMap<object, ImageBlock[]>();
|
||||
|
||||
export function extractImages(message: unknown): ImageBlock[] {
|
||||
const m = message as Record<string, unknown>;
|
||||
const content = m.content;
|
||||
if (!Array.isArray(content)) return [];
|
||||
const images: ImageBlock[] = [];
|
||||
for (const p of content) {
|
||||
const item = p as Record<string, unknown>;
|
||||
if (item.type === "image" && typeof item.dataUrl === "string") {
|
||||
images.push({
|
||||
dataUrl: item.dataUrl as string,
|
||||
mimeType: (item.mimeType as string) ?? "image/png",
|
||||
});
|
||||
}
|
||||
}
|
||||
return images;
|
||||
}
|
||||
|
||||
export function extractImagesCached(message: unknown): ImageBlock[] {
|
||||
if (!message || typeof message !== "object") return extractImages(message);
|
||||
const obj = message as object;
|
||||
if (imageCache.has(obj)) return imageCache.get(obj)!;
|
||||
const value = extractImages(message);
|
||||
imageCache.set(obj, value);
|
||||
return value;
|
||||
}
|
||||
|
||||
export function formatReasoningMarkdown(text: string): string {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return "";
|
||||
|
||||
@ -2,6 +2,14 @@ import type { GatewayBrowserClient } from "../gateway";
|
||||
import { extractText } from "../chat/message-extract";
|
||||
import { generateUUID } from "../uuid";
|
||||
|
||||
export type ChatAttachment = {
|
||||
id: string;
|
||||
dataUrl: string;
|
||||
mimeType: string;
|
||||
fileName: string;
|
||||
size: number;
|
||||
};
|
||||
|
||||
export type ChatState = {
|
||||
client: GatewayBrowserClient | null;
|
||||
connected: boolean;
|
||||
@ -43,21 +51,46 @@ export async function loadChatHistory(state: ChatState) {
|
||||
}
|
||||
}
|
||||
|
||||
export async function sendChatMessage(state: ChatState, message: string): Promise<boolean> {
|
||||
export async function sendChatMessage(
|
||||
state: ChatState,
|
||||
message: string,
|
||||
attachments?: ChatAttachment[],
|
||||
): Promise<boolean> {
|
||||
if (!state.client || !state.connected) return false;
|
||||
const msg = message.trim();
|
||||
if (!msg) return false;
|
||||
const hasAttachments = attachments && attachments.length > 0;
|
||||
if (!msg && !hasAttachments) return false;
|
||||
|
||||
// Build local content blocks for chat history
|
||||
const contentBlocks: unknown[] = [];
|
||||
if (msg) contentBlocks.push({ type: "text", text: msg });
|
||||
if (hasAttachments) {
|
||||
for (const att of attachments) {
|
||||
contentBlocks.push({ type: "image", dataUrl: att.dataUrl, mimeType: att.mimeType });
|
||||
}
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
state.chatMessages = [
|
||||
...state.chatMessages,
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: msg }],
|
||||
content: contentBlocks,
|
||||
timestamp: now,
|
||||
},
|
||||
];
|
||||
|
||||
// Backend requires NonEmptyString for message; use space fallback for image-only sends
|
||||
const wireMessage = msg || " ";
|
||||
|
||||
// Convert attachments to gateway format (base64 content without data URL prefix)
|
||||
const wireAttachments = hasAttachments
|
||||
? attachments.map((att) => {
|
||||
const base64 = att.dataUrl.replace(/^data:[^;]+;base64,/, "");
|
||||
return { type: "image", mimeType: att.mimeType, fileName: att.fileName, content: base64 };
|
||||
})
|
||||
: undefined;
|
||||
|
||||
state.chatSending = true;
|
||||
state.lastError = null;
|
||||
const runId = generateUUID();
|
||||
@ -67,9 +100,10 @@ export async function sendChatMessage(state: ChatState, message: string): Promis
|
||||
try {
|
||||
await state.client.request("chat.send", {
|
||||
sessionKey: state.sessionKey,
|
||||
message: msg,
|
||||
message: wireMessage,
|
||||
deliver: false,
|
||||
idempotencyKey: runId,
|
||||
...(wireAttachments ? { attachments: wireAttachments } : {}),
|
||||
});
|
||||
return true;
|
||||
} catch (err) {
|
||||
|
||||
@ -3,7 +3,9 @@ import { repeat } from "lit/directives/repeat.js";
|
||||
import type { SessionsListResult } from "../types";
|
||||
import type { ChatQueueItem } from "../ui-types";
|
||||
import type { ChatItem, MessageGroup } from "../types/chat-types";
|
||||
import type { ChatAttachment } from "../controllers/chat";
|
||||
import { icons } from "../icons";
|
||||
import { generateUUID } from "../uuid";
|
||||
import {
|
||||
normalizeMessage,
|
||||
normalizeRoleForGrouping,
|
||||
@ -37,6 +39,7 @@ export type ChatProps = {
|
||||
streamStartedAt: number | null;
|
||||
assistantAvatarUrl?: string | null;
|
||||
draft: string;
|
||||
attachments: ChatAttachment[];
|
||||
queue: ChatQueueItem[];
|
||||
connected: boolean;
|
||||
canSend: boolean;
|
||||
@ -56,6 +59,7 @@ export type ChatProps = {
|
||||
onRefresh: () => void;
|
||||
onToggleFocusMode: () => void;
|
||||
onDraftChange: (next: string) => void;
|
||||
onAttachmentsChange: (next: ChatAttachment[]) => void;
|
||||
onSend: () => void;
|
||||
onAbort?: () => void;
|
||||
onQueueRemove: (id: string) => void;
|
||||
@ -66,6 +70,45 @@ export type ChatProps = {
|
||||
onChatScroll?: (event: Event) => void;
|
||||
};
|
||||
|
||||
const MAX_ATTACHMENT_BYTES = 5_000_000; // 5 MB
|
||||
|
||||
function handlePaste(e: ClipboardEvent, props: ChatProps) {
|
||||
const items = e.clipboardData?.items;
|
||||
if (!items) return;
|
||||
const imageFiles: File[] = [];
|
||||
for (const item of items) {
|
||||
if (item.kind === "file" && item.type.startsWith("image/")) {
|
||||
const file = item.getAsFile();
|
||||
if (file) imageFiles.push(file);
|
||||
}
|
||||
}
|
||||
if (imageFiles.length === 0) return;
|
||||
e.preventDefault();
|
||||
for (const file of imageFiles) {
|
||||
if (file.size > MAX_ATTACHMENT_BYTES) {
|
||||
console.warn(`Skipped image "${file.name}": exceeds 5 MB limit (${file.size} bytes)`);
|
||||
continue;
|
||||
}
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => {
|
||||
const dataUrl = reader.result as string;
|
||||
const attachment: ChatAttachment = {
|
||||
id: generateUUID(),
|
||||
dataUrl,
|
||||
mimeType: file.type,
|
||||
fileName: file.name || "pasted-image",
|
||||
size: file.size,
|
||||
};
|
||||
props.onAttachmentsChange([...props.attachments, attachment]);
|
||||
};
|
||||
reader.readAsDataURL(file);
|
||||
}
|
||||
}
|
||||
|
||||
function removeAttachment(props: ChatProps, id: string) {
|
||||
props.onAttachmentsChange(props.attachments.filter((a) => a.id !== id));
|
||||
}
|
||||
|
||||
const COMPACTION_TOAST_DURATION_MS = 5000;
|
||||
|
||||
function renderCompactionIndicator(status: CompactionIndicatorStatus | null | undefined) {
|
||||
@ -235,39 +278,63 @@ export function renderChat(props: ChatProps) {
|
||||
: nothing}
|
||||
|
||||
<div class="chat-compose">
|
||||
<label class="field chat-compose__field">
|
||||
<span>Message</span>
|
||||
<textarea
|
||||
.value=${props.draft}
|
||||
?disabled=${!props.connected}
|
||||
@keydown=${(e: KeyboardEvent) => {
|
||||
if (e.key !== "Enter") return;
|
||||
if (e.isComposing || e.keyCode === 229) return;
|
||||
if (e.shiftKey) return; // Allow Shift+Enter for line breaks
|
||||
if (!props.connected) return;
|
||||
e.preventDefault();
|
||||
if (canCompose) props.onSend();
|
||||
}}
|
||||
@input=${(e: Event) =>
|
||||
props.onDraftChange((e.target as HTMLTextAreaElement).value)}
|
||||
placeholder=${composePlaceholder}
|
||||
></textarea>
|
||||
</label>
|
||||
<div class="chat-compose__actions">
|
||||
<button
|
||||
class="btn"
|
||||
?disabled=${!props.connected || (!canAbort && props.sending)}
|
||||
@click=${canAbort ? props.onAbort : props.onNewSession}
|
||||
>
|
||||
${canAbort ? "Stop" : "New session"}
|
||||
</button>
|
||||
<button
|
||||
class="btn primary"
|
||||
?disabled=${!props.connected}
|
||||
@click=${props.onSend}
|
||||
>
|
||||
${isBusy ? "Queue" : "Send"}<kbd class="btn-kbd">↵</kbd>
|
||||
</button>
|
||||
${props.attachments.length > 0
|
||||
? html`
|
||||
<div class="chat-attachments">
|
||||
${props.attachments.map(
|
||||
(att) => html`
|
||||
<div class="chat-attachment">
|
||||
<img class="chat-attachment__img" src=${att.dataUrl} alt=${att.fileName} />
|
||||
<button
|
||||
class="chat-attachment__remove"
|
||||
type="button"
|
||||
aria-label="Remove attachment"
|
||||
@click=${() => removeAttachment(props, att.id)}
|
||||
>
|
||||
${icons.x}
|
||||
</button>
|
||||
</div>
|
||||
`,
|
||||
)}
|
||||
</div>
|
||||
`
|
||||
: nothing}
|
||||
<div class="chat-compose__row">
|
||||
<label class="field chat-compose__field">
|
||||
<span>Message</span>
|
||||
<textarea
|
||||
.value=${props.draft}
|
||||
?disabled=${!props.connected}
|
||||
@keydown=${(e: KeyboardEvent) => {
|
||||
if (e.key !== "Enter") return;
|
||||
if (e.isComposing || e.keyCode === 229) return;
|
||||
if (e.shiftKey) return; // Allow Shift+Enter for line breaks
|
||||
if (!props.connected) return;
|
||||
e.preventDefault();
|
||||
if (canCompose) props.onSend();
|
||||
}}
|
||||
@input=${(e: Event) =>
|
||||
props.onDraftChange((e.target as HTMLTextAreaElement).value)}
|
||||
@paste=${(e: ClipboardEvent) => handlePaste(e, props)}
|
||||
placeholder=${composePlaceholder}
|
||||
></textarea>
|
||||
</label>
|
||||
<div class="chat-compose__actions">
|
||||
<button
|
||||
class="btn"
|
||||
?disabled=${!props.connected || (!canAbort && props.sending)}
|
||||
@click=${canAbort ? props.onAbort : props.onNewSession}
|
||||
>
|
||||
${canAbort ? "Stop" : "New session"}
|
||||
</button>
|
||||
<button
|
||||
class="btn primary"
|
||||
?disabled=${!props.connected}
|
||||
@click=${props.onSend}
|
||||
>
|
||||
${isBusy ? "Queue" : "Send"}<kbd class="btn-kbd">↵</kbd>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user