Merge upstream/main into feat/windows-shell-compat
This commit is contained in:
Nathan Hangen
commit
5139637723
@ -13,6 +13,7 @@ Status: beta.
|
||||
- Branding: update launchd labels, mobile bundle IDs, and logging subsystems to bot.molt (legacy com.clawdbot migrations). Thanks @thewilloftheshadow.
|
||||
- Tools: add per-sender group tool policies and fix precedence. (#1757) Thanks @adam91holt.
|
||||
- Agents: summarize dropped messages during compaction safeguard pruning. (#2509) Thanks @jogi47.
|
||||
- Memory Search: allow extra paths for memory indexing (ignores symlinks). (#3600) Thanks @kira-ariaki.
|
||||
- Skills: add multi-image input support to Nano Banana Pro skill. (#1958) Thanks @tyler6204.
|
||||
- Agents: honor tools.exec.safeBins in exec allowlist checks. (#2281)
|
||||
- Matrix: switch plugin SDK to @vector-im/matrix-bot-sdk.
|
||||
@ -77,6 +78,7 @@ Status: beta.
|
||||
- Agents: prevent retries on oversized image errors and surface size limits. (#2871) Thanks @Suksham-sharma.
|
||||
- Agents: inherit provider baseUrl/api for inline models. (#2740) Thanks @lploc94.
|
||||
- Memory Search: keep auto provider model defaults and only include remote when configured. (#2576) Thanks @papago2355.
|
||||
- TTS: read OPENAI_TTS_BASE_URL at runtime instead of module load to honor config.env. (#3341) Thanks @hclsys.
|
||||
- macOS: auto-scroll to bottom when sending a new message while scrolled up. (#2471) Thanks @kennyklee.
|
||||
- Web UI: auto-expand the chat compose textarea while typing (with sensible max height). (#2950) Thanks @shivamraut101.
|
||||
- Gateway: prevent crashes on transient network errors (fetch failures, timeouts, DNS). Added fatal error detection to only exit on truly critical errors. Fixes #2895, #2879, #2873. (#2980) Thanks @elliotsecops.
|
||||
@ -106,6 +108,7 @@ Status: beta.
|
||||
- Telegram: centralize API error logging for delivery and bot calls. (#2492) Thanks @altryne.
|
||||
- Voice Call: enforce Twilio webhook signature verification for ngrok URLs; disable ngrok free tier bypass by default.
|
||||
- Security: harden Tailscale Serve auth by validating identity via local tailscaled before trusting headers.
|
||||
- Media: fix text attachment MIME misclassification with CSV/TSV inference and UTF-16 detection; add XML attribute escaping for file output. (#3628) Thanks @frankekn.
|
||||
- Build: align memory-core peer dependency with lockfile.
|
||||
- Security: add mDNS discovery mode with minimal default to reduce information disclosure. (#1882) Thanks @orlyjamie.
|
||||
- Security: harden URL fetches with DNS pinning to reduce rebinding risk. Thanks Chris Zheng.
|
||||
|
||||
@ -125,7 +125,7 @@ the prefix (use `""` to remove it).
|
||||
- **DM policy**: `channels.whatsapp.dmPolicy` controls direct chat access (default: `pairing`).
|
||||
- Pairing: unknown senders get a pairing code (approve via `moltbot pairing approve whatsapp <code>`; codes expire after 1 hour).
|
||||
- Open: requires `channels.whatsapp.allowFrom` to include `"*"`.
|
||||
- Self messages are always allowed; “self-chat mode” still requires `channels.whatsapp.allowFrom` to include your own number.
|
||||
- Your linked WhatsApp number is implicitly trusted, so self messages skip `channels.whatsapp.dmPolicy` and `channels.whatsapp.allowFrom` checks.
|
||||
|
||||
### Personal-number mode (fallback)
|
||||
If you run Moltbot on your **personal WhatsApp number**, enable `channels.whatsapp.selfChatMode` (see sample above).
|
||||
|
||||
@ -39,3 +39,4 @@ Notes:
|
||||
- `memory status --deep` probes vector + embedding availability.
|
||||
- `memory status --deep --index` runs a reindex if the store is dirty.
|
||||
- `memory index --verbose` prints per-phase details (provider, model, sources, batch activity).
|
||||
- `memory status` includes any extra paths configured via `memorySearch.extraPaths`.
|
||||
|
||||
@ -75,8 +75,9 @@ For the full compaction lifecycle, see
|
||||
|
||||
## Vector memory search
|
||||
|
||||
Moltbot can build a small vector index over `MEMORY.md` and `memory/*.md` so
|
||||
semantic queries can find related notes even when wording differs.
|
||||
Moltbot can build a small vector index over `MEMORY.md` and `memory/*.md` (plus
|
||||
any extra directories or files you opt in) so semantic queries can find related
|
||||
notes even when wording differs.
|
||||
|
||||
Defaults:
|
||||
- Enabled by default.
|
||||
@ -96,6 +97,27 @@ embeddings for memory search. For Gemini, use `GEMINI_API_KEY` or
|
||||
`models.providers.google.apiKey`. When using a custom OpenAI-compatible endpoint,
|
||||
set `memorySearch.remote.apiKey` (and optional `memorySearch.remote.headers`).
|
||||
|
||||
### Additional memory paths
|
||||
|
||||
If you want to index Markdown files outside the default workspace layout, add
|
||||
explicit paths:
|
||||
|
||||
```json5
|
||||
agents: {
|
||||
defaults: {
|
||||
memorySearch: {
|
||||
extraPaths: ["../team-docs", "/srv/shared-notes/overview.md"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Notes:
|
||||
- Paths can be absolute or workspace-relative.
|
||||
- Directories are scanned recursively for `.md` files.
|
||||
- Only Markdown files are indexed.
|
||||
- Symlinks are ignored (files or directories).
|
||||
|
||||
### Gemini embeddings (native)
|
||||
|
||||
Set the provider to `gemini` to use the Gemini embeddings API directly:
|
||||
@ -189,14 +211,14 @@ Local mode:
|
||||
### How the memory tools work
|
||||
|
||||
- `memory_search` semantically searches Markdown chunks (~400 token target, 80-token overlap) from `MEMORY.md` + `memory/**/*.md`. It returns snippet text (capped ~700 chars), file path, line range, score, provider/model, and whether we fell back from local → remote embeddings. No full file payload is returned.
|
||||
- `memory_get` reads a specific memory Markdown file (workspace-relative), optionally from a starting line and for N lines. Paths outside `MEMORY.md` / `memory/` are rejected.
|
||||
- `memory_get` reads a specific memory Markdown file (workspace-relative), optionally from a starting line and for N lines. Paths outside `MEMORY.md` / `memory/` are allowed only when explicitly listed in `memorySearch.extraPaths`.
|
||||
- Both tools are enabled only when `memorySearch.enabled` resolves true for the agent.
|
||||
|
||||
### What gets indexed (and when)
|
||||
|
||||
- File type: Markdown only (`MEMORY.md`, `memory/**/*.md`).
|
||||
- File type: Markdown only (`MEMORY.md`, `memory/**/*.md`, plus any `.md` files under `memorySearch.extraPaths`).
|
||||
- Index storage: per-agent SQLite at `~/.clawdbot/memory/<agentId>.sqlite` (configurable via `agents.defaults.memorySearch.store.path`, supports `{agentId}` token).
|
||||
- Freshness: watcher on `MEMORY.md` + `memory/` marks the index dirty (debounce 1.5s). Sync is scheduled on session start, on search, or on an interval and runs asynchronously. Session transcripts use delta thresholds to trigger background sync.
|
||||
- Freshness: watcher on `MEMORY.md`, `memory/`, and `memorySearch.extraPaths` marks the index dirty (debounce 1.5s). Sync is scheduled on session start, on search, or on an interval and runs asynchronously. Session transcripts use delta thresholds to trigger background sync.
|
||||
- Reindex triggers: the index stores the embedding **provider/model + endpoint fingerprint + chunking params**. If any of those change, Moltbot automatically resets and reindexes the entire store.
|
||||
|
||||
### Hybrid search (BM25 + vector)
|
||||
|
||||
@ -267,7 +267,8 @@ Save to `~/.clawdbot/moltbot.json` and you can DM the bot from that number.
|
||||
model: "gemini-embedding-001",
|
||||
remote: {
|
||||
apiKey: "${GEMINI_API_KEY}"
|
||||
}
|
||||
},
|
||||
extraPaths: ["../team-docs", "/srv/shared-notes"]
|
||||
},
|
||||
sandbox: {
|
||||
mode: "non-main",
|
||||
|
||||
@ -4,9 +4,9 @@ read_when:
|
||||
- You want privacy-focused inference in Moltbot
|
||||
- You want Venice AI setup guidance
|
||||
---
|
||||
# Venice AI (Venius highlight)
|
||||
# Venice AI (Venice highlight)
|
||||
|
||||
**Venius** is our highlight Venice setup for privacy-first inference with optional anonymized access to proprietary models.
|
||||
**Venice** is our highlight Venice setup for privacy-first inference with optional anonymized access to proprietary models.
|
||||
|
||||
Venice AI provides privacy-focused AI inference with support for uncensored models and access to major proprietary models through their anonymized proxy. All inference is private by default—no training on your data, no logging.
|
||||
|
||||
|
||||
@ -82,6 +82,29 @@ describe("memory search config", () => {
|
||||
expect(resolved?.store.vector.extensionPath).toBe("/opt/sqlite-vec.dylib");
|
||||
});
|
||||
|
||||
it("merges extra memory paths from defaults and overrides", () => {
|
||||
const cfg = {
|
||||
agents: {
|
||||
defaults: {
|
||||
memorySearch: {
|
||||
extraPaths: ["/shared/notes", " docs "],
|
||||
},
|
||||
},
|
||||
list: [
|
||||
{
|
||||
id: "main",
|
||||
default: true,
|
||||
memorySearch: {
|
||||
extraPaths: ["/shared/notes", "../team-notes"],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
const resolved = resolveMemorySearchConfig(cfg, "main");
|
||||
expect(resolved?.extraPaths).toEqual(["/shared/notes", "docs", "../team-notes"]);
|
||||
});
|
||||
|
||||
it("includes batch defaults for openai without remote overrides", () => {
|
||||
const cfg = {
|
||||
agents: {
|
||||
|
||||
@ -9,6 +9,7 @@ import { resolveAgentConfig } from "./agent-scope.js";
|
||||
export type ResolvedMemorySearchConfig = {
|
||||
enabled: boolean;
|
||||
sources: Array<"memory" | "sessions">;
|
||||
extraPaths: string[];
|
||||
provider: "openai" | "local" | "gemini" | "auto";
|
||||
remote?: {
|
||||
baseUrl?: string;
|
||||
@ -162,6 +163,10 @@ function mergeConfig(
|
||||
modelCacheDir: overrides?.local?.modelCacheDir ?? defaults?.local?.modelCacheDir,
|
||||
};
|
||||
const sources = normalizeSources(overrides?.sources ?? defaults?.sources, sessionMemory);
|
||||
const rawPaths = [...(defaults?.extraPaths ?? []), ...(overrides?.extraPaths ?? [])]
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean);
|
||||
const extraPaths = Array.from(new Set(rawPaths));
|
||||
const vector = {
|
||||
enabled: overrides?.store?.vector?.enabled ?? defaults?.store?.vector?.enabled ?? true,
|
||||
extensionPath:
|
||||
@ -236,6 +241,7 @@ function mergeConfig(
|
||||
return {
|
||||
enabled,
|
||||
sources,
|
||||
extraPaths,
|
||||
provider,
|
||||
remote,
|
||||
experimental: {
|
||||
|
||||
@ -83,7 +83,7 @@ export function createMemoryGetTool(options: {
|
||||
label: "Memory Get",
|
||||
name: "memory_get",
|
||||
description:
|
||||
"Safe snippet read from MEMORY.md or memory/*.md with optional from/lines; use after memory_search to pull only the needed lines and keep context small.",
|
||||
"Safe snippet read from MEMORY.md, memory/*.md, or configured memorySearch.extraPaths with optional from/lines; use after memory_search to pull only the needed lines and keep context small.",
|
||||
parameters: MemoryGetSchema,
|
||||
execute: async (_toolCallId, params) => {
|
||||
const relPath = readStringParam(params, "path", { required: true });
|
||||
|
||||
@ -12,7 +12,7 @@ import { setVerbose } from "../globals.js";
|
||||
import { withProgress, withProgressTotals } from "./progress.js";
|
||||
import { formatErrorMessage, withManager } from "./cli-utils.js";
|
||||
import { getMemorySearchManager, type MemorySearchManagerResult } from "../memory/index.js";
|
||||
import { listMemoryFiles } from "../memory/internal.js";
|
||||
import { listMemoryFiles, normalizeExtraMemoryPaths } from "../memory/internal.js";
|
||||
import { defaultRuntime } from "../runtime.js";
|
||||
import { formatDocsLink } from "../terminal/links.js";
|
||||
import { colorize, isRich, theme } from "../terminal/theme.js";
|
||||
@ -74,6 +74,10 @@ function resolveAgentIds(cfg: ReturnType<typeof loadConfig>, agent?: string): st
|
||||
return [resolveDefaultAgentId(cfg)];
|
||||
}
|
||||
|
||||
function formatExtraPaths(workspaceDir: string, extraPaths: string[]): string[] {
|
||||
return normalizeExtraMemoryPaths(workspaceDir, extraPaths).map((entry) => shortenHomePath(entry));
|
||||
}
|
||||
|
||||
async function checkReadableFile(pathname: string): Promise<{ exists: boolean; issue?: string }> {
|
||||
try {
|
||||
await fs.access(pathname, fsSync.constants.R_OK);
|
||||
@ -110,7 +114,10 @@ async function scanSessionFiles(agentId: string): Promise<SourceScan> {
|
||||
}
|
||||
}
|
||||
|
||||
async function scanMemoryFiles(workspaceDir: string): Promise<SourceScan> {
|
||||
async function scanMemoryFiles(
|
||||
workspaceDir: string,
|
||||
extraPaths: string[] = [],
|
||||
): Promise<SourceScan> {
|
||||
const issues: string[] = [];
|
||||
const memoryFile = path.join(workspaceDir, "MEMORY.md");
|
||||
const altMemoryFile = path.join(workspaceDir, "memory.md");
|
||||
@ -121,6 +128,25 @@ async function scanMemoryFiles(workspaceDir: string): Promise<SourceScan> {
|
||||
if (primary.issue) issues.push(primary.issue);
|
||||
if (alt.issue) issues.push(alt.issue);
|
||||
|
||||
const resolvedExtraPaths = normalizeExtraMemoryPaths(workspaceDir, extraPaths);
|
||||
for (const extraPath of resolvedExtraPaths) {
|
||||
try {
|
||||
const stat = await fs.lstat(extraPath);
|
||||
if (stat.isSymbolicLink()) continue;
|
||||
const extraCheck = await checkReadableFile(extraPath);
|
||||
if (extraCheck.issue) issues.push(extraCheck.issue);
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code === "ENOENT") {
|
||||
issues.push(`additional memory path missing (${shortenHomePath(extraPath)})`);
|
||||
} else {
|
||||
issues.push(
|
||||
`additional memory path not accessible (${shortenHomePath(extraPath)}): ${code ?? "error"}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let dirReadable: boolean | null = null;
|
||||
try {
|
||||
await fs.access(memoryDir, fsSync.constants.R_OK);
|
||||
@ -141,7 +167,7 @@ async function scanMemoryFiles(workspaceDir: string): Promise<SourceScan> {
|
||||
let listed: string[] = [];
|
||||
let listedOk = false;
|
||||
try {
|
||||
listed = await listMemoryFiles(workspaceDir);
|
||||
listed = await listMemoryFiles(workspaceDir, resolvedExtraPaths);
|
||||
listedOk = true;
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
@ -176,11 +202,13 @@ async function scanMemorySources(params: {
|
||||
workspaceDir: string;
|
||||
agentId: string;
|
||||
sources: MemorySourceName[];
|
||||
extraPaths?: string[];
|
||||
}): Promise<MemorySourceScan> {
|
||||
const scans: SourceScan[] = [];
|
||||
const extraPaths = params.extraPaths ?? [];
|
||||
for (const source of params.sources) {
|
||||
if (source === "memory") {
|
||||
scans.push(await scanMemoryFiles(params.workspaceDir));
|
||||
scans.push(await scanMemoryFiles(params.workspaceDir, extraPaths));
|
||||
}
|
||||
if (source === "sessions") {
|
||||
scans.push(await scanSessionFiles(params.agentId));
|
||||
@ -268,6 +296,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) {
|
||||
workspaceDir: status.workspaceDir,
|
||||
agentId,
|
||||
sources,
|
||||
extraPaths: status.extraPaths,
|
||||
});
|
||||
allResults.push({ agentId, status, embeddingProbe, indexError, scan });
|
||||
},
|
||||
@ -299,6 +328,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) {
|
||||
const line = indexError ? `Memory index failed: ${indexError}` : "Memory index complete.";
|
||||
defaultRuntime.log(line);
|
||||
}
|
||||
const extraPaths = formatExtraPaths(status.workspaceDir, status.extraPaths ?? []);
|
||||
const lines = [
|
||||
`${heading("Memory Search")} ${muted(`(${agentId})`)}`,
|
||||
`${label("Provider")} ${info(status.provider)} ${muted(
|
||||
@ -306,6 +336,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) {
|
||||
)}`,
|
||||
`${label("Model")} ${info(status.model)}`,
|
||||
status.sources?.length ? `${label("Sources")} ${info(status.sources.join(", "))}` : null,
|
||||
extraPaths.length ? `${label("Extra paths")} ${info(extraPaths.join(", "))}` : null,
|
||||
`${label("Indexed")} ${success(indexedLabel)}`,
|
||||
`${label("Dirty")} ${status.dirty ? warn("yes") : muted("no")}`,
|
||||
`${label("Store")} ${info(shortenHomePath(status.dbPath))}`,
|
||||
@ -469,6 +500,7 @@ export function registerMemoryCli(program: Command) {
|
||||
const sourceLabels = status.sources.map((source) =>
|
||||
formatSourceLabel(source, status.workspaceDir, agentId),
|
||||
);
|
||||
const extraPaths = formatExtraPaths(status.workspaceDir, status.extraPaths ?? []);
|
||||
const lines = [
|
||||
`${heading("Memory Index")} ${muted(`(${agentId})`)}`,
|
||||
`${label("Provider")} ${info(status.provider)} ${muted(
|
||||
@ -478,6 +510,9 @@ export function registerMemoryCli(program: Command) {
|
||||
sourceLabels.length
|
||||
? `${label("Sources")} ${info(sourceLabels.join(", "))}`
|
||||
: null,
|
||||
extraPaths.length
|
||||
? `${label("Extra paths")} ${info(extraPaths.join(", "))}`
|
||||
: null,
|
||||
].filter(Boolean) as string[];
|
||||
if (status.fallback) {
|
||||
lines.push(`${label("Fallback")} ${warn(status.fallback.from)}`);
|
||||
|
||||
@ -222,6 +222,7 @@ const FIELD_LABELS: Record<string, string> = {
|
||||
"agents.defaults.memorySearch": "Memory Search",
|
||||
"agents.defaults.memorySearch.enabled": "Enable Memory Search",
|
||||
"agents.defaults.memorySearch.sources": "Memory Search Sources",
|
||||
"agents.defaults.memorySearch.extraPaths": "Extra Memory Paths",
|
||||
"agents.defaults.memorySearch.experimental.sessionMemory":
|
||||
"Memory Search Session Index (Experimental)",
|
||||
"agents.defaults.memorySearch.provider": "Memory Search Provider",
|
||||
@ -499,6 +500,8 @@ const FIELD_HELP: Record<string, string> = {
|
||||
"Vector search over MEMORY.md and memory/*.md (per-agent overrides supported).",
|
||||
"agents.defaults.memorySearch.sources":
|
||||
'Sources to index for memory search (default: ["memory"]; add "sessions" to include session transcripts).',
|
||||
"agents.defaults.memorySearch.extraPaths":
|
||||
"Extra paths to include in memory search (directories or .md files; relative paths resolved from workspace).",
|
||||
"agents.defaults.memorySearch.experimental.sessionMemory":
|
||||
"Enable experimental session transcript indexing for memory search (default: false).",
|
||||
"agents.defaults.memorySearch.provider": 'Embedding provider ("openai", "gemini", or "local").',
|
||||
|
||||
@ -226,6 +226,8 @@ export type MemorySearchConfig = {
|
||||
enabled?: boolean;
|
||||
/** Sources to index and search (default: ["memory"]). */
|
||||
sources?: Array<"memory" | "sessions">;
|
||||
/** Extra paths to include in memory search (directories or .md files). */
|
||||
extraPaths?: string[];
|
||||
/** Experimental memory search settings. */
|
||||
experimental?: {
|
||||
/** Enable session transcript indexing (experimental, default: false). */
|
||||
|
||||
@ -304,6 +304,7 @@ export const MemorySearchSchema = z
|
||||
.object({
|
||||
enabled: z.boolean().optional(),
|
||||
sources: z.array(z.union([z.literal("memory"), z.literal("sessions")])).optional(),
|
||||
extraPaths: z.array(z.string()).optional(),
|
||||
experimental: z
|
||||
.object({
|
||||
sessionMemory: z.boolean().optional(),
|
||||
|
||||
@ -41,7 +41,7 @@ describe("applyMediaUnderstanding", () => {
|
||||
mockedResolveApiKey.mockClear();
|
||||
mockedFetchRemoteMedia.mockReset();
|
||||
mockedFetchRemoteMedia.mockResolvedValue({
|
||||
buffer: Buffer.from("audio-bytes"),
|
||||
buffer: Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]),
|
||||
contentType: "audio/ogg",
|
||||
fileName: "note.ogg",
|
||||
});
|
||||
@ -51,7 +51,7 @@ describe("applyMediaUnderstanding", () => {
|
||||
const { applyMediaUnderstanding } = await loadApply();
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
|
||||
const audioPath = path.join(dir, "note.ogg");
|
||||
await fs.writeFile(audioPath, "hello");
|
||||
await fs.writeFile(audioPath, Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8]));
|
||||
|
||||
const ctx: MsgContext = {
|
||||
Body: "<media:audio>",
|
||||
@ -94,7 +94,7 @@ describe("applyMediaUnderstanding", () => {
|
||||
const { applyMediaUnderstanding } = await loadApply();
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
|
||||
const audioPath = path.join(dir, "note.ogg");
|
||||
await fs.writeFile(audioPath, "hello");
|
||||
await fs.writeFile(audioPath, Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8]));
|
||||
|
||||
const ctx: MsgContext = {
|
||||
Body: "<media:audio> /capture status",
|
||||
@ -176,7 +176,7 @@ describe("applyMediaUnderstanding", () => {
|
||||
const { applyMediaUnderstanding } = await loadApply();
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
|
||||
const audioPath = path.join(dir, "large.wav");
|
||||
await fs.writeFile(audioPath, "0123456789");
|
||||
await fs.writeFile(audioPath, Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]));
|
||||
|
||||
const ctx: MsgContext = {
|
||||
Body: "<media:audio>",
|
||||
@ -211,7 +211,7 @@ describe("applyMediaUnderstanding", () => {
|
||||
const { applyMediaUnderstanding } = await loadApply();
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
|
||||
const audioPath = path.join(dir, "note.ogg");
|
||||
await fs.writeFile(audioPath, "hello");
|
||||
await fs.writeFile(audioPath, Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8]));
|
||||
|
||||
const ctx: MsgContext = {
|
||||
Body: "<media:audio>",
|
||||
@ -352,7 +352,7 @@ describe("applyMediaUnderstanding", () => {
|
||||
const { applyMediaUnderstanding } = await loadApply();
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
|
||||
const audioPath = path.join(dir, "fallback.ogg");
|
||||
await fs.writeFile(audioPath, "hello");
|
||||
await fs.writeFile(audioPath, Buffer.from([0, 255, 0, 1, 2, 3, 4, 5, 6]));
|
||||
|
||||
const ctx: MsgContext = {
|
||||
Body: "<media:audio>",
|
||||
@ -390,8 +390,8 @@ describe("applyMediaUnderstanding", () => {
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
|
||||
const audioPathA = path.join(dir, "note-a.ogg");
|
||||
const audioPathB = path.join(dir, "note-b.ogg");
|
||||
await fs.writeFile(audioPathA, "hello");
|
||||
await fs.writeFile(audioPathB, "world");
|
||||
await fs.writeFile(audioPathA, Buffer.from([200, 201, 202, 203, 204, 205, 206, 207, 208]));
|
||||
await fs.writeFile(audioPathB, Buffer.from([200, 201, 202, 203, 204, 205, 206, 207, 208]));
|
||||
|
||||
const ctx: MsgContext = {
|
||||
Body: "<media:audio>",
|
||||
@ -435,7 +435,7 @@ describe("applyMediaUnderstanding", () => {
|
||||
const audioPath = path.join(dir, "note.ogg");
|
||||
const videoPath = path.join(dir, "clip.mp4");
|
||||
await fs.writeFile(imagePath, "image-bytes");
|
||||
await fs.writeFile(audioPath, "audio-bytes");
|
||||
await fs.writeFile(audioPath, Buffer.from([200, 201, 202, 203, 204, 205, 206, 207, 208]));
|
||||
await fs.writeFile(videoPath, "video-bytes");
|
||||
|
||||
const ctx: MsgContext = {
|
||||
@ -487,4 +487,187 @@ describe("applyMediaUnderstanding", () => {
|
||||
expect(ctx.CommandBody).toBe("audio ok");
|
||||
expect(ctx.BodyForCommands).toBe("audio ok");
|
||||
});
|
||||
|
||||
it("treats text-like audio attachments as CSV (comma wins over tabs)", async () => {
|
||||
const { applyMediaUnderstanding } = await loadApply();
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
|
||||
const csvPath = path.join(dir, "data.mp3");
|
||||
const csvText = '"a","b"\t"c"\n"1","2"\t"3"';
|
||||
const csvBuffer = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(csvText, "utf16le")]);
|
||||
await fs.writeFile(csvPath, csvBuffer);
|
||||
|
||||
const ctx: MsgContext = {
|
||||
Body: "<media:audio>",
|
||||
MediaPath: csvPath,
|
||||
MediaType: "audio/mpeg",
|
||||
};
|
||||
const cfg: MoltbotConfig = {
|
||||
tools: {
|
||||
media: {
|
||||
audio: { enabled: false },
|
||||
image: { enabled: false },
|
||||
video: { enabled: false },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = await applyMediaUnderstanding({ ctx, cfg });
|
||||
|
||||
expect(result.appliedFile).toBe(true);
|
||||
expect(ctx.Body).toContain('<file name="data.mp3" mime="text/csv">');
|
||||
expect(ctx.Body).toContain('"a","b"\t"c"');
|
||||
});
|
||||
|
||||
it("infers TSV when tabs are present without commas", async () => {
|
||||
const { applyMediaUnderstanding } = await loadApply();
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
|
||||
const tsvPath = path.join(dir, "report.mp3");
|
||||
const tsvText = "a\tb\tc\n1\t2\t3";
|
||||
await fs.writeFile(tsvPath, tsvText);
|
||||
|
||||
const ctx: MsgContext = {
|
||||
Body: "<media:audio>",
|
||||
MediaPath: tsvPath,
|
||||
MediaType: "audio/mpeg",
|
||||
};
|
||||
const cfg: MoltbotConfig = {
|
||||
tools: {
|
||||
media: {
|
||||
audio: { enabled: false },
|
||||
image: { enabled: false },
|
||||
video: { enabled: false },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = await applyMediaUnderstanding({ ctx, cfg });
|
||||
|
||||
expect(result.appliedFile).toBe(true);
|
||||
expect(ctx.Body).toContain('<file name="report.mp3" mime="text/tab-separated-values">');
|
||||
expect(ctx.Body).toContain("a\tb\tc");
|
||||
});
|
||||
|
||||
it("escapes XML special characters in filenames to prevent injection", async () => {
|
||||
const { applyMediaUnderstanding } = await loadApply();
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
|
||||
// Create file with XML special characters in the name (what filesystem allows)
|
||||
// Note: The sanitizeFilename in store.ts would strip most dangerous chars,
|
||||
// but we test that even if some slip through, they get escaped in output
|
||||
const filePath = path.join(dir, "file<test>.txt");
|
||||
await fs.writeFile(filePath, "safe content");
|
||||
|
||||
const ctx: MsgContext = {
|
||||
Body: "<media:document>",
|
||||
MediaPath: filePath,
|
||||
MediaType: "text/plain",
|
||||
};
|
||||
const cfg: MoltbotConfig = {
|
||||
tools: {
|
||||
media: {
|
||||
audio: { enabled: false },
|
||||
image: { enabled: false },
|
||||
video: { enabled: false },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = await applyMediaUnderstanding({ ctx, cfg });
|
||||
|
||||
expect(result.appliedFile).toBe(true);
|
||||
// Verify XML special chars are escaped in the output
|
||||
expect(ctx.Body).toContain("<");
|
||||
expect(ctx.Body).toContain(">");
|
||||
// The raw < and > should not appear unescaped in the name attribute
|
||||
expect(ctx.Body).not.toMatch(/name="[^"]*<[^"]*"/);
|
||||
});
|
||||
|
||||
it("normalizes MIME types to prevent attribute injection", async () => {
|
||||
const { applyMediaUnderstanding } = await loadApply();
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
|
||||
const filePath = path.join(dir, "data.txt");
|
||||
await fs.writeFile(filePath, "test content");
|
||||
|
||||
const ctx: MsgContext = {
|
||||
Body: "<media:document>",
|
||||
MediaPath: filePath,
|
||||
// Attempt to inject via MIME type with quotes - normalization should strip this
|
||||
MediaType: 'text/plain" onclick="alert(1)',
|
||||
};
|
||||
const cfg: MoltbotConfig = {
|
||||
tools: {
|
||||
media: {
|
||||
audio: { enabled: false },
|
||||
image: { enabled: false },
|
||||
video: { enabled: false },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = await applyMediaUnderstanding({ ctx, cfg });
|
||||
|
||||
expect(result.appliedFile).toBe(true);
|
||||
// MIME normalization strips everything after first ; or " - verify injection is blocked
|
||||
expect(ctx.Body).not.toContain("onclick=");
|
||||
expect(ctx.Body).not.toContain("alert(1)");
|
||||
// Verify the MIME type is normalized to just "text/plain"
|
||||
expect(ctx.Body).toContain('mime="text/plain"');
|
||||
});
|
||||
|
||||
it("handles path traversal attempts in filenames safely", async () => {
|
||||
const { applyMediaUnderstanding } = await loadApply();
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
|
||||
// Even if a file somehow got a path-like name, it should be handled safely
|
||||
const filePath = path.join(dir, "normal.txt");
|
||||
await fs.writeFile(filePath, "legitimate content");
|
||||
|
||||
const ctx: MsgContext = {
|
||||
Body: "<media:document>",
|
||||
MediaPath: filePath,
|
||||
MediaType: "text/plain",
|
||||
};
|
||||
const cfg: MoltbotConfig = {
|
||||
tools: {
|
||||
media: {
|
||||
audio: { enabled: false },
|
||||
image: { enabled: false },
|
||||
video: { enabled: false },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = await applyMediaUnderstanding({ ctx, cfg });
|
||||
|
||||
expect(result.appliedFile).toBe(true);
|
||||
// Verify the file was processed and output contains expected structure
|
||||
expect(ctx.Body).toContain('<file name="');
|
||||
expect(ctx.Body).toContain('mime="text/plain"');
|
||||
expect(ctx.Body).toContain("legitimate content");
|
||||
});
|
||||
|
||||
it("handles files with non-ASCII Unicode filenames", async () => {
|
||||
const { applyMediaUnderstanding } = await loadApply();
|
||||
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-media-"));
|
||||
const filePath = path.join(dir, "文档.txt");
|
||||
await fs.writeFile(filePath, "中文内容");
|
||||
|
||||
const ctx: MsgContext = {
|
||||
Body: "<media:document>",
|
||||
MediaPath: filePath,
|
||||
MediaType: "text/plain",
|
||||
};
|
||||
const cfg: MoltbotConfig = {
|
||||
tools: {
|
||||
media: {
|
||||
audio: { enabled: false },
|
||||
image: { enabled: false },
|
||||
video: { enabled: false },
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const result = await applyMediaUnderstanding({ ctx, cfg });
|
||||
|
||||
expect(result.appliedFile).toBe(true);
|
||||
expect(ctx.Body).toContain("中文内容");
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,6 +1,22 @@
|
||||
import path from "node:path";
|
||||
|
||||
import type { MoltbotConfig } from "../config/config.js";
|
||||
import type { MsgContext } from "../auto-reply/templating.js";
|
||||
import { finalizeInboundContext } from "../auto-reply/reply/inbound-context.js";
|
||||
import { logVerbose, shouldLogVerbose } from "../globals.js";
|
||||
import {
|
||||
DEFAULT_INPUT_FILE_MAX_BYTES,
|
||||
DEFAULT_INPUT_FILE_MAX_CHARS,
|
||||
DEFAULT_INPUT_FILE_MIMES,
|
||||
DEFAULT_INPUT_MAX_REDIRECTS,
|
||||
DEFAULT_INPUT_PDF_MAX_PAGES,
|
||||
DEFAULT_INPUT_PDF_MAX_PIXELS,
|
||||
DEFAULT_INPUT_PDF_MIN_TEXT_CHARS,
|
||||
DEFAULT_INPUT_TIMEOUT_MS,
|
||||
extractFileContentFromSource,
|
||||
normalizeMimeList,
|
||||
normalizeMimeType,
|
||||
} from "../media/input-files.js";
|
||||
import {
|
||||
extractMediaUserText,
|
||||
formatAudioTranscripts,
|
||||
@ -14,6 +30,7 @@ import type {
|
||||
} from "./types.js";
|
||||
import { runWithConcurrency } from "./concurrency.js";
|
||||
import { resolveConcurrency } from "./resolve.js";
|
||||
import { resolveAttachmentKind } from "./attachments.js";
|
||||
import {
|
||||
type ActiveMediaModel,
|
||||
buildProviderRegistry,
|
||||
@ -28,9 +45,279 @@ export type ApplyMediaUnderstandingResult = {
|
||||
appliedImage: boolean;
|
||||
appliedAudio: boolean;
|
||||
appliedVideo: boolean;
|
||||
appliedFile: boolean;
|
||||
};
|
||||
|
||||
const CAPABILITY_ORDER: MediaUnderstandingCapability[] = ["image", "audio", "video"];
|
||||
const EXTRA_TEXT_MIMES = [
|
||||
"application/xml",
|
||||
"text/xml",
|
||||
"application/x-yaml",
|
||||
"text/yaml",
|
||||
"application/yaml",
|
||||
"application/javascript",
|
||||
"text/javascript",
|
||||
"text/tab-separated-values",
|
||||
];
|
||||
const TEXT_EXT_MIME = new Map<string, string>([
|
||||
[".csv", "text/csv"],
|
||||
[".tsv", "text/tab-separated-values"],
|
||||
[".txt", "text/plain"],
|
||||
[".md", "text/markdown"],
|
||||
[".log", "text/plain"],
|
||||
[".ini", "text/plain"],
|
||||
[".cfg", "text/plain"],
|
||||
[".conf", "text/plain"],
|
||||
[".env", "text/plain"],
|
||||
[".json", "application/json"],
|
||||
[".yaml", "text/yaml"],
|
||||
[".yml", "text/yaml"],
|
||||
[".xml", "application/xml"],
|
||||
]);
|
||||
|
||||
const XML_ESCAPE_MAP: Record<string, string> = {
|
||||
"<": "<",
|
||||
">": ">",
|
||||
"&": "&",
|
||||
'"': """,
|
||||
"'": "'",
|
||||
};
|
||||
|
||||
/**
|
||||
* Escapes special XML characters in attribute values to prevent injection.
|
||||
*/
|
||||
function xmlEscapeAttr(value: string): string {
|
||||
return value.replace(/[<>&"']/g, (char) => XML_ESCAPE_MAP[char] ?? char);
|
||||
}
|
||||
|
||||
function resolveFileLimits(cfg: MoltbotConfig) {
|
||||
const files = cfg.gateway?.http?.endpoints?.responses?.files;
|
||||
return {
|
||||
allowUrl: files?.allowUrl ?? true,
|
||||
allowedMimes: normalizeMimeList(files?.allowedMimes, DEFAULT_INPUT_FILE_MIMES),
|
||||
maxBytes: files?.maxBytes ?? DEFAULT_INPUT_FILE_MAX_BYTES,
|
||||
maxChars: files?.maxChars ?? DEFAULT_INPUT_FILE_MAX_CHARS,
|
||||
maxRedirects: files?.maxRedirects ?? DEFAULT_INPUT_MAX_REDIRECTS,
|
||||
timeoutMs: files?.timeoutMs ?? DEFAULT_INPUT_TIMEOUT_MS,
|
||||
pdf: {
|
||||
maxPages: files?.pdf?.maxPages ?? DEFAULT_INPUT_PDF_MAX_PAGES,
|
||||
maxPixels: files?.pdf?.maxPixels ?? DEFAULT_INPUT_PDF_MAX_PIXELS,
|
||||
minTextChars: files?.pdf?.minTextChars ?? DEFAULT_INPUT_PDF_MIN_TEXT_CHARS,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function appendFileBlocks(body: string | undefined, blocks: string[]): string {
|
||||
if (!blocks || blocks.length === 0) {
|
||||
return body ?? "";
|
||||
}
|
||||
const base = typeof body === "string" ? body.trim() : "";
|
||||
const suffix = blocks.join("\n\n").trim();
|
||||
if (!base) {
|
||||
return suffix;
|
||||
}
|
||||
return `${base}\n\n${suffix}`.trim();
|
||||
}
|
||||
|
||||
function resolveUtf16Charset(buffer?: Buffer): "utf-16le" | "utf-16be" | undefined {
|
||||
if (!buffer || buffer.length < 2) return undefined;
|
||||
const b0 = buffer[0];
|
||||
const b1 = buffer[1];
|
||||
if (b0 === 0xff && b1 === 0xfe) {
|
||||
return "utf-16le";
|
||||
}
|
||||
if (b0 === 0xfe && b1 === 0xff) {
|
||||
return "utf-16be";
|
||||
}
|
||||
const sampleLen = Math.min(buffer.length, 2048);
|
||||
let zeroCount = 0;
|
||||
for (let i = 0; i < sampleLen; i += 1) {
|
||||
if (buffer[i] === 0) zeroCount += 1;
|
||||
}
|
||||
if (zeroCount / sampleLen > 0.2) {
|
||||
return "utf-16le";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function looksLikeUtf8Text(buffer?: Buffer): boolean {
|
||||
if (!buffer || buffer.length === 0) return false;
|
||||
const sampleLen = Math.min(buffer.length, 4096);
|
||||
let printable = 0;
|
||||
let other = 0;
|
||||
for (let i = 0; i < sampleLen; i += 1) {
|
||||
const byte = buffer[i];
|
||||
if (byte === 0) {
|
||||
other += 1;
|
||||
continue;
|
||||
}
|
||||
if (byte === 9 || byte === 10 || byte === 13 || (byte >= 32 && byte <= 126)) {
|
||||
printable += 1;
|
||||
} else {
|
||||
other += 1;
|
||||
}
|
||||
}
|
||||
const total = printable + other;
|
||||
if (total === 0) return false;
|
||||
return printable / total > 0.85;
|
||||
}
|
||||
|
||||
function decodeTextSample(buffer?: Buffer): string {
|
||||
if (!buffer || buffer.length === 0) return "";
|
||||
const sample = buffer.subarray(0, Math.min(buffer.length, 8192));
|
||||
const utf16Charset = resolveUtf16Charset(sample);
|
||||
if (utf16Charset === "utf-16be") {
|
||||
const swapped = Buffer.alloc(sample.length);
|
||||
for (let i = 0; i + 1 < sample.length; i += 2) {
|
||||
swapped[i] = sample[i + 1];
|
||||
swapped[i + 1] = sample[i];
|
||||
}
|
||||
return new TextDecoder("utf-16le").decode(swapped);
|
||||
}
|
||||
if (utf16Charset === "utf-16le") {
|
||||
return new TextDecoder("utf-16le").decode(sample);
|
||||
}
|
||||
return new TextDecoder("utf-8").decode(sample);
|
||||
}
|
||||
|
||||
function guessDelimitedMime(text: string): string | undefined {
|
||||
if (!text) return undefined;
|
||||
const line = text.split(/\r?\n/)[0] ?? "";
|
||||
const tabs = (line.match(/\t/g) ?? []).length;
|
||||
const commas = (line.match(/,/g) ?? []).length;
|
||||
if (commas > 0) {
|
||||
return "text/csv";
|
||||
}
|
||||
if (tabs > 0) {
|
||||
return "text/tab-separated-values";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function resolveTextMimeFromName(name?: string): string | undefined {
|
||||
if (!name) return undefined;
|
||||
const ext = path.extname(name).toLowerCase();
|
||||
return TEXT_EXT_MIME.get(ext);
|
||||
}
|
||||
|
||||
async function extractFileBlocks(params: {
|
||||
attachments: ReturnType<typeof normalizeMediaAttachments>;
|
||||
cache: ReturnType<typeof createMediaAttachmentCache>;
|
||||
limits: ReturnType<typeof resolveFileLimits>;
|
||||
}): Promise<string[]> {
|
||||
const { attachments, cache, limits } = params;
|
||||
if (!attachments || attachments.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const blocks: string[] = [];
|
||||
for (const attachment of attachments) {
|
||||
if (!attachment) {
|
||||
continue;
|
||||
}
|
||||
const forcedTextMime = resolveTextMimeFromName(attachment.path ?? attachment.url ?? "");
|
||||
const kind = forcedTextMime ? "document" : resolveAttachmentKind(attachment);
|
||||
if (!forcedTextMime && (kind === "image" || kind === "video")) {
|
||||
continue;
|
||||
}
|
||||
if (!limits.allowUrl && attachment.url && !attachment.path) {
|
||||
if (shouldLogVerbose()) {
|
||||
logVerbose(`media: file attachment skipped (url disabled) index=${attachment.index}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let bufferResult: Awaited<ReturnType<typeof cache.getBuffer>>;
|
||||
try {
|
||||
bufferResult = await cache.getBuffer({
|
||||
attachmentIndex: attachment.index,
|
||||
maxBytes: limits.maxBytes,
|
||||
timeoutMs: limits.timeoutMs,
|
||||
});
|
||||
} catch (err) {
|
||||
if (shouldLogVerbose()) {
|
||||
logVerbose(`media: file attachment skipped (buffer): ${String(err)}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const nameHint = bufferResult?.fileName ?? attachment.path ?? attachment.url;
|
||||
const forcedTextMimeResolved = forcedTextMime ?? resolveTextMimeFromName(nameHint ?? "");
|
||||
const utf16Charset = resolveUtf16Charset(bufferResult?.buffer);
|
||||
const textSample = decodeTextSample(bufferResult?.buffer);
|
||||
const textLike = Boolean(utf16Charset) || looksLikeUtf8Text(bufferResult?.buffer);
|
||||
if (!forcedTextMimeResolved && kind === "audio" && !textLike) {
|
||||
continue;
|
||||
}
|
||||
const guessedDelimited = textLike ? guessDelimitedMime(textSample) : undefined;
|
||||
const textHint =
|
||||
forcedTextMimeResolved ?? guessedDelimited ?? (textLike ? "text/plain" : undefined);
|
||||
const rawMime = bufferResult?.mime ?? attachment.mime;
|
||||
const mimeType = textHint ?? normalizeMimeType(rawMime);
|
||||
// Log when MIME type is overridden from non-text to text for auditability
|
||||
if (textHint && rawMime && !rawMime.startsWith("text/")) {
|
||||
logVerbose(
|
||||
`media: MIME override from "${rawMime}" to "${textHint}" for index=${attachment.index}`,
|
||||
);
|
||||
}
|
||||
if (!mimeType) {
|
||||
if (shouldLogVerbose()) {
|
||||
logVerbose(`media: file attachment skipped (unknown mime) index=${attachment.index}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const allowedMimes = new Set(limits.allowedMimes);
|
||||
for (const extra of EXTRA_TEXT_MIMES) {
|
||||
allowedMimes.add(extra);
|
||||
}
|
||||
if (mimeType.startsWith("text/")) {
|
||||
allowedMimes.add(mimeType);
|
||||
}
|
||||
if (!allowedMimes.has(mimeType)) {
|
||||
if (shouldLogVerbose()) {
|
||||
logVerbose(
|
||||
`media: file attachment skipped (unsupported mime ${mimeType}) index=${attachment.index}`,
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let extracted: Awaited<ReturnType<typeof extractFileContentFromSource>>;
|
||||
try {
|
||||
const mediaType = utf16Charset ? `${mimeType}; charset=${utf16Charset}` : mimeType;
|
||||
extracted = await extractFileContentFromSource({
|
||||
source: {
|
||||
type: "base64",
|
||||
data: bufferResult.buffer.toString("base64"),
|
||||
mediaType,
|
||||
filename: bufferResult.fileName,
|
||||
},
|
||||
limits: {
|
||||
...limits,
|
||||
allowedMimes,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
if (shouldLogVerbose()) {
|
||||
logVerbose(`media: file attachment skipped (extract): ${String(err)}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const text = extracted?.text?.trim() ?? "";
|
||||
let blockText = text;
|
||||
if (!blockText) {
|
||||
if (extracted?.images && extracted.images.length > 0) {
|
||||
blockText = "[PDF content rendered to images; images not forwarded to model]";
|
||||
} else {
|
||||
blockText = "[No extractable text]";
|
||||
}
|
||||
}
|
||||
const safeName = (bufferResult.fileName ?? `file-${attachment.index + 1}`)
|
||||
.replace(/[\r\n\t]+/g, " ")
|
||||
.trim();
|
||||
// Escape XML special characters in attributes to prevent injection
|
||||
blocks.push(
|
||||
`<file name="${xmlEscapeAttr(safeName)}" mime="${xmlEscapeAttr(mimeType)}">\n${blockText}\n</file>`,
|
||||
);
|
||||
}
|
||||
return blocks;
|
||||
}
|
||||
|
||||
export async function applyMediaUnderstanding(params: {
|
||||
ctx: MsgContext;
|
||||
@ -51,6 +338,12 @@ export async function applyMediaUnderstanding(params: {
|
||||
const cache = createMediaAttachmentCache(attachments);
|
||||
|
||||
try {
|
||||
const fileBlocks = await extractFileBlocks({
|
||||
attachments,
|
||||
cache,
|
||||
limits: resolveFileLimits(cfg),
|
||||
});
|
||||
|
||||
const tasks = CAPABILITY_ORDER.map((capability) => async () => {
|
||||
const config = cfg.tools?.media?.[capability];
|
||||
return await runCapability({
|
||||
@ -99,7 +392,15 @@ export async function applyMediaUnderstanding(params: {
|
||||
ctx.RawBody = originalUserText;
|
||||
}
|
||||
ctx.MediaUnderstanding = [...(ctx.MediaUnderstanding ?? []), ...outputs];
|
||||
finalizeInboundContext(ctx, { forceBodyForAgent: true, forceBodyForCommands: true });
|
||||
}
|
||||
if (fileBlocks.length > 0) {
|
||||
ctx.Body = appendFileBlocks(ctx.Body, fileBlocks);
|
||||
}
|
||||
if (outputs.length > 0 || fileBlocks.length > 0) {
|
||||
finalizeInboundContext(ctx, {
|
||||
forceBodyForAgent: true,
|
||||
forceBodyForCommands: outputs.length > 0,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
@ -108,6 +409,7 @@ export async function applyMediaUnderstanding(params: {
|
||||
appliedImage: outputs.some((output) => output.kind === "image.description"),
|
||||
appliedAudio: outputs.some((output) => output.kind === "audio.transcription"),
|
||||
appliedVideo: outputs.some((output) => output.kind === "video.description"),
|
||||
appliedFile: fileBlocks.length > 0,
|
||||
};
|
||||
} finally {
|
||||
await cache.cleanup();
|
||||
|
||||
@ -412,4 +412,52 @@ describe("memory index", () => {
|
||||
manager = result.manager;
|
||||
await expect(result.manager.readFile({ relPath: "NOTES.md" })).rejects.toThrow("path required");
|
||||
});
|
||||
|
||||
it("allows reading from additional memory paths and blocks symlinks", async () => {
|
||||
const extraDir = path.join(workspaceDir, "extra");
|
||||
await fs.mkdir(extraDir, { recursive: true });
|
||||
await fs.writeFile(path.join(extraDir, "extra.md"), "Extra content.");
|
||||
|
||||
const cfg = {
|
||||
agents: {
|
||||
defaults: {
|
||||
workspace: workspaceDir,
|
||||
memorySearch: {
|
||||
provider: "openai",
|
||||
model: "mock-embed",
|
||||
store: { path: indexPath },
|
||||
sync: { watch: false, onSessionStart: false, onSearch: true },
|
||||
extraPaths: [extraDir],
|
||||
},
|
||||
},
|
||||
list: [{ id: "main", default: true }],
|
||||
},
|
||||
};
|
||||
const result = await getMemorySearchManager({ cfg, agentId: "main" });
|
||||
expect(result.manager).not.toBeNull();
|
||||
if (!result.manager) throw new Error("manager missing");
|
||||
manager = result.manager;
|
||||
await expect(result.manager.readFile({ relPath: "extra/extra.md" })).resolves.toEqual({
|
||||
path: "extra/extra.md",
|
||||
text: "Extra content.",
|
||||
});
|
||||
|
||||
const linkPath = path.join(extraDir, "linked.md");
|
||||
let symlinkOk = true;
|
||||
try {
|
||||
await fs.symlink(path.join(extraDir, "extra.md"), linkPath, "file");
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code === "EPERM" || code === "EACCES") {
|
||||
symlinkOk = false;
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
if (symlinkOk) {
|
||||
await expect(result.manager.readFile({ relPath: "extra/linked.md" })).rejects.toThrow(
|
||||
"path required",
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@ -1,6 +1,117 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { chunkMarkdown } from "./internal.js";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { chunkMarkdown, listMemoryFiles, normalizeExtraMemoryPaths } from "./internal.js";
|
||||
|
||||
describe("normalizeExtraMemoryPaths", () => {
|
||||
it("trims, resolves, and dedupes paths", () => {
|
||||
const workspaceDir = path.join(os.tmpdir(), "memory-test-workspace");
|
||||
const absPath = path.resolve(path.sep, "shared-notes");
|
||||
const result = normalizeExtraMemoryPaths(workspaceDir, [
|
||||
" notes ",
|
||||
"./notes",
|
||||
absPath,
|
||||
absPath,
|
||||
"",
|
||||
]);
|
||||
expect(result).toEqual([path.resolve(workspaceDir, "notes"), absPath]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("listMemoryFiles", () => {
|
||||
let tmpDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-test-"));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("includes files from additional paths (directory)", async () => {
|
||||
await fs.writeFile(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
const extraDir = path.join(tmpDir, "extra-notes");
|
||||
await fs.mkdir(extraDir, { recursive: true });
|
||||
await fs.writeFile(path.join(extraDir, "note1.md"), "# Note 1");
|
||||
await fs.writeFile(path.join(extraDir, "note2.md"), "# Note 2");
|
||||
await fs.writeFile(path.join(extraDir, "ignore.txt"), "Not a markdown file");
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, [extraDir]);
|
||||
expect(files).toHaveLength(3);
|
||||
expect(files.some((file) => file.endsWith("MEMORY.md"))).toBe(true);
|
||||
expect(files.some((file) => file.endsWith("note1.md"))).toBe(true);
|
||||
expect(files.some((file) => file.endsWith("note2.md"))).toBe(true);
|
||||
expect(files.some((file) => file.endsWith("ignore.txt"))).toBe(false);
|
||||
});
|
||||
|
||||
it("includes files from additional paths (single file)", async () => {
|
||||
await fs.writeFile(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
const singleFile = path.join(tmpDir, "standalone.md");
|
||||
await fs.writeFile(singleFile, "# Standalone");
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, [singleFile]);
|
||||
expect(files).toHaveLength(2);
|
||||
expect(files.some((file) => file.endsWith("standalone.md"))).toBe(true);
|
||||
});
|
||||
|
||||
it("handles relative paths in additional paths", async () => {
|
||||
await fs.writeFile(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
const extraDir = path.join(tmpDir, "subdir");
|
||||
await fs.mkdir(extraDir, { recursive: true });
|
||||
await fs.writeFile(path.join(extraDir, "nested.md"), "# Nested");
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, ["subdir"]);
|
||||
expect(files).toHaveLength(2);
|
||||
expect(files.some((file) => file.endsWith("nested.md"))).toBe(true);
|
||||
});
|
||||
|
||||
it("ignores non-existent additional paths", async () => {
|
||||
await fs.writeFile(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, ["/does/not/exist"]);
|
||||
expect(files).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("ignores symlinked files and directories", async () => {
|
||||
await fs.writeFile(path.join(tmpDir, "MEMORY.md"), "# Default memory");
|
||||
const extraDir = path.join(tmpDir, "extra");
|
||||
await fs.mkdir(extraDir, { recursive: true });
|
||||
await fs.writeFile(path.join(extraDir, "note.md"), "# Note");
|
||||
|
||||
const targetFile = path.join(tmpDir, "target.md");
|
||||
await fs.writeFile(targetFile, "# Target");
|
||||
const linkFile = path.join(extraDir, "linked.md");
|
||||
|
||||
const targetDir = path.join(tmpDir, "target-dir");
|
||||
await fs.mkdir(targetDir, { recursive: true });
|
||||
await fs.writeFile(path.join(targetDir, "nested.md"), "# Nested");
|
||||
const linkDir = path.join(tmpDir, "linked-dir");
|
||||
|
||||
let symlinksOk = true;
|
||||
try {
|
||||
await fs.symlink(targetFile, linkFile, "file");
|
||||
await fs.symlink(targetDir, linkDir, "dir");
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code === "EPERM" || code === "EACCES") {
|
||||
symlinksOk = false;
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
const files = await listMemoryFiles(tmpDir, [extraDir, linkDir]);
|
||||
expect(files.some((file) => file.endsWith("note.md"))).toBe(true);
|
||||
if (symlinksOk) {
|
||||
expect(files.some((file) => file.endsWith("linked.md"))).toBe(false);
|
||||
expect(files.some((file) => file.endsWith("nested.md"))).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("chunkMarkdown", () => {
|
||||
it("splits overly long lines into max-sized chunks", () => {
|
||||
|
||||
@ -30,6 +30,17 @@ export function normalizeRelPath(value: string): string {
|
||||
return trimmed.replace(/\\/g, "/");
|
||||
}
|
||||
|
||||
export function normalizeExtraMemoryPaths(workspaceDir: string, extraPaths?: string[]): string[] {
|
||||
if (!extraPaths?.length) return [];
|
||||
const resolved = extraPaths
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean)
|
||||
.map((value) =>
|
||||
path.isAbsolute(value) ? path.resolve(value) : path.resolve(workspaceDir, value),
|
||||
);
|
||||
return Array.from(new Set(resolved));
|
||||
}
|
||||
|
||||
export function isMemoryPath(relPath: string): boolean {
|
||||
const normalized = normalizeRelPath(relPath);
|
||||
if (!normalized) return false;
|
||||
@ -37,19 +48,11 @@ export function isMemoryPath(relPath: string): boolean {
|
||||
return normalized.startsWith("memory/");
|
||||
}
|
||||
|
||||
async function exists(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
await fs.access(filePath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function walkDir(dir: string, files: string[]) {
|
||||
const entries = await fs.readdir(dir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isSymbolicLink()) continue;
|
||||
if (entry.isDirectory()) {
|
||||
await walkDir(full, files);
|
||||
continue;
|
||||
@ -60,15 +63,48 @@ async function walkDir(dir: string, files: string[]) {
|
||||
}
|
||||
}
|
||||
|
||||
export async function listMemoryFiles(workspaceDir: string): Promise<string[]> {
|
||||
export async function listMemoryFiles(
|
||||
workspaceDir: string,
|
||||
extraPaths?: string[],
|
||||
): Promise<string[]> {
|
||||
const result: string[] = [];
|
||||
const memoryFile = path.join(workspaceDir, "MEMORY.md");
|
||||
const altMemoryFile = path.join(workspaceDir, "memory.md");
|
||||
if (await exists(memoryFile)) result.push(memoryFile);
|
||||
if (await exists(altMemoryFile)) result.push(altMemoryFile);
|
||||
const memoryDir = path.join(workspaceDir, "memory");
|
||||
if (await exists(memoryDir)) {
|
||||
await walkDir(memoryDir, result);
|
||||
|
||||
const addMarkdownFile = async (absPath: string) => {
|
||||
try {
|
||||
const stat = await fs.lstat(absPath);
|
||||
if (stat.isSymbolicLink() || !stat.isFile()) return;
|
||||
if (!absPath.endsWith(".md")) return;
|
||||
result.push(absPath);
|
||||
} catch {}
|
||||
};
|
||||
|
||||
await addMarkdownFile(memoryFile);
|
||||
await addMarkdownFile(altMemoryFile);
|
||||
try {
|
||||
const dirStat = await fs.lstat(memoryDir);
|
||||
if (!dirStat.isSymbolicLink() && dirStat.isDirectory()) {
|
||||
await walkDir(memoryDir, result);
|
||||
}
|
||||
} catch {}
|
||||
|
||||
const normalizedExtraPaths = normalizeExtraMemoryPaths(workspaceDir, extraPaths);
|
||||
if (normalizedExtraPaths.length > 0) {
|
||||
for (const inputPath of normalizedExtraPaths) {
|
||||
try {
|
||||
const stat = await fs.lstat(inputPath);
|
||||
if (stat.isSymbolicLink()) continue;
|
||||
if (stat.isDirectory()) {
|
||||
await walkDir(inputPath, result);
|
||||
continue;
|
||||
}
|
||||
if (stat.isFile() && inputPath.endsWith(".md")) {
|
||||
result.push(inputPath);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
if (result.length <= 1) return result;
|
||||
const seen = new Set<string>();
|
||||
|
||||
@ -13,6 +13,7 @@ export function computeMemoryManagerCacheKey(params: {
|
||||
JSON.stringify({
|
||||
enabled: settings.enabled,
|
||||
sources: [...settings.sources].sort((a, b) => a.localeCompare(b)),
|
||||
extraPaths: [...settings.extraPaths].sort((a, b) => a.localeCompare(b)),
|
||||
provider: settings.provider,
|
||||
model: settings.model,
|
||||
fallback: settings.fallback,
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fsSync from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
@ -35,9 +36,9 @@ import {
|
||||
hashText,
|
||||
isMemoryPath,
|
||||
listMemoryFiles,
|
||||
normalizeExtraMemoryPaths,
|
||||
type MemoryChunk,
|
||||
type MemoryFileEntry,
|
||||
normalizeRelPath,
|
||||
parseEmbedding,
|
||||
} from "./internal.js";
|
||||
import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "./hybrid.js";
|
||||
@ -396,13 +397,52 @@ export class MemoryIndexManager {
|
||||
from?: number;
|
||||
lines?: number;
|
||||
}): Promise<{ text: string; path: string }> {
|
||||
const relPath = normalizeRelPath(params.relPath);
|
||||
if (!relPath || !isMemoryPath(relPath)) {
|
||||
const rawPath = params.relPath.trim();
|
||||
if (!rawPath) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
const absPath = path.resolve(this.workspaceDir, relPath);
|
||||
if (!absPath.startsWith(this.workspaceDir)) {
|
||||
throw new Error("path escapes workspace");
|
||||
const absPath = path.isAbsolute(rawPath)
|
||||
? path.resolve(rawPath)
|
||||
: path.resolve(this.workspaceDir, rawPath);
|
||||
const relPath = path.relative(this.workspaceDir, absPath).replace(/\\/g, "/");
|
||||
const inWorkspace =
|
||||
relPath.length > 0 && !relPath.startsWith("..") && !path.isAbsolute(relPath);
|
||||
const allowedWorkspace = inWorkspace && isMemoryPath(relPath);
|
||||
let allowedAdditional = false;
|
||||
if (!allowedWorkspace && this.settings.extraPaths.length > 0) {
|
||||
const additionalPaths = normalizeExtraMemoryPaths(
|
||||
this.workspaceDir,
|
||||
this.settings.extraPaths,
|
||||
);
|
||||
for (const additionalPath of additionalPaths) {
|
||||
try {
|
||||
const stat = await fs.lstat(additionalPath);
|
||||
if (stat.isSymbolicLink()) continue;
|
||||
if (stat.isDirectory()) {
|
||||
if (absPath === additionalPath || absPath.startsWith(`${additionalPath}${path.sep}`)) {
|
||||
allowedAdditional = true;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (stat.isFile()) {
|
||||
if (absPath === additionalPath && absPath.endsWith(".md")) {
|
||||
allowedAdditional = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
if (!allowedWorkspace && !allowedAdditional) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
if (!absPath.endsWith(".md")) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
const stat = await fs.lstat(absPath);
|
||||
if (stat.isSymbolicLink() || !stat.isFile()) {
|
||||
throw new Error("path required");
|
||||
}
|
||||
const content = await fs.readFile(absPath, "utf-8");
|
||||
if (!params.from && !params.lines) {
|
||||
@ -425,6 +465,7 @@ export class MemoryIndexManager {
|
||||
model: string;
|
||||
requestedProvider: string;
|
||||
sources: MemorySource[];
|
||||
extraPaths: string[];
|
||||
sourceCounts: Array<{ source: MemorySource; files: number; chunks: number }>;
|
||||
cache?: { enabled: boolean; entries?: number; maxEntries?: number };
|
||||
fts?: { enabled: boolean; available: boolean; error?: string };
|
||||
@ -498,6 +539,7 @@ export class MemoryIndexManager {
|
||||
model: this.provider.model,
|
||||
requestedProvider: this.requestedProvider,
|
||||
sources: Array.from(this.sources),
|
||||
extraPaths: this.settings.extraPaths,
|
||||
sourceCounts,
|
||||
cache: this.cache.enabled
|
||||
? {
|
||||
@ -769,11 +811,23 @@ export class MemoryIndexManager {
|
||||
|
||||
private ensureWatcher() {
|
||||
if (!this.sources.has("memory") || !this.settings.sync.watch || this.watcher) return;
|
||||
const watchPaths = [
|
||||
const additionalPaths = normalizeExtraMemoryPaths(this.workspaceDir, this.settings.extraPaths)
|
||||
.map((entry) => {
|
||||
try {
|
||||
const stat = fsSync.lstatSync(entry);
|
||||
return stat.isSymbolicLink() ? null : entry;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter((entry): entry is string => Boolean(entry));
|
||||
const watchPaths = new Set<string>([
|
||||
path.join(this.workspaceDir, "MEMORY.md"),
|
||||
path.join(this.workspaceDir, "memory.md"),
|
||||
path.join(this.workspaceDir, "memory"),
|
||||
];
|
||||
this.watcher = chokidar.watch(watchPaths, {
|
||||
...additionalPaths,
|
||||
]);
|
||||
this.watcher = chokidar.watch(Array.from(watchPaths), {
|
||||
ignoreInitial: true,
|
||||
awaitWriteFinish: {
|
||||
stabilityThreshold: this.settings.sync.watchDebounceMs,
|
||||
@ -975,7 +1029,7 @@ export class MemoryIndexManager {
|
||||
needsFullReindex: boolean;
|
||||
progress?: MemorySyncProgressState;
|
||||
}) {
|
||||
const files = await listMemoryFiles(this.workspaceDir);
|
||||
const files = await listMemoryFiles(this.workspaceDir, this.settings.extraPaths);
|
||||
const fileEntries = await Promise.all(
|
||||
files.map(async (file) => buildFileEntry(file, this.workspaceDir)),
|
||||
);
|
||||
|
||||
@ -14,6 +14,7 @@ type ProgressState = {
|
||||
|
||||
export async function syncMemoryFiles(params: {
|
||||
workspaceDir: string;
|
||||
extraPaths?: string[];
|
||||
db: DatabaseSync;
|
||||
needsFullReindex: boolean;
|
||||
progress?: ProgressState;
|
||||
@ -27,7 +28,7 @@ export async function syncMemoryFiles(params: {
|
||||
ftsAvailable: boolean;
|
||||
model: string;
|
||||
}) {
|
||||
const files = await listMemoryFiles(params.workspaceDir);
|
||||
const files = await listMemoryFiles(params.workspaceDir, params.extraPaths);
|
||||
const fileEntries = await Promise.all(
|
||||
files.map(async (file) => buildFileEntry(file, params.workspaceDir)),
|
||||
);
|
||||
|
||||
@ -310,7 +310,14 @@ export async function resolveMedia(
|
||||
fetchImpl,
|
||||
filePathHint: file.file_path,
|
||||
});
|
||||
const saved = await saveMediaBuffer(fetched.buffer, fetched.contentType, "inbound", maxBytes);
|
||||
const originalName = fetched.fileName ?? file.file_path;
|
||||
const saved = await saveMediaBuffer(
|
||||
fetched.buffer,
|
||||
fetched.contentType,
|
||||
"inbound",
|
||||
maxBytes,
|
||||
originalName,
|
||||
);
|
||||
|
||||
// Check sticker cache for existing description
|
||||
const cached = sticker.file_unique_id ? getCachedSticker(sticker.file_unique_id) : null;
|
||||
@ -377,7 +384,14 @@ export async function resolveMedia(
|
||||
fetchImpl,
|
||||
filePathHint: file.file_path,
|
||||
});
|
||||
const saved = await saveMediaBuffer(fetched.buffer, fetched.contentType, "inbound", maxBytes);
|
||||
const originalName = fetched.fileName ?? file.file_path;
|
||||
const saved = await saveMediaBuffer(
|
||||
fetched.buffer,
|
||||
fetched.contentType,
|
||||
"inbound",
|
||||
maxBytes,
|
||||
originalName,
|
||||
);
|
||||
let placeholder = "<media:document>";
|
||||
if (msg.photo) placeholder = "<media:image>";
|
||||
else if (msg.video) placeholder = "<media:video>";
|
||||
|
||||
@ -40,7 +40,7 @@ export async function downloadTelegramFile(
|
||||
filePath: info.file_path,
|
||||
});
|
||||
// save with inbound subdir
|
||||
const saved = await saveMediaBuffer(array, mime, "inbound", maxBytes);
|
||||
const saved = await saveMediaBuffer(array, mime, "inbound", maxBytes, info.file_path);
|
||||
// Ensure extension matches mime if possible
|
||||
if (!saved.contentType && mime) saved.contentType = mime;
|
||||
return saved;
|
||||
|
||||
@ -757,11 +757,19 @@ export const OPENAI_TTS_MODELS = ["gpt-4o-mini-tts", "tts-1", "tts-1-hd"] as con
|
||||
* Custom OpenAI-compatible TTS endpoint.
|
||||
* When set, model/voice validation is relaxed to allow non-OpenAI models.
|
||||
* Example: OPENAI_TTS_BASE_URL=http://localhost:8880/v1
|
||||
*
|
||||
* Note: Read at runtime (not module load) to support config.env loading.
|
||||
*/
|
||||
const OPENAI_TTS_BASE_URL = (
|
||||
process.env.OPENAI_TTS_BASE_URL?.trim() || "https://api.openai.com/v1"
|
||||
).replace(/\/+$/, "");
|
||||
const isCustomOpenAIEndpoint = OPENAI_TTS_BASE_URL !== "https://api.openai.com/v1";
|
||||
function getOpenAITtsBaseUrl(): string {
|
||||
return (process.env.OPENAI_TTS_BASE_URL?.trim() || "https://api.openai.com/v1").replace(
|
||||
/\/+$/,
|
||||
"",
|
||||
);
|
||||
}
|
||||
|
||||
function isCustomOpenAIEndpoint(): boolean {
|
||||
return getOpenAITtsBaseUrl() !== "https://api.openai.com/v1";
|
||||
}
|
||||
export const OPENAI_TTS_VOICES = [
|
||||
"alloy",
|
||||
"ash",
|
||||
@ -778,13 +786,13 @@ type OpenAiTtsVoice = (typeof OPENAI_TTS_VOICES)[number];
|
||||
|
||||
function isValidOpenAIModel(model: string): boolean {
|
||||
// Allow any model when using custom endpoint (e.g., Kokoro, LocalAI)
|
||||
if (isCustomOpenAIEndpoint) return true;
|
||||
if (isCustomOpenAIEndpoint()) return true;
|
||||
return OPENAI_TTS_MODELS.includes(model as (typeof OPENAI_TTS_MODELS)[number]);
|
||||
}
|
||||
|
||||
function isValidOpenAIVoice(voice: string): voice is OpenAiTtsVoice {
|
||||
// Allow any voice when using custom endpoint (e.g., Kokoro Chinese voices)
|
||||
if (isCustomOpenAIEndpoint) return true;
|
||||
if (isCustomOpenAIEndpoint()) return true;
|
||||
return OPENAI_TTS_VOICES.includes(voice as OpenAiTtsVoice);
|
||||
}
|
||||
|
||||
@ -1011,7 +1019,7 @@ async function openaiTTS(params: {
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
try {
|
||||
const response = await fetch(`${OPENAI_TTS_BASE_URL}/audio/speech`, {
|
||||
const response = await fetch(`${getOpenAITtsBaseUrl()}/audio/speech`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
|
||||
@ -42,6 +42,7 @@ import { renderNodes } from "./views/nodes";
|
||||
import { renderOverview } from "./views/overview";
|
||||
import { renderSessions } from "./views/sessions";
|
||||
import { renderExecApprovalPrompt } from "./views/exec-approval";
|
||||
import { renderGatewayUrlConfirmation } from "./views/gateway-url-confirmation";
|
||||
import {
|
||||
approveDevicePairing,
|
||||
loadDevices,
|
||||
@ -578,6 +579,7 @@ export function renderApp(state: AppViewState) {
|
||||
: nothing}
|
||||
</main>
|
||||
${renderExecApprovalPrompt(state)}
|
||||
${renderGatewayUrlConfirmation(state)}
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
@ -33,6 +33,7 @@ type SettingsHost = {
|
||||
basePath: string;
|
||||
themeMedia: MediaQueryList | null;
|
||||
themeMediaHandler: ((event: MediaQueryListEvent) => void) | null;
|
||||
pendingGatewayUrl?: string | null;
|
||||
};
|
||||
|
||||
export function applySettings(host: SettingsHost, next: UiSettings) {
|
||||
@ -98,7 +99,7 @@ export function applySettingsFromUrl(host: SettingsHost) {
|
||||
if (gatewayUrlRaw != null) {
|
||||
const gatewayUrl = gatewayUrlRaw.trim();
|
||||
if (gatewayUrl && gatewayUrl !== host.settings.gatewayUrl) {
|
||||
applySettings(host, { ...host.settings, gatewayUrl });
|
||||
host.pendingGatewayUrl = gatewayUrl;
|
||||
}
|
||||
params.delete("gatewayUrl");
|
||||
shouldCleanUrl = true;
|
||||
|
||||
@ -73,6 +73,7 @@ export type AppViewState = {
|
||||
execApprovalQueue: ExecApprovalRequest[];
|
||||
execApprovalBusy: boolean;
|
||||
execApprovalError: string | null;
|
||||
pendingGatewayUrl: string | null;
|
||||
configLoading: boolean;
|
||||
configRaw: string;
|
||||
configRawOriginal: string;
|
||||
@ -165,6 +166,8 @@ export type AppViewState = {
|
||||
handleNostrProfileImport: () => Promise<void>;
|
||||
handleNostrProfileToggleAdvanced: () => void;
|
||||
handleExecApprovalDecision: (decision: "allow-once" | "allow-always" | "deny") => Promise<void>;
|
||||
handleGatewayUrlConfirm: () => void;
|
||||
handleGatewayUrlCancel: () => void;
|
||||
handleConfigLoad: () => Promise<void>;
|
||||
handleConfigSave: () => Promise<void>;
|
||||
handleConfigApply: () => Promise<void>;
|
||||
|
||||
@ -152,6 +152,7 @@ export class MoltbotApp extends LitElement {
|
||||
@state() execApprovalQueue: ExecApprovalRequest[] = [];
|
||||
@state() execApprovalBusy = false;
|
||||
@state() execApprovalError: string | null = null;
|
||||
@state() pendingGatewayUrl: string | null = null;
|
||||
|
||||
@state() configLoading = false;
|
||||
@state() configRaw = "{\n}\n";
|
||||
@ -448,6 +449,21 @@ export class MoltbotApp extends LitElement {
|
||||
}
|
||||
}
|
||||
|
||||
handleGatewayUrlConfirm() {
|
||||
const nextGatewayUrl = this.pendingGatewayUrl;
|
||||
if (!nextGatewayUrl) return;
|
||||
this.pendingGatewayUrl = null;
|
||||
applySettingsInternal(
|
||||
this as unknown as Parameters<typeof applySettingsInternal>[0],
|
||||
{ ...this.settings, gatewayUrl: nextGatewayUrl },
|
||||
);
|
||||
this.connect();
|
||||
}
|
||||
|
||||
handleGatewayUrlCancel() {
|
||||
this.pendingGatewayUrl = null;
|
||||
}
|
||||
|
||||
// Sidebar handlers for tool output viewing
|
||||
handleOpenSidebar(content: string) {
|
||||
if (this.sidebarCloseTimer != null) {
|
||||
|
||||
@ -260,6 +260,11 @@ function renderTextInput(params: {
|
||||
}
|
||||
onPatch(path, raw);
|
||||
}}
|
||||
@change=${(e: Event) => {
|
||||
if (inputType === "number") return;
|
||||
const raw = (e.target as HTMLInputElement).value;
|
||||
onPatch(path, raw.trim());
|
||||
}}
|
||||
/>
|
||||
${schema.default !== undefined ? html`
|
||||
<button
|
||||
|
||||
39
ui/src/ui/views/gateway-url-confirmation.ts
Normal file
39
ui/src/ui/views/gateway-url-confirmation.ts
Normal file
@ -0,0 +1,39 @@
|
||||
import { html, nothing } from "lit";
|
||||
|
||||
import type { AppViewState } from "../app-view-state";
|
||||
|
||||
export function renderGatewayUrlConfirmation(state: AppViewState) {
|
||||
const { pendingGatewayUrl } = state;
|
||||
if (!pendingGatewayUrl) return nothing;
|
||||
|
||||
return html`
|
||||
<div class="exec-approval-overlay" role="dialog" aria-modal="true" aria-live="polite">
|
||||
<div class="exec-approval-card">
|
||||
<div class="exec-approval-header">
|
||||
<div>
|
||||
<div class="exec-approval-title">Change Gateway URL</div>
|
||||
<div class="exec-approval-sub">This will reconnect to a different gateway server</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="exec-approval-command mono">${pendingGatewayUrl}</div>
|
||||
<div class="callout danger" style="margin-top: 12px;">
|
||||
Only confirm if you trust this URL. Malicious URLs can compromise your system.
|
||||
</div>
|
||||
<div class="exec-approval-actions">
|
||||
<button
|
||||
class="btn primary"
|
||||
@click=${() => state.handleGatewayUrlConfirm()}
|
||||
>
|
||||
Confirm
|
||||
</button>
|
||||
<button
|
||||
class="btn"
|
||||
@click=${() => state.handleGatewayUrlCancel()}
|
||||
>
|
||||
Cancel
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user