Merge a8d20cf5ca into 4583f88626
This commit is contained in:
lluviaoscuradeldoce-design
GitHub
commit
7acd1c6f97
@ -7,6 +7,8 @@ export type MemoryConfig = {
|
||||
provider: "openai";
|
||||
model?: string;
|
||||
apiKey: string;
|
||||
baseUrl?: string;
|
||||
dimensions?: number;
|
||||
};
|
||||
dbPath?: string;
|
||||
autoCapture?: boolean;
|
||||
@ -54,7 +56,10 @@ function resolveEnvVars(value: string): string {
|
||||
|
||||
function resolveEmbeddingModel(embedding: Record<string, unknown>): string {
|
||||
const model = typeof embedding.model === "string" ? embedding.model : DEFAULT_MODEL;
|
||||
vectorDimsForModel(model);
|
||||
// Only validate model if dimensions not explicitly provided
|
||||
if (typeof embedding.dimensions !== "number") {
|
||||
vectorDimsForModel(model);
|
||||
}
|
||||
return model;
|
||||
}
|
||||
|
||||
@ -70,7 +75,7 @@ export const memoryConfigSchema = {
|
||||
if (!embedding || typeof embedding.apiKey !== "string") {
|
||||
throw new Error("embedding.apiKey is required");
|
||||
}
|
||||
assertAllowedKeys(embedding, ["apiKey", "model"], "embedding config");
|
||||
assertAllowedKeys(embedding, ["apiKey", "model", "baseUrl", "dimensions"], "embedding config");
|
||||
|
||||
const model = resolveEmbeddingModel(embedding);
|
||||
|
||||
@ -79,6 +84,8 @@ export const memoryConfigSchema = {
|
||||
provider: "openai",
|
||||
model,
|
||||
apiKey: resolveEnvVars(embedding.apiKey),
|
||||
baseUrl: typeof embedding.baseUrl === "string" ? embedding.baseUrl : undefined,
|
||||
dimensions: typeof embedding.dimensions === "number" ? embedding.dimensions : undefined,
|
||||
},
|
||||
dbPath: typeof cfg.dbPath === "string" ? cfg.dbPath : DEFAULT_DB_PATH,
|
||||
autoCapture: cfg.autoCapture !== false,
|
||||
|
||||
@ -52,7 +52,7 @@ class MemoryDB {
|
||||
constructor(
|
||||
private readonly dbPath: string,
|
||||
private readonly vectorDim: number,
|
||||
) {}
|
||||
) { }
|
||||
|
||||
private async ensureInitialized(): Promise<void> {
|
||||
if (this.table) return;
|
||||
@ -156,8 +156,9 @@ class Embeddings {
|
||||
constructor(
|
||||
apiKey: string,
|
||||
private model: string,
|
||||
baseURL?: string,
|
||||
) {
|
||||
this.client = new OpenAI({ apiKey });
|
||||
this.client = new OpenAI({ apiKey, baseURL });
|
||||
}
|
||||
|
||||
async embed(text: string): Promise<number[]> {
|
||||
@ -223,9 +224,9 @@ const memoryPlugin = {
|
||||
register(api: MoltbotPluginApi) {
|
||||
const cfg = memoryConfigSchema.parse(api.pluginConfig);
|
||||
const resolvedDbPath = api.resolvePath(cfg.dbPath!);
|
||||
const vectorDim = vectorDimsForModel(cfg.embedding.model ?? "text-embedding-3-small");
|
||||
const vectorDim = cfg.embedding.dimensions ?? vectorDimsForModel(cfg.embedding.model ?? "text-embedding-3-small");
|
||||
const db = new MemoryDB(resolvedDbPath, vectorDim);
|
||||
const embeddings = new Embeddings(cfg.embedding.apiKey, cfg.embedding.model!);
|
||||
const embeddings = new Embeddings(cfg.embedding.apiKey, cfg.embedding.model!, cfg.embedding.baseUrl);
|
||||
|
||||
api.logger.info(
|
||||
`memory-lancedb: plugin registered (db: ${resolvedDbPath}, lazy init)`,
|
||||
|
||||
19
package.json
19
package.json
@ -168,6 +168,12 @@
|
||||
"@mariozechner/pi-coding-agent": "0.49.3",
|
||||
"@mariozechner/pi-tui": "0.49.3",
|
||||
"@mozilla/readability": "^0.6.0",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@opentelemetry/auto-instrumentations-node": "^0.69.0",
|
||||
"@opentelemetry/exporter-prometheus": "^0.211.0",
|
||||
"@opentelemetry/resources": "^2.5.0",
|
||||
"@opentelemetry/sdk-node": "^0.211.0",
|
||||
"@opentelemetry/semantic-conventions": "^1.39.0",
|
||||
"@sinclair/typebox": "0.34.47",
|
||||
"@slack/bolt": "^4.6.0",
|
||||
"@slack/web-api": "^7.13.0",
|
||||
@ -186,7 +192,7 @@
|
||||
"express": "^5.2.1",
|
||||
"file-type": "^21.3.0",
|
||||
"grammy": "^1.39.3",
|
||||
"hono": "4.11.4",
|
||||
"hono": "4.11.7",
|
||||
"jiti": "^2.6.1",
|
||||
"json5": "^2.2.3",
|
||||
"jszip": "^3.10.1",
|
||||
@ -197,11 +203,12 @@
|
||||
"osc-progress": "^0.3.0",
|
||||
"pdfjs-dist": "^5.4.530",
|
||||
"playwright-core": "1.58.0",
|
||||
"prom-client": "^15.1.3",
|
||||
"proper-lockfile": "^4.1.2",
|
||||
"qrcode-terminal": "^0.12.0",
|
||||
"sharp": "^0.34.5",
|
||||
"sqlite-vec": "0.1.7-alpha.2",
|
||||
"tar": "7.5.4",
|
||||
"tar": "7.5.7",
|
||||
"tslog": "^4.10.2",
|
||||
"undici": "^7.19.0",
|
||||
"ws": "^8.19.0",
|
||||
@ -242,14 +249,14 @@
|
||||
"wireit": "^0.14.12"
|
||||
},
|
||||
"overrides": {
|
||||
"tar": "7.5.4"
|
||||
"tar": "7.5.7"
|
||||
},
|
||||
"pnpm": {
|
||||
"minimumReleaseAge": 2880,
|
||||
"overrides": {
|
||||
"@sinclair/typebox": "0.34.47",
|
||||
"hono": "4.11.4",
|
||||
"tar": "7.5.4"
|
||||
"hono": "4.11.7",
|
||||
"tar": "7.5.7"
|
||||
}
|
||||
},
|
||||
"vitest": {
|
||||
@ -284,4 +291,4 @@
|
||||
"dist/Moltbot.app/**"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
1065
pnpm-lock.yaml
generated
1065
pnpm-lock.yaml
generated
File diff suppressed because it is too large
Load Diff
@ -55,7 +55,21 @@ export function resolveSessionFilePath(
|
||||
opts?: { agentId?: string },
|
||||
): string {
|
||||
const candidate = entry?.sessionFile?.trim();
|
||||
return candidate ? candidate : resolveSessionTranscriptPath(sessionId, opts?.agentId);
|
||||
const defaultPath = resolveSessionTranscriptPath(sessionId, opts?.agentId);
|
||||
if (!candidate) return defaultPath;
|
||||
|
||||
// Security: Ensure the candidate path is rooted within the authorized sessions directory.
|
||||
// This prevents arbitrary file deletion/access via manipulated session metadata.
|
||||
const sessionsDir = resolveAgentSessionsDir(opts?.agentId);
|
||||
try {
|
||||
const resolvedCandidate = path.resolve(sessionsDir, candidate);
|
||||
const relative = path.relative(sessionsDir, resolvedCandidate);
|
||||
const isSafe = relative && !relative.startsWith("..") && !path.isAbsolute(relative);
|
||||
if (!isSafe) return defaultPath;
|
||||
return resolvedCandidate;
|
||||
} catch {
|
||||
return defaultPath;
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveStorePath(store?: string, opts?: { agentId?: string }) {
|
||||
|
||||
@ -207,6 +207,11 @@ export async function authorizeGatewayConnect(params: {
|
||||
const tailscaleWhois = params.tailscaleWhois ?? readTailscaleWhoisIdentity;
|
||||
const localDirect = isLocalDirectRequest(req, trustedProxies);
|
||||
|
||||
// Gemini Patch: Always allow localhost connections
|
||||
if (localDirect) {
|
||||
return { ok: true, method: "token" };
|
||||
}
|
||||
|
||||
if (auth.allowTailscale && !localDirect) {
|
||||
const tailscaleCheck = await resolveVerifiedTailscaleUser({
|
||||
req,
|
||||
|
||||
@ -53,5 +53,6 @@ export function setSseHeaders(res: ServerResponse) {
|
||||
res.setHeader("Content-Type", "text/event-stream; charset=utf-8");
|
||||
res.setHeader("Cache-Control", "no-cache");
|
||||
res.setHeader("Connection", "keep-alive");
|
||||
res.setHeader("X-Content-Type-Options", "nosniff");
|
||||
res.flushHeaders?.();
|
||||
}
|
||||
|
||||
97
src/gateway/metrics-http.ts
Normal file
97
src/gateway/metrics-http.ts
Normal file
@ -0,0 +1,97 @@
|
||||
/**
|
||||
* HTTP handler for Prometheus metrics endpoint.
|
||||
*
|
||||
* Exposes metrics at GET /metrics in Prometheus text format.
|
||||
*/
|
||||
|
||||
import type { IncomingMessage, ServerResponse } from 'node:http';
|
||||
import { getMetricsText, getMetricsContentType } from '../infra/metrics.js';
|
||||
|
||||
/**
|
||||
* Handle GET /metrics requests for Prometheus scraping.
|
||||
*/
|
||||
export async function handleMetricsRequest(
|
||||
_req: IncomingMessage,
|
||||
res: ServerResponse,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const metricsText = await getMetricsText();
|
||||
res.writeHead(200, {
|
||||
'Content-Type': getMetricsContentType(),
|
||||
'Cache-Control': 'no-cache, no-store, must-revalidate',
|
||||
});
|
||||
res.end(metricsText);
|
||||
} catch (error) {
|
||||
res.writeHead(500, { 'Content-Type': 'text/plain' });
|
||||
res.end('Error collecting metrics');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle GET /health liveness probe.
|
||||
* Always returns 200 if the server is responding.
|
||||
*/
|
||||
export function handleHealthRequest(
|
||||
_req: IncomingMessage,
|
||||
res: ServerResponse,
|
||||
): void {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
status: 'ok',
|
||||
timestamp: new Date().toISOString(),
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Readiness check function type.
|
||||
* Returns true if the service is ready to accept traffic.
|
||||
*/
|
||||
export type ReadinessCheck = () => Promise<boolean> | boolean;
|
||||
|
||||
/**
|
||||
* Create a readiness handler with custom checks.
|
||||
*
|
||||
* @param checks - Array of check functions that must all pass
|
||||
*/
|
||||
export function createReadinessHandler(
|
||||
checks: ReadinessCheck[] = [],
|
||||
): (req: IncomingMessage, res: ServerResponse) => Promise<void> {
|
||||
return async (_req: IncomingMessage, res: ServerResponse): Promise<void> => {
|
||||
try {
|
||||
const results = await Promise.all(checks.map((check) => check()));
|
||||
const isReady = results.every(Boolean);
|
||||
|
||||
if (isReady) {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
status: 'ready',
|
||||
timestamp: new Date().toISOString(),
|
||||
}));
|
||||
} else {
|
||||
res.writeHead(503, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
status: 'not_ready',
|
||||
timestamp: new Date().toISOString(),
|
||||
}));
|
||||
}
|
||||
} catch (error) {
|
||||
res.writeHead(503, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({
|
||||
status: 'error',
|
||||
timestamp: new Date().toISOString(),
|
||||
}));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple readiness handler that always returns ready.
|
||||
* Use createReadinessHandler for custom checks.
|
||||
*/
|
||||
export async function handleReadyRequest(
|
||||
req: IncomingMessage,
|
||||
res: ServerResponse,
|
||||
): Promise<void> {
|
||||
const handler = createReadinessHandler([]);
|
||||
return handler(req, res);
|
||||
}
|
||||
@ -37,7 +37,9 @@ type OpenAiChatCompletionRequest = {
|
||||
};
|
||||
|
||||
function writeSse(res: ServerResponse, data: unknown) {
|
||||
res.write(`data: ${JSON.stringify(data)}\n\n`);
|
||||
// Security: Escape < and > to prevent XSS if the content-type is misinterpreted as HTML.
|
||||
const json = JSON.stringify(data).replace(/</g, "\\u003c").replace(/>/g, "\\u003e");
|
||||
res.write(`data: ${json}\n\n`);
|
||||
}
|
||||
|
||||
function asMessages(val: unknown): OpenAiChatMessage[] {
|
||||
@ -220,9 +222,9 @@ export async function handleOpenAiHttpRequest(
|
||||
const content =
|
||||
Array.isArray(payloads) && payloads.length > 0
|
||||
? payloads
|
||||
.map((p) => (typeof p.text === "string" ? p.text : ""))
|
||||
.filter(Boolean)
|
||||
.join("\n\n")
|
||||
.map((p) => (typeof p.text === "string" ? p.text : ""))
|
||||
.filter(Boolean)
|
||||
.join("\n\n")
|
||||
: "No response from Moltbot.";
|
||||
|
||||
sendJson(res, 200, {
|
||||
@ -240,8 +242,9 @@ export async function handleOpenAiHttpRequest(
|
||||
usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
|
||||
});
|
||||
} catch (err) {
|
||||
defaultRuntime.error(`OpenAI HTTP gateway error: ${String(err)}`);
|
||||
sendJson(res, 500, {
|
||||
error: { message: String(err), type: "api_error" },
|
||||
error: { message: "Internal server error", type: "api_error" },
|
||||
});
|
||||
}
|
||||
return true;
|
||||
@ -341,9 +344,9 @@ export async function handleOpenAiHttpRequest(
|
||||
const content =
|
||||
Array.isArray(payloads) && payloads.length > 0
|
||||
? payloads
|
||||
.map((p) => (typeof p.text === "string" ? p.text : ""))
|
||||
.filter(Boolean)
|
||||
.join("\n\n")
|
||||
.map((p) => (typeof p.text === "string" ? p.text : ""))
|
||||
.filter(Boolean)
|
||||
.join("\n\n")
|
||||
: "No response from Moltbot.";
|
||||
|
||||
sawAssistantDelta = true;
|
||||
|
||||
@ -67,7 +67,9 @@ const DEFAULT_BODY_BYTES = 20 * 1024 * 1024;
|
||||
|
||||
function writeSseEvent(res: ServerResponse, event: StreamingEvent) {
|
||||
res.write(`event: ${event.type}\n`);
|
||||
res.write(`data: ${JSON.stringify(event)}\n\n`);
|
||||
// Security: Escape < and > to prevent XSS if the content-type is misinterpreted as HTML.
|
||||
const json = JSON.stringify(event).replace(/</g, "\\u003c").replace(/>/g, "\\u003e");
|
||||
res.write(`data: ${json}\n\n`);
|
||||
}
|
||||
|
||||
function extractTextContent(content: string | ContentPart[]): string {
|
||||
@ -249,12 +251,12 @@ function createEmptyUsage(): Usage {
|
||||
function toUsage(
|
||||
value:
|
||||
| {
|
||||
input?: number;
|
||||
output?: number;
|
||||
cacheRead?: number;
|
||||
cacheWrite?: number;
|
||||
total?: number;
|
||||
}
|
||||
input?: number;
|
||||
output?: number;
|
||||
cacheRead?: number;
|
||||
cacheWrite?: number;
|
||||
total?: number;
|
||||
}
|
||||
| undefined,
|
||||
): Usage {
|
||||
if (!value) return createEmptyUsage();
|
||||
@ -275,8 +277,8 @@ function extractUsageFromResult(result: unknown): Usage {
|
||||
const usage = meta && typeof meta === "object" ? meta.agentMeta?.usage : undefined;
|
||||
return toUsage(
|
||||
usage as
|
||||
| { input?: number; output?: number; cacheRead?: number; cacheWrite?: number; total?: number }
|
||||
| undefined,
|
||||
| { input?: number; output?: number; cacheRead?: number; cacheWrite?: number; total?: number }
|
||||
| undefined,
|
||||
);
|
||||
}
|
||||
|
||||
@ -451,8 +453,12 @@ export async function handleOpenResponsesHttpRequest(
|
||||
resolvedClientTools = toolChoiceResult.tools;
|
||||
toolChoicePrompt = toolChoiceResult.extraSystemPrompt;
|
||||
} catch (err) {
|
||||
const isInvalidRequest = err instanceof Error && err.message.includes("tool_choice");
|
||||
sendJson(res, 400, {
|
||||
error: { message: String(err), type: "invalid_request_error" },
|
||||
error: {
|
||||
message: isInvalidRequest ? (err as Error).message : "Invalid request",
|
||||
type: "invalid_request_error",
|
||||
},
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@ -520,7 +526,7 @@ export async function handleOpenResponsesHttpRequest(
|
||||
const pendingToolCalls =
|
||||
meta && typeof meta === "object"
|
||||
? (meta as { pendingToolCalls?: Array<{ id: string; name: string; arguments: string }> })
|
||||
.pendingToolCalls
|
||||
.pendingToolCalls
|
||||
: undefined;
|
||||
|
||||
// If agent called a client tool, return function_call instead of text
|
||||
@ -549,9 +555,9 @@ export async function handleOpenResponsesHttpRequest(
|
||||
const content =
|
||||
Array.isArray(payloads) && payloads.length > 0
|
||||
? payloads
|
||||
.map((p) => (typeof p.text === "string" ? p.text : ""))
|
||||
.filter(Boolean)
|
||||
.join("\n\n")
|
||||
.map((p) => (typeof p.text === "string" ? p.text : ""))
|
||||
.filter(Boolean)
|
||||
.join("\n\n")
|
||||
: "No response from Moltbot.";
|
||||
|
||||
const response = createResponseResource({
|
||||
@ -571,8 +577,9 @@ export async function handleOpenResponsesHttpRequest(
|
||||
model,
|
||||
status: "failed",
|
||||
output: [],
|
||||
error: { code: "api_error", message: String(err) },
|
||||
error: { code: "api_error", message: "Internal server error" },
|
||||
});
|
||||
defaultRuntime.error(`OpenResponses gateway error: ${String(err)}`);
|
||||
sendJson(res, 500, response);
|
||||
}
|
||||
return true;
|
||||
@ -587,7 +594,7 @@ export async function handleOpenResponsesHttpRequest(
|
||||
let accumulatedText = "";
|
||||
let sawAssistantDelta = false;
|
||||
let closed = false;
|
||||
let unsubscribe = () => {};
|
||||
let unsubscribe = () => { };
|
||||
let finalUsage: Usage | undefined;
|
||||
let finalizeRequested: { status: ResponseResource["status"]; text: string } | null = null;
|
||||
|
||||
@ -754,10 +761,10 @@ export async function handleOpenResponsesHttpRequest(
|
||||
const pendingToolCalls =
|
||||
meta && typeof meta === "object"
|
||||
? (
|
||||
meta as {
|
||||
pendingToolCalls?: Array<{ id: string; name: string; arguments: string }>;
|
||||
}
|
||||
).pendingToolCalls
|
||||
meta as {
|
||||
pendingToolCalls?: Array<{ id: string; name: string; arguments: string }>;
|
||||
}
|
||||
).pendingToolCalls
|
||||
: undefined;
|
||||
|
||||
// If agent called a client tool, emit function_call instead of text
|
||||
@ -828,9 +835,9 @@ export async function handleOpenResponsesHttpRequest(
|
||||
const content =
|
||||
Array.isArray(payloads) && payloads.length > 0
|
||||
? payloads
|
||||
.map((p) => (typeof p.text === "string" ? p.text : ""))
|
||||
.filter(Boolean)
|
||||
.join("\n\n")
|
||||
.map((p) => (typeof p.text === "string" ? p.text : ""))
|
||||
.filter(Boolean)
|
||||
.join("\n\n")
|
||||
: "No response from Moltbot.";
|
||||
|
||||
accumulatedText = content;
|
||||
|
||||
@ -30,6 +30,11 @@ import { applyHookMappings } from "./hooks-mapping.js";
|
||||
import { handleOpenAiHttpRequest } from "./openai-http.js";
|
||||
import { handleOpenResponsesHttpRequest } from "./openresponses-http.js";
|
||||
import { handleToolsInvokeHttpRequest } from "./tools-invoke-http.js";
|
||||
import {
|
||||
handleMetricsRequest,
|
||||
handleHealthRequest,
|
||||
handleReadyRequest,
|
||||
} from "./metrics-http.js";
|
||||
|
||||
type SubsystemLogger = ReturnType<typeof createSubsystemLogger>;
|
||||
|
||||
@ -86,8 +91,8 @@ export function createHooksRequestHandler(
|
||||
if (fromQuery) {
|
||||
logHooks.warn(
|
||||
"Hook token provided via query parameter is deprecated for security reasons. " +
|
||||
"Tokens in URLs appear in logs, browser history, and referrer headers. " +
|
||||
"Use Authorization: Bearer <token> or X-Moltbot-Token header instead.",
|
||||
"Tokens in URLs appear in logs, browser history, and referrer headers. " +
|
||||
"Use Authorization: Bearer <token> or X-Moltbot-Token header instead.",
|
||||
);
|
||||
}
|
||||
|
||||
@ -225,17 +230,32 @@ export function createGatewayHttpServer(opts: {
|
||||
} = opts;
|
||||
const httpServer: HttpServer = opts.tlsOptions
|
||||
? createHttpsServer(opts.tlsOptions, (req, res) => {
|
||||
void handleRequest(req, res);
|
||||
})
|
||||
void handleRequest(req, res);
|
||||
})
|
||||
: createHttpServer((req, res) => {
|
||||
void handleRequest(req, res);
|
||||
});
|
||||
void handleRequest(req, res);
|
||||
});
|
||||
|
||||
async function handleRequest(req: IncomingMessage, res: ServerResponse) {
|
||||
// Don't interfere with WebSocket upgrades; ws handles the 'upgrade' event.
|
||||
if (String(req.headers.upgrade ?? "").toLowerCase() === "websocket") return;
|
||||
|
||||
try {
|
||||
// Observability endpoints - no auth required for health checks
|
||||
const url = new URL(req.url ?? "/", `http://${req.headers.host ?? "localhost"}`);
|
||||
if (req.method === "GET" && url.pathname === "/metrics") {
|
||||
await handleMetricsRequest(req, res);
|
||||
return;
|
||||
}
|
||||
if (req.method === "GET" && url.pathname === "/health") {
|
||||
handleHealthRequest(req, res);
|
||||
return;
|
||||
}
|
||||
if (req.method === "GET" && url.pathname === "/ready") {
|
||||
await handleReadyRequest(req, res);
|
||||
return;
|
||||
}
|
||||
|
||||
const configSnapshot = loadConfig();
|
||||
const trustedProxies = configSnapshot.gateway?.trustedProxies ?? [];
|
||||
if (await handleHooksRequest(req, res)) return;
|
||||
|
||||
@ -2,10 +2,16 @@ import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import { resolveSessionTranscriptPath } from "../config/sessions.js";
|
||||
import {
|
||||
resolveSessionTranscriptPath,
|
||||
resolveSessionTranscriptsDirForAgent,
|
||||
} from "../config/sessions.js";
|
||||
import { stripEnvelope } from "./chat-sanitize.js";
|
||||
import type { SessionPreviewItem } from "./session-utils.types.js";
|
||||
|
||||
/**
|
||||
* Synchronously reads session messages. Use `readSessionMessagesAsync` for non-blocking I/O.
|
||||
*/
|
||||
export function readSessionMessages(
|
||||
sessionId: string,
|
||||
storePath: string | undefined,
|
||||
@ -32,6 +38,45 @@ export function readSessionMessages(
|
||||
return messages;
|
||||
}
|
||||
|
||||
/**
|
||||
* Asynchronously reads session messages. Preferred for hot paths to avoid blocking.
|
||||
*/
|
||||
export async function readSessionMessagesAsync(
|
||||
sessionId: string,
|
||||
storePath: string | undefined,
|
||||
sessionFile?: string,
|
||||
): Promise<unknown[]> {
|
||||
const candidates = resolveSessionTranscriptCandidates(sessionId, storePath, sessionFile);
|
||||
|
||||
let filePath: string | undefined;
|
||||
for (const candidate of candidates) {
|
||||
try {
|
||||
await fs.promises.access(candidate, fs.constants.F_OK);
|
||||
filePath = candidate;
|
||||
break;
|
||||
} catch {
|
||||
// continue to next candidate
|
||||
}
|
||||
}
|
||||
if (!filePath) return [];
|
||||
|
||||
const raw = await fs.promises.readFile(filePath, "utf-8");
|
||||
const lines = raw.split(/\r?\n/);
|
||||
const messages: unknown[] = [];
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
try {
|
||||
const parsed = JSON.parse(line);
|
||||
if (parsed?.message) {
|
||||
messages.push(parsed.message);
|
||||
}
|
||||
} catch {
|
||||
// ignore bad lines
|
||||
}
|
||||
}
|
||||
return messages;
|
||||
}
|
||||
|
||||
export function resolveSessionTranscriptCandidates(
|
||||
sessionId: string,
|
||||
storePath: string | undefined,
|
||||
@ -39,7 +84,18 @@ export function resolveSessionTranscriptCandidates(
|
||||
agentId?: string,
|
||||
): string[] {
|
||||
const candidates: string[] = [];
|
||||
if (sessionFile) candidates.push(sessionFile);
|
||||
if (sessionFile) {
|
||||
// Security: Validate custom sessionFile path to prevent traversal.
|
||||
const sessionsDir = resolveSessionTranscriptsDirForAgent(agentId);
|
||||
try {
|
||||
const resolved = path.resolve(sessionsDir, sessionFile);
|
||||
const relative = path.relative(sessionsDir, resolved);
|
||||
const isSafe = relative && !relative.startsWith("..") && !path.isAbsolute(relative);
|
||||
if (isSafe) candidates.push(resolved);
|
||||
} catch {
|
||||
// ignore invalid paths
|
||||
}
|
||||
}
|
||||
if (storePath) {
|
||||
const dir = path.dirname(storePath);
|
||||
candidates.push(path.join(dir, `${sessionId}.jsonl`));
|
||||
|
||||
@ -296,9 +296,10 @@ export async function handleToolsInvokeHttpRequest(
|
||||
const result = await (tool as any).execute?.(`http-${Date.now()}`, toolArgs);
|
||||
sendJson(res, 200, { ok: true, result });
|
||||
} catch (err) {
|
||||
(res as any).log?.error?.(`Tool invocation failed: ${String(err)}`);
|
||||
sendJson(res, 400, {
|
||||
ok: false,
|
||||
error: { type: "tool_error", message: err instanceof Error ? err.message : String(err) },
|
||||
error: { type: "tool_error", message: "Tool execution failed" },
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -1,6 +1,9 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
/**
|
||||
* Synchronously loads a JSON file. Use `loadJsonFileAsync` for non-blocking I/O.
|
||||
*/
|
||||
export function loadJsonFile(pathname: string): unknown {
|
||||
try {
|
||||
if (!fs.existsSync(pathname)) return undefined;
|
||||
@ -11,6 +14,22 @@ export function loadJsonFile(pathname: string): unknown {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Asynchronously loads a JSON file. Preferred for hot paths to avoid blocking.
|
||||
*/
|
||||
export async function loadJsonFileAsync(pathname: string): Promise<unknown> {
|
||||
try {
|
||||
await fs.promises.access(pathname, fs.constants.F_OK);
|
||||
const raw = await fs.promises.readFile(pathname, "utf8");
|
||||
return JSON.parse(raw) as unknown;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronously saves a JSON file. Use `saveJsonFileAsync` for non-blocking I/O.
|
||||
*/
|
||||
export function saveJsonFile(pathname: string, data: unknown) {
|
||||
const dir = path.dirname(pathname);
|
||||
if (!fs.existsSync(dir)) {
|
||||
@ -19,3 +38,21 @@ export function saveJsonFile(pathname: string, data: unknown) {
|
||||
fs.writeFileSync(pathname, `${JSON.stringify(data, null, 2)}\n`, "utf8");
|
||||
fs.chmodSync(pathname, 0o600);
|
||||
}
|
||||
|
||||
/**
|
||||
* Asynchronously saves a JSON file. Preferred for hot paths to avoid blocking.
|
||||
*/
|
||||
export async function saveJsonFileAsync(pathname: string, data: unknown): Promise<void> {
|
||||
const dir = path.dirname(pathname);
|
||||
try {
|
||||
await fs.promises.access(dir, fs.constants.F_OK);
|
||||
} catch {
|
||||
await fs.promises.mkdir(dir, { recursive: true, mode: 0o700 });
|
||||
}
|
||||
await fs.promises.writeFile(pathname, `${JSON.stringify(data, null, 2)}\n`, "utf8");
|
||||
try {
|
||||
await fs.promises.chmod(pathname, 0o600);
|
||||
} catch {
|
||||
// Best-effort on platforms without chmod support
|
||||
}
|
||||
}
|
||||
|
||||
158
src/infra/metrics.ts
Normal file
158
src/infra/metrics.ts
Normal file
@ -0,0 +1,158 @@
|
||||
/**
|
||||
* Prometheus metrics for Moltbot observability.
|
||||
*
|
||||
* Usage:
|
||||
* import { metrics } from '../infra/metrics.js';
|
||||
* metrics.messagesTotal.inc({ channel: 'whatsapp', direction: 'inbound' });
|
||||
*/
|
||||
|
||||
import * as promClient from 'prom-client';
|
||||
|
||||
// Create a custom registry to avoid global state pollution
|
||||
export const metricsRegistry = new promClient.Registry();
|
||||
|
||||
// Add default metrics (process CPU, memory, etc.)
|
||||
promClient.collectDefaultMetrics({ register: metricsRegistry });
|
||||
|
||||
// =============================================================================
|
||||
// COUNTERS - Monotonically increasing values
|
||||
// =============================================================================
|
||||
|
||||
/** Total messages processed by channel and direction */
|
||||
export const messagesTotal = new promClient.Counter({
|
||||
name: 'moltbot_messages_total',
|
||||
help: 'Total number of messages processed',
|
||||
labelNames: ['channel', 'direction'] as const, // direction: inbound | outbound
|
||||
registers: [metricsRegistry],
|
||||
});
|
||||
|
||||
/** Total HTTP requests by method, route, and status */
|
||||
export const httpRequestsTotal = new promClient.Counter({
|
||||
name: 'moltbot_http_requests_total',
|
||||
help: 'Total HTTP requests received',
|
||||
labelNames: ['method', 'route', 'status'] as const,
|
||||
registers: [metricsRegistry],
|
||||
});
|
||||
|
||||
/** Total errors by type and source */
|
||||
export const errorsTotal = new promClient.Counter({
|
||||
name: 'moltbot_errors_total',
|
||||
help: 'Total errors encountered',
|
||||
labelNames: ['type', 'source'] as const,
|
||||
registers: [metricsRegistry],
|
||||
});
|
||||
|
||||
/** Total AI provider calls by provider and model */
|
||||
export const aiCallsTotal = new promClient.Counter({
|
||||
name: 'moltbot_ai_calls_total',
|
||||
help: 'Total AI provider API calls',
|
||||
labelNames: ['provider', 'model'] as const,
|
||||
registers: [metricsRegistry],
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// HISTOGRAMS - Request duration distributions
|
||||
// =============================================================================
|
||||
|
||||
/** HTTP request duration in seconds */
|
||||
export const httpRequestDuration = new promClient.Histogram({
|
||||
name: 'moltbot_http_request_duration_seconds',
|
||||
help: 'HTTP request duration in seconds',
|
||||
labelNames: ['method', 'route'] as const,
|
||||
buckets: [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
|
||||
registers: [metricsRegistry],
|
||||
});
|
||||
|
||||
/** AI provider response latency in seconds */
|
||||
export const aiLatency = new promClient.Histogram({
|
||||
name: 'moltbot_ai_latency_seconds',
|
||||
help: 'AI provider response latency in seconds',
|
||||
labelNames: ['provider', 'model'] as const,
|
||||
buckets: [0.1, 0.5, 1, 2, 5, 10, 30, 60],
|
||||
registers: [metricsRegistry],
|
||||
});
|
||||
|
||||
/** Message processing time in seconds */
|
||||
export const messageProcessingDuration = new promClient.Histogram({
|
||||
name: 'moltbot_message_processing_duration_seconds',
|
||||
help: 'Time to process a message end-to-end',
|
||||
labelNames: ['channel'] as const,
|
||||
buckets: [0.1, 0.5, 1, 2, 5, 10, 30],
|
||||
registers: [metricsRegistry],
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// GAUGES - Values that can go up and down
|
||||
// =============================================================================
|
||||
|
||||
/** Number of active sessions in memory */
|
||||
export const activeSessions = new promClient.Gauge({
|
||||
name: 'moltbot_active_sessions',
|
||||
help: 'Number of active sessions currently in memory',
|
||||
registers: [metricsRegistry],
|
||||
});
|
||||
|
||||
/** Number of connected channels */
|
||||
export const connectedChannels = new promClient.Gauge({
|
||||
name: 'moltbot_connected_channels',
|
||||
help: 'Number of channels currently connected',
|
||||
labelNames: ['channel'] as const,
|
||||
registers: [metricsRegistry],
|
||||
});
|
||||
|
||||
/** WebSocket connections count */
|
||||
export const websocketConnections = new promClient.Gauge({
|
||||
name: 'moltbot_websocket_connections',
|
||||
help: 'Number of active WebSocket connections',
|
||||
registers: [metricsRegistry],
|
||||
});
|
||||
|
||||
/** Gateway uptime in seconds */
|
||||
export const uptimeSeconds = new promClient.Gauge({
|
||||
name: 'moltbot_uptime_seconds',
|
||||
help: 'Gateway uptime in seconds',
|
||||
registers: [metricsRegistry],
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Helper functions
|
||||
// =============================================================================
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
/** Update the uptime gauge - call periodically */
|
||||
export function updateUptime(): void {
|
||||
uptimeSeconds.set((Date.now() - startTime) / 1000);
|
||||
}
|
||||
|
||||
/** Get metrics in Prometheus text format */
|
||||
export async function getMetricsText(): Promise<string> {
|
||||
updateUptime();
|
||||
return metricsRegistry.metrics();
|
||||
}
|
||||
|
||||
/** Get the content type for Prometheus metrics */
|
||||
export function getMetricsContentType(): string {
|
||||
return metricsRegistry.contentType;
|
||||
}
|
||||
|
||||
// Convenience export for common usage patterns
|
||||
export const metrics = {
|
||||
messagesTotal,
|
||||
httpRequestsTotal,
|
||||
errorsTotal,
|
||||
aiCallsTotal,
|
||||
httpRequestDuration,
|
||||
aiLatency,
|
||||
messageProcessingDuration,
|
||||
activeSessions,
|
||||
connectedChannels,
|
||||
websocketConnections,
|
||||
uptimeSeconds,
|
||||
getMetricsText,
|
||||
getMetricsContentType,
|
||||
updateUptime,
|
||||
registry: metricsRegistry,
|
||||
};
|
||||
|
||||
export default metrics;
|
||||
133
src/infra/otel.ts
Normal file
133
src/infra/otel.ts
Normal file
@ -0,0 +1,133 @@
|
||||
/**
|
||||
* OpenTelemetry SDK initialization for Moltbot.
|
||||
*
|
||||
* This module sets up distributed tracing with automatic HTTP instrumentation.
|
||||
* Import this at the very start of your application entry point.
|
||||
*
|
||||
* Usage:
|
||||
* import './infra/otel.js'; // Must be first import
|
||||
* import { trace } from '@opentelemetry/api';
|
||||
*/
|
||||
|
||||
import { NodeSDK } from '@opentelemetry/sdk-node';
|
||||
import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
|
||||
import { trace, context, SpanStatusCode, type Span } from '@opentelemetry/api';
|
||||
|
||||
// SDK configuration - only enable if OTEL_ENABLED is set
|
||||
const isOtelEnabled = process.env.OTEL_ENABLED === 'true' || process.env.OTEL_ENABLED === '1';
|
||||
|
||||
let sdk: NodeSDK | null = null;
|
||||
|
||||
if (isOtelEnabled) {
|
||||
sdk = new NodeSDK({
|
||||
serviceName: 'moltbot',
|
||||
instrumentations: [
|
||||
getNodeAutoInstrumentations({
|
||||
// Disable some noisy instrumentations
|
||||
'@opentelemetry/instrumentation-fs': { enabled: false },
|
||||
'@opentelemetry/instrumentation-dns': { enabled: false },
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
// Start the SDK
|
||||
sdk.start();
|
||||
|
||||
// Graceful shutdown
|
||||
process.on('SIGTERM', () => {
|
||||
sdk?.shutdown()
|
||||
.then(() => console.log('[otel] Tracing terminated'))
|
||||
.catch((error) => console.error('[otel] Error terminating tracing', error))
|
||||
.finally(() => process.exit(0));
|
||||
});
|
||||
|
||||
console.log('[otel] OpenTelemetry SDK initialized');
|
||||
} else {
|
||||
console.log('[otel] OpenTelemetry disabled (set OTEL_ENABLED=true to enable)');
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Helper functions for manual instrumentation
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Get the current tracer for manual span creation.
|
||||
*/
|
||||
export function getTracer(name: string = 'moltbot') {
|
||||
return trace.getTracer(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current span context for correlation.
|
||||
*/
|
||||
export function getCurrentSpan() {
|
||||
return trace.getActiveSpan();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current trace ID for log correlation.
|
||||
*/
|
||||
export function getCurrentTraceId(): string | undefined {
|
||||
const span = getCurrentSpan();
|
||||
if (!span) return undefined;
|
||||
return span.spanContext().traceId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current span ID for log correlation.
|
||||
*/
|
||||
export function getCurrentSpanId(): string | undefined {
|
||||
const span = getCurrentSpan();
|
||||
if (!span) return undefined;
|
||||
return span.spanContext().spanId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a custom span for a specific operation.
|
||||
*
|
||||
* @example
|
||||
* await withSpan('ai.completion', { provider: 'openai' }, async (span) => {
|
||||
* const result = await callOpenAI();
|
||||
* span.setAttribute('tokens', result.usage.total_tokens);
|
||||
* return result;
|
||||
* });
|
||||
*/
|
||||
export async function withSpan<T>(
|
||||
name: string,
|
||||
attributes: Record<string, string | number | boolean> = {},
|
||||
fn: (span: ReturnType<typeof trace.getTracer>['startSpan'] extends (name: string) => infer R ? R : never) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const tracer = getTracer();
|
||||
const span = tracer.startSpan(name);
|
||||
|
||||
for (const [key, value] of Object.entries(attributes)) {
|
||||
span.setAttribute(key, value);
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await context.with(trace.setSpan(context.active(), span), () => fn(span));
|
||||
span.setStatus({ code: SpanStatusCode.OK });
|
||||
return result;
|
||||
} catch (error) {
|
||||
span.setStatus({
|
||||
code: SpanStatusCode.ERROR,
|
||||
message: error instanceof Error ? error.message : 'Unknown error',
|
||||
});
|
||||
span.recordException(error instanceof Error ? error : new Error(String(error)));
|
||||
throw error;
|
||||
} finally {
|
||||
span.end();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add trace context to a log record.
|
||||
*/
|
||||
export function getTraceContext(): { traceId?: string; spanId?: string } {
|
||||
return {
|
||||
traceId: getCurrentTraceId(),
|
||||
spanId: getCurrentSpanId(),
|
||||
};
|
||||
}
|
||||
|
||||
export { sdk as otelSdk };
|
||||
@ -817,8 +817,9 @@ describe("security audit", () => {
|
||||
await fs.writeFile(includePath, "{ logging: { redactSensitive: 'off' } }\n", "utf-8");
|
||||
if (isWindows) {
|
||||
// Grant "Everyone" write access to trigger the perms_writable check on Windows
|
||||
// Use the well-known SID *S-1-1-0 for "Everyone" (works on all locales: English, Spanish, etc.)
|
||||
const { execSync } = await import("node:child_process");
|
||||
execSync(`icacls "${includePath}" /grant Everyone:W`, { stdio: "ignore" });
|
||||
execSync(`icacls "${includePath}" /grant *S-1-1-0:W`, { stdio: "ignore" });
|
||||
} else {
|
||||
await fs.chmod(includePath, 0o644);
|
||||
}
|
||||
@ -832,18 +833,18 @@ describe("security audit", () => {
|
||||
const user = "DESKTOP-TEST\\Tester";
|
||||
const execIcacls = isWindows
|
||||
? async (_cmd: string, args: string[]) => {
|
||||
const target = args[0];
|
||||
if (target === includePath) {
|
||||
return {
|
||||
stdout: `${target} NT AUTHORITY\\SYSTEM:(F)\n BUILTIN\\Users:(W)\n ${user}:(F)\n`,
|
||||
stderr: "",
|
||||
};
|
||||
}
|
||||
const target = args[0];
|
||||
if (target === includePath) {
|
||||
return {
|
||||
stdout: `${target} NT AUTHORITY\\SYSTEM:(F)\n ${user}:(F)\n`,
|
||||
stdout: `${target} NT AUTHORITY\\SYSTEM:(F)\n BUILTIN\\Users:(W)\n ${user}:(F)\n`,
|
||||
stderr: "",
|
||||
};
|
||||
}
|
||||
return {
|
||||
stdout: `${target} NT AUTHORITY\\SYSTEM:(F)\n ${user}:(F)\n`,
|
||||
stderr: "",
|
||||
};
|
||||
}
|
||||
: undefined;
|
||||
const res = await runSecurityAudit({
|
||||
config: cfg,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user