Merge d5dc6eb443 into da71eaebd2
This commit is contained in:
commit
6e346b7ac4
@ -227,7 +227,7 @@ async function fetchWithRedirects(params: {
|
|||||||
throw new Error("Redirect loop detected");
|
throw new Error("Redirect loop detected");
|
||||||
}
|
}
|
||||||
visited.add(nextUrl);
|
visited.add(nextUrl);
|
||||||
void res.body?.cancel();
|
res.body?.cancel().catch(() => {});
|
||||||
await closeDispatcher(dispatcher);
|
await closeDispatcher(dispatcher);
|
||||||
currentUrl = nextUrl;
|
currentUrl = nextUrl;
|
||||||
continue;
|
continue;
|
||||||
@ -282,15 +282,21 @@ export async function fetchFirecrawlContent(params: {
|
|||||||
storeInCache: params.storeInCache,
|
storeInCache: params.storeInCache,
|
||||||
};
|
};
|
||||||
|
|
||||||
const res = await fetch(endpoint, {
|
let res: Response;
|
||||||
method: "POST",
|
try {
|
||||||
headers: {
|
res = await fetch(endpoint, {
|
||||||
Authorization: `Bearer ${params.apiKey}`,
|
method: "POST",
|
||||||
"Content-Type": "application/json",
|
headers: {
|
||||||
},
|
Authorization: `Bearer ${params.apiKey}`,
|
||||||
body: JSON.stringify(body),
|
"Content-Type": "application/json",
|
||||||
signal: withTimeout(undefined, params.timeoutSeconds * 1000),
|
},
|
||||||
});
|
body: JSON.stringify(body),
|
||||||
|
signal: withTimeout(undefined, params.timeoutSeconds * 1000),
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
const detail = err instanceof Error ? err.message : String(err);
|
||||||
|
throw new Error(`Firecrawl fetch failed (network): ${detail}`);
|
||||||
|
}
|
||||||
|
|
||||||
const payload = (await res.json()) as {
|
const payload = (await res.json()) as {
|
||||||
success?: boolean;
|
success?: boolean;
|
||||||
@ -600,25 +606,36 @@ export function createWebFetchTool(options?: {
|
|||||||
const url = readStringParam(params, "url", { required: true });
|
const url = readStringParam(params, "url", { required: true });
|
||||||
const extractMode = readStringParam(params, "extractMode") === "text" ? "text" : "markdown";
|
const extractMode = readStringParam(params, "extractMode") === "text" ? "text" : "markdown";
|
||||||
const maxChars = readNumberParam(params, "maxChars", { integer: true });
|
const maxChars = readNumberParam(params, "maxChars", { integer: true });
|
||||||
const result = await runWebFetch({
|
try {
|
||||||
url,
|
const result = await runWebFetch({
|
||||||
extractMode,
|
url,
|
||||||
maxChars: resolveMaxChars(maxChars ?? fetch?.maxChars, DEFAULT_FETCH_MAX_CHARS),
|
extractMode,
|
||||||
maxRedirects: resolveMaxRedirects(fetch?.maxRedirects, DEFAULT_FETCH_MAX_REDIRECTS),
|
maxChars: resolveMaxChars(maxChars ?? fetch?.maxChars, DEFAULT_FETCH_MAX_CHARS),
|
||||||
timeoutSeconds: resolveTimeoutSeconds(fetch?.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS),
|
maxRedirects: resolveMaxRedirects(fetch?.maxRedirects, DEFAULT_FETCH_MAX_REDIRECTS),
|
||||||
cacheTtlMs: resolveCacheTtlMs(fetch?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES),
|
timeoutSeconds: resolveTimeoutSeconds(fetch?.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS),
|
||||||
userAgent,
|
cacheTtlMs: resolveCacheTtlMs(fetch?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES),
|
||||||
readabilityEnabled,
|
userAgent,
|
||||||
firecrawlEnabled,
|
readabilityEnabled,
|
||||||
firecrawlApiKey,
|
firecrawlEnabled,
|
||||||
firecrawlBaseUrl,
|
firecrawlApiKey,
|
||||||
firecrawlOnlyMainContent,
|
firecrawlBaseUrl,
|
||||||
firecrawlMaxAgeMs,
|
firecrawlOnlyMainContent,
|
||||||
firecrawlProxy: "auto",
|
firecrawlMaxAgeMs,
|
||||||
firecrawlStoreInCache: true,
|
firecrawlProxy: "auto",
|
||||||
firecrawlTimeoutSeconds,
|
firecrawlStoreInCache: true,
|
||||||
});
|
firecrawlTimeoutSeconds,
|
||||||
return jsonResult(result);
|
});
|
||||||
|
return jsonResult(result);
|
||||||
|
} catch (err) {
|
||||||
|
const message =
|
||||||
|
err instanceof Error ? err.message : typeof err === "string" ? err : "Unknown error";
|
||||||
|
return jsonResult({
|
||||||
|
status: "error",
|
||||||
|
tool: "web_fetch",
|
||||||
|
url,
|
||||||
|
error: `Web fetch failed: ${message}`,
|
||||||
|
});
|
||||||
|
}
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@ -125,7 +125,7 @@ describe("web_fetch extraction fallbacks", () => {
|
|||||||
expect(details.text).toContain("firecrawl content");
|
expect(details.text).toContain("firecrawl content");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("throws when readability is disabled and firecrawl is unavailable", async () => {
|
it("returns error result when readability is disabled and firecrawl is unavailable", async () => {
|
||||||
const mockFetch = vi.fn((input: RequestInfo) =>
|
const mockFetch = vi.fn((input: RequestInfo) =>
|
||||||
Promise.resolve(htmlResponse("<html><body>hi</body></html>", requestUrl(input))),
|
Promise.resolve(htmlResponse("<html><body>hi</body></html>", requestUrl(input))),
|
||||||
);
|
);
|
||||||
@ -143,12 +143,15 @@ describe("web_fetch extraction fallbacks", () => {
|
|||||||
sandboxed: false,
|
sandboxed: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
await expect(
|
const result = await tool?.execute?.("call", {
|
||||||
tool?.execute?.("call", { url: "https://example.com/readability-off" }),
|
url: "https://example.com/readability-off",
|
||||||
).rejects.toThrow("Readability disabled");
|
});
|
||||||
|
const details = result?.details as { status?: string; error?: string };
|
||||||
|
expect(details.status).toBe("error");
|
||||||
|
expect(details.error).toContain("Readability disabled");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("throws when readability is empty and firecrawl fails", async () => {
|
it("returns error result when readability is empty and firecrawl fails", async () => {
|
||||||
const mockFetch = vi.fn((input: RequestInfo) => {
|
const mockFetch = vi.fn((input: RequestInfo) => {
|
||||||
const url = requestUrl(input);
|
const url = requestUrl(input);
|
||||||
if (url.includes("api.firecrawl.dev")) {
|
if (url.includes("api.firecrawl.dev")) {
|
||||||
@ -172,9 +175,12 @@ describe("web_fetch extraction fallbacks", () => {
|
|||||||
sandboxed: false,
|
sandboxed: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
await expect(
|
const result = await tool?.execute?.("call", {
|
||||||
tool?.execute?.("call", { url: "https://example.com/readability-empty" }),
|
url: "https://example.com/readability-empty",
|
||||||
).rejects.toThrow("Readability and Firecrawl returned no content");
|
});
|
||||||
|
const details = result?.details as { status?: string; error?: string };
|
||||||
|
expect(details.status).toBe("error");
|
||||||
|
expect(details.error).toContain("Readability and Firecrawl returned no content");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("uses firecrawl when direct fetch fails", async () => {
|
it("uses firecrawl when direct fetch fails", async () => {
|
||||||
@ -232,17 +238,14 @@ describe("web_fetch extraction fallbacks", () => {
|
|||||||
sandboxed: false,
|
sandboxed: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
let message = "";
|
const result = await tool?.execute?.("call", { url: "https://example.com/missing" });
|
||||||
try {
|
const details = result?.details as { status?: string; error?: string };
|
||||||
await tool?.execute?.("call", { url: "https://example.com/missing" });
|
expect(details.status).toBe("error");
|
||||||
} catch (error) {
|
expect(details.error).toContain("Web fetch failed");
|
||||||
message = (error as Error).message;
|
expect(details.error).toContain("404");
|
||||||
}
|
expect(details.error).toContain("Not Found");
|
||||||
|
expect(details.error).not.toContain("<html");
|
||||||
expect(message).toContain("Web fetch failed (404):");
|
expect(details.error!.length).toBeLessThan(5_000);
|
||||||
expect(message).toContain("Not Found");
|
|
||||||
expect(message).not.toContain("<html");
|
|
||||||
expect(message.length).toBeLessThan(5_000);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("strips HTML errors when content-type is missing", async () => {
|
it("strips HTML errors when content-type is missing", async () => {
|
||||||
@ -265,8 +268,32 @@ describe("web_fetch extraction fallbacks", () => {
|
|||||||
sandboxed: false,
|
sandboxed: false,
|
||||||
});
|
});
|
||||||
|
|
||||||
await expect(tool?.execute?.("call", { url: "https://example.com/oops" })).rejects.toThrow(
|
const result = await tool?.execute?.("call", { url: "https://example.com/oops" });
|
||||||
/Web fetch failed \(500\):.*Oops/,
|
const details = result?.details as { status?: string; error?: string };
|
||||||
);
|
expect(details.status).toBe("error");
|
||||||
|
expect(details.error).toMatch(/Web fetch failed.*500.*Oops/);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns error result for network-level TypeError instead of crashing", async () => {
|
||||||
|
const mockFetch = vi.fn(() => Promise.reject(new TypeError("fetch failed")));
|
||||||
|
// @ts-expect-error mock fetch
|
||||||
|
global.fetch = mockFetch;
|
||||||
|
|
||||||
|
const tool = createWebFetchTool({
|
||||||
|
config: {
|
||||||
|
tools: {
|
||||||
|
web: {
|
||||||
|
fetch: { cacheTtlMinutes: 0, firecrawl: { enabled: false } },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
sandboxed: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
const result = await tool?.execute?.("call", { url: "https://unreachable.example.com/" });
|
||||||
|
const details = result?.details as { status?: string; error?: string; url?: string };
|
||||||
|
expect(details.status).toBe("error");
|
||||||
|
expect(details.error).toContain("fetch failed");
|
||||||
|
expect(details.url).toBe("https://unreachable.example.com/");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@ -12,6 +12,7 @@ import {
|
|||||||
resolveAgentWorkspaceDir,
|
resolveAgentWorkspaceDir,
|
||||||
resolveAgentDir,
|
resolveAgentDir,
|
||||||
} from "../agents/agent-scope.js";
|
} from "../agents/agent-scope.js";
|
||||||
|
import { resolveDefaultModelForAgent } from "../agents/model-selection.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generate a short 1-2 word filename slug from session content using LLM
|
* Generate a short 1-2 word filename slug from session content using LLM
|
||||||
@ -38,6 +39,11 @@ ${params.sessionContent.slice(0, 2000)}
|
|||||||
|
|
||||||
Reply with ONLY the slug, nothing else. Examples: "vendor-pitch", "api-design", "bug-fix"`;
|
Reply with ONLY the slug, nothing else. Examples: "vendor-pitch", "api-design", "bug-fix"`;
|
||||||
|
|
||||||
|
// Resolve user's configured default model instead of hardcoded Opus
|
||||||
|
const { provider, model } = resolveDefaultModelForAgent({
|
||||||
|
cfg: params.cfg,
|
||||||
|
});
|
||||||
|
|
||||||
const result = await runEmbeddedPiAgent({
|
const result = await runEmbeddedPiAgent({
|
||||||
sessionId: `slug-generator-${Date.now()}`,
|
sessionId: `slug-generator-${Date.now()}`,
|
||||||
sessionKey: "temp:slug-generator",
|
sessionKey: "temp:slug-generator",
|
||||||
@ -46,6 +52,8 @@ Reply with ONLY the slug, nothing else. Examples: "vendor-pitch", "api-design",
|
|||||||
agentDir,
|
agentDir,
|
||||||
config: params.cfg,
|
config: params.cfg,
|
||||||
prompt,
|
prompt,
|
||||||
|
provider,
|
||||||
|
model,
|
||||||
timeoutMs: 15_000, // 15 second timeout
|
timeoutMs: 15_000, // 15 second timeout
|
||||||
runId: `slug-gen-${Date.now()}`,
|
runId: `slug-gen-${Date.now()}`,
|
||||||
});
|
});
|
||||||
@ -75,7 +83,10 @@ Reply with ONLY the slug, nothing else. Examples: "vendor-pitch", "api-design",
|
|||||||
// Clean up temporary session file
|
// Clean up temporary session file
|
||||||
if (tempSessionFile) {
|
if (tempSessionFile) {
|
||||||
try {
|
try {
|
||||||
await fs.rm(path.dirname(tempSessionFile), { recursive: true, force: true });
|
await fs.rm(path.dirname(tempSessionFile), {
|
||||||
|
recursive: true,
|
||||||
|
force: true,
|
||||||
|
});
|
||||||
} catch {
|
} catch {
|
||||||
// Ignore cleanup errors
|
// Ignore cleanup errors
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user