feat(workspace): add workspace sync config types and rclone infra
Foundation for bidirectional workspace sync with cloud storage providers. - Add WorkspaceSyncConfig types and Zod schema (types.workspace.ts) - Add rclone wrapper with bisync support (rclone.ts) - Binary detection and installation prompts - Config file generation from moltbot.json credentials - OAuth authorization flow - Bisync and one-way sync operations - Remote listing and health checks - Wire up workspace config in MoltbotConfig Part 1 of 4 for #4012. Extracted from #3505. Co-authored-by: ashbrener <ash@ashbrener.com>
This commit is contained in:
Ash Brener
parent
6af205a13a
commit
81f4e3dc8f
@ -23,6 +23,7 @@ import type { NodeHostConfig } from "./types.node-host.js";
|
||||
import type { PluginsConfig } from "./types.plugins.js";
|
||||
import type { SkillsConfig } from "./types.skills.js";
|
||||
import type { ToolsConfig } from "./types.tools.js";
|
||||
import type { WorkspaceConfig } from "./types.workspace.js";
|
||||
|
||||
export type OpenClawConfig = {
|
||||
meta?: {
|
||||
@ -95,6 +96,7 @@ export type OpenClawConfig = {
|
||||
canvasHost?: CanvasHostConfig;
|
||||
talk?: TalkConfig;
|
||||
gateway?: GatewayConfig;
|
||||
workspace?: WorkspaceConfig;
|
||||
};
|
||||
|
||||
export type ConfigValidationIssue = {
|
||||
|
||||
129
src/config/types.workspace.ts
Normal file
129
src/config/types.workspace.ts
Normal file
@ -0,0 +1,129 @@
|
||||
/**
|
||||
* Workspace sync provider modes.
|
||||
* - off: no sync
|
||||
* - dropbox: Dropbox via rclone
|
||||
* - gdrive: Google Drive via rclone
|
||||
* - onedrive: OneDrive via rclone
|
||||
* - s3: S3-compatible storage via rclone
|
||||
* - custom: custom rclone remote (user-configured)
|
||||
*/
|
||||
export type WorkspaceSyncProvider = "off" | "dropbox" | "gdrive" | "onedrive" | "s3" | "custom";
|
||||
|
||||
/**
|
||||
* Workspace sync configuration.
|
||||
* Enables bidirectional sync between the agent workspace and cloud storage.
|
||||
*/
|
||||
export type WorkspaceSyncConfig = {
|
||||
/**
|
||||
* Sync provider mode.
|
||||
* - off: disabled (default)
|
||||
* - dropbox: Dropbox via rclone
|
||||
* - gdrive: Google Drive via rclone
|
||||
* - onedrive: OneDrive via rclone
|
||||
* - s3: S3-compatible storage via rclone
|
||||
* - custom: custom rclone remote
|
||||
*/
|
||||
provider?: WorkspaceSyncProvider;
|
||||
|
||||
/**
|
||||
* Remote path/folder in cloud storage (e.g., "moltbot-share").
|
||||
* For Dropbox App folders, this is relative to the app folder root.
|
||||
*/
|
||||
remotePath?: string;
|
||||
|
||||
/**
|
||||
* Local subfolder within workspace to sync (default: "shared").
|
||||
* Files outside this folder are not synced.
|
||||
*/
|
||||
localPath?: string;
|
||||
|
||||
/**
|
||||
* Sync interval in seconds (0 = manual only, default: 0).
|
||||
* When > 0, the gateway runs rclone bisync in the background at this interval.
|
||||
* This is a pure file operation - it does NOT wake the bot or incur LLM costs.
|
||||
*/
|
||||
interval?: number;
|
||||
|
||||
/**
|
||||
* Sync on session start (default: false).
|
||||
*/
|
||||
onSessionStart?: boolean;
|
||||
|
||||
/**
|
||||
* Sync on session end (default: false).
|
||||
*/
|
||||
onSessionEnd?: boolean;
|
||||
|
||||
/**
|
||||
* rclone remote name (default: "cloud").
|
||||
* Used when provider is "custom" or to override the auto-generated name.
|
||||
*/
|
||||
remoteName?: string;
|
||||
|
||||
/**
|
||||
* Path to rclone config file.
|
||||
* Default: $CLAWDBOT_STATE_DIR/.config/rclone/rclone.conf
|
||||
*/
|
||||
configPath?: string;
|
||||
|
||||
/**
|
||||
* Conflict resolution strategy.
|
||||
* - newer: keep the newer file, rename older with .conflict suffix
|
||||
* - local: local wins, remote gets .conflict suffix
|
||||
* - remote: remote wins, local gets .conflict suffix
|
||||
*/
|
||||
conflictResolve?: "newer" | "local" | "remote";
|
||||
|
||||
/**
|
||||
* File patterns to exclude from sync (glob patterns).
|
||||
* Defaults include: .git/**, node_modules/**, .venv/**, *.log, .DS_Store
|
||||
*/
|
||||
exclude?: string[];
|
||||
|
||||
/**
|
||||
* Follow symlinks during sync (default: false).
|
||||
* When false, symlinks are skipped with a notice.
|
||||
* When true, symlinks are followed and their targets are copied.
|
||||
*/
|
||||
copySymlinks?: boolean;
|
||||
|
||||
/**
|
||||
* S3-specific configuration (when provider is "s3").
|
||||
*/
|
||||
s3?: {
|
||||
/** S3 endpoint URL (for non-AWS S3-compatible services). */
|
||||
endpoint?: string;
|
||||
/** S3 bucket name. */
|
||||
bucket?: string;
|
||||
/** S3 region. */
|
||||
region?: string;
|
||||
/** Access key ID (prefer env var S3_ACCESS_KEY_ID). */
|
||||
accessKeyId?: string;
|
||||
/** Secret access key (prefer env var S3_SECRET_ACCESS_KEY). */
|
||||
secretAccessKey?: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* Dropbox-specific configuration.
|
||||
*/
|
||||
dropbox?: {
|
||||
/** Use app folder access (more secure, limited to Apps/<app-name>/). */
|
||||
appFolder?: boolean;
|
||||
/** Dropbox app key / client_id. */
|
||||
appKey?: string;
|
||||
/** Dropbox app secret / client_secret. */
|
||||
appSecret?: string;
|
||||
/** OAuth token JSON (prefer env var ${DROPBOX_TOKEN}). */
|
||||
token?: string;
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Top-level workspace configuration.
|
||||
*/
|
||||
export type WorkspaceConfig = {
|
||||
/**
|
||||
* Cloud sync configuration for bidirectional workspace sync.
|
||||
*/
|
||||
sync?: WorkspaceSyncConfig;
|
||||
};
|
||||
277
src/infra/rclone.test.ts
Normal file
277
src/infra/rclone.test.ts
Normal file
@ -0,0 +1,277 @@
|
||||
import path from "node:path";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import { describe, expect, it, beforeEach, afterEach } from "vitest";
|
||||
|
||||
import {
|
||||
resolveSyncConfig,
|
||||
generateRcloneConfig,
|
||||
isRcloneConfigured,
|
||||
ensureRcloneConfigFromConfig,
|
||||
} from "./rclone.js";
|
||||
|
||||
describe("rclone helpers", () => {
|
||||
describe("resolveSyncConfig", () => {
|
||||
it("uses defaults when config is minimal", () => {
|
||||
const config = { provider: "dropbox" as const, remotePath: "test-folder" };
|
||||
const workspaceDir = path.join("/home", "user", "workspace");
|
||||
const stateDir = path.join("/home", "user", ".moltbot");
|
||||
|
||||
const resolved = resolveSyncConfig(config, workspaceDir, stateDir);
|
||||
|
||||
expect(resolved.provider).toBe("dropbox");
|
||||
expect(resolved.remotePath).toBe("test-folder");
|
||||
expect(resolved.localPath).toBe(path.join(workspaceDir, "shared"));
|
||||
expect(resolved.remoteName).toBe("cloud");
|
||||
expect(resolved.conflictResolve).toBe("newer");
|
||||
expect(resolved.interval).toBe(0);
|
||||
expect(resolved.onSessionStart).toBe(false);
|
||||
expect(resolved.onSessionEnd).toBe(false);
|
||||
});
|
||||
|
||||
it("respects custom localPath", () => {
|
||||
const config = {
|
||||
provider: "dropbox" as const,
|
||||
remotePath: "test-folder",
|
||||
localPath: "sync",
|
||||
};
|
||||
const workspaceDir = path.join("/home", "user", "workspace");
|
||||
const stateDir = path.join("/home", "user", ".moltbot");
|
||||
|
||||
const resolved = resolveSyncConfig(config, workspaceDir, stateDir);
|
||||
|
||||
expect(resolved.localPath).toBe(path.join(workspaceDir, "sync"));
|
||||
});
|
||||
|
||||
it("respects custom remoteName", () => {
|
||||
const config = {
|
||||
provider: "dropbox" as const,
|
||||
remotePath: "test-folder",
|
||||
remoteName: "my-dropbox",
|
||||
};
|
||||
const workspaceDir = path.join("/home", "user", "workspace");
|
||||
const stateDir = path.join("/home", "user", ".moltbot");
|
||||
|
||||
const resolved = resolveSyncConfig(config, workspaceDir, stateDir);
|
||||
|
||||
expect(resolved.remoteName).toBe("my-dropbox");
|
||||
});
|
||||
|
||||
it("respects interval and session hooks", () => {
|
||||
const config = {
|
||||
provider: "dropbox" as const,
|
||||
remotePath: "test-folder",
|
||||
interval: 300,
|
||||
onSessionStart: true,
|
||||
onSessionEnd: true,
|
||||
};
|
||||
const workspaceDir = path.join("/home", "user", "workspace");
|
||||
const stateDir = path.join("/home", "user", ".moltbot");
|
||||
|
||||
const resolved = resolveSyncConfig(config, workspaceDir, stateDir);
|
||||
|
||||
expect(resolved.interval).toBe(300);
|
||||
expect(resolved.onSessionStart).toBe(true);
|
||||
expect(resolved.onSessionEnd).toBe(true);
|
||||
});
|
||||
|
||||
it("applies default excludes", () => {
|
||||
const config = { provider: "dropbox" as const, remotePath: "test" };
|
||||
const resolved = resolveSyncConfig(config, "/workspace", "/state");
|
||||
|
||||
expect(resolved.exclude).toContain(".git/**");
|
||||
expect(resolved.exclude).toContain("node_modules/**");
|
||||
});
|
||||
|
||||
it("respects custom excludes", () => {
|
||||
const config = {
|
||||
provider: "dropbox" as const,
|
||||
remotePath: "test",
|
||||
exclude: ["*.tmp", "cache/**"],
|
||||
};
|
||||
const resolved = resolveSyncConfig(config, "/workspace", "/state");
|
||||
|
||||
expect(resolved.exclude).toEqual(["*.tmp", "cache/**"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("generateRcloneConfig", () => {
|
||||
it("generates dropbox config with token", () => {
|
||||
const config = generateRcloneConfig("dropbox", "cloud", '{"access_token":"abc123"}');
|
||||
|
||||
expect(config).toContain("[cloud]");
|
||||
expect(config).toContain("type = dropbox");
|
||||
expect(config).toContain('token = {"access_token":"abc123"}');
|
||||
});
|
||||
|
||||
it("generates gdrive config", () => {
|
||||
const config = generateRcloneConfig("gdrive", "drive", '{"access_token":"xyz"}');
|
||||
|
||||
expect(config).toContain("[drive]");
|
||||
expect(config).toContain("type = drive");
|
||||
expect(config).toContain('token = {"access_token":"xyz"}');
|
||||
});
|
||||
|
||||
it("generates onedrive config", () => {
|
||||
const config = generateRcloneConfig("onedrive", "od", '{"access_token":"123"}');
|
||||
|
||||
expect(config).toContain("[od]");
|
||||
expect(config).toContain("type = onedrive");
|
||||
});
|
||||
|
||||
it("includes app key/secret for dropbox app folder", () => {
|
||||
const config = generateRcloneConfig("dropbox", "cloud", '{"access_token":"abc"}', {
|
||||
dropbox: { appKey: "key123", appSecret: "secret456" },
|
||||
});
|
||||
|
||||
expect(config).toContain("client_id = key123");
|
||||
expect(config).toContain("client_secret = secret456");
|
||||
});
|
||||
|
||||
it("generates s3 config with endpoint", () => {
|
||||
const config = generateRcloneConfig("s3", "r2", "", {
|
||||
s3: {
|
||||
endpoint: "https://xxx.r2.cloudflarestorage.com",
|
||||
accessKeyId: "AKID",
|
||||
secretAccessKey: "SECRET",
|
||||
},
|
||||
});
|
||||
|
||||
expect(config).toContain("[r2]");
|
||||
expect(config).toContain("type = s3");
|
||||
expect(config).toContain("endpoint = https://xxx.r2.cloudflarestorage.com");
|
||||
expect(config).toContain("access_key_id = AKID");
|
||||
expect(config).toContain("secret_access_key = SECRET");
|
||||
});
|
||||
|
||||
it("includes region for s3 when provided", () => {
|
||||
const config = generateRcloneConfig("s3", "aws", "", {
|
||||
s3: {
|
||||
region: "us-east-1",
|
||||
bucket: "my-bucket",
|
||||
},
|
||||
});
|
||||
|
||||
expect(config).toContain("region = us-east-1");
|
||||
});
|
||||
});
|
||||
|
||||
describe("isRcloneConfigured", () => {
|
||||
it("returns false when config file does not exist", () => {
|
||||
const result = isRcloneConfigured("/nonexistent/path/rclone.conf", "cloud");
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ensureRcloneConfigFromConfig", () => {
|
||||
let tempDir: string;
|
||||
let configPath: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "rclone-test-"));
|
||||
configPath = path.join(tempDir, "rclone.conf");
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("returns false when provider is off", () => {
|
||||
const result = ensureRcloneConfigFromConfig({ provider: "off" }, configPath, "cloud");
|
||||
expect(result).toBe(false);
|
||||
expect(fs.existsSync(configPath)).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false when provider is undefined", () => {
|
||||
const result = ensureRcloneConfigFromConfig(undefined, configPath, "cloud");
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false when dropbox has no token", () => {
|
||||
const result = ensureRcloneConfigFromConfig(
|
||||
{ provider: "dropbox", dropbox: { appKey: "key", appSecret: "secret" } },
|
||||
configPath,
|
||||
"cloud",
|
||||
);
|
||||
expect(result).toBe(false);
|
||||
expect(fs.existsSync(configPath)).toBe(false);
|
||||
});
|
||||
|
||||
it("generates config when dropbox has token", () => {
|
||||
const result = ensureRcloneConfigFromConfig(
|
||||
{
|
||||
provider: "dropbox",
|
||||
dropbox: {
|
||||
token: '{"access_token":"test123"}',
|
||||
appKey: "mykey",
|
||||
appSecret: "mysecret",
|
||||
},
|
||||
},
|
||||
configPath,
|
||||
"cloud",
|
||||
);
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(fs.existsSync(configPath)).toBe(true);
|
||||
|
||||
const content = fs.readFileSync(configPath, "utf-8");
|
||||
expect(content).toContain("[cloud]");
|
||||
expect(content).toContain("type = dropbox");
|
||||
expect(content).toContain('token = {"access_token":"test123"}');
|
||||
expect(content).toContain("client_id = mykey");
|
||||
expect(content).toContain("client_secret = mysecret");
|
||||
});
|
||||
|
||||
it("returns true without regenerating when config already exists", () => {
|
||||
// Create existing config
|
||||
fs.mkdirSync(path.dirname(configPath), { recursive: true });
|
||||
fs.writeFileSync(configPath, "[cloud]\ntype = dropbox\ntoken = old");
|
||||
|
||||
const result = ensureRcloneConfigFromConfig(
|
||||
{ provider: "dropbox", dropbox: { token: '{"new":"token"}' } },
|
||||
configPath,
|
||||
"cloud",
|
||||
);
|
||||
|
||||
expect(result).toBe(true);
|
||||
// Should NOT overwrite existing config
|
||||
const content = fs.readFileSync(configPath, "utf-8");
|
||||
expect(content).toContain("token = old");
|
||||
});
|
||||
|
||||
it("returns false when s3 has no credentials", () => {
|
||||
const result = ensureRcloneConfigFromConfig(
|
||||
{ provider: "s3", s3: { endpoint: "https://example.com" } },
|
||||
configPath,
|
||||
"cloud",
|
||||
);
|
||||
expect(result).toBe(false);
|
||||
});
|
||||
|
||||
it("generates config when s3 has credentials", () => {
|
||||
const result = ensureRcloneConfigFromConfig(
|
||||
{
|
||||
provider: "s3",
|
||||
s3: {
|
||||
endpoint: "https://r2.example.com",
|
||||
accessKeyId: "AKID123",
|
||||
secretAccessKey: "SECRET456",
|
||||
region: "auto",
|
||||
},
|
||||
},
|
||||
configPath,
|
||||
"r2",
|
||||
);
|
||||
|
||||
expect(result).toBe(true);
|
||||
expect(fs.existsSync(configPath)).toBe(true);
|
||||
|
||||
const content = fs.readFileSync(configPath, "utf-8");
|
||||
expect(content).toContain("[r2]");
|
||||
expect(content).toContain("type = s3");
|
||||
expect(content).toContain("endpoint = https://r2.example.com");
|
||||
expect(content).toContain("access_key_id = AKID123");
|
||||
expect(content).toContain("secret_access_key = SECRET456");
|
||||
});
|
||||
});
|
||||
});
|
||||
598
src/infra/rclone.ts
Normal file
598
src/infra/rclone.ts
Normal file
@ -0,0 +1,598 @@
|
||||
import { existsSync, mkdirSync, writeFileSync, readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { homedir, platform } from "node:os";
|
||||
import { logVerbose } from "../globals.js";
|
||||
import { runExec } from "../process/exec.js";
|
||||
import type { WorkspaceSyncConfig, WorkspaceSyncProvider } from "../config/types.workspace.js";
|
||||
import type { RuntimeEnv } from "../runtime.js";
|
||||
import { defaultRuntime } from "../runtime.js";
|
||||
|
||||
const DEFAULT_REMOTE_NAME = "cloud";
|
||||
const DEFAULT_LOCAL_PATH = "shared";
|
||||
const DEFAULT_REMOTE_PATH = "moltbot-share";
|
||||
const DEFAULT_CONFLICT_RESOLVE = "newer";
|
||||
const DEFAULT_EXCLUDES = [
|
||||
".git/**",
|
||||
"node_modules/**",
|
||||
".venv/**",
|
||||
"__pycache__/**",
|
||||
"*.log",
|
||||
".DS_Store",
|
||||
];
|
||||
|
||||
export type RcloneSyncResult = {
|
||||
ok: boolean;
|
||||
error?: string;
|
||||
filesTransferred?: number;
|
||||
bytesTransferred?: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Find rclone binary in PATH or common locations.
|
||||
*/
|
||||
export async function findRcloneBinary(): Promise<string | null> {
|
||||
const checkBinary = async (path: string): Promise<boolean> => {
|
||||
if (!path || (path.startsWith("/") && !existsSync(path))) return false;
|
||||
try {
|
||||
await runExec(path, ["--version"], { timeoutMs: 3000 });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// Strategy 1: which command
|
||||
try {
|
||||
const { stdout } = await runExec("which", ["rclone"]);
|
||||
const fromPath = stdout.trim();
|
||||
if (fromPath && (await checkBinary(fromPath))) {
|
||||
return fromPath;
|
||||
}
|
||||
} catch {
|
||||
// which failed, continue
|
||||
}
|
||||
|
||||
// Strategy 2: Common install locations
|
||||
const commonPaths = ["/usr/local/bin/rclone", "/usr/bin/rclone", "/opt/homebrew/bin/rclone"];
|
||||
for (const path of commonPaths) {
|
||||
if (await checkBinary(path)) {
|
||||
return path;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
let cachedRcloneBinary: string | null = null;
|
||||
|
||||
export async function getRcloneBinary(): Promise<string> {
|
||||
if (cachedRcloneBinary) return cachedRcloneBinary;
|
||||
cachedRcloneBinary = await findRcloneBinary();
|
||||
return cachedRcloneBinary ?? "rclone";
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if rclone is installed.
|
||||
*/
|
||||
export async function isRcloneInstalled(): Promise<boolean> {
|
||||
const binary = await findRcloneBinary();
|
||||
return binary !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure rclone is installed, offering to install if missing.
|
||||
* Returns true if rclone is available, false if user declined install.
|
||||
*/
|
||||
export async function ensureRcloneInstalled(
|
||||
prompt: (message: string, defaultValue: boolean) => Promise<boolean>,
|
||||
exec: typeof runExec = runExec,
|
||||
runtime: RuntimeEnv = defaultRuntime,
|
||||
): Promise<boolean> {
|
||||
const installed = await isRcloneInstalled();
|
||||
if (installed) return true;
|
||||
|
||||
const isMac = platform() === "darwin";
|
||||
const isLinux = platform() === "linux";
|
||||
|
||||
if (isMac) {
|
||||
// Check if Homebrew is available
|
||||
const hasBrew = await exec("which", ["brew"]).then(
|
||||
() => true,
|
||||
() => false,
|
||||
);
|
||||
|
||||
if (hasBrew) {
|
||||
const install = await prompt(
|
||||
"rclone not found. Install via Homebrew (brew install rclone)?",
|
||||
true,
|
||||
);
|
||||
if (!install) {
|
||||
return false;
|
||||
}
|
||||
logVerbose("Installing rclone via Homebrew...");
|
||||
try {
|
||||
await exec("brew", ["install", "rclone"], { timeoutMs: 120_000 });
|
||||
// Clear cached binary so we find the new one
|
||||
cachedRcloneBinary = null;
|
||||
return true;
|
||||
} catch (err) {
|
||||
runtime.error(
|
||||
`Failed to install rclone: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (isLinux || isMac) {
|
||||
const install = await prompt(
|
||||
"rclone not found. Install via official script (curl https://rclone.org/install.sh)?",
|
||||
true,
|
||||
);
|
||||
if (!install) {
|
||||
return false;
|
||||
}
|
||||
logVerbose("Installing rclone via official script...");
|
||||
try {
|
||||
// Download and run the install script
|
||||
const { stdout } = await exec("curl", ["-s", "https://rclone.org/install.sh"], {
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
await exec("sudo", ["bash", "-c", stdout], { timeoutMs: 120_000 });
|
||||
// Clear cached binary so we find the new one
|
||||
cachedRcloneBinary = null;
|
||||
return true;
|
||||
} catch (err) {
|
||||
runtime.error(
|
||||
`Failed to install rclone: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
runtime.error("Try installing manually: https://rclone.org/install/");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
runtime.error("rclone not found. Please install manually: https://rclone.org/install/");
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the default rclone config path.
|
||||
*/
|
||||
export function getDefaultRcloneConfigPath(stateDir?: string): string {
|
||||
const base = stateDir ?? process.env.CLAWDBOT_STATE_DIR ?? join(homedir(), ".clawdbot");
|
||||
return join(base, ".config", "rclone", "rclone.conf");
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve sync config with defaults.
|
||||
*/
|
||||
export function resolveSyncConfig(
|
||||
config: WorkspaceSyncConfig | undefined,
|
||||
workspace: string,
|
||||
stateDir?: string,
|
||||
): {
|
||||
provider: WorkspaceSyncProvider;
|
||||
remoteName: string;
|
||||
remotePath: string;
|
||||
localPath: string;
|
||||
configPath: string;
|
||||
conflictResolve: "newer" | "local" | "remote";
|
||||
exclude: string[];
|
||||
copySymlinks: boolean;
|
||||
interval: number;
|
||||
onSessionStart: boolean;
|
||||
onSessionEnd: boolean;
|
||||
} {
|
||||
return {
|
||||
provider: config?.provider ?? "off",
|
||||
remoteName: config?.remoteName ?? DEFAULT_REMOTE_NAME,
|
||||
remotePath: config?.remotePath ?? DEFAULT_REMOTE_PATH,
|
||||
localPath: join(workspace, config?.localPath ?? DEFAULT_LOCAL_PATH),
|
||||
configPath: config?.configPath ?? getDefaultRcloneConfigPath(stateDir),
|
||||
conflictResolve: config?.conflictResolve ?? DEFAULT_CONFLICT_RESOLVE,
|
||||
exclude: config?.exclude ?? DEFAULT_EXCLUDES,
|
||||
copySymlinks: config?.copySymlinks ?? false,
|
||||
interval: config?.interval ?? 0,
|
||||
onSessionStart: config?.onSessionStart ?? false,
|
||||
onSessionEnd: config?.onSessionEnd ?? false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get rclone type string for a provider.
|
||||
*/
|
||||
function getRcloneType(provider: WorkspaceSyncProvider): string {
|
||||
switch (provider) {
|
||||
case "dropbox":
|
||||
return "dropbox";
|
||||
case "gdrive":
|
||||
return "drive";
|
||||
case "onedrive":
|
||||
return "onedrive";
|
||||
case "s3":
|
||||
return "s3";
|
||||
default:
|
||||
return "unknown";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate rclone config content for a provider.
|
||||
*/
|
||||
export function generateRcloneConfig(
|
||||
provider: WorkspaceSyncProvider,
|
||||
remoteName: string,
|
||||
token: string,
|
||||
options?: {
|
||||
dropbox?: { appKey?: string; appSecret?: string };
|
||||
s3?: {
|
||||
endpoint?: string;
|
||||
bucket?: string;
|
||||
region?: string;
|
||||
accessKeyId?: string;
|
||||
secretAccessKey?: string;
|
||||
};
|
||||
},
|
||||
): string {
|
||||
const type = getRcloneType(provider);
|
||||
let config = `[${remoteName}]\ntype = ${type}\n`;
|
||||
|
||||
if (provider === "dropbox") {
|
||||
config += `token = ${token}\n`;
|
||||
if (options?.dropbox?.appKey) {
|
||||
config += `client_id = ${options.dropbox.appKey}\n`;
|
||||
}
|
||||
if (options?.dropbox?.appSecret) {
|
||||
config += `client_secret = ${options.dropbox.appSecret}\n`;
|
||||
}
|
||||
} else if (provider === "gdrive") {
|
||||
config += `token = ${token}\n`;
|
||||
} else if (provider === "onedrive") {
|
||||
config += `token = ${token}\n`;
|
||||
} else if (provider === "s3") {
|
||||
if (options?.s3?.endpoint) {
|
||||
config += `endpoint = ${options.s3.endpoint}\n`;
|
||||
}
|
||||
if (options?.s3?.region) {
|
||||
config += `region = ${options.s3.region}\n`;
|
||||
}
|
||||
if (options?.s3?.accessKeyId) {
|
||||
config += `access_key_id = ${options.s3.accessKeyId}\n`;
|
||||
}
|
||||
if (options?.s3?.secretAccessKey) {
|
||||
config += `secret_access_key = ${options.s3.secretAccessKey}\n`;
|
||||
}
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write rclone config to disk.
|
||||
*/
|
||||
export function writeRcloneConfig(configPath: string, content: string): void {
|
||||
const dir = dirname(configPath);
|
||||
if (!existsSync(dir)) {
|
||||
mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
writeFileSync(configPath, content, { mode: 0o600 });
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if rclone config exists and has the remote configured.
|
||||
*/
|
||||
export function isRcloneConfigured(configPath: string, remoteName: string): boolean {
|
||||
if (!existsSync(configPath)) return false;
|
||||
try {
|
||||
const content = readFileSync(configPath, "utf-8");
|
||||
return content.includes(`[${remoteName}]`);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure rclone config exists, auto-generating from moltbot.json config if credentials are present.
|
||||
* This allows users to configure sync entirely via moltbot.json + env vars without manual rclone setup.
|
||||
*
|
||||
* @returns true if config exists or was generated, false if credentials are missing
|
||||
*/
|
||||
export function ensureRcloneConfigFromConfig(
|
||||
syncConfig: WorkspaceSyncConfig | undefined,
|
||||
configPath: string,
|
||||
remoteName: string,
|
||||
): boolean {
|
||||
// If config already exists with this remote, we're good
|
||||
if (isRcloneConfigured(configPath, remoteName)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!syncConfig?.provider || syncConfig.provider === "off") {
|
||||
return false;
|
||||
}
|
||||
|
||||
// For Dropbox: need token (appKey/appSecret optional but recommended)
|
||||
if (syncConfig.provider === "dropbox") {
|
||||
const token = syncConfig.dropbox?.token;
|
||||
if (!token) {
|
||||
return false;
|
||||
}
|
||||
|
||||
logVerbose(`[rclone] Auto-generating config for ${remoteName} from moltbot.json credentials`);
|
||||
|
||||
const configContent = generateRcloneConfig(syncConfig.provider, remoteName, token, {
|
||||
dropbox: {
|
||||
appKey: syncConfig.dropbox?.appKey,
|
||||
appSecret: syncConfig.dropbox?.appSecret,
|
||||
},
|
||||
});
|
||||
|
||||
writeRcloneConfig(configPath, configContent);
|
||||
return true;
|
||||
}
|
||||
|
||||
// For S3: need accessKeyId and secretAccessKey
|
||||
if (syncConfig.provider === "s3") {
|
||||
const { accessKeyId, secretAccessKey, endpoint, bucket, region } = syncConfig.s3 ?? {};
|
||||
if (!accessKeyId || !secretAccessKey) {
|
||||
return false;
|
||||
}
|
||||
|
||||
logVerbose(`[rclone] Auto-generating config for ${remoteName} from moltbot.json credentials`);
|
||||
|
||||
const configContent = generateRcloneConfig(syncConfig.provider, remoteName, "", {
|
||||
s3: { endpoint, bucket, region, accessKeyId, secretAccessKey },
|
||||
});
|
||||
|
||||
writeRcloneConfig(configPath, configContent);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Other providers require manual rclone config
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run rclone authorize command (returns the token).
|
||||
* This must be run on a machine with a browser.
|
||||
*/
|
||||
export async function authorizeRclone(
|
||||
provider: WorkspaceSyncProvider,
|
||||
appKey?: string,
|
||||
appSecret?: string,
|
||||
): Promise<{ ok: true; token: string } | { ok: false; error: string }> {
|
||||
const rcloneBin = await getRcloneBinary();
|
||||
const type = getRcloneType(provider);
|
||||
|
||||
const args = ["authorize", type];
|
||||
if (appKey && appSecret) {
|
||||
args.push(appKey, appSecret);
|
||||
}
|
||||
|
||||
try {
|
||||
const { stdout, stderr } = await runExec(rcloneBin, args, {
|
||||
timeoutMs: 300_000, // 5 minutes for OAuth flow
|
||||
maxBuffer: 1_000_000,
|
||||
});
|
||||
|
||||
// Extract token from output
|
||||
const combined = stdout + stderr;
|
||||
const tokenMatch = combined.match(/\{[^}]*"access_token"[^}]*\}/);
|
||||
if (tokenMatch) {
|
||||
return { ok: true, token: tokenMatch[0] };
|
||||
}
|
||||
|
||||
// Try to find JSON object in output
|
||||
const jsonMatch = combined.match(
|
||||
/Paste the following into your remote machine[\s\S]*?(\{[\s\S]*?\})\s*$/m,
|
||||
);
|
||||
if (jsonMatch) {
|
||||
return { ok: true, token: jsonMatch[1] };
|
||||
}
|
||||
|
||||
return { ok: false, error: "Could not extract token from rclone output" };
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
return { ok: false, error: `Authorization failed: ${message}` };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run bidirectional sync using rclone bisync.
|
||||
*/
|
||||
export async function runBisync(params: {
|
||||
configPath: string;
|
||||
remoteName: string;
|
||||
remotePath: string;
|
||||
localPath: string;
|
||||
conflictResolve: "newer" | "local" | "remote";
|
||||
exclude: string[];
|
||||
copySymlinks?: boolean;
|
||||
resync?: boolean;
|
||||
dryRun?: boolean;
|
||||
verbose?: boolean;
|
||||
}): Promise<RcloneSyncResult> {
|
||||
const rcloneBin = await getRcloneBinary();
|
||||
|
||||
// Ensure local directory exists
|
||||
if (!existsSync(params.localPath)) {
|
||||
mkdirSync(params.localPath, { recursive: true });
|
||||
}
|
||||
|
||||
const args = [
|
||||
"bisync",
|
||||
`${params.remoteName}:${params.remotePath}`,
|
||||
params.localPath,
|
||||
"--config",
|
||||
params.configPath,
|
||||
"--conflict-resolve",
|
||||
params.conflictResolve,
|
||||
"--conflict-suffix",
|
||||
".conflict",
|
||||
];
|
||||
|
||||
// Add excludes
|
||||
for (const pattern of params.exclude) {
|
||||
args.push("--exclude", pattern);
|
||||
}
|
||||
|
||||
// Follow symlinks if configured
|
||||
if (params.copySymlinks) {
|
||||
args.push("--copy-links");
|
||||
}
|
||||
|
||||
if (params.resync) {
|
||||
args.push("--resync");
|
||||
}
|
||||
|
||||
if (params.dryRun) {
|
||||
args.push("--dry-run");
|
||||
}
|
||||
|
||||
if (params.verbose) {
|
||||
args.push("--verbose");
|
||||
} else {
|
||||
// Suppress NOTICE messages (e.g., symlink warnings) unless verbose
|
||||
args.push("--log-level", "WARNING");
|
||||
}
|
||||
|
||||
logVerbose(`Running: ${rcloneBin} ${args.join(" ")}`);
|
||||
|
||||
try {
|
||||
const { stdout, stderr } = await runExec(rcloneBin, args, {
|
||||
timeoutMs: 600_000, // 10 minutes
|
||||
maxBuffer: 10_000_000,
|
||||
});
|
||||
|
||||
// Parse output for stats
|
||||
const combined = stdout + stderr;
|
||||
const transferredMatch = combined.match(/Transferred:\s*(\d+)\s*\/\s*(\d+)/);
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
filesTransferred: transferredMatch ? parseInt(transferredMatch[1], 10) : undefined,
|
||||
};
|
||||
} catch (err) {
|
||||
const errObj = err as { stdout?: string; stderr?: string; message?: string };
|
||||
const message =
|
||||
errObj.stderr?.trim() || errObj.stdout?.trim() || errObj.message || "Unknown sync error";
|
||||
|
||||
// Check for common errors
|
||||
if (message.includes("bisync requires --resync")) {
|
||||
return {
|
||||
ok: false,
|
||||
error: "First sync requires --resync flag to establish baseline",
|
||||
};
|
||||
}
|
||||
|
||||
return { ok: false, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one-way sync (copy) from remote to local or vice versa.
|
||||
*/
|
||||
export async function runSync(params: {
|
||||
configPath: string;
|
||||
remoteName: string;
|
||||
remotePath: string;
|
||||
localPath: string;
|
||||
direction: "pull" | "push";
|
||||
exclude: string[];
|
||||
dryRun?: boolean;
|
||||
verbose?: boolean;
|
||||
}): Promise<RcloneSyncResult> {
|
||||
const rcloneBin = await getRcloneBinary();
|
||||
|
||||
// Ensure local directory exists
|
||||
if (!existsSync(params.localPath)) {
|
||||
mkdirSync(params.localPath, { recursive: true });
|
||||
}
|
||||
|
||||
const remote = `${params.remoteName}:${params.remotePath}`;
|
||||
const [source, dest] =
|
||||
params.direction === "pull" ? [remote, params.localPath] : [params.localPath, remote];
|
||||
|
||||
const args = ["sync", source, dest, "--config", params.configPath];
|
||||
|
||||
for (const pattern of params.exclude) {
|
||||
args.push("--exclude", pattern);
|
||||
}
|
||||
|
||||
if (params.dryRun) {
|
||||
args.push("--dry-run");
|
||||
}
|
||||
|
||||
if (params.verbose) {
|
||||
args.push("--verbose");
|
||||
} else {
|
||||
// Suppress NOTICE messages (e.g., symlink warnings) unless verbose
|
||||
args.push("--log-level", "WARNING");
|
||||
}
|
||||
|
||||
logVerbose(`Running: ${rcloneBin} ${args.join(" ")}`);
|
||||
|
||||
try {
|
||||
await runExec(rcloneBin, args, {
|
||||
timeoutMs: 600_000,
|
||||
maxBuffer: 10_000_000,
|
||||
});
|
||||
return { ok: true };
|
||||
} catch (err) {
|
||||
const errObj = err as { stderr?: string; message?: string };
|
||||
const message = errObj.stderr?.trim() || errObj.message || "Unknown sync error";
|
||||
return { ok: false, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List files in remote.
|
||||
*/
|
||||
export async function listRemote(params: {
|
||||
configPath: string;
|
||||
remoteName: string;
|
||||
remotePath: string;
|
||||
}): Promise<{ ok: true; files: string[] } | { ok: false; error: string }> {
|
||||
const rcloneBin = await getRcloneBinary();
|
||||
|
||||
try {
|
||||
const { stdout } = await runExec(
|
||||
rcloneBin,
|
||||
["lsf", `${params.remoteName}:${params.remotePath}`, "--config", params.configPath],
|
||||
{ timeoutMs: 30_000, maxBuffer: 1_000_000 },
|
||||
);
|
||||
|
||||
const files = stdout
|
||||
.trim()
|
||||
.split("\n")
|
||||
.filter((f) => f.length > 0);
|
||||
return { ok: true, files };
|
||||
} catch (err) {
|
||||
const errObj = err as { stderr?: string; message?: string };
|
||||
const message = errObj.stderr?.trim() || errObj.message || "Unknown error";
|
||||
return { ok: false, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check remote connection.
|
||||
*/
|
||||
export async function checkRemote(params: {
|
||||
configPath: string;
|
||||
remoteName: string;
|
||||
}): Promise<{ ok: true } | { ok: false; error: string }> {
|
||||
const rcloneBin = await getRcloneBinary();
|
||||
|
||||
try {
|
||||
await runExec(
|
||||
rcloneBin,
|
||||
["about", `${params.remoteName}:`, "--config", params.configPath, "--json"],
|
||||
{ timeoutMs: 30_000, maxBuffer: 100_000 },
|
||||
);
|
||||
return { ok: true };
|
||||
} catch (err) {
|
||||
const errObj = err as { stderr?: string; message?: string };
|
||||
const message = errObj.stderr?.trim() || errObj.message || "Connection failed";
|
||||
return { ok: false, error: message };
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user