From 7e0cd651561b808d3b0779543554566287c2f57f Mon Sep 17 00:00:00 2001 From: Ash Brener Date: Wed, 28 Jan 2026 10:22:17 +0200 Subject: [PATCH] feat: add workspace cloud sync via rclone Add bidirectional workspace synchronization with cloud storage providers (Dropbox, Google Drive, OneDrive, S3/R2) using rclone. Features: - Config-driven setup via workspace.sync in moltbot.json - Native CLI: moltbot workspace setup/sync/status/authorize/list - Session hooks: onSessionStart/onSessionEnd triggers (no LLM cost) - Background interval sync via gateway (no LLM cost) - Auto-install rclone when missing (Homebrew/official script) - Dropbox app folder support for ringfenced security - Secure defaults: syncs shared/ subdirectory only New files: - src/infra/rclone.ts - rclone wrapper - src/cli/workspace-cli.ts - CLI commands - src/hooks/bundled/workspace-sync/ - session hooks - src/gateway/workspace-sync-manager.ts - background sync - src/config/types.workspace.ts - config types - docs/gateway/workspace-sync.md - full documentation Tests: 23 unit tests for rclone helpers and hook handler --- docs/concepts/agent-workspace.md | 15 + docs/docs.json | 3 +- docs/gateway/workspace-sync.md | 400 ++++++++++ scripts/workspace-sync.sh | 123 +++ src/cli/program/command-registry.ts | 5 + src/cli/workspace-cli.ts | 701 ++++++++++++++++++ src/config/types.clawdbot.ts | 2 + src/config/types.workspace.ts | 119 +++ src/config/zod-schema.ts | 49 ++ src/gateway/server-close.ts | 2 + src/gateway/server-startup.ts | 12 + src/gateway/workspace-sync-manager.ts | 190 +++++ src/hooks/bundled/workspace-sync/HOOK.md | 64 ++ .../bundled/workspace-sync/handler.test.ts | 327 ++++++++ src/hooks/bundled/workspace-sync/handler.ts | 118 +++ src/infra/rclone.test.ts | 157 ++++ src/infra/rclone.ts | 516 +++++++++++++ 17 files changed, 2802 insertions(+), 1 deletion(-) create mode 100644 docs/gateway/workspace-sync.md create mode 100755 scripts/workspace-sync.sh create mode 100644 src/cli/workspace-cli.ts create mode 100644 src/config/types.workspace.ts create mode 100644 src/gateway/workspace-sync-manager.ts create mode 100644 src/hooks/bundled/workspace-sync/HOOK.md create mode 100644 src/hooks/bundled/workspace-sync/handler.test.ts create mode 100644 src/hooks/bundled/workspace-sync/handler.ts create mode 100644 src/infra/rclone.test.ts create mode 100644 src/infra/rclone.ts diff --git a/docs/concepts/agent-workspace.md b/docs/concepts/agent-workspace.md index ace2ad4eb..333ff43a0 100644 --- a/docs/concepts/agent-workspace.md +++ b/docs/concepts/agent-workspace.md @@ -223,6 +223,21 @@ Suggested `.gitignore` starter: 4. If you need sessions, copy `~/.clawdbot/agents//sessions/` from the old machine separately. +## Cloud sync (remote Gateways) + +If your Gateway runs on a remote server (Fly.io, Hetzner, VPS), you can sync a +workspace subfolder with your local machine using cloud storage (Dropbox, Google +Drive, OneDrive, S3, etc.): + +``` +Local Machine Cloud Provider Remote Gateway +~/Dropbox/clawd/ ←→ Dropbox/GDrive/etc ←→ /shared/ +``` + +This lets you drop files locally and have them appear on the remote Gateway. + +See [Workspace cloud sync](/gateway/workspace-sync) for setup instructions. + ## Advanced notes - Multi-agent routing can use different workspaces per agent. See diff --git a/docs/docs.json b/docs/docs.json index a463479aa..93a157a48 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -965,7 +965,8 @@ "gateway/remote-gateway-readme", "gateway/discovery", "gateway/bonjour", - "gateway/tailscale" + "gateway/tailscale", + "gateway/workspace-sync" ] }, { diff --git a/docs/gateway/workspace-sync.md b/docs/gateway/workspace-sync.md new file mode 100644 index 000000000..98ce24cbb --- /dev/null +++ b/docs/gateway/workspace-sync.md @@ -0,0 +1,400 @@ +--- +summary: "Sync your agent workspace with cloud storage (Dropbox, Google Drive, OneDrive, S3)" +read_when: + - Setting up workspace sync on a remote/cloud Gateway + - Sharing files between local machine and remote Moltbot +--- + +# Workspace Cloud Sync + +Sync your agent workspace between a remote Gateway (Fly.io, Hetzner, VPS) and your local machine using cloud storage. + +## How it works + +``` +Local Machine Cloud Provider Remote Gateway +~/Dropbox/moltbot/ ←→ Dropbox/GDrive/etc ←→ /shared/ + (native app) (any provider) (rclone bisync) +``` + +- **Local**: Native cloud app syncs `~/Dropbox/moltbot/` (or equivalent) +- **Remote**: rclone bisync keeps `/shared/` in sync with the cloud +- **Result**: Drop a file locally, it appears on the remote Gateway (and vice versa) + +## Quick start + +```bash +# Interactive setup wizard (recommended) +moltbot workspace setup +``` + +The setup wizard guides you through: +1. ✅ Checking rclone installation +2. 📦 Selecting cloud provider (Dropbox, Google Drive, OneDrive, S3) +3. 🔐 Dropbox app folder option (for scoped access) +4. ⏱️ Background sync interval +5. 🔑 OAuth authorization +6. 🔄 First sync + +Or configure manually: + +**1) Add minimal config:** + +```json5 +{ + workspace: { + sync: { + provider: "dropbox", + remotePath: "moltbot-share" + } + } +} +``` + +**2) Authorize and sync:** + +```bash +moltbot workspace authorize # opens browser for OAuth +moltbot workspace sync --resync # first sync (establishes baseline) +``` + +## Configuration + +Add to `~/.clawdbot/moltbot.json`: + +```json5 +{ + workspace: { + sync: { + provider: "dropbox", // dropbox | gdrive | onedrive | s3 | custom + remotePath: "moltbot-share", // folder in cloud storage + localPath: "shared", // subfolder in workspace (default: shared) + interval: 300, // background sync every 5 minutes (0 = disabled) + onSessionStart: true, // sync when session starts + onSessionEnd: false, // sync when session ends + conflictResolve: "newer", // newer | local | remote + exclude: [".git/**", "node_modules/**", "*.log"] + } + } +} +``` + + +**Zero LLM cost.** The `interval` setting runs pure rclone in the background. +It does NOT wake the bot or trigger any LLM calls - it's just file synchronization. + + +### Provider-specific options + +**Dropbox with app folder (recommended):** + +```json5 +{ + workspace: { + sync: { + provider: "dropbox", + remotePath: "", // empty = app folder root + dropbox: { + appFolder: true, + appKey: "your-app-key", + appSecret: "your-app-secret" + } + } + } +} +``` + +**S3/R2/Minio:** + +```json5 +{ + workspace: { + sync: { + provider: "s3", + remotePath: "moltbot-sync", // path within bucket + s3: { + // Cloudflare R2: https://.r2.cloudflarestorage.com + // AWS S3: https://s3..amazonaws.com + // Minio: https://your-minio-host:9000 + endpoint: "https://abc123def456.r2.cloudflarestorage.com", + bucket: "your-bucket", + region: "auto" // "auto" for R2, or specific region for AWS + // accessKeyId and secretAccessKey via env vars recommended + } + } + } +} +``` + +## CLI commands + +```bash +# Interactive setup wizard +moltbot workspace setup + +# Check sync status +moltbot workspace status + +# Sync bidirectionally +moltbot workspace sync + +# First sync (required to establish baseline) +moltbot workspace sync --resync + +# Preview changes without syncing +moltbot workspace sync --dry-run + +# One-way sync +moltbot workspace sync --direction pull # remote → local +moltbot workspace sync --direction push # local → remote + +# Authorize with cloud provider (use 'setup' for guided flow) +moltbot workspace authorize +moltbot workspace authorize --provider gdrive + +# List remote files +moltbot workspace list +``` + +## Auto-sync hooks + +Enable automatic sync on session start/end. These hooks run during existing agent activity, +so they **don't wake the bot** or incur extra LLM costs: + +```json5 +{ + workspace: { + sync: { + provider: "dropbox", + remotePath: "moltbot-share", + onSessionStart: true, // sync when session starts (no LLM cost) + onSessionEnd: false // sync when session ends (no LLM cost) + } + }, + hooks: { + internal: { + entries: { + "workspace-sync": { enabled: true } + } + } + } +} +``` + +## Periodic background sync + +Set `interval` to enable automatic background sync (in seconds): + +```json5 +{ + workspace: { + sync: { + provider: "dropbox", + remotePath: "moltbot-share", + interval: 300 // sync every 5 minutes (minimum: 60s) + } + } +} +``` + +The gateway runs rclone bisync in the background at this interval. +This is a **pure file operation** - it does NOT wake the bot or incur any LLM costs. + +### Alternative: External cron + +If you prefer external scheduling (e.g., for more control or logging): + +```bash +# Add to crontab (crontab -e) +*/5 * * * * moltbot workspace sync >> /var/log/moltbot-sync.log 2>&1 +``` + +## Supported providers + +| Provider | Config value | Auth method | +|----------|--------------|-------------| +| Dropbox | `dropbox` | OAuth token | +| Google Drive | `gdrive` | OAuth token | +| OneDrive | `onedrive` | OAuth token | +| S3/R2/Minio | `s3` | Access keys | +| Custom rclone | `custom` | Varies | + +For the full list of 70+ providers, see [rclone overview](https://rclone.org/overview/). + +## Manual setup (without wizard) + +If you prefer manual configuration: + +### 1. Install rclone + +rclone is **auto-installed** when you run `moltbot workspace setup`. + +For manual installation: +- **macOS**: `brew install rclone` +- **Linux**: `curl -s https://rclone.org/install.sh | sudo bash` +- **Docker**: Add to Dockerfile: `RUN curl -s https://rclone.org/install.sh | bash` + +### 2. Authorize rclone (from your local machine) + +Run this on your **local machine** (where you have a browser): + +```bash +# Install rclone locally if needed +brew install rclone # or: curl -s https://rclone.org/install.sh | bash + +# Authorize with your cloud provider +rclone authorize "dropbox" # or: gdrive, onedrive, s3, etc. +``` + +Copy the JSON token it outputs. + +### 3. Configure rclone on the Gateway + +SSH into your Gateway and create the config: + +```bash +mkdir -p /data/workspace/.config/rclone + +cat > /data/workspace/.config/rclone/rclone.conf << 'EOF' +[cloud] +type = dropbox +token = {"access_token":"YOUR_TOKEN_HERE","token_type":"bearer","expiry":"..."} +EOF +``` + +For other providers, see [rclone config docs](https://rclone.org/docs/). + +### 4. Create the sync folder + +**On your local machine:** + +Create the folder your cloud app syncs (e.g., `~/Dropbox/moltbot-share/`). + +**On the Gateway:** + +```bash +mkdir -p /data/workspace/shared +``` + +### 5. Run the first sync + +```bash +# First sync needs --resync to establish baseline +rclone bisync cloud:moltbot-share /data/workspace/shared --resync + +# Subsequent syncs +rclone bisync cloud:moltbot-share /data/workspace/shared +``` + +## Troubleshooting + +### Token expired + +Re-authorize on your local machine and update the config: + +```bash +moltbot workspace authorize +# Or manually: +rclone authorize "dropbox" +# Copy new token to Gateway's rclone.conf +``` + +### Conflicts + +Files modified on both sides get `.conflict` suffix. Check and resolve manually: + +```bash +find /data/workspace/shared -name "*.conflict" +``` + +### First sync fails + +Use `--resync` flag to establish baseline: + +```bash +moltbot workspace sync --resync +``` + +### Permission errors + +Ensure the workspace directory is writable: + +```bash +chmod -R 755 /data/workspace/shared +``` + +## Security notes + +- **Token storage**: rclone tokens are stored in `rclone.conf`. Keep this file secure. +- **Sensitive files**: Don't sync secrets, API keys, or credentials. +- **Encryption**: Consider using rclone's [crypt](https://rclone.org/crypt/) for sensitive data. + +### Dropbox: App folder access (recommended) + +By default, `rclone authorize "dropbox"` requests **full Dropbox access**. For better security, create an app-scoped token that only accesses a single folder: + +**1. Create a Dropbox App** + +1. Go to [Dropbox App Console](https://www.dropbox.com/developers/apps) +2. Click **Create app** +3. Choose: + - **Scoped access** (not "Dropbox Business API") + - **App folder** — only access to `Apps//` +4. Name it (e.g., `moltbot-sync`) +5. Click **Create app** + +**2. Configure permissions** + +In your app's **Permissions** tab, enable: +- `files.metadata.read` +- `files.metadata.write` +- `files.content.read` +- `files.content.write` + +Click **Submit** to save. + +**3. Generate token** + +In the **Settings** tab: +- Note your **App key** and **App secret** +- Under **OAuth 2**, click **Generate** to create an access token + +**4. Configure in moltbot.json** + +```json5 +{ + workspace: { + sync: { + provider: "dropbox", + remotePath: "", // empty = app folder root + dropbox: { + appFolder: true, + appKey: "your-app-key", + appSecret: "your-app-secret" + } + } + } +} +``` + +Then authorize: + +```bash +moltbot workspace authorize +``` + +**Benefits of app folder access:** +- 🔒 Token only accesses one folder, not your entire Dropbox +- 🛡️ If token is compromised, blast radius is limited +- 📁 Clean separation — sync folder lives under `Apps/` + +### Google Drive: Limited folder access + +Similar scoping is possible with Google Drive using a service account + shared folder, but setup is more involved. See [rclone Google Drive docs](https://rclone.org/drive/). + +## See also + +- [Agent workspace](/concepts/agent-workspace) — workspace layout and backup +- [Fly.io deployment](/platforms/fly) — Docker-based cloud deployment +- [Hetzner deployment](/platforms/hetzner) — VPS deployment +- [rclone docs](https://rclone.org/docs/) — full rclone documentation diff --git a/scripts/workspace-sync.sh b/scripts/workspace-sync.sh new file mode 100755 index 000000000..ada7861fe --- /dev/null +++ b/scripts/workspace-sync.sh @@ -0,0 +1,123 @@ +#!/bin/bash +# Workspace cloud sync using rclone bisync +# Docs: https://docs.molt.bot/gateway/workspace-sync + +set -e + +# Configuration (override via environment) +RCLONE_CONFIG="${RCLONE_CONFIG:-$HOME/.config/rclone/rclone.conf}" +RCLONE_REMOTE="${RCLONE_REMOTE:-cloud}" +REMOTE_PATH="${REMOTE_PATH:-moltbot-share}" +LOCAL_PATH="${LOCAL_PATH:-${MOLTBOT_STATE_DIR:-${CLAWDBOT_STATE_DIR:-$HOME/.clawdbot}}/workspace/shared}" + +# Flags +RESYNC="${RESYNC:-false}" +VERBOSE="${VERBOSE:-false}" +DRY_RUN="${DRY_RUN:-false}" + +usage() { + cat << 'EOF' +Usage: workspace-sync.sh [OPTIONS] + +Bidirectional sync between cloud storage and workspace. + +Options: + --resync Force resync (required on first run) + --dry-run Show what would be synced without doing it + --verbose Show detailed output + --help Show this help + +Environment: + RCLONE_CONFIG Path to rclone.conf (default: ~/.config/rclone/rclone.conf) + RCLONE_REMOTE rclone remote name (default: cloud) + REMOTE_PATH Folder in cloud storage (default: moltbot-share) + LOCAL_PATH Local folder to sync (default: /workspace/shared) + +Examples: + # First sync (establishes baseline) + ./scripts/workspace-sync.sh --resync + + # Regular sync + ./scripts/workspace-sync.sh + + # Preview changes + ./scripts/workspace-sync.sh --dry-run --verbose + +Setup: https://docs.molt.bot/gateway/workspace-sync +EOF + exit 0 +} + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --resync) + RESYNC=true + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + --verbose|-v) + VERBOSE=true + shift + ;; + --help|-h) + usage + ;; + *) + echo "Unknown option: $1" + usage + ;; + esac +done + +# Check rclone is installed +if ! command -v rclone &> /dev/null; then + echo "Error: rclone not installed" + echo "Install: curl -s https://rclone.org/install.sh | bash" + exit 1 +fi + +# Check config exists +if [[ ! -f "$RCLONE_CONFIG" ]]; then + echo "Error: rclone config not found at $RCLONE_CONFIG" + echo "Setup: https://docs.molt.bot/gateway/workspace-sync" + exit 1 +fi + +# Ensure local directory exists +mkdir -p "$LOCAL_PATH" + +# Build rclone command +RCLONE_ARGS=( + bisync + "${RCLONE_REMOTE}:${REMOTE_PATH}" + "$LOCAL_PATH" + --config "$RCLONE_CONFIG" + --conflict-resolve newer + --conflict-suffix .conflict + --exclude ".git/**" + --exclude "node_modules/**" + --exclude "*.log" + --exclude ".DS_Store" +) + +if [[ "$RESYNC" == "true" ]]; then + RCLONE_ARGS+=(--resync) +fi + +if [[ "$DRY_RUN" == "true" ]]; then + RCLONE_ARGS+=(--dry-run) +fi + +if [[ "$VERBOSE" == "true" ]]; then + RCLONE_ARGS+=(--verbose) +fi + +# Run sync +echo "Syncing: ${RCLONE_REMOTE}:${REMOTE_PATH} <-> $LOCAL_PATH" +rclone "${RCLONE_ARGS[@]}" + +echo "Sync complete: $(date)" diff --git a/src/cli/program/command-registry.ts b/src/cli/program/command-registry.ts index 0b4618ef0..10359c2ef 100644 --- a/src/cli/program/command-registry.ts +++ b/src/cli/program/command-registry.ts @@ -9,6 +9,7 @@ import { getFlagValue, getPositiveIntFlagValue, getVerboseFlag, hasFlag } from " import { registerBrowserCli } from "../browser-cli.js"; import { registerConfigCli } from "../config-cli.js"; import { registerMemoryCli, runMemoryStatus } from "../memory-cli.js"; +import { registerWorkspaceCli } from "../workspace-cli.js"; import { registerAgentCommands } from "./register.agent.js"; import { registerConfigureCommand } from "./register.configure.js"; import { registerMaintenanceCommands } from "./register.maintenance.js"; @@ -152,6 +153,10 @@ export const commandRegistry: CommandRegistration[] = [ id: "browser", register: ({ program }) => registerBrowserCli(program), }, + { + id: "workspace", + register: ({ program }) => registerWorkspaceCli(program), + }, ]; export function registerProgramCommands( diff --git a/src/cli/workspace-cli.ts b/src/cli/workspace-cli.ts new file mode 100644 index 000000000..5e0c18bcb --- /dev/null +++ b/src/cli/workspace-cli.ts @@ -0,0 +1,701 @@ +import { homedir } from "node:os"; +import type { Command } from "commander"; +import * as clack from "@clack/prompts"; + +import { loadConfig, writeConfigFile } from "../config/config.js"; +import { resolveDefaultAgentId, resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; +import { resolveStateDir } from "../config/paths.js"; +import { danger, setVerbose } from "../globals.js"; +import { defaultRuntime } from "../runtime.js"; +import { formatCliCommand } from "./command-format.js"; +import { colorize, isRich, theme } from "../terminal/theme.js"; +import { formatDocsLink } from "../terminal/links.js"; +import { shortenHomePath } from "../utils.js"; +import { + isRcloneInstalled, + ensureRcloneInstalled, + isRcloneConfigured, + resolveSyncConfig, + runBisync, + runSync, + checkRemote, + listRemote, + authorizeRclone, + writeRcloneConfig, + generateRcloneConfig, + type RcloneSyncResult, +} from "../infra/rclone.js"; +import type { WorkspaceSyncProvider } from "../config/types.workspace.js"; +import type { MoltbotConfig } from "../config/types.clawdbot.js"; + +type WorkspaceSyncOptions = { + agent?: string; + resync?: boolean; + dryRun?: boolean; + verbose?: boolean; + direction?: "pull" | "push"; +}; + +type WorkspaceStatusOptions = { + agent?: string; + verbose?: boolean; +}; + +type WorkspaceAuthorizeOptions = { + provider?: string; + appKey?: string; + appSecret?: string; +}; + +function resolveAgent(cfg: ReturnType, agent?: string) { + const trimmed = agent?.trim(); + if (trimmed) return trimmed; + return resolveDefaultAgentId(cfg); +} + +export function registerWorkspaceCli(program: Command): void { + const workspace = program + .command("workspace") + .description("Workspace management and cloud sync") + .addHelpText( + "after", + () => + `\n${theme.muted("Docs:")} ${formatDocsLink( + "/gateway/workspace-sync", + "docs.molt.bot/gateway/workspace-sync", + )}\n`, + ) + .action(() => { + workspace.outputHelp(); + defaultRuntime.error( + danger(`Missing subcommand. Try: "${formatCliCommand("moltbot workspace setup")}"`), + ); + defaultRuntime.exit(1); + }); + + // moltbot workspace sync + workspace + .command("sync") + .description("Sync workspace with cloud storage") + .option("--agent ", "Agent ID (default: main)") + .option("--resync", "Force resync (required for first sync)") + .option("--dry-run", "Preview changes without syncing") + .option("--direction ", "One-way sync: pull or push") + .option("-v, --verbose", "Verbose output") + .action(async (opts: WorkspaceSyncOptions) => { + try { + if (opts.verbose) setVerbose(true); + + const cfg = loadConfig(); + const agentId = resolveAgent(cfg, opts.agent); + const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId); + const stateDir = resolveStateDir(process.env, homedir); + const syncConfig = cfg.workspace?.sync; + + const rich = isRich(); + + if (!syncConfig?.provider || syncConfig.provider === "off") { + defaultRuntime.error(colorize(rich, theme.error, "Workspace sync not configured.")); + defaultRuntime.error(""); + defaultRuntime.error(`Run: ${formatCliCommand("moltbot workspace setup")}`); + defaultRuntime.error(`Docs: ${formatDocsLink("/gateway/workspace-sync")}`); + defaultRuntime.exit(1); + return; + } + + // Check rclone + const installed = await isRcloneInstalled(); + if (!installed) { + defaultRuntime.error(colorize(rich, theme.error, "rclone not installed.")); + defaultRuntime.error(""); + defaultRuntime.error(`Run: ${formatCliCommand("moltbot workspace setup")}`); + defaultRuntime.exit(1); + return; + } + + const resolved = resolveSyncConfig(syncConfig, workspaceDir, stateDir); + + // Check config + if (!isRcloneConfigured(resolved.configPath, resolved.remoteName)) { + console.error( + colorize( + rich, + theme.error, + `rclone not configured for remote "${resolved.remoteName}".`, + ), + ); + console.error(""); + console.error("Run: moltbot workspace authorize"); + console.error(`Or manually configure: ${shortenHomePath(resolved.configPath)}`); + defaultRuntime.exit(1); + } + + console.log( + colorize(rich, theme.info, `Syncing ${resolved.remoteName}:${resolved.remotePath}`), + ); + console.log(colorize(rich, theme.muted, `Local: ${shortenHomePath(resolved.localPath)}`)); + + let result: RcloneSyncResult; + + if (opts.direction) { + // One-way sync + result = await runSync({ + configPath: resolved.configPath, + remoteName: resolved.remoteName, + remotePath: resolved.remotePath, + localPath: resolved.localPath, + direction: opts.direction, + exclude: resolved.exclude, + dryRun: opts.dryRun, + verbose: opts.verbose, + }); + } else { + // Bidirectional sync + result = await runBisync({ + configPath: resolved.configPath, + remoteName: resolved.remoteName, + remotePath: resolved.remotePath, + localPath: resolved.localPath, + conflictResolve: resolved.conflictResolve, + exclude: resolved.exclude, + resync: opts.resync, + dryRun: opts.dryRun, + verbose: opts.verbose, + }); + } + + if (result.ok) { + defaultRuntime.log(colorize(rich, theme.success, "✓ Sync completed")); + if (result.filesTransferred) { + defaultRuntime.log( + colorize(rich, theme.muted, `Files transferred: ${result.filesTransferred}`), + ); + } + } else { + defaultRuntime.error(colorize(rich, theme.error, `✗ Sync failed: ${result.error}`)); + if (result.error?.includes("--resync")) { + defaultRuntime.error(""); + defaultRuntime.error( + `First sync requires --resync: ${formatCliCommand("moltbot workspace sync --resync")}`, + ); + } + defaultRuntime.exit(1); + } + } catch (err) { + defaultRuntime.error( + `${theme.error("Error:")} ${err instanceof Error ? err.message : String(err)}`, + ); + defaultRuntime.exit(1); + } + }); + + // moltbot workspace status + workspace + .command("status") + .description("Show workspace sync status") + .option("--agent ", "Agent ID (default: main)") + .option("-v, --verbose", "Verbose output") + .action(async (opts: WorkspaceStatusOptions) => { + if (opts.verbose) setVerbose(true); + + const cfg = loadConfig(); + const agentId = resolveAgent(cfg, opts.agent); + const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId); + const stateDir = resolveStateDir(process.env, homedir); + const syncConfig = cfg.workspace?.sync; + + const rich = isRich(); + + console.log(colorize(rich, theme.heading, "Workspace Sync Status")); + console.log(""); + + // Check config + if (!syncConfig?.provider || syncConfig.provider === "off") { + console.log(colorize(rich, theme.muted, "Provider: not configured")); + console.log(""); + console.log(`Configure in ~/.clawdbot/moltbot.json`); + console.log(`Docs: ${formatDocsLink("/gateway/workspace-sync")}`); + return; + } + + const resolved = resolveSyncConfig(syncConfig, workspaceDir, stateDir); + + console.log(`Provider: ${colorize(rich, theme.info, syncConfig.provider)}`); + console.log(`Remote: ${resolved.remoteName}:${resolved.remotePath}`); + console.log(`Local: ${shortenHomePath(resolved.localPath)}`); + console.log(`Config: ${shortenHomePath(resolved.configPath)}`); + console.log(""); + + // Check rclone + const installed = await isRcloneInstalled(); + if (!installed) { + console.log(colorize(rich, theme.error, "✗ rclone not installed")); + return; + } + console.log(colorize(rich, theme.success, "✓ rclone installed")); + + // Check config + const configured = isRcloneConfigured(resolved.configPath, resolved.remoteName); + if (!configured) { + console.log(colorize(rich, theme.error, "✗ rclone not configured")); + console.log(""); + console.log("Run: moltbot workspace authorize"); + return; + } + console.log(colorize(rich, theme.success, "✓ rclone configured")); + + // Check connection + const check = await checkRemote({ + configPath: resolved.configPath, + remoteName: resolved.remoteName, + }); + if (!check.ok) { + console.log(colorize(rich, theme.error, `✗ Connection failed: ${check.error}`)); + return; + } + console.log(colorize(rich, theme.success, "✓ Remote connected")); + + // List remote files + const list = await listRemote({ + configPath: resolved.configPath, + remoteName: resolved.remoteName, + remotePath: resolved.remotePath, + }); + if (list.ok) { + console.log(""); + console.log(`Remote files: ${list.files.length}`); + if (opts.verbose && list.files.length > 0) { + for (const file of list.files.slice(0, 10)) { + console.log(colorize(rich, theme.muted, ` ${file}`)); + } + if (list.files.length > 10) { + console.log(colorize(rich, theme.muted, ` ... and ${list.files.length - 10} more`)); + } + } + } + + // Show auto-sync settings + console.log(""); + console.log("Auto-sync:"); + console.log(` On session start: ${resolved.onSessionStart ? "yes" : "no"}`); + console.log(` On session end: ${resolved.onSessionEnd ? "yes" : "no"}`); + if (resolved.interval > 0) { + console.log(` Background interval: ${resolved.interval}s (pure rclone, zero LLM cost)`); + } else { + console.log(` Background interval: disabled`); + } + }); + + // moltbot workspace setup - interactive wizard + workspace + .command("setup") + .description("Interactive setup wizard for cloud sync") + .action(async () => { + const rich = isRich(); + + clack.intro(colorize(rich, theme.heading, "Workspace Cloud Sync Setup")); + + // Step 1: Check/install rclone + const rcloneInstalled = await isRcloneInstalled(); + if (!rcloneInstalled) { + // Offer to install + const installed = await ensureRcloneInstalled(async (message, defaultValue) => { + const result = await clack.confirm({ message, initialValue: defaultValue }); + return !clack.isCancel(result) && result; + }); + if (!installed) { + clack.note( + "Install rclone manually:\n\n" + + " macOS: brew install rclone\n" + + " Linux: curl -s https://rclone.org/install.sh | sudo bash\n" + + " Docker: Add to Dockerfile: RUN curl -s https://rclone.org/install.sh | bash", + "Installation required", + ); + clack.outro("Install rclone and run this command again."); + defaultRuntime.exit(1); + return; + } + } + clack.log.success("rclone is installed"); + + // Step 2: Select provider + const provider = (await clack.select({ + message: "Select cloud provider", + options: [ + { value: "dropbox", label: "Dropbox", hint: "Recommended - easy setup" }, + { + value: "gdrive", + label: "Google Drive", + hint: "Requires service account for scoped access", + }, + { value: "onedrive", label: "OneDrive", hint: "Microsoft 365" }, + { value: "s3", label: "S3 / R2 / Minio", hint: "Access key authentication" }, + ], + })) as WorkspaceSyncProvider; + + if (clack.isCancel(provider)) { + clack.cancel("Setup cancelled."); + defaultRuntime.exit(0); + return; + } + + // Step 3: Remote folder name + const remotePath = (await clack.text({ + message: "Remote folder name", + placeholder: "moltbot-share", + initialValue: "moltbot-share", + validate: (value) => { + if (!value.trim()) return "Folder name is required"; + if (value.includes("/")) return "Use a simple folder name, not a path"; + return undefined; + }, + })) as string; + + if (clack.isCancel(remotePath)) { + clack.cancel("Setup cancelled."); + defaultRuntime.exit(0); + return; + } + + // Step 4: Dropbox-specific: app folder option + let useAppFolder = false; + let appKey: string | undefined; + let appSecret: string | undefined; + + if (provider === "dropbox") { + const accessType = (await clack.select({ + message: "Dropbox access type", + options: [ + { + value: "full", + label: "Full Dropbox", + hint: "Access entire Dropbox (simpler setup)", + }, + { + value: "app", + label: "App Folder only", + hint: "Restricted to Apps// (more secure)", + }, + ], + })) as "full" | "app"; + + if (clack.isCancel(accessType)) { + clack.cancel("Setup cancelled."); + defaultRuntime.exit(0); + return; + } + + useAppFolder = accessType === "app"; + + if (useAppFolder) { + clack.note( + "1. Go to https://www.dropbox.com/developers/apps\n" + + "2. Click 'Create app'\n" + + "3. Choose 'Scoped access' → 'App folder'\n" + + "4. Name it (e.g., 'moltbot-sync')\n" + + "5. In Permissions tab, enable:\n" + + " - files.metadata.read/write\n" + + " - files.content.read/write\n" + + "6. Copy the App key and App secret from Settings", + "Create Dropbox App", + ); + + appKey = (await clack.text({ + message: "Dropbox App key", + placeholder: "your-app-key", + })) as string; + + if (clack.isCancel(appKey)) { + clack.cancel("Setup cancelled."); + defaultRuntime.exit(0); + return; + } + + appSecret = (await clack.text({ + message: "Dropbox App secret", + placeholder: "your-app-secret", + })) as string; + + if (clack.isCancel(appSecret)) { + clack.cancel("Setup cancelled."); + defaultRuntime.exit(0); + return; + } + } + } + + // Step 5: Background sync interval + const intervalChoice = (await clack.select({ + message: "Background sync interval", + options: [ + { value: "0", label: "Manual only", hint: "Run 'moltbot workspace sync' when needed" }, + { value: "300", label: "Every 5 minutes", hint: "Recommended" }, + { value: "600", label: "Every 10 minutes" }, + { value: "1800", label: "Every 30 minutes" }, + { value: "3600", label: "Every hour" }, + ], + })) as string; + + if (clack.isCancel(intervalChoice)) { + clack.cancel("Setup cancelled."); + defaultRuntime.exit(0); + return; + } + + const interval = parseInt(intervalChoice, 10); + + // Step 6: Session hooks + const onSessionStart = (await clack.confirm({ + message: "Sync when session starts?", + initialValue: true, + })) as boolean; + + if (clack.isCancel(onSessionStart)) { + clack.cancel("Setup cancelled."); + defaultRuntime.exit(0); + return; + } + + // Step 7: Save config + const spinner = clack.spinner(); + spinner.start("Saving configuration..."); + + try { + const cfg = loadConfig(); + const newConfig: MoltbotConfig = { + ...cfg, + workspace: { + ...cfg.workspace, + sync: { + provider, + remotePath: remotePath.trim(), + localPath: "shared", + interval, + onSessionStart, + onSessionEnd: false, + ...(useAppFolder && appKey && appSecret + ? { + dropbox: { + appFolder: true, + appKey: appKey.trim(), + appSecret: appSecret.trim(), + }, + } + : {}), + }, + }, + }; + + await writeConfigFile(newConfig); + spinner.stop("Configuration saved"); + } catch (err) { + spinner.stop("Failed to save configuration"); + clack.log.error(`Error: ${err instanceof Error ? err.message : String(err)}`); + defaultRuntime.exit(1); + return; + } + + // Step 8: OAuth authorization + clack.log.info("Starting OAuth authorization..."); + clack.note( + "A browser window will open.\n" + "Log in and authorize access, then return here.", + "Authorization", + ); + + const authResult = await authorizeRclone(provider, appKey, appSecret); + + if (!authResult.ok) { + clack.log.error(`Authorization failed: ${authResult.error}`); + clack.outro("Fix the error and run 'moltbot workspace setup' again."); + defaultRuntime.exit(1); + return; + } + + clack.log.success("Authorization successful"); + + // Step 9: Save rclone config + const stateDir = resolveStateDir(process.env, homedir); + const configPath = `${stateDir}/.config/rclone/rclone.conf`; + const remoteName = "cloud"; + + const configContent = generateRcloneConfig(provider, remoteName, authResult.token, { + dropbox: useAppFolder ? { appKey, appSecret } : undefined, + }); + + writeRcloneConfig(configPath, configContent); + clack.log.success(`rclone config saved to ${shortenHomePath(configPath)}`); + + // Step 10: Create local folder info + clack.note( + `Create this folder on your local machine:\n\n` + + ` ~/Dropbox/${remotePath}/\n\n` + + `Or wherever your cloud app syncs files.`, + "Local folder", + ); + + // Step 11: First sync + const runFirstSync = (await clack.confirm({ + message: "Run first sync now? (--resync)", + initialValue: true, + })) as boolean; + + if (runFirstSync && !clack.isCancel(runFirstSync)) { + const syncSpinner = clack.spinner(); + syncSpinner.start("Running first sync..."); + + const cfg = loadConfig(); + const agentId = resolveDefaultAgentId(cfg); + const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId); + const resolved = resolveSyncConfig(cfg.workspace?.sync, workspaceDir, stateDir); + + const syncResult = await runBisync({ + configPath: resolved.configPath, + remoteName: resolved.remoteName, + remotePath: resolved.remotePath, + localPath: resolved.localPath, + conflictResolve: resolved.conflictResolve, + exclude: resolved.exclude, + resync: true, + }); + + if (syncResult.ok) { + syncSpinner.stop("First sync completed"); + } else { + syncSpinner.stop("First sync failed"); + clack.log.warn(`Error: ${syncResult.error}`); + clack.log.info("You can retry with: moltbot workspace sync --resync"); + } + } + + // Done! + clack.outro(colorize(rich, theme.success, "✓ Workspace sync configured!")); + + console.log(""); + console.log("Commands:"); + console.log(" moltbot workspace sync Sync now"); + console.log(" moltbot workspace status Check status"); + console.log(" moltbot workspace list List remote files"); + console.log(""); + console.log(`Docs: ${formatDocsLink("/gateway/workspace-sync")}`); + }); + + // moltbot workspace authorize + workspace + .command("authorize") + .description("Authorize rclone with cloud provider (use 'setup' for guided flow)") + .option("--provider ", "Provider: dropbox, gdrive, onedrive, s3") + .option("--app-key ", "Dropbox app key (for app folder access)") + .option("--app-secret ", "Dropbox app secret (for app folder access)") + .action(async (opts: WorkspaceAuthorizeOptions) => { + const cfg = loadConfig(); + const syncConfig = cfg.workspace?.sync; + const rich = isRich(); + + // Determine provider + let provider: WorkspaceSyncProvider = + (opts.provider as WorkspaceSyncProvider) || syncConfig?.provider || "dropbox"; + + if (provider === "off" || provider === "custom") { + console.error(colorize(rich, theme.error, "Please specify a provider: --provider dropbox")); + defaultRuntime.exit(1); + } + + // Check rclone + const installed = await isRcloneInstalled(); + if (!installed) { + console.error(colorize(rich, theme.error, "rclone not installed.")); + console.error(""); + console.error("Install: curl -s https://rclone.org/install.sh | bash"); + defaultRuntime.exit(1); + } + + console.log(colorize(rich, theme.info, `Authorizing with ${provider}...`)); + console.log(""); + console.log("A browser window will open for authentication."); + console.log("Complete the OAuth flow and return here."); + console.log(""); + + const result = await authorizeRclone( + provider, + opts.appKey || syncConfig?.dropbox?.appKey, + opts.appSecret || syncConfig?.dropbox?.appSecret, + ); + + if (!result.ok) { + console.error(colorize(rich, theme.error, `Authorization failed: ${result.error}`)); + defaultRuntime.exit(1); + } + + console.log(colorize(rich, theme.success, "✓ Authorization successful")); + console.log(""); + + // Generate and save config + const stateDir = resolveStateDir(process.env, homedir); + const remoteName = syncConfig?.remoteName || "cloud"; + const configPath = syncConfig?.configPath || `${stateDir}/.config/rclone/rclone.conf`; + + const configContent = generateRcloneConfig(provider, remoteName, result.token, { + dropbox: syncConfig?.dropbox, + s3: syncConfig?.s3, + }); + + writeRcloneConfig(configPath, configContent); + + console.log(`Config saved to: ${shortenHomePath(configPath)}`); + console.log(""); + console.log("Next steps:"); + console.log(" 1. Create the remote folder (e.g., ~/Dropbox/moltbot-share/)"); + console.log(" 2. Run first sync: moltbot workspace sync --resync"); + }); + + // moltbot workspace list + workspace + .command("list") + .description("List files in remote storage") + .option("--agent ", "Agent ID (default: main)") + .action(async (opts: { agent?: string }) => { + const cfg = loadConfig(); + const agentId = resolveAgent(cfg, opts.agent); + const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId); + const stateDir = resolveStateDir(process.env, homedir); + const syncConfig = cfg.workspace?.sync; + + const rich = isRich(); + + if (!syncConfig?.provider || syncConfig.provider === "off") { + console.error(colorize(rich, theme.error, "Workspace sync not configured.")); + defaultRuntime.exit(1); + } + + const resolved = resolveSyncConfig(syncConfig, workspaceDir, stateDir); + + if (!isRcloneConfigured(resolved.configPath, resolved.remoteName)) { + console.error(colorize(rich, theme.error, "rclone not configured.")); + console.error("Run: moltbot workspace authorize"); + defaultRuntime.exit(1); + } + + const result = await listRemote({ + configPath: resolved.configPath, + remoteName: resolved.remoteName, + remotePath: resolved.remotePath, + }); + + if (!result.ok) { + console.error(colorize(rich, theme.error, `Failed to list: ${result.error}`)); + defaultRuntime.exit(1); + } + + if (result.files.length === 0) { + console.log(colorize(rich, theme.muted, "No files in remote.")); + return; + } + + console.log(`${resolved.remoteName}:${resolved.remotePath}/`); + for (const file of result.files) { + console.log(` ${file}`); + } + console.log(""); + console.log(colorize(rich, theme.muted, `${result.files.length} files`)); + }); +} diff --git a/src/config/types.clawdbot.ts b/src/config/types.clawdbot.ts index c2b2f05e4..d78902c1d 100644 --- a/src/config/types.clawdbot.ts +++ b/src/config/types.clawdbot.ts @@ -23,6 +23,7 @@ import type { NodeHostConfig } from "./types.node-host.js"; import type { PluginsConfig } from "./types.plugins.js"; import type { SkillsConfig } from "./types.skills.js"; import type { ToolsConfig } from "./types.tools.js"; +import type { WorkspaceConfig } from "./types.workspace.js"; export type MoltbotConfig = { meta?: { @@ -95,6 +96,7 @@ export type MoltbotConfig = { canvasHost?: CanvasHostConfig; talk?: TalkConfig; gateway?: GatewayConfig; + workspace?: WorkspaceConfig; }; export type ConfigValidationIssue = { diff --git a/src/config/types.workspace.ts b/src/config/types.workspace.ts new file mode 100644 index 000000000..212e394cd --- /dev/null +++ b/src/config/types.workspace.ts @@ -0,0 +1,119 @@ +/** + * Workspace sync provider modes. + * - off: no sync + * - dropbox: Dropbox via rclone + * - gdrive: Google Drive via rclone + * - onedrive: OneDrive via rclone + * - s3: S3-compatible storage via rclone + * - custom: custom rclone remote (user-configured) + */ +export type WorkspaceSyncProvider = "off" | "dropbox" | "gdrive" | "onedrive" | "s3" | "custom"; + +/** + * Workspace sync configuration. + * Enables bidirectional sync between the agent workspace and cloud storage. + */ +export type WorkspaceSyncConfig = { + /** + * Sync provider mode. + * - off: disabled (default) + * - dropbox: Dropbox via rclone + * - gdrive: Google Drive via rclone + * - onedrive: OneDrive via rclone + * - s3: S3-compatible storage via rclone + * - custom: custom rclone remote + */ + provider?: WorkspaceSyncProvider; + + /** + * Remote path/folder in cloud storage (e.g., "moltbot-share"). + * For Dropbox App folders, this is relative to the app folder root. + */ + remotePath?: string; + + /** + * Local subfolder within workspace to sync (default: "shared"). + * Files outside this folder are not synced. + */ + localPath?: string; + + /** + * Sync interval in seconds (0 = manual only, default: 0). + * When > 0, the gateway runs rclone bisync in the background at this interval. + * This is a pure file operation - it does NOT wake the bot or incur LLM costs. + */ + interval?: number; + + /** + * Sync on session start (default: false). + */ + onSessionStart?: boolean; + + /** + * Sync on session end (default: false). + */ + onSessionEnd?: boolean; + + /** + * rclone remote name (default: "cloud"). + * Used when provider is "custom" or to override the auto-generated name. + */ + remoteName?: string; + + /** + * Path to rclone config file. + * Default: $CLAWDBOT_STATE_DIR/.config/rclone/rclone.conf + */ + configPath?: string; + + /** + * Conflict resolution strategy. + * - newer: keep the newer file, rename older with .conflict suffix + * - local: local wins, remote gets .conflict suffix + * - remote: remote wins, local gets .conflict suffix + */ + conflictResolve?: "newer" | "local" | "remote"; + + /** + * File patterns to exclude from sync (glob patterns). + */ + exclude?: string[]; + + /** + * S3-specific configuration (when provider is "s3"). + */ + s3?: { + /** S3 endpoint URL (for non-AWS S3-compatible services). */ + endpoint?: string; + /** S3 bucket name. */ + bucket?: string; + /** S3 region. */ + region?: string; + /** Access key ID (prefer env var S3_ACCESS_KEY_ID). */ + accessKeyId?: string; + /** Secret access key (prefer env var S3_SECRET_ACCESS_KEY). */ + secretAccessKey?: string; + }; + + /** + * Dropbox-specific configuration. + */ + dropbox?: { + /** Use app folder access (more secure, limited to Apps//). */ + appFolder?: boolean; + /** Dropbox app key (for app folder access). */ + appKey?: string; + /** Dropbox app secret (for app folder access). */ + appSecret?: string; + }; +}; + +/** + * Top-level workspace configuration. + */ +export type WorkspaceConfig = { + /** + * Cloud sync configuration for bidirectional workspace sync. + */ + sync?: WorkspaceSyncConfig; +}; diff --git a/src/config/zod-schema.ts b/src/config/zod-schema.ts index ce4115517..6802faf42 100644 --- a/src/config/zod-schema.ts +++ b/src/config/zod-schema.ts @@ -527,6 +527,55 @@ export const MoltbotSchema = z }) .strict() .optional(), + workspace: z + .object({ + sync: z + .object({ + provider: z + .union([ + z.literal("off"), + z.literal("dropbox"), + z.literal("gdrive"), + z.literal("onedrive"), + z.literal("s3"), + z.literal("custom"), + ]) + .optional(), + remotePath: z.string().optional(), + localPath: z.string().optional(), + interval: z.number().int().nonnegative().optional(), + onSessionStart: z.boolean().optional(), + onSessionEnd: z.boolean().optional(), + remoteName: z.string().optional(), + configPath: z.string().optional(), + conflictResolve: z + .union([z.literal("newer"), z.literal("local"), z.literal("remote")]) + .optional(), + exclude: z.array(z.string()).optional(), + s3: z + .object({ + endpoint: z.string().optional(), + bucket: z.string().optional(), + region: z.string().optional(), + accessKeyId: z.string().optional(), + secretAccessKey: z.string().optional(), + }) + .strict() + .optional(), + dropbox: z + .object({ + appFolder: z.boolean().optional(), + appKey: z.string().optional(), + appSecret: z.string().optional(), + }) + .strict() + .optional(), + }) + .strict() + .optional(), + }) + .strict() + .optional(), }) .strict() .superRefine((cfg, ctx) => { diff --git a/src/gateway/server-close.ts b/src/gateway/server-close.ts index da9f5a39e..2071c5ab0 100644 --- a/src/gateway/server-close.ts +++ b/src/gateway/server-close.ts @@ -5,6 +5,7 @@ import { type ChannelId, listChannelPlugins } from "../channels/plugins/index.js import { stopGmailWatcher } from "../hooks/gmail-watcher.js"; import type { HeartbeatRunner } from "../infra/heartbeat-runner.js"; import type { PluginServicesHandle } from "../plugins/services.js"; +import { stopWorkspaceSyncManager } from "./workspace-sync-manager.js"; export function createGatewayCloseHandler(params: { bonjourStop: (() => Promise) | null; @@ -68,6 +69,7 @@ export function createGatewayCloseHandler(params: { await params.pluginServices.stop().catch(() => {}); } await stopGmailWatcher(); + stopWorkspaceSyncManager(); params.cron.stop(); params.heartbeatRunner.stop(); for (const timer of params.nodePresenceTimers.values()) { diff --git a/src/gateway/server-startup.ts b/src/gateway/server-startup.ts index e6bdbb0dc..62321aef2 100644 --- a/src/gateway/server-startup.ts +++ b/src/gateway/server-startup.ts @@ -22,6 +22,7 @@ import { scheduleRestartSentinelWake, shouldWakeFromRestartSentinel, } from "./server-restart-sentinel.js"; +import { startWorkspaceSyncManager } from "./workspace-sync-manager.js"; export async function startGatewaySidecars(params: { cfg: ReturnType; @@ -139,6 +140,17 @@ export async function startGatewaySidecars(params: { }, 250); } + // Start workspace sync manager (runs rclone in background - no LLM cost). + try { + startWorkspaceSyncManager(params.cfg, { + info: (msg) => params.logHooks.info(msg), + warn: (msg) => params.logHooks.warn(msg), + error: (msg) => params.logHooks.error(msg), + }); + } catch (err) { + params.logHooks.warn(`workspace sync manager failed to start: ${String(err)}`); + } + let pluginServices: PluginServicesHandle | null = null; try { pluginServices = await startPluginServices({ diff --git a/src/gateway/workspace-sync-manager.ts b/src/gateway/workspace-sync-manager.ts new file mode 100644 index 000000000..fdabde206 --- /dev/null +++ b/src/gateway/workspace-sync-manager.ts @@ -0,0 +1,190 @@ +/** + * Background workspace sync manager for the gateway. + * + * Runs rclone bisync at configured intervals WITHOUT involving the agent/LLM. + * This is a pure file operation that incurs zero token cost. + */ + +import type { MoltbotConfig } from "../config/config.js"; +import { resolveAgentWorkspaceDir, resolveDefaultAgentId } from "../agents/agent-scope.js"; +import { + isRcloneInstalled, + isRcloneConfigured, + resolveSyncConfig, + runBisync, +} from "../infra/rclone.js"; + +type SyncManagerLogger = { + info: (msg: string) => void; + warn: (msg: string) => void; + error: (msg: string) => void; +}; + +type SyncManagerState = { + intervalId: ReturnType | null; + lastSyncAt: Date | null; + lastSyncOk: boolean | null; + syncCount: number; + errorCount: number; +}; + +const state: SyncManagerState = { + intervalId: null, + lastSyncAt: null, + lastSyncOk: null, + syncCount: 0, + errorCount: 0, +}; + +let currentConfig: MoltbotConfig | null = null; +let currentLogger: SyncManagerLogger | null = null; + +/** + * Run a single sync operation. + * This is a pure rclone operation - no agent/LLM involvement. + */ +async function runSync(): Promise { + if (!currentConfig || !currentLogger) return; + + const syncConfig = currentConfig.workspace?.sync; + if (!syncConfig?.provider || syncConfig.provider === "off") return; + + const logger = currentLogger; + + try { + // Check if rclone is available + const installed = await isRcloneInstalled(); + if (!installed) { + logger.warn("[workspace-sync] rclone not installed, skipping periodic sync"); + return; + } + + // Resolve workspace and config + const agentId = resolveDefaultAgentId(currentConfig); + const workspaceDir = resolveAgentWorkspaceDir(currentConfig, agentId); + const resolved = resolveSyncConfig(syncConfig, workspaceDir); + + // Check if configured + if (!isRcloneConfigured(resolved.configPath, resolved.remoteName)) { + logger.warn(`[workspace-sync] rclone not configured for "${resolved.remoteName}", skipping`); + return; + } + + logger.info( + `[workspace-sync] Running periodic sync: ${resolved.remoteName}:${resolved.remotePath}`, + ); + + // Run bisync - pure file operation, no LLM involvement + const result = await runBisync({ + configPath: resolved.configPath, + remoteName: resolved.remoteName, + remotePath: resolved.remotePath, + localPath: resolved.localPath, + conflictResolve: resolved.conflictResolve, + exclude: resolved.exclude, + }); + + state.lastSyncAt = new Date(); + state.syncCount++; + + if (result.ok) { + state.lastSyncOk = true; + logger.info("[workspace-sync] Periodic sync completed"); + } else { + state.lastSyncOk = false; + state.errorCount++; + logger.warn(`[workspace-sync] Periodic sync failed: ${result.error}`); + } + } catch (err) { + state.lastSyncOk = false; + state.errorCount++; + logger.error( + `[workspace-sync] Periodic sync error: ${err instanceof Error ? err.message : String(err)}`, + ); + } +} + +/** + * Start the background sync manager. + * Called when the gateway starts. + */ +export function startWorkspaceSyncManager(cfg: MoltbotConfig, logger: SyncManagerLogger): void { + // Stop any existing interval + stopWorkspaceSyncManager(); + + currentConfig = cfg; + currentLogger = logger; + + const syncConfig = cfg.workspace?.sync; + if (!syncConfig?.provider || syncConfig.provider === "off") { + logger.info("[workspace-sync] Workspace sync not configured"); + return; + } + + const intervalSeconds = syncConfig.interval ?? 0; + if (intervalSeconds <= 0) { + logger.info("[workspace-sync] Periodic sync disabled (interval=0)"); + return; + } + + // Minimum interval: 60 seconds to prevent thrashing + const effectiveInterval = Math.max(intervalSeconds, 60); + if (effectiveInterval !== intervalSeconds) { + logger.warn( + `[workspace-sync] Interval increased from ${intervalSeconds}s to ${effectiveInterval}s (minimum)`, + ); + } + + logger.info( + `[workspace-sync] Starting periodic sync every ${effectiveInterval}s (pure file sync, zero LLM cost)`, + ); + + // Run initial sync after a short delay (let gateway fully start) + setTimeout(() => { + runSync().catch(() => {}); + }, 5000); + + // Set up periodic sync + state.intervalId = setInterval(() => { + runSync().catch(() => {}); + }, effectiveInterval * 1000); +} + +/** + * Stop the background sync manager. + * Called when the gateway stops. + */ +export function stopWorkspaceSyncManager(): void { + if (state.intervalId) { + clearInterval(state.intervalId); + state.intervalId = null; + } + currentConfig = null; + currentLogger = null; +} + +/** + * Get current sync manager status. + */ +export function getWorkspaceSyncStatus(): { + running: boolean; + lastSyncAt: Date | null; + lastSyncOk: boolean | null; + syncCount: number; + errorCount: number; +} { + return { + running: state.intervalId !== null, + lastSyncAt: state.lastSyncAt, + lastSyncOk: state.lastSyncOk, + syncCount: state.syncCount, + errorCount: state.errorCount, + }; +} + +/** + * Trigger an immediate sync (for CLI use). + */ +export async function triggerImmediateSync(): Promise { + await runSync(); +} diff --git a/src/hooks/bundled/workspace-sync/HOOK.md b/src/hooks/bundled/workspace-sync/HOOK.md new file mode 100644 index 000000000..4dcb8b948 --- /dev/null +++ b/src/hooks/bundled/workspace-sync/HOOK.md @@ -0,0 +1,64 @@ +--- +name: workspace-sync +description: Sync workspace with cloud storage (Dropbox, Google Drive, etc.) on session start/end +metadata: + events: + - session:start + - session:end + requires: + bins: + - rclone + config: + - workspace.sync.provider +--- + +# Workspace Sync Hook + +Automatically syncs your agent workspace with cloud storage when sessions start or end. + +## Configuration + +Enable in `~/.clawdbot/moltbot.json`: + +```json5 +{ + workspace: { + sync: { + provider: "dropbox", // dropbox | gdrive | onedrive | s3 | custom + remotePath: "moltbot-share", // folder in cloud storage + localPath: "shared", // subfolder in workspace + onSessionStart: true, // sync when session starts + onSessionEnd: false // sync when session ends + } + }, + hooks: { + internal: { + entries: { + "workspace-sync": { enabled: true } + } + } + } +} +``` + +## Requirements + +- rclone (auto-installed via `moltbot workspace setup` if missing) +- Cloud provider account (Dropbox, Google Drive, OneDrive, or S3) + +## Setup + +Run the interactive setup wizard: + +```bash +moltbot workspace setup +``` + +The wizard will: +1. Install rclone (if needed) +2. Guide you through provider selection +3. Handle OAuth authorization +4. Configure sync settings +5. Run first sync + +See [Workspace Cloud Sync](/gateway/workspace-sync) for full documentation. diff --git a/src/hooks/bundled/workspace-sync/handler.test.ts b/src/hooks/bundled/workspace-sync/handler.test.ts new file mode 100644 index 000000000..be5fcc30b --- /dev/null +++ b/src/hooks/bundled/workspace-sync/handler.test.ts @@ -0,0 +1,327 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import workspaceSyncHandler from "./handler.js"; +import * as rclone from "../../../infra/rclone.js"; + +vi.mock("../../../infra/rclone.js", () => ({ + isRcloneInstalled: vi.fn(), + isRcloneConfigured: vi.fn(), + resolveSyncConfig: vi.fn(), + runBisync: vi.fn(), +})); + +vi.mock("../../../agents/agent-scope.js", () => ({ + resolveAgentWorkspaceDir: vi.fn(() => "/workspace"), +})); + +vi.mock("../../../routing/session-key.js", () => ({ + resolveAgentIdFromSessionKey: vi.fn(() => "default"), +})); + +describe("workspace-sync hook handler", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("ignores non-session events", async () => { + const event = { + type: "command", + action: "new", + sessionKey: "test-session", + context: {}, + }; + + await workspaceSyncHandler(event as never); + + expect(rclone.isRcloneInstalled).not.toHaveBeenCalled(); + }); + + it("ignores session events when sync is not configured", async () => { + const event = { + type: "session", + action: "start", + sessionKey: "test-session", + context: { cfg: {} }, + }; + + await workspaceSyncHandler(event as never); + + expect(rclone.isRcloneInstalled).not.toHaveBeenCalled(); + }); + + it("ignores session start when onSessionStart is false", async () => { + const event = { + type: "session", + action: "start", + sessionKey: "test-session", + context: { + cfg: { + workspace: { + sync: { + provider: "dropbox", + remotePath: "moltbot-share", + onSessionStart: false, + onSessionEnd: true, + }, + }, + }, + }, + }; + + await workspaceSyncHandler(event as never); + + expect(rclone.isRcloneInstalled).not.toHaveBeenCalled(); + }); + + it("ignores session end when onSessionEnd is false", async () => { + const event = { + type: "session", + action: "end", + sessionKey: "test-session", + context: { + cfg: { + workspace: { + sync: { + provider: "dropbox", + remotePath: "moltbot-share", + onSessionStart: true, + onSessionEnd: false, + }, + }, + }, + }, + }; + + await workspaceSyncHandler(event as never); + + expect(rclone.isRcloneInstalled).not.toHaveBeenCalled(); + }); + + it("ignores provider: off", async () => { + const event = { + type: "session", + action: "start", + sessionKey: "test-session", + context: { + cfg: { + workspace: { + sync: { + provider: "off", + remotePath: "moltbot-share", + onSessionStart: true, + }, + }, + }, + }, + }; + + await workspaceSyncHandler(event as never); + + expect(rclone.isRcloneInstalled).not.toHaveBeenCalled(); + }); + + it("warns when rclone is not installed", async () => { + vi.mocked(rclone.isRcloneInstalled).mockResolvedValue(false); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + const event = { + type: "session", + action: "start", + sessionKey: "test-session", + context: { + cfg: { + workspace: { + sync: { + provider: "dropbox", + remotePath: "moltbot-share", + onSessionStart: true, + }, + }, + }, + }, + }; + + await workspaceSyncHandler(event as never); + + expect(rclone.isRcloneInstalled).toHaveBeenCalled(); + expect(warnSpy).toHaveBeenCalledWith("[workspace-sync] rclone not installed, skipping sync"); + }); + + it("warns when rclone is not configured", async () => { + vi.mocked(rclone.isRcloneInstalled).mockResolvedValue(true); + vi.mocked(rclone.isRcloneConfigured).mockReturnValue(false); + vi.mocked(rclone.resolveSyncConfig).mockReturnValue({ + provider: "dropbox", + remoteName: "cloud", + remotePath: "moltbot-share", + localPath: "/workspace/shared", + configPath: "/home/.config/rclone/rclone.conf", + conflictResolve: "newer", + exclude: [], + interval: 0, + onSessionStart: true, + onSessionEnd: false, + }); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + const event = { + type: "session", + action: "start", + sessionKey: "test-session", + context: { + cfg: { + workspace: { + sync: { + provider: "dropbox", + remotePath: "moltbot-share", + onSessionStart: true, + }, + }, + }, + }, + }; + + await workspaceSyncHandler(event as never); + + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('rclone not configured for remote "cloud"'), + ); + }); + + it("runs bisync on session start when configured", async () => { + vi.mocked(rclone.isRcloneInstalled).mockResolvedValue(true); + vi.mocked(rclone.isRcloneConfigured).mockReturnValue(true); + vi.mocked(rclone.resolveSyncConfig).mockReturnValue({ + provider: "dropbox", + remoteName: "cloud", + remotePath: "moltbot-share", + localPath: "/workspace/shared", + configPath: "/home/.config/rclone/rclone.conf", + conflictResolve: "newer", + exclude: [".git/**"], + interval: 0, + onSessionStart: true, + onSessionEnd: false, + }); + vi.mocked(rclone.runBisync).mockResolvedValue({ ok: true }); + vi.spyOn(console, "log").mockImplementation(() => {}); + + const event = { + type: "session", + action: "start", + sessionKey: "test-session", + context: { + cfg: { + workspace: { + sync: { + provider: "dropbox", + remotePath: "moltbot-share", + onSessionStart: true, + }, + }, + }, + }, + }; + + await workspaceSyncHandler(event as never); + + expect(rclone.runBisync).toHaveBeenCalledWith( + expect.objectContaining({ + remoteName: "cloud", + remotePath: "moltbot-share", + localPath: "/workspace/shared", + conflictResolve: "newer", + }), + ); + }); + + it("handles session stop as end event", async () => { + vi.mocked(rclone.isRcloneInstalled).mockResolvedValue(true); + vi.mocked(rclone.isRcloneConfigured).mockReturnValue(true); + vi.mocked(rclone.resolveSyncConfig).mockReturnValue({ + provider: "dropbox", + remoteName: "cloud", + remotePath: "moltbot-share", + localPath: "/workspace/shared", + configPath: "/home/.config/rclone/rclone.conf", + conflictResolve: "newer", + exclude: [], + interval: 0, + onSessionStart: false, + onSessionEnd: true, + }); + vi.mocked(rclone.runBisync).mockResolvedValue({ ok: true }); + vi.spyOn(console, "log").mockImplementation(() => {}); + + const event = { + type: "session", + action: "stop", + sessionKey: "test-session", + context: { + cfg: { + workspace: { + sync: { + provider: "dropbox", + remotePath: "moltbot-share", + onSessionEnd: true, + }, + }, + }, + }, + }; + + await workspaceSyncHandler(event as never); + + expect(rclone.runBisync).toHaveBeenCalled(); + }); + + it("warns about --resync on first run error", async () => { + vi.mocked(rclone.isRcloneInstalled).mockResolvedValue(true); + vi.mocked(rclone.isRcloneConfigured).mockReturnValue(true); + vi.mocked(rclone.resolveSyncConfig).mockReturnValue({ + provider: "dropbox", + remoteName: "cloud", + remotePath: "moltbot-share", + localPath: "/workspace/shared", + configPath: "/home/.config/rclone/rclone.conf", + conflictResolve: "newer", + exclude: [], + interval: 0, + onSessionStart: true, + onSessionEnd: false, + }); + vi.mocked(rclone.runBisync).mockResolvedValue({ + ok: false, + error: "bisync requires --resync on first run", + }); + vi.spyOn(console, "log").mockImplementation(() => {}); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + + const event = { + type: "session", + action: "start", + sessionKey: "test-session", + context: { + cfg: { + workspace: { + sync: { + provider: "dropbox", + remotePath: "moltbot-share", + onSessionStart: true, + }, + }, + }, + }, + }; + + await workspaceSyncHandler(event as never); + + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining("First sync requires manual --resync"), + ); + }); +}); diff --git a/src/hooks/bundled/workspace-sync/handler.ts b/src/hooks/bundled/workspace-sync/handler.ts new file mode 100644 index 000000000..9edff53cb --- /dev/null +++ b/src/hooks/bundled/workspace-sync/handler.ts @@ -0,0 +1,118 @@ +/** + * Workspace sync hook handler + * + * Syncs workspace with cloud storage on session start/end + * when workspace.sync is configured. + */ + +import type { MoltbotConfig } from "../../../config/config.js"; +import { resolveAgentWorkspaceDir } from "../../../agents/agent-scope.js"; +import { resolveAgentIdFromSessionKey } from "../../../routing/session-key.js"; +import type { HookHandler } from "../../hooks.js"; +import { + isRcloneInstalled, + isRcloneConfigured, + resolveSyncConfig, + runBisync, +} from "../../../infra/rclone.js"; + +/** + * Sync workspace on session start or end + */ +const workspaceSyncHandler: HookHandler = async (event) => { + // Only handle session events + if (event.type !== "session") { + return; + } + + const context = event.context || {}; + const cfg = context.cfg as MoltbotConfig | undefined; + + if (!cfg?.workspace?.sync) { + return; + } + + const syncConfig = cfg.workspace.sync; + + // Check if sync is enabled for this event + const isStart = event.action === "start"; + const isEnd = event.action === "end" || event.action === "stop"; + + if (isStart && !syncConfig.onSessionStart) { + return; + } + + if (isEnd && !syncConfig.onSessionEnd) { + return; + } + + if (!isStart && !isEnd) { + return; + } + + // Check if provider is configured + if (!syncConfig.provider || syncConfig.provider === "off") { + return; + } + + console.log(`[workspace-sync] Triggered on session ${event.action}`); + + try { + // Check if rclone is installed + const installed = await isRcloneInstalled(); + if (!installed) { + console.warn("[workspace-sync] rclone not installed, skipping sync"); + return; + } + + // Resolve workspace and config + const agentId = resolveAgentIdFromSessionKey(event.sessionKey); + const workspaceDir = resolveAgentWorkspaceDir(cfg, agentId); + const stateDir = context.stateDir as string | undefined; + + const resolved = resolveSyncConfig(syncConfig, workspaceDir, stateDir); + + // Check if rclone is configured + if (!isRcloneConfigured(resolved.configPath, resolved.remoteName)) { + console.warn( + `[workspace-sync] rclone not configured for remote "${resolved.remoteName}", skipping sync`, + ); + return; + } + + console.log( + `[workspace-sync] Syncing ${resolved.remoteName}:${resolved.remotePath} <-> ${resolved.localPath}`, + ); + + // Run sync + const result = await runBisync({ + configPath: resolved.configPath, + remoteName: resolved.remoteName, + remotePath: resolved.remotePath, + localPath: resolved.localPath, + conflictResolve: resolved.conflictResolve, + exclude: resolved.exclude, + verbose: false, + }); + + if (result.ok) { + console.log("[workspace-sync] Sync completed successfully"); + if (result.filesTransferred) { + console.log(`[workspace-sync] Files transferred: ${result.filesTransferred}`); + } + } else { + // Check if this is a first-run issue + if (result.error?.includes("--resync")) { + console.warn( + "[workspace-sync] First sync requires manual --resync. Run: moltbot workspace sync --resync", + ); + } else { + console.error(`[workspace-sync] Sync failed: ${result.error}`); + } + } + } catch (err) { + console.error("[workspace-sync] Error:", err instanceof Error ? err.message : String(err)); + } +}; + +export default workspaceSyncHandler; diff --git a/src/infra/rclone.test.ts b/src/infra/rclone.test.ts new file mode 100644 index 000000000..e9ff73dc6 --- /dev/null +++ b/src/infra/rclone.test.ts @@ -0,0 +1,157 @@ +import { describe, expect, it } from "vitest"; + +import { resolveSyncConfig, generateRcloneConfig, isRcloneConfigured } from "./rclone.js"; + +describe("rclone helpers", () => { + describe("resolveSyncConfig", () => { + it("uses defaults when config is minimal", () => { + const config = { provider: "dropbox" as const, remotePath: "test-folder" }; + const workspaceDir = "/home/user/workspace"; + const stateDir = "/home/user/.moltbot"; + + const resolved = resolveSyncConfig(config, workspaceDir, stateDir); + + expect(resolved.provider).toBe("dropbox"); + expect(resolved.remotePath).toBe("test-folder"); + expect(resolved.localPath).toBe("/home/user/workspace/shared"); + expect(resolved.remoteName).toBe("cloud"); + expect(resolved.conflictResolve).toBe("newer"); + expect(resolved.interval).toBe(0); + expect(resolved.onSessionStart).toBe(false); + expect(resolved.onSessionEnd).toBe(false); + }); + + it("respects custom localPath", () => { + const config = { + provider: "dropbox" as const, + remotePath: "test-folder", + localPath: "sync", + }; + const workspaceDir = "/home/user/workspace"; + const stateDir = "/home/user/.moltbot"; + + const resolved = resolveSyncConfig(config, workspaceDir, stateDir); + + expect(resolved.localPath).toBe("/home/user/workspace/sync"); + }); + + it("respects custom remoteName", () => { + const config = { + provider: "dropbox" as const, + remotePath: "test-folder", + remoteName: "my-dropbox", + }; + const workspaceDir = "/home/user/workspace"; + const stateDir = "/home/user/.moltbot"; + + const resolved = resolveSyncConfig(config, workspaceDir, stateDir); + + expect(resolved.remoteName).toBe("my-dropbox"); + }); + + it("respects interval and session hooks", () => { + const config = { + provider: "dropbox" as const, + remotePath: "test-folder", + interval: 300, + onSessionStart: true, + onSessionEnd: true, + }; + const workspaceDir = "/home/user/workspace"; + const stateDir = "/home/user/.moltbot"; + + const resolved = resolveSyncConfig(config, workspaceDir, stateDir); + + expect(resolved.interval).toBe(300); + expect(resolved.onSessionStart).toBe(true); + expect(resolved.onSessionEnd).toBe(true); + }); + + it("applies default excludes", () => { + const config = { provider: "dropbox" as const, remotePath: "test" }; + const resolved = resolveSyncConfig(config, "/workspace", "/state"); + + expect(resolved.exclude).toContain(".git/**"); + expect(resolved.exclude).toContain("node_modules/**"); + }); + + it("respects custom excludes", () => { + const config = { + provider: "dropbox" as const, + remotePath: "test", + exclude: ["*.tmp", "cache/**"], + }; + const resolved = resolveSyncConfig(config, "/workspace", "/state"); + + expect(resolved.exclude).toEqual(["*.tmp", "cache/**"]); + }); + }); + + describe("generateRcloneConfig", () => { + it("generates dropbox config with token", () => { + const config = generateRcloneConfig("dropbox", "cloud", '{"access_token":"abc123"}'); + + expect(config).toContain("[cloud]"); + expect(config).toContain("type = dropbox"); + expect(config).toContain('token = {"access_token":"abc123"}'); + }); + + it("generates gdrive config", () => { + const config = generateRcloneConfig("gdrive", "drive", '{"access_token":"xyz"}'); + + expect(config).toContain("[drive]"); + expect(config).toContain("type = drive"); + expect(config).toContain('token = {"access_token":"xyz"}'); + }); + + it("generates onedrive config", () => { + const config = generateRcloneConfig("onedrive", "od", '{"access_token":"123"}'); + + expect(config).toContain("[od]"); + expect(config).toContain("type = onedrive"); + }); + + it("includes app key/secret for dropbox app folder", () => { + const config = generateRcloneConfig("dropbox", "cloud", '{"access_token":"abc"}', { + dropbox: { appKey: "key123", appSecret: "secret456" }, + }); + + expect(config).toContain("client_id = key123"); + expect(config).toContain("client_secret = secret456"); + }); + + it("generates s3 config with endpoint", () => { + const config = generateRcloneConfig("s3", "r2", "", { + s3: { + endpoint: "https://xxx.r2.cloudflarestorage.com", + accessKeyId: "AKID", + secretAccessKey: "SECRET", + }, + }); + + expect(config).toContain("[r2]"); + expect(config).toContain("type = s3"); + expect(config).toContain("endpoint = https://xxx.r2.cloudflarestorage.com"); + expect(config).toContain("access_key_id = AKID"); + expect(config).toContain("secret_access_key = SECRET"); + }); + + it("includes region for s3 when provided", () => { + const config = generateRcloneConfig("s3", "aws", "", { + s3: { + region: "us-east-1", + bucket: "my-bucket", + }, + }); + + expect(config).toContain("region = us-east-1"); + }); + }); + + describe("isRcloneConfigured", () => { + it("returns false when config file does not exist", () => { + const result = isRcloneConfigured("/nonexistent/path/rclone.conf", "cloud"); + expect(result).toBe(false); + }); + }); +}); diff --git a/src/infra/rclone.ts b/src/infra/rclone.ts new file mode 100644 index 000000000..e667bd02f --- /dev/null +++ b/src/infra/rclone.ts @@ -0,0 +1,516 @@ +import { existsSync, mkdirSync, writeFileSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { homedir, platform } from "node:os"; +import { logVerbose } from "../globals.js"; +import { runExec } from "../process/exec.js"; +import type { WorkspaceSyncConfig, WorkspaceSyncProvider } from "../config/types.workspace.js"; +import type { RuntimeEnv } from "../runtime.js"; +import { defaultRuntime } from "../runtime.js"; + +const DEFAULT_REMOTE_NAME = "cloud"; +const DEFAULT_LOCAL_PATH = "shared"; +const DEFAULT_REMOTE_PATH = "moltbot-share"; +const DEFAULT_CONFLICT_RESOLVE = "newer"; +const DEFAULT_EXCLUDES = [".git/**", "node_modules/**", "*.log", ".DS_Store"]; + +export type RcloneSyncResult = { + ok: boolean; + error?: string; + filesTransferred?: number; + bytesTransferred?: number; +}; + +/** + * Find rclone binary in PATH or common locations. + */ +export async function findRcloneBinary(): Promise { + const checkBinary = async (path: string): Promise => { + if (!path || (path.startsWith("/") && !existsSync(path))) return false; + try { + await runExec(path, ["--version"], { timeoutMs: 3000 }); + return true; + } catch { + return false; + } + }; + + // Strategy 1: which command + try { + const { stdout } = await runExec("which", ["rclone"]); + const fromPath = stdout.trim(); + if (fromPath && (await checkBinary(fromPath))) { + return fromPath; + } + } catch { + // which failed, continue + } + + // Strategy 2: Common install locations + const commonPaths = ["/usr/local/bin/rclone", "/usr/bin/rclone", "/opt/homebrew/bin/rclone"]; + for (const path of commonPaths) { + if (await checkBinary(path)) { + return path; + } + } + + return null; +} + +let cachedRcloneBinary: string | null = null; + +export async function getRcloneBinary(): Promise { + if (cachedRcloneBinary) return cachedRcloneBinary; + cachedRcloneBinary = await findRcloneBinary(); + return cachedRcloneBinary ?? "rclone"; +} + +/** + * Check if rclone is installed. + */ +export async function isRcloneInstalled(): Promise { + const binary = await findRcloneBinary(); + return binary !== null; +} + +/** + * Ensure rclone is installed, offering to install if missing. + * Returns true if rclone is available, false if user declined install. + */ +export async function ensureRcloneInstalled( + prompt: (message: string, defaultValue: boolean) => Promise, + exec: typeof runExec = runExec, + runtime: RuntimeEnv = defaultRuntime, +): Promise { + const installed = await isRcloneInstalled(); + if (installed) return true; + + const isMac = platform() === "darwin"; + const isLinux = platform() === "linux"; + + if (isMac) { + // Check if Homebrew is available + const hasBrew = await exec("which", ["brew"]).then( + () => true, + () => false, + ); + + if (hasBrew) { + const install = await prompt( + "rclone not found. Install via Homebrew (brew install rclone)?", + true, + ); + if (!install) { + return false; + } + logVerbose("Installing rclone via Homebrew..."); + try { + await exec("brew", ["install", "rclone"], { timeoutMs: 120_000 }); + // Clear cached binary so we find the new one + cachedRcloneBinary = null; + return true; + } catch (err) { + runtime.error( + `Failed to install rclone: ${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } + } + } + + if (isLinux || isMac) { + const install = await prompt( + "rclone not found. Install via official script (curl https://rclone.org/install.sh)?", + true, + ); + if (!install) { + return false; + } + logVerbose("Installing rclone via official script..."); + try { + // Download and run the install script + const { stdout } = await exec("curl", ["-s", "https://rclone.org/install.sh"], { + timeoutMs: 30_000, + }); + await exec("sudo", ["bash", "-c", stdout], { timeoutMs: 120_000 }); + // Clear cached binary so we find the new one + cachedRcloneBinary = null; + return true; + } catch (err) { + runtime.error( + `Failed to install rclone: ${err instanceof Error ? err.message : String(err)}`, + ); + runtime.error("Try installing manually: https://rclone.org/install/"); + return false; + } + } + + runtime.error("rclone not found. Please install manually: https://rclone.org/install/"); + return false; +} + +/** + * Get the default rclone config path. + */ +export function getDefaultRcloneConfigPath(stateDir?: string): string { + const base = stateDir ?? process.env.CLAWDBOT_STATE_DIR ?? join(homedir(), ".clawdbot"); + return join(base, ".config", "rclone", "rclone.conf"); +} + +/** + * Resolve sync config with defaults. + */ +export function resolveSyncConfig( + config: WorkspaceSyncConfig | undefined, + workspace: string, + stateDir?: string, +): { + provider: WorkspaceSyncProvider; + remoteName: string; + remotePath: string; + localPath: string; + configPath: string; + conflictResolve: "newer" | "local" | "remote"; + exclude: string[]; + interval: number; + onSessionStart: boolean; + onSessionEnd: boolean; +} { + return { + provider: config?.provider ?? "off", + remoteName: config?.remoteName ?? DEFAULT_REMOTE_NAME, + remotePath: config?.remotePath ?? DEFAULT_REMOTE_PATH, + localPath: join(workspace, config?.localPath ?? DEFAULT_LOCAL_PATH), + configPath: config?.configPath ?? getDefaultRcloneConfigPath(stateDir), + conflictResolve: config?.conflictResolve ?? DEFAULT_CONFLICT_RESOLVE, + exclude: config?.exclude ?? DEFAULT_EXCLUDES, + interval: config?.interval ?? 0, + onSessionStart: config?.onSessionStart ?? false, + onSessionEnd: config?.onSessionEnd ?? false, + }; +} + +/** + * Get rclone type string for a provider. + */ +function getRcloneType(provider: WorkspaceSyncProvider): string { + switch (provider) { + case "dropbox": + return "dropbox"; + case "gdrive": + return "drive"; + case "onedrive": + return "onedrive"; + case "s3": + return "s3"; + default: + return "unknown"; + } +} + +/** + * Generate rclone config content for a provider. + */ +export function generateRcloneConfig( + provider: WorkspaceSyncProvider, + remoteName: string, + token: string, + options?: { + dropbox?: { appKey?: string; appSecret?: string }; + s3?: { + endpoint?: string; + bucket?: string; + region?: string; + accessKeyId?: string; + secretAccessKey?: string; + }; + }, +): string { + const type = getRcloneType(provider); + let config = `[${remoteName}]\ntype = ${type}\n`; + + if (provider === "dropbox") { + config += `token = ${token}\n`; + if (options?.dropbox?.appKey) { + config += `client_id = ${options.dropbox.appKey}\n`; + } + if (options?.dropbox?.appSecret) { + config += `client_secret = ${options.dropbox.appSecret}\n`; + } + } else if (provider === "gdrive") { + config += `token = ${token}\n`; + } else if (provider === "onedrive") { + config += `token = ${token}\n`; + } else if (provider === "s3") { + if (options?.s3?.endpoint) { + config += `endpoint = ${options.s3.endpoint}\n`; + } + if (options?.s3?.region) { + config += `region = ${options.s3.region}\n`; + } + if (options?.s3?.accessKeyId) { + config += `access_key_id = ${options.s3.accessKeyId}\n`; + } + if (options?.s3?.secretAccessKey) { + config += `secret_access_key = ${options.s3.secretAccessKey}\n`; + } + } + + return config; +} + +/** + * Write rclone config to disk. + */ +export function writeRcloneConfig(configPath: string, content: string): void { + const dir = dirname(configPath); + if (!existsSync(dir)) { + mkdirSync(dir, { recursive: true }); + } + writeFileSync(configPath, content, { mode: 0o600 }); +} + +/** + * Check if rclone config exists and has the remote configured. + */ +export function isRcloneConfigured(configPath: string, remoteName: string): boolean { + if (!existsSync(configPath)) return false; + try { + const content = readFileSync(configPath, "utf-8"); + return content.includes(`[${remoteName}]`); + } catch { + return false; + } +} + +/** + * Run rclone authorize command (returns the token). + * This must be run on a machine with a browser. + */ +export async function authorizeRclone( + provider: WorkspaceSyncProvider, + appKey?: string, + appSecret?: string, +): Promise<{ ok: true; token: string } | { ok: false; error: string }> { + const rcloneBin = await getRcloneBinary(); + const type = getRcloneType(provider); + + const args = ["authorize", type]; + if (appKey && appSecret) { + args.push(appKey, appSecret); + } + + try { + const { stdout, stderr } = await runExec(rcloneBin, args, { + timeoutMs: 300_000, // 5 minutes for OAuth flow + maxBuffer: 1_000_000, + }); + + // Extract token from output + const combined = stdout + stderr; + const tokenMatch = combined.match(/\{[^}]*"access_token"[^}]*\}/); + if (tokenMatch) { + return { ok: true, token: tokenMatch[0] }; + } + + // Try to find JSON object in output + const jsonMatch = combined.match( + /Paste the following into your remote machine[\s\S]*?(\{[\s\S]*?\})\s*$/m, + ); + if (jsonMatch) { + return { ok: true, token: jsonMatch[1] }; + } + + return { ok: false, error: "Could not extract token from rclone output" }; + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { ok: false, error: `Authorization failed: ${message}` }; + } +} + +/** + * Run bidirectional sync using rclone bisync. + */ +export async function runBisync(params: { + configPath: string; + remoteName: string; + remotePath: string; + localPath: string; + conflictResolve: "newer" | "local" | "remote"; + exclude: string[]; + resync?: boolean; + dryRun?: boolean; + verbose?: boolean; +}): Promise { + const rcloneBin = await getRcloneBinary(); + + // Ensure local directory exists + if (!existsSync(params.localPath)) { + mkdirSync(params.localPath, { recursive: true }); + } + + const args = [ + "bisync", + `${params.remoteName}:${params.remotePath}`, + params.localPath, + "--config", + params.configPath, + "--conflict-resolve", + params.conflictResolve, + "--conflict-suffix", + ".conflict", + ]; + + // Add excludes + for (const pattern of params.exclude) { + args.push("--exclude", pattern); + } + + if (params.resync) { + args.push("--resync"); + } + + if (params.dryRun) { + args.push("--dry-run"); + } + + if (params.verbose) { + args.push("--verbose"); + } + + logVerbose(`Running: ${rcloneBin} ${args.join(" ")}`); + + try { + const { stdout, stderr } = await runExec(rcloneBin, args, { + timeoutMs: 600_000, // 10 minutes + maxBuffer: 10_000_000, + }); + + // Parse output for stats + const combined = stdout + stderr; + const transferredMatch = combined.match(/Transferred:\s*(\d+)\s*\/\s*(\d+)/); + + return { + ok: true, + filesTransferred: transferredMatch ? parseInt(transferredMatch[1], 10) : undefined, + }; + } catch (err) { + const errObj = err as { stdout?: string; stderr?: string; message?: string }; + const message = + errObj.stderr?.trim() || errObj.stdout?.trim() || errObj.message || "Unknown sync error"; + + // Check for common errors + if (message.includes("bisync requires --resync")) { + return { + ok: false, + error: "First sync requires --resync flag to establish baseline", + }; + } + + return { ok: false, error: message }; + } +} + +/** + * Run one-way sync (copy) from remote to local or vice versa. + */ +export async function runSync(params: { + configPath: string; + remoteName: string; + remotePath: string; + localPath: string; + direction: "pull" | "push"; + exclude: string[]; + dryRun?: boolean; + verbose?: boolean; +}): Promise { + const rcloneBin = await getRcloneBinary(); + + // Ensure local directory exists + if (!existsSync(params.localPath)) { + mkdirSync(params.localPath, { recursive: true }); + } + + const remote = `${params.remoteName}:${params.remotePath}`; + const [source, dest] = + params.direction === "pull" ? [remote, params.localPath] : [params.localPath, remote]; + + const args = ["sync", source, dest, "--config", params.configPath]; + + for (const pattern of params.exclude) { + args.push("--exclude", pattern); + } + + if (params.dryRun) { + args.push("--dry-run"); + } + + if (params.verbose) { + args.push("--verbose"); + } + + logVerbose(`Running: ${rcloneBin} ${args.join(" ")}`); + + try { + await runExec(rcloneBin, args, { + timeoutMs: 600_000, + maxBuffer: 10_000_000, + }); + return { ok: true }; + } catch (err) { + const errObj = err as { stderr?: string; message?: string }; + const message = errObj.stderr?.trim() || errObj.message || "Unknown sync error"; + return { ok: false, error: message }; + } +} + +/** + * List files in remote. + */ +export async function listRemote(params: { + configPath: string; + remoteName: string; + remotePath: string; +}): Promise<{ ok: true; files: string[] } | { ok: false; error: string }> { + const rcloneBin = await getRcloneBinary(); + + try { + const { stdout } = await runExec( + rcloneBin, + ["lsf", `${params.remoteName}:${params.remotePath}`, "--config", params.configPath], + { timeoutMs: 30_000, maxBuffer: 1_000_000 }, + ); + + const files = stdout + .trim() + .split("\n") + .filter((f) => f.length > 0); + return { ok: true, files }; + } catch (err) { + const errObj = err as { stderr?: string; message?: string }; + const message = errObj.stderr?.trim() || errObj.message || "Unknown error"; + return { ok: false, error: message }; + } +} + +/** + * Check remote connection. + */ +export async function checkRemote(params: { + configPath: string; + remoteName: string; +}): Promise<{ ok: true } | { ok: false; error: string }> { + const rcloneBin = await getRcloneBinary(); + + try { + await runExec( + rcloneBin, + ["about", `${params.remoteName}:`, "--config", params.configPath, "--json"], + { timeoutMs: 30_000, maxBuffer: 100_000 }, + ); + return { ok: true }; + } catch (err) { + const errObj = err as { stderr?: string; message?: string }; + const message = errObj.stderr?.trim() || errObj.message || "Connection failed"; + return { ok: false, error: message }; + } +}