From 6867d5218a6fd9b9c8b257b9c8e857237676e401 Mon Sep 17 00:00:00 2001 From: Cheese Date: Tue, 27 Jan 2026 17:41:40 +0800 Subject: [PATCH] feat(database): add TiDB tool (mysql CLI) --- .gitignore | 1 + docs/tools/index.md | 6 + docs/tools/tidb.md | 88 ++++ skills/tidb-sql/SKILL.md | 53 ++ skills/tidb-sql/references/auto-random.md | 40 ++ skills/tidb-sql/references/explain.md | 80 +++ skills/tidb-sql/references/flashback.md | 86 ++++ .../tidb-sql/references/full-text-search.md | 63 +++ .../references/mysql-compatibility-notes.md | 99 ++++ skills/tidb-sql/references/tidb-cloud-ssl.md | 91 ++++ skills/tidb-sql/references/transactions.md | 40 ++ skills/tidb-sql/references/vector.md | 95 ++++ skills/tidb-sql/scripts/render_dot_png.sh | 28 ++ src/agents/clawdbot-tools.tidb.test.ts | 44 ++ src/agents/clawdbot-tools.ts | 3 + src/agents/tool-display.json | 5 + src/agents/tool-policy.ts | 2 + src/agents/tools/tidb-tool.test.ts | 39 ++ src/agents/tools/tidb-tool.ts | 474 ++++++++++++++++++ src/config/schema.ts | 13 + src/config/types.tools.ts | 20 + src/config/zod-schema.agent-runtime.ts | 10 + 22 files changed, 1380 insertions(+) create mode 100644 docs/tools/tidb.md create mode 100644 skills/tidb-sql/SKILL.md create mode 100644 skills/tidb-sql/references/auto-random.md create mode 100644 skills/tidb-sql/references/explain.md create mode 100644 skills/tidb-sql/references/flashback.md create mode 100644 skills/tidb-sql/references/full-text-search.md create mode 100644 skills/tidb-sql/references/mysql-compatibility-notes.md create mode 100644 skills/tidb-sql/references/tidb-cloud-ssl.md create mode 100644 skills/tidb-sql/references/transactions.md create mode 100644 skills/tidb-sql/references/vector.md create mode 100755 skills/tidb-sql/scripts/render_dot_png.sh create mode 100644 src/agents/clawdbot-tools.tidb.test.ts create mode 100644 src/agents/tools/tidb-tool.test.ts create mode 100644 src/agents/tools/tidb-tool.ts diff --git a/.gitignore b/.gitignore index d3fdee6b5..1eae1f177 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,7 @@ apps/ios/*.mobileprovision # Local untracked files .local/ +ref/* .vscode/ IDENTITY.md USER.md diff --git a/docs/tools/index.md b/docs/tools/index.md index 3ada44edd..ee55644be 100644 --- a/docs/tools/index.md +++ b/docs/tools/index.md @@ -137,6 +137,7 @@ Available groups: - `group:sessions`: `sessions_list`, `sessions_history`, `sessions_send`, `sessions_spawn`, `session_status` - `group:memory`: `memory_search`, `memory_get` - `group:web`: `web_search`, `web_fetch` +- `group:db`: `tidb` - `group:ui`: `browser`, `canvas` - `group:automation`: `cron`, `gateway` - `group:messaging`: `message` @@ -199,6 +200,11 @@ Manage background exec sessions. Core actions: - `list`, `poll`, `log`, `write`, `kill`, `clear`, `remove` +### `tidb` +Run SQL against TiDB (MySQL protocol) using the local `mysql` CLI. + +Docs: [TiDB tool](/tools/tidb). + Notes: - `poll` returns new output and exit status when complete. - `log` supports line-based `offset`/`limit` (omit `offset` to grab the last N lines). diff --git a/docs/tools/tidb.md b/docs/tools/tidb.md new file mode 100644 index 000000000..d3597d383 --- /dev/null +++ b/docs/tools/tidb.md @@ -0,0 +1,88 @@ +--- +summary: "TiDB tool (mysql CLI) for durable structured storage and analysis" +read_when: + - You want to store/query large structured data in TiDB + - You use TiDB Cloud and have a Connect panel .env snippet +--- + +# TiDB tool + +Clawdbot ships a `tidb` tool that talks to **TiDB over the MySQL protocol** by invoking the local `mysql` CLI. + +Use it when you want results to be **durable and queryable** (analysis, reporting, large tables), not for small ephemeral notes. + +## Quick setup with TiDB Cloud (recommended) + +In TiDB Cloud, open your cluster **Connect** panel: + +1) Set **Connect With** ā†’ **.env** +2) Copy the block like: + +```bash +DB_HOST=gateway01.us-west-2.prod.aws.tidbcloud.com +DB_PORT=4000 +DB_USERNAME='your.cluster.id.root' +DB_PASSWORD='' +DB_DATABASE='test' +``` + +3) Paste it into the **gateway host** env file: + +```bash +cat >> ~/.clawdbot/.env <<'EOF' +DB_HOST=gateway01.us-west-2.prod.aws.tidbcloud.com +DB_PORT=4000 +DB_USERNAME='your.cluster.id.root' +DB_PASSWORD='' +DB_DATABASE='test' +EOF +``` + +4) Enable the tool in `~/.clawdbot/clawdbot.json`: + +```json5 +{ + tools: { + tidb: { enabled: true } + } +} +``` + +That is it. Clawdbot will read `DB_HOST/DB_PORT/DB_USERNAME/DB_PASSWORD/DB_DATABASE` automatically. + +Notes: +- For `*.tidbcloud.com` hosts, Clawdbot defaults to `mysql --ssl-mode=VERIFY_IDENTITY`. +- Credentials in env vars are visible to the gateway process; use a dedicated database user with minimal privileges. + +## Alternative: Connection string (copy, do not build by hand) + +If TiDB Cloud shows a **General** connection string like: + +```text +mysql://your.cluster.id.root:@gateway01.us-west-2.prod.aws.tidbcloud.com:4000/test +``` + +Put it directly into `~/.clawdbot/.env` as `TIDB_URL`: + +```bash +cat >> ~/.clawdbot/.env <<'EOF' +TIDB_URL=mysql://your.cluster.id.root:@gateway01.us-west-2.prod.aws.tidbcloud.com:4000/test +EOF +``` + +## Using the tool + +Call the `tidb` tool with: +- `sql` (required) +- `database` (optional override) +- `format`: `rows` (default, parses first result set) or `raw` +- `timeoutSeconds` (optional) + +Example: + +```sql +SELECT VERSION(); +``` + +See also: [Env vars](/help/faq#env-vars-and-env-loading). + diff --git a/skills/tidb-sql/SKILL.md b/skills/tidb-sql/SKILL.md new file mode 100644 index 000000000..cf7f7d443 --- /dev/null +++ b/skills/tidb-sql/SKILL.md @@ -0,0 +1,53 @@ +--- +name: tidb-sql +description: Write, review, and adapt SQL for TiDB with correct handling of TiDB-vs-MySQL differences (VECTOR type + vector indexes/functions, full-text search, AUTO_RANDOM, optimistic/pessimistic transactions, foreign keys, views, DDL limitations, and unsupported MySQL features like procedures/triggers/events/GEOMETRY/SPATIAL). Use when generating SQL that must run on TiDB, migrating MySQL SQL to TiDB, or debugging TiDB SQL compatibility errors. +metadata: {"clawdbot":{"emoji":"šŸ¬","requires":{"bins":["mysql"]}}} +--- + +# TiDB SQL (MySQL-compat-focused) + +Goal: generate SQL that runs correctly on TiDB by default, and avoid "works on MySQL but breaks on TiDB" constructs. + +## Workflow (use every time) + +1. Identify the target engine and version: + - Run `SELECT VERSION();` + - If the result contains `TiDB`, treat it as TiDB and parse the version (needed for feature gates like Vector / Foreign Key). + - If connecting to TiDB Cloud, ensure the client enables SSL with certificate + identity verification (see `skills/tidb-sql/references/tidb-cloud-ssl.md`). +2. Ask 2 quick capability questions if the request depends on them: + - "Do you have TiFlash?" (needed for vector indexes) + - "Is this TiDB Cloud Starter/Essential in a supported region for Full-Text Search?" (availability is limited) +3. Generate SQL using TiDB-safe defaults: + - Avoid unsupported MySQL features (procedures/triggers/events/UDF/GEOMETRY/SPATIAL, etc.) + - Treat views as read-only + - Treat primary key changes as migration/rebuild work +4. If the user provides MySQL SQL, do a compatibility pass: + - Replace unsupported features with TiDB alternatives + - Call out behavior differences and version prerequisites explicitly +5. If SQL is slow or fails unexpectedly, use TiDB-native diagnostics: + - Use `EXPLAIN FORMAT = "tidb_json"` for structured plans and operator trees. + - Use `EXPLAIN ANALYZE` to compare `estRows` vs `actRows` (it executes the query). + - If the plan looks wrong, consider `ANALYZE TABLE ...` to refresh statistics. + +## High-signal differences (keep in mind) + +- **Vector**: TiDB supports `VECTOR` / `VECTOR(D)` types and vector functions/indexes; MySQL does not. +- **No GEOMETRY/SPATIAL**: avoid `GEOMETRY`, spatial functions, and `SPATIAL` indexes. +- **No procedures / functions / triggers / events**: move logic to the application layer or an external scheduler. +- **Full-text search (TiDB feature)**: use TiDB full-text search SQL when available; don't assume MySQL `FULLTEXT` works everywhere. +- **Views are read-only**: no `UPDATE/INSERT/DELETE` against views. +- **Foreign keys**: supported in TiDB v6.6.0+; otherwise, don't rely on FK enforcement. +- **Primary key changes are restricted**: assume "create new table + backfill + swap" for PK changes. +- **AUTO_RANDOM**: prefer `AUTO_RANDOM` over `AUTO_INCREMENT` for write-hotspot avoidance when appropriate. +- **Transactions**: TiDB supports pessimistic and optimistic modes; handle optimistic `COMMIT` failures in application logic. + +## Use these references (inside this skill) + +- `skills/tidb-sql/references/vector.md` - VECTOR types, functions, vector index DDL, and query patterns. +- `skills/tidb-sql/references/full-text-search.md` - Full-text search SQL patterns and availability gotchas. +- `skills/tidb-sql/references/auto-random.md` - `AUTO_RANDOM` rules, DDL patterns, and restrictions. +- `skills/tidb-sql/references/transactions.md` - pessimistic vs optimistic mode and session/global knobs. +- `skills/tidb-sql/references/mysql-compatibility-notes.md` - other "MySQL vs TiDB" differences that commonly break SQL. +- `skills/tidb-sql/references/explain.md` - EXPLAIN / EXPLAIN ANALYZE usage, tidb_json and dot formats. +- `skills/tidb-sql/references/flashback.md` - FLASHBACK TABLE/DATABASE and FLASHBACK CLUSTER recovery playbooks. +- `skills/tidb-sql/references/tidb-cloud-ssl.md` - TiDB Cloud SSL verification requirements and client flags. diff --git a/skills/tidb-sql/references/auto-random.md b/skills/tidb-sql/references/auto-random.md new file mode 100644 index 000000000..444a76190 --- /dev/null +++ b/skills/tidb-sql/references/auto-random.md @@ -0,0 +1,40 @@ +--- +title: TiDB AUTO_RANDOM (SQL) +--- + +# TiDB AUTO_RANDOM (SQL) + +`AUTO_RANDOM` is used to avoid write hotspots that can happen with sequential keys in distributed storage. + +## When to prefer AUTO_RANDOM + +- When you would otherwise use `BIGINT AUTO_INCREMENT` as the primary key in a write-heavy workload. +- When you do not require strictly increasing IDs. + +## DDL patterns + +Valid forms (must be `BIGINT` and part of the primary key; typically first PK column): + +```sql +CREATE TABLE t (a BIGINT PRIMARY KEY AUTO_RANDOM, b VARCHAR(255)); +CREATE TABLE t (a BIGINT AUTO_RANDOM(6), b VARCHAR(255), PRIMARY KEY (a)); +``` + +Insert behavior: + +- If you omit the `AUTO_RANDOM` column in `INSERT`, TiDB generates a random unique value. +- If you specify it explicitly, TiDB inserts it as provided (but this is usually discouraged). + +## Operational gotchas + +- Explicit inserts can require enabling `@@allow_auto_random_explicit_insert = 1`. +- After explicit inserts in multi-node setups, you might need to "rebase" to avoid collisions: + +```sql +ALTER TABLE t AUTO_RANDOM_BASE = 0; +``` + +## Restrictions to remember + +- You cannot add/remove/modify the `AUTO_RANDOM` attribute later with `ALTER TABLE`. +- You cannot combine `AUTO_RANDOM` with `AUTO_INCREMENT` or `DEFAULT` on the same column. diff --git a/skills/tidb-sql/references/explain.md b/skills/tidb-sql/references/explain.md new file mode 100644 index 000000000..b6e3e98a8 --- /dev/null +++ b/skills/tidb-sql/references/explain.md @@ -0,0 +1,80 @@ +--- +title: TiDB EXPLAIN and EXPLAIN ANALYZE (Troubleshooting) +--- + +# TiDB EXPLAIN and EXPLAIN ANALYZE (Troubleshooting) + +Use `EXPLAIN` to see the plan without executing the query, and `EXPLAIN ANALYZE` to execute it and capture runtime stats. + +## Quick rules + +- If `EXPLAIN` looks "obviously wrong", run `ANALYZE TABLE ` on involved tables and re-check. +- Prefer `EXPLAIN FORMAT = "tidb_json"` when you need to programmatically inspect the operator tree. +- Prefer `EXPLAIN FORMAT = "dot"` when you need a visual operator graph (Graphviz). +- `EXPLAIN ANALYZE` executes the statement. Use carefully on production / heavy queries. +- TiDB does not support MySQL `FORMAT=JSON` or `FORMAT=TREE`. Use `FORMAT="tidb_json"` instead. + +## Default EXPLAIN columns (row format) + +TiDB `EXPLAIN` outputs these columns by default: `id`, `estRows`, `task`, `access object`, `operator info`. + +## Structured plan: FORMAT = "tidb_json" + +```sql +EXPLAIN FORMAT = "tidb_json" +SELECT /* your query */ 1; +``` + +The output is a JSON array. Each object can include: + +- `id`, `estRows`, `taskType`, `accessObject`, `operatorInfo` +- `subOperators`: array of child operators (tree structure) + +Tip: If a field is missing, it is empty. + +## Visual plan: FORMAT = "dot" (Graphviz) + +```sql +EXPLAIN FORMAT = "dot" +SELECT /* your query */ 1; +``` + +This returns a DOT graph string (starting with `digraph ... {`). + +### Render DOT to an image (optional) + +If `dot` (Graphviz) is installed locally: + +```bash +dot plan.dot -T png -O +``` + +If you want a helper script, see `skills/tidb-sql/scripts/render_dot_png.sh`. + +## EXPLAIN ANALYZE + +```sql +EXPLAIN ANALYZE +SELECT /* your query */ 1; +``` + +Compared to `EXPLAIN`, `EXPLAIN ANALYZE` adds runtime columns such as: + +- `actRows` +- `execution info` (time, loops, etc.) +- `memory`, `disk` + +Use it to compare `estRows` vs `actRows`. Large gaps usually indicate stale or missing statistics, skewed data, or predicates the optimizer cannot estimate well. + +Note: When you use `EXPLAIN ANALYZE` to execute DML statements, the data modifications are normally executed, and the execution plan for DML statements cannot be shown yet. + +## EXPLAIN FOR CONNECTION (advanced) + +TiDB supports: + +```sql +EXPLAIN FOR CONNECTION ; +``` + +Privilege note: in TiDB, to explain another connection you typically need `SUPER` (or be the same user/session). + diff --git a/skills/tidb-sql/references/flashback.md b/skills/tidb-sql/references/flashback.md new file mode 100644 index 000000000..415340253 --- /dev/null +++ b/skills/tidb-sql/references/flashback.md @@ -0,0 +1,86 @@ +--- +title: TiDB Flashback (Recover from Drops/Truncates) +--- + +# TiDB Flashback (Recover from Drops/Truncates) + +Use flashback to recover from accidental `DROP` / `TRUNCATE` if you catch it before GC permanently removes the historical versions. + +Important: Flashback is constrained by GC. Default `tidb_gc_life_time` is often short (for example, 10 minutes). Act quickly. + +## Before you try to recover + +1. Confirm you are on TiDB (not MySQL): `SELECT VERSION();` +2. Check the GC safe point: + +```sql +SELECT * FROM mysql.tidb WHERE variable_name = 'tikv_gc_safe_point'; +``` + +If the drop/truncate happened before the safe point, flashback cannot recover it. + +## FLASHBACK TABLE (TiDB v4.0+) + +Recover a dropped table: + +```sql +FLASHBACK TABLE t; +``` + +Recover a truncated table: + +- After `TRUNCATE`, the table name still exists, so you must recover to a new name: + +```sql +FLASHBACK TABLE t TO t_recovered; +``` + +Notes: + +- You cannot restore the same deleted table multiple times (the restored table reuses the same table ID). + +## FLASHBACK DATABASE (TiDB v6.4.0+) + +Recover a dropped database: + +```sql +FLASHBACK DATABASE test; +``` + +Recover and rename: + +```sql +FLASHBACK DATABASE test TO test_recovered; +``` + +Notes: + +- You cannot restore the same database multiple times (schema IDs must be globally unique). + +## FLASHBACK CLUSTER TO TIMESTAMP / TSO (high impact) + +Use this to restore the whole cluster to a specific point in time. + +Availability / safety gates: + +- Not applicable to TiDB Cloud Starter/Essential clusters. +- Requires `SUPER` privilege. +- Must be within GC lifetime. +- Do not specify a future timestamp/TSO. +- During execution, TiDB disconnects related connections and blocks reads/writes. It cannot be canceled once started. +- It writes old data forward with a new timestamp (it does not delete current data). Ensure enough storage space. +- If you use TiCDC, metadata rollbacks are not replicated; plan to pause changefeeds and reconcile schemas after. + +Syntax: + +```sql +FLASHBACK CLUSTER TO TIMESTAMP '2022-09-21 16:02:50'; +FLASHBACK CLUSTER TO TSO 445494839813079041; +``` + +Get a TSO for a precise point: + +```sql +SELECT @@tidb_current_ts; +``` + diff --git a/skills/tidb-sql/references/full-text-search.md b/skills/tidb-sql/references/full-text-search.md new file mode 100644 index 000000000..d75634ad4 --- /dev/null +++ b/skills/tidb-sql/references/full-text-search.md @@ -0,0 +1,63 @@ +--- +title: TiDB Full-Text Search (SQL) +--- + +# TiDB Full-Text Search (SQL) + +TiDB provides a full-text search feature that can replace MySQL-style keyword search use cases. + +## Availability gate (important) + +Full-text search availability can depend on TiDB Cloud tier/region and TiDB version. Always confirm your deployment's capability before relying on it. + +## Create a full-text index + +Create with table: + +```sql +CREATE TABLE stock_items( + id INT, + title TEXT, + FULLTEXT INDEX (title) WITH PARSER MULTILINGUAL +); +``` + +Or add to an existing table: + +```sql +ALTER TABLE stock_items + ADD FULLTEXT INDEX (title) WITH PARSER MULTILINGUAL + ADD_COLUMNAR_REPLICA_ON_DEMAND; +``` + +Parsers: + +- `STANDARD`: best for space/punctuation-delimited languages (often English) +- `MULTILINGUAL`: supports mixed languages (including CJK) + +Note: `ADD_COLUMNAR_REPLICA_ON_DEMAND` is used in the official examples for enabling full-text search indexing. If your TiDB deployment rejects this clause, remove it and follow the deployment-specific guidance. + +## Query with ranking + +Use `FTS_MATCH_WORD(query, column)` in both `WHERE` and `ORDER BY`: + +```sql +SELECT * +FROM stock_items +WHERE FTS_MATCH_WORD('bluetooth earbuds', title) +ORDER BY FTS_MATCH_WORD('bluetooth earbuds', title) DESC +LIMIT 10; +``` + +Count matches: + +```sql +SELECT COUNT(*) +FROM stock_items +WHERE FTS_MATCH_WORD('bluetooth earbuds', title); +``` + +## Migration note (MySQL FULLTEXT) + +Do not assume MySQL `FULLTEXT` behavior/availability carries over to TiDB in all deployments. +If the user says "use FULLTEXT", clarify whether they mean "MySQL FULLTEXT index" or "TiDB full-text search feature". diff --git a/skills/tidb-sql/references/mysql-compatibility-notes.md b/skills/tidb-sql/references/mysql-compatibility-notes.md new file mode 100644 index 000000000..602dd36a1 --- /dev/null +++ b/skills/tidb-sql/references/mysql-compatibility-notes.md @@ -0,0 +1,99 @@ +--- +title: MySQL to TiDB SQL Compatibility Notes (Common Breaks) +--- + +# MySQL to TiDB SQL Compatibility Notes (Common Breaks) + +Use this as a quick "lint list" when adapting MySQL SQL to TiDB. + +## Detect TiDB vs MySQL quickly + +```sql +SELECT VERSION(); +``` + +If the returned string contains `TiDB`, you are connected to TiDB and can infer the TiDB version from that string. + +## Unsupported or commonly unavailable features (avoid generating by default) + +Always confirm your TiDB version and deployment (TiDB Cloud tier/region vs self-managed) before relying on borderline features. + +- Stored procedures and stored functions +- Triggers +- Events (event scheduler) +- User-defined functions (UDF) +- `SPATIAL` / `GEOMETRY` functions, data types, and indexes +- XML functions +- `XA` syntax (TiDB uses 2PC internally but does not expose XA over SQL) +- `CREATE TABLE ... AS SELECT ...` (CTAS) +- `CHECK TABLE`, `CHECKSUM TABLE`, `REPAIR TABLE`, `OPTIMIZE TABLE` +- `HANDLER`, `CREATE TABLESPACE` +- Some advanced query syntaxes might be unsupported depending on TiDB version (examples seen in TiDB docs include `SKIP LOCKED`, lateral derived tables, and `JOIN ... ON (subquery)` patterns) + +## FULLTEXT: clarify intent + +- Do not assume MySQL `FULLTEXT` indexes work everywhere on TiDB. +- If the user needs keyword search, prefer TiDB full-text search when their deployment supports it (see `skills/tidb-sql/references/full-text-search.md`). + +## Views + +- Views are not updatable: do not emit `UPDATE/INSERT/DELETE` against views. + +## SELECT syntax edge cases + +- Do not emit `SELECT ... INTO @variable` (unsupported). +- In TiDB, `SELECT ... GROUP BY expr` does not imply `ORDER BY expr` (MySQL 5.7 behavior differs). If ordering matters, add an explicit `ORDER BY`. + +## Built-in functions (be defensive) + +- TiDB supports most MySQL built-ins, but not all. When porting SQL that uses non-trivial built-ins, validate availability with: + +```sql +SHOW BUILTINS; +``` + +## Charset/collation pitfalls + +- TiDB supports a limited set of character sets. If you see errors around charset/collation, stick to commonly supported sets like `utf8mb4` (and avoid exotic charsets). +- Default charset/collation can differ from MySQL: TiDB defaults are typically `utf8mb4` and `utf8mb4_bin`. If you depend on case-insensitive comparisons, set the collation explicitly. + +## Name casing pitfalls + +- TiDB supports `lower_case_table_names = 2` only (case-insensitive lookup behavior). Do not rely on two objects whose names differ only by letter case. + +## AUTO_INCREMENT pitfalls (and why AUTO_RANDOM is common on TiDB) + +- AUTO_INCREMENT IDs are globally unique in TiDB, but not necessarily sequential across nodes. Avoid mixing implicit IDs with custom explicit values. +- Removing `AUTO_INCREMENT` is possible (guarded by `tidb_allow_remove_auto_inc`), but adding it later is not supported. +- If you do not define a primary key, TiDB uses `_tidb_rowid`. Its allocator can interact with AUTO_INCREMENT in ways that surprise MySQL users. +- If you are designing a write-heavy schema, prefer `AUTO_RANDOM` for BIGINT PKs when it fits (see `skills/tidb-sql/references/auto-random.md`). + +## DDL / schema changes (be conservative, TiDB has extra restrictions) + +- Avoid "multi-change" `ALTER TABLE` that references the same column/index more than once in one statement. +- Avoid packing multiple TiDB-specific schema changes into one `ALTER TABLE` when possible (split them). +- Not all type changes are supported via `ALTER TABLE` (for unsupported changes, plan a backfill/migration). +- `ALGORITHM={INSTANT,INPLACE,COPY}` is treated as an assertion, not as an algorithm selector. +- Adding/dropping a clustered primary key can be unsupported; in practice, treat PK changes as "new table + backfill + swap". +- Index type decorations like `USING HASH|BTREE|RTREE|FULLTEXT` can be parsed but ignored. Do not rely on them to change behavior. + +## Partitioning notes (avoid fancy operations unless you know TiDB supports them) + +- Supported partitioning types include `HASH`, `RANGE`, `LIST`, and `KEY`. +- Some partition DDL operations are ignored, and `SUBPARTITION` is not supported. If you need advanced partition DDL, confirm support on your TiDB version first. + +## Optimizer / plan differences + +- `optimizer_switch` is read-only and does not affect TiDB plans. +- Optimizer hints are not a drop-in replacement for MySQL hints. Use `EXPLAIN` on TiDB to validate critical queries. + - For structured plans, use `EXPLAIN FORMAT = "tidb_json"` (see `skills/tidb-sql/references/explain.md`). + +## Timezone and timestamp defaults + +- TiDB supports named timezones based on system timezone rules; MySQL often requires timezone tables for named timezones. +- TiDB only supports `explicit_defaults_for_timestamp = ON`. If you are porting MySQL 5.7-era SQL that relies on implicit TIMESTAMP defaults, test carefully and set defaults explicitly. + +## Deprecated MySQL features you should avoid porting + +- Floating-point type precision specifiers (prefer `DECIMAL` when you need fixed precision). +- `ZEROFILL` (pad in the application layer instead). diff --git a/skills/tidb-sql/references/tidb-cloud-ssl.md b/skills/tidb-sql/references/tidb-cloud-ssl.md new file mode 100644 index 000000000..28df95787 --- /dev/null +++ b/skills/tidb-sql/references/tidb-cloud-ssl.md @@ -0,0 +1,91 @@ +--- +title: TiDB Cloud SSL Verification (Connection Gotchas) +--- + +# TiDB Cloud SSL Verification (Connection Gotchas) + +When connecting to TiDB Cloud over the MySQL protocol, enable TLS and enforce: + +- server certificate verification +- server identity (hostname) verification + +If SSL verification is missing or misconfigured, you might see connection failures even before you can run SQL. + +## Detect TiDB Cloud gateway hosts + +If the host matches this pattern (Python regex): + +```python +r"gateway\\d{2}\\.(.+)\\.(prod|dev|staging)\\.(shared\\.)?(aws|alicloud)\\.tidbcloud\\.com" +``` + +assume TiDB Cloud requirements apply. + +## Common client settings + +MySQL CLI: + +```bash +mysql ... --ssl-mode=VERIFY_IDENTITY +``` + +MariaDB CLI: + +```bash +mariadb ... --ssl-verify-server-cert +``` + +Node.js (mysql2): + +```js +ssl: { + minVersion: 'TLSv1.2', + rejectUnauthorized: true, +}, +``` + +Prisma (connection string query param): + +```text +sslaccept=strict +``` + +Go (database/sql + go-sql-driver/mysql): + +Register a TLS config and reference it via `tls=` in the DSN: + +```go +mysql.RegisterTLSConfig("tidb", &tls.Config{ + MinVersion: tls.VersionTLS12, + ServerName: "gateway01.ap-southeast-1.prod.aws.tidbcloud.com", +}) + +db, err := sql.Open( + "mysql", + "USER:PASSWORD@tcp(gateway01.ap-southeast-1.prod.aws.tidbcloud.com:4000)/DB?tls=tidb", +) +``` + +Rails (ActiveRecord + mysql2, database.yml URL): + +```yaml +development: + adapter: mysql2 + url: mysql2://USER:PASSWORD@gateway01.ap-southeast-1.prod.aws.tidbcloud.com:4000/DB?ssl_mode=verify_identity +``` + +MySQL Connector/J (JDBC URL params): + +```text +sslMode=VERIFY_IDENTITY +``` + +## DSN / URL query parameters (generic) + +If you are building a connection string and can only toggle via query parameters, append: + +```text +ssl_verify_cert=true&ssl_verify_identity=true +``` + +Use this especially when the host matches the gateway pattern above. diff --git a/skills/tidb-sql/references/transactions.md b/skills/tidb-sql/references/transactions.md new file mode 100644 index 000000000..be236f37f --- /dev/null +++ b/skills/tidb-sql/references/transactions.md @@ -0,0 +1,40 @@ +--- +title: TiDB Transactions (Optimistic vs Pessimistic) +--- + +# TiDB Transactions (Optimistic vs Pessimistic) + +TiDB supports both pessimistic and optimistic transaction modes. +MySQL/InnoDB users typically expect pessimistic behavior by default. + +## Choose a mode + +- Prefer **pessimistic** when conflicts are common or when the application cannot safely retry on commit failures. +- Consider **optimistic** when write-write conflicts are rare and you can handle commit failures in the app. + +## Set default mode (cluster-wide) + +```sql +SET GLOBAL tidb_txn_mode = 'pessimistic'; +-- or: +SET GLOBAL tidb_txn_mode = 'optimistic'; +``` + +## Force mode per transaction + +```sql +BEGIN PESSIMISTIC; +-- ... DML ... +COMMIT; +``` + +```sql +BEGIN OPTIMISTIC; +-- ... DML ... +COMMIT; +``` + +## App-level rule (optimistic) + +If you generate SQL intended for optimistic transactions, require the caller/application to handle `COMMIT` errors and retry the whole transaction safely (idempotency + retry loop). + diff --git a/skills/tidb-sql/references/vector.md b/skills/tidb-sql/references/vector.md new file mode 100644 index 000000000..9c20d0da1 --- /dev/null +++ b/skills/tidb-sql/references/vector.md @@ -0,0 +1,95 @@ +--- +title: TiDB Vector SQL (Types, Functions, Indexes) +--- + +# TiDB Vector SQL (Types, Functions, Indexes) + +## Feature gate + +- Vector data types and functions require TiDB v8.4.0+ (v8.5.0+ recommended for self-managed/dedicated deployments). +- Confirm with `SELECT VERSION();`. + +## Data types + +- `VECTOR`: variable dimension (cannot build a vector index on it) +- `VECTOR(D)`: fixed dimension `D` (required for vector index) + +Example: + +```sql +CREATE TABLE embedded_documents ( + id INT PRIMARY KEY, + document TEXT, + embedding VECTOR(3) +); +``` + +Insert vector literals as strings: + +```sql +INSERT INTO embedded_documents VALUES (1, 'dog', '[1,2,1]'); +``` + +## Distance functions (common) + +- `VEC_COSINE_DISTANCE(v1, v2)` +- `VEC_L2_DISTANCE(v1, v2)` +- (Also exists: `VEC_L1_DISTANCE`, `VEC_NEGATIVE_INNER_PRODUCT`) + +Example query (exact scan): + +```sql +SELECT id, document, VEC_COSINE_DISTANCE(embedding, '[1,2,3]') AS distance +FROM embedded_documents +ORDER BY distance +LIMIT 10; +``` + +## Cast / parsing helpers + +- `VEC_FROM_TEXT('[...]')` - string -> vector +- `VEC_AS_TEXT(vec)` - vector -> string +- `CAST('[...]' AS VECTOR)` - string -> vector + +Tip: If you compare vector constants, cast explicitly to avoid string-based comparisons. + +## Vector index (HNSW) essentials + +Prerequisites / constraints: + +- Requires TiFlash nodes (and TiFlash replica for the table). +- Cannot be `PRIMARY KEY` or `UNIQUE`. +- Single vector column only (no composite vector+other-column index). +- Must use the same distance function in both index definition and query ordering. + +Create index at table creation time: + +```sql +CREATE TABLE foo ( + id INT PRIMARY KEY, + embedding VECTOR(3), + VECTOR INDEX idx_embedding ((VEC_COSINE_DISTANCE(embedding))) +); +``` + +Create index on an existing table: + +```sql +CREATE VECTOR INDEX idx_embedding ON foo ((VEC_COSINE_DISTANCE(embedding))) USING HNSW; +-- or: +ALTER TABLE foo ADD VECTOR INDEX idx_embedding ((VEC_COSINE_DISTANCE(embedding))) USING HNSW; +``` + +Query pattern to use the ANN index: + +- Use `ORDER BY VEC_COSINE_DISTANCE(...) ASC LIMIT ` (Top-K must be present) +- Desc order or mismatched distance function prevents index usage + +Validate index usage: + +```sql +EXPLAIN SELECT * FROM foo +ORDER BY VEC_COSINE_DISTANCE(embedding, '[1,2,3]') +LIMIT 10; +SHOW WARNINGS; +``` diff --git a/skills/tidb-sql/scripts/render_dot_png.sh b/skills/tidb-sql/scripts/render_dot_png.sh new file mode 100755 index 000000000..e44fe43b6 --- /dev/null +++ b/skills/tidb-sql/scripts/render_dot_png.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 || $# -gt 2 ]]; then + echo "Usage: $0 [output.png]" >&2 + exit 2 +fi + +dot_file="$1" +out_png="${2:-}" + +if [[ ! -f "$dot_file" ]]; then + echo "File not found: $dot_file" >&2 + exit 2 +fi + +if ! command -v dot >/dev/null 2>&1; then + echo "Graphviz 'dot' not found. Install graphviz, then re-run." >&2 + echo "Alternatively: copy DOT contents to a web-based Graphviz renderer." >&2 + exit 1 +fi + +if [[ -z "$out_png" ]]; then + dot "$dot_file" -T png -O +else + dot "$dot_file" -T png -o "$out_png" +fi + diff --git a/src/agents/clawdbot-tools.tidb.test.ts b/src/agents/clawdbot-tools.tidb.test.ts new file mode 100644 index 000000000..156b6870e --- /dev/null +++ b/src/agents/clawdbot-tools.tidb.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; + +import type { ClawdbotConfig } from "../config/config.js"; +import { createClawdbotTools } from "./clawdbot-tools.js"; + +describe("createClawdbotTools (tidb)", () => { + it("omits tidb tool when disabled", () => { + const cfg: ClawdbotConfig = { + tools: { + tidb: { + enabled: false, + url: "tidb://user:pass@example.com/mydb", + }, + }, + }; + const tools = createClawdbotTools({ config: cfg }); + expect(tools.some((tool) => tool.name === "tidb")).toBe(false); + }); + + it("adds tidb tool when enabled + configured", () => { + const cfg: ClawdbotConfig = { + tools: { + tidb: { + enabled: true, + url: "tidb://user:pass@example.com/mydb", + }, + }, + }; + const tools = createClawdbotTools({ config: cfg }); + expect(tools.some((tool) => tool.name === "tidb")).toBe(true); + }); + + it("adds tidb tool when enabled (even if url is missing)", () => { + const cfg: ClawdbotConfig = { + tools: { + tidb: { + enabled: true, + }, + }, + }; + const tools = createClawdbotTools({ config: cfg }); + expect(tools.some((tool) => tool.name === "tidb")).toBe(true); + }); +}); diff --git a/src/agents/clawdbot-tools.ts b/src/agents/clawdbot-tools.ts index 995925cfe..0388833b7 100644 --- a/src/agents/clawdbot-tools.ts +++ b/src/agents/clawdbot-tools.ts @@ -16,6 +16,7 @@ import { createSessionsHistoryTool } from "./tools/sessions-history-tool.js"; import { createSessionsListTool } from "./tools/sessions-list-tool.js"; import { createSessionsSendTool } from "./tools/sessions-send-tool.js"; import { createSessionsSpawnTool } from "./tools/sessions-spawn-tool.js"; +import { createTiDbTool } from "./tools/tidb-tool.js"; import { createWebFetchTool, createWebSearchTool } from "./tools/web-tools.js"; import { createTtsTool } from "./tools/tts-tool.js"; @@ -70,6 +71,7 @@ export function createClawdbotTools(options?: { config: options?.config, sandboxed: options?.sandboxed, }); + const tidbTool = createTiDbTool({ config: options?.config }); const tools: AnyAgentTool[] = [ createBrowserTool({ sandboxBridgeUrl: options?.sandboxBrowserBridgeUrl, @@ -136,6 +138,7 @@ export function createClawdbotTools(options?: { }), ...(webSearchTool ? [webSearchTool] : []), ...(webFetchTool ? [webFetchTool] : []), + ...(tidbTool ? [tidbTool] : []), ...(imageTool ? [imageTool] : []), ]; diff --git a/src/agents/tool-display.json b/src/agents/tool-display.json index 3fea81405..6db1dd65d 100644 --- a/src/agents/tool-display.json +++ b/src/agents/tool-display.json @@ -296,6 +296,11 @@ "title": "Web Fetch", "detailKeys": ["url", "extractMode", "maxChars"] }, + "tidb": { + "emoji": "šŸ¬", + "title": "TiDB", + "detailKeys": ["database", "format", "timeoutSeconds"] + }, "whatsapp_login": { "emoji": "šŸŸ¢", "title": "WhatsApp Login", diff --git a/src/agents/tool-policy.ts b/src/agents/tool-policy.ts index 85152069e..86a9b8b62 100644 --- a/src/agents/tool-policy.ts +++ b/src/agents/tool-policy.ts @@ -14,6 +14,7 @@ export const TOOL_GROUPS: Record = { // NOTE: Keep canonical (lowercase) tool names here. "group:memory": ["memory_search", "memory_get"], "group:web": ["web_search", "web_fetch"], + "group:db": ["tidb"], // Basic workspace/file tools "group:fs": ["read", "write", "edit", "apply_patch"], // Host/runtime execution tools @@ -52,6 +53,7 @@ export const TOOL_GROUPS: Record = { "memory_get", "web_search", "web_fetch", + "tidb", "image", ], }; diff --git a/src/agents/tools/tidb-tool.test.ts b/src/agents/tools/tidb-tool.test.ts new file mode 100644 index 000000000..30d3546e7 --- /dev/null +++ b/src/agents/tools/tidb-tool.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; + +import { parseTidbUrl, resolveTidbSslMode } from "./tidb-tool.js"; + +describe("tidb tool url parsing", () => { + it("parses tidb:// with default port 4000", () => { + const parsed = parseTidbUrl("tidb://user:pass@example.com/mydb"); + expect(parsed.scheme).toBe("tidb"); + expect(parsed.host).toBe("example.com"); + expect(parsed.port).toBe(4000); + expect(parsed.user).toBe("user"); + expect(parsed.password).toBe("pass"); + expect(parsed.database).toBe("mydb"); + }); + + it("parses mysql:// with default port 3306", () => { + const parsed = parseTidbUrl("mysql://user@example.com/mydb"); + expect(parsed.scheme).toBe("mysql"); + expect(parsed.port).toBe(3306); + }); + + it("passes through query params and resolves sslMode", () => { + const parsed = parseTidbUrl( + "tidb://user:pass@example.com:4000/mydb?sslMode=verify_identity&connectTimeout=10", + ); + expect(parsed.params.sslMode).toBe("verify_identity"); + expect(resolveTidbSslMode(parsed)).toBe("VERIFY_IDENTITY"); + }); + + it("infers TiDB Cloud SSL defaults when unset", () => { + const parsed = parseTidbUrl("tidb://user:pass@gateway01.foo.prod.aws.tidbcloud.com/mydb"); + expect(resolveTidbSslMode(parsed)).toBe("VERIFY_IDENTITY"); + }); + + it("infers TiDB Cloud SSL defaults for mysql:// urls too", () => { + const parsed = parseTidbUrl("mysql://user:pass@gateway01.foo.prod.aws.tidbcloud.com:4000/mydb"); + expect(resolveTidbSslMode(parsed)).toBe("VERIFY_IDENTITY"); + }); +}); diff --git a/src/agents/tools/tidb-tool.ts b/src/agents/tools/tidb-tool.ts new file mode 100644 index 000000000..e2df370c8 --- /dev/null +++ b/src/agents/tools/tidb-tool.ts @@ -0,0 +1,474 @@ +import { spawn } from "node:child_process"; + +import { Type } from "@sinclair/typebox"; + +import type { ClawdbotConfig } from "../../config/config.js"; +import { sanitizeBinaryOutput } from "../shell-utils.js"; +import { optionalStringEnum } from "../schema/typebox.js"; +import type { AnyAgentTool } from "./common.js"; +import { jsonResult, readNumberParam, readStringParam } from "./common.js"; + +const OUTPUT_FORMATS = ["rows", "raw"] as const; + +const TiDbSchema = Type.Object({ + sql: Type.String({ + description: + "SQL to execute on TiDB via the mysql CLI. Use this for saving/querying large structured datasets (analysis, reporting, durable storage).", + }), + database: Type.Optional( + Type.String({ + description: "Optional database/schema name override (defaults to URL pathname).", + }), + ), + format: optionalStringEnum(OUTPUT_FORMATS, { + description: + 'Output format: "rows" parses the first result set as TSV into JSON rows; "raw" returns stdout/stderr strings.', + default: "rows", + }), + timeoutSeconds: Type.Optional( + Type.Number({ + description: + "Timeout for this mysql invocation in seconds (overrides tools.tidb.timeoutSeconds).", + minimum: 1, + maximum: 600, + }), + ), +}); + +export type ParsedTidbUrl = { + scheme: "tidb" | "mysql"; + host: string; + port: number; + user?: string; + password?: string; + database?: string; + params: Record; +}; + +function decodeUrlComponent(value: string): string { + try { + return decodeURIComponent(value); + } catch { + return value; + } +} + +export function parseTidbUrl(raw: string): ParsedTidbUrl { + const trimmed = raw.trim(); + if (!trimmed) throw new Error("TiDB URL is empty."); + + let parsed: URL; + try { + parsed = new URL(trimmed); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new Error(`Invalid TiDB URL: ${message}`); + } + + const protocol = parsed.protocol.toLowerCase(); + const scheme: ParsedTidbUrl["scheme"] = + protocol === "tidb:" ? "tidb" : protocol === "mysql:" ? "mysql" : "mysql"; + if (protocol !== "tidb:" && protocol !== "mysql:") { + throw new Error(`Unsupported TiDB URL scheme "${parsed.protocol}" (use tidb:// or mysql://).`); + } + + const host = parsed.hostname.trim(); + if (!host) throw new Error("TiDB URL is missing hostname."); + + const defaultPort = scheme === "tidb" ? 4000 : 3306; + const port = parsed.port ? Number.parseInt(parsed.port, 10) : defaultPort; + if (!Number.isFinite(port) || port <= 0) throw new Error("TiDB URL has an invalid port."); + + const user = parsed.username ? decodeUrlComponent(parsed.username) : undefined; + const password = parsed.password ? decodeUrlComponent(parsed.password) : undefined; + const database = + parsed.pathname && parsed.pathname !== "/" + ? decodeUrlComponent(parsed.pathname.slice(1)) + : undefined; + + const params: Record = {}; + for (const [key, value] of parsed.searchParams.entries()) { + const k = key.trim(); + if (!k) continue; + params[k] = value; + } + + return { scheme, host, port, user, password, database, params }; +} + +function readBoolParam(params: Record, ...keys: string[]): boolean | undefined { + for (const key of keys) { + if (!(key in params)) continue; + const raw = params[key]?.trim().toLowerCase(); + if (!raw) return undefined; + if (raw === "1" || raw === "true" || raw === "yes" || raw === "on") return true; + if (raw === "0" || raw === "false" || raw === "no" || raw === "off") return false; + } + return undefined; +} + +function readStringParamFromMap( + params: Record, + ...keys: string[] +): string | undefined { + for (const key of keys) { + const value = params[key]; + if (typeof value !== "string") continue; + const trimmed = value.trim(); + if (trimmed) return trimmed; + } + return undefined; +} + +export function resolveTidbSslMode(parsed: ParsedTidbUrl): string | undefined { + const explicit = readStringParamFromMap( + parsed.params, + "ssl-mode", + "sslMode", + "sslmode", + "ssl_mode", + ); + if (explicit) return explicit.toUpperCase(); + + const verifyIdentity = readBoolParam( + parsed.params, + "ssl_verify_identity", + "sslVerifyIdentity", + "ssl-verify-identity", + ); + if (verifyIdentity === true) return "VERIFY_IDENTITY"; + + const verifyCert = readBoolParam( + parsed.params, + "ssl_verify_cert", + "sslVerifyCert", + "ssl-verify-cert", + ); + if (verifyCert === true) return "VERIFY_CA"; + + const ssl = readBoolParam(parsed.params, "ssl", "tls"); + if (ssl === true) return "REQUIRED"; + + // TiDB Cloud gateways commonly require TLS + hostname verification. + if (parsed.host.toLowerCase().endsWith(".tidbcloud.com")) { + return "VERIFY_IDENTITY"; + } + + return undefined; +} + +function truncate(text: string, maxChars: number): string { + const normalized = sanitizeBinaryOutput(text); + if (normalized.length <= maxChars) return normalized; + return `${normalized.slice(0, maxChars)}\nā€¦ (truncated ${normalized.length - maxChars} chars)`; +} + +function truncateSqlPreview(sql: string, maxChars = 4000): string { + const normalized = sanitizeBinaryOutput(sql); + if (normalized.length <= maxChars) return normalized; + return `${normalized.slice(0, maxChars)}\nā€¦ (sql truncated ${normalized.length - maxChars} chars)`; +} + +function parseTsvToRows(stdout: string): { + columns: string[]; + rows: Array>; +} { + const lines = stdout + .split(/\r?\n/g) + .map((line) => line.trimEnd()) + .filter((line) => line.length > 0); + if (lines.length === 0) return { columns: [], rows: [] }; + + const header = lines[0].split("\t"); + const columns = header.map((c) => c.trim()); + const rows: Array> = []; + + for (const line of lines.slice(1)) { + const values = line.split("\t"); + const row: Record = {}; + for (let index = 0; index < columns.length; index += 1) { + const key = columns[index] || `col_${index + 1}`; + const value = values[index]; + row[key] = value === undefined ? null : value; + } + rows.push(row); + } + + return { columns, rows }; +} + +type TiDbToolConfig = NonNullable["tidb"]; + +function resolveTiDbToolConfig(cfg?: ClawdbotConfig): TiDbToolConfig | undefined { + const tidb = cfg?.tools?.tidb; + if (!tidb || typeof tidb !== "object") return undefined; + return tidb as TiDbToolConfig; +} + +function resolveTidbUrlFromConfig(cfg?: TiDbToolConfig): string | undefined { + const fromConfig = typeof cfg?.url === "string" ? cfg.url.trim() : ""; + return fromConfig || undefined; +} + +function stripOuterQuotes(value: string): string { + const trimmed = value.trim(); + if (trimmed.length < 2) return trimmed; + const first = trimmed[0]; + const last = trimmed[trimmed.length - 1]; + if ((first === "'" && last === "'") || (first === '"' && last === '"')) { + return trimmed.slice(1, -1); + } + return trimmed; +} + +function readEnvString(...keys: string[]): string | undefined { + for (const key of keys) { + const raw = process.env[key]; + if (typeof raw !== "string") continue; + const cleaned = stripOuterQuotes(raw); + if (cleaned) return cleaned; + } + return undefined; +} + +function buildTidbUrlFromEnv(): string | undefined { + // TiDB Cloud "Connect with .env" defaults to these keys. + const host = readEnvString("DB_HOST", "TIDB_HOST", "MYSQL_HOST"); + const portRaw = readEnvString("DB_PORT", "TIDB_PORT", "MYSQL_PORT"); + const user = readEnvString("DB_USERNAME", "TIDB_USERNAME", "MYSQL_USER", "MYSQL_USERNAME"); + const password = readEnvString("DB_PASSWORD", "TIDB_PASSWORD", "MYSQL_PASSWORD"); + const database = readEnvString("DB_DATABASE", "TIDB_DATABASE", "MYSQL_DATABASE", "MYSQL_DB"); + + if (!host || !user) return undefined; + + const port = portRaw ? Number.parseInt(portRaw, 10) : 4000; + if (!Number.isFinite(port) || port <= 0) return undefined; + + const auth = + password && password.length > 0 + ? `${encodeURIComponent(user)}:${encodeURIComponent(password)}` + : encodeURIComponent(user); + const dbPath = database ? `/${encodeURIComponent(database)}` : ""; + + // Use mysql:// since TiDB Cloud provides a MySQL-protocol DSN. + return `mysql://${auth}@${host}:${port}${dbPath}`; +} + +function resolveTidbUrl(cfg?: TiDbToolConfig): string | undefined { + const fromConfig = resolveTidbUrlFromConfig(cfg); + if (fromConfig) return fromConfig; + + const fromEnvUrl = stripOuterQuotes(process.env.TIDB_URL ?? "").trim(); + if (fromEnvUrl) return fromEnvUrl; + + return buildTidbUrlFromEnv(); +} + +function resolveTidbCommand(cfg?: TiDbToolConfig): string { + const raw = typeof cfg?.command === "string" ? cfg.command.trim() : ""; + return raw || "mysql"; +} + +function resolveDefaultTimeoutSeconds(cfg?: TiDbToolConfig): number { + const raw = typeof cfg?.timeoutSeconds === "number" ? cfg.timeoutSeconds : undefined; + if (typeof raw === "number" && Number.isFinite(raw) && raw > 0) return Math.trunc(raw); + return 30; +} + +function resolveMaxOutputChars(cfg?: TiDbToolConfig): number { + const raw = typeof cfg?.maxOutputChars === "number" ? cfg.maxOutputChars : undefined; + if (typeof raw === "number" && Number.isFinite(raw) && raw > 0) return Math.trunc(raw); + return 60_000; +} + +export function createTiDbTool(options: { config?: ClawdbotConfig }): AnyAgentTool | null { + const cfg = options.config; + if (!cfg) return null; + + const tidbCfg = resolveTiDbToolConfig(cfg); + if (!tidbCfg?.enabled) return null; + + const command = resolveTidbCommand(tidbCfg); + const maxOutputChars = resolveMaxOutputChars(tidbCfg); + + return { + label: "TiDB", + name: "tidb", + description: + "Query or persist large structured data in TiDB (MySQL protocol) using the mysql CLI. Use this when results should be durable/queryable (analysis, reporting, large tables), not for small ephemeral notes.", + parameters: TiDbSchema, + execute: async (_toolCallId, params) => { + const url = resolveTidbUrl(tidbCfg); + if (!url) { + return jsonResult({ + error: "missing_tidb_config", + message: + "TiDB tool is enabled but not configured. Use TiDB Cloud Connect panel and paste its .env output into the gateway environment (recommended: ~/.clawdbot/.env), or set tools.tidb.url / TIDB_URL.", + acceptedEnv: + "DB_HOST, DB_PORT, DB_USERNAME, DB_PASSWORD, DB_DATABASE (TiDB Cloud .env) or TIDB_URL.", + docs: "https://docs.clawd.bot/tools/tidb", + }); + } + + const sql = readStringParam(params, "sql", { required: true, trim: false }); + const databaseOverride = readStringParam(params, "database"); + const format = readStringParam(params, "format") as + | (typeof OUTPUT_FORMATS)[number] + | undefined; + const timeoutSecondsOverride = readNumberParam(params, "timeoutSeconds", { integer: true }); + + const parsed = parseTidbUrl(url); + if (!parsed.user) { + throw new Error( + "TiDB connection must include a username (from tools.tidb.url/TIDB_URL or DB_USERNAME/TIDB_USERNAME).", + ); + } + + const database = (databaseOverride?.trim() || parsed.database || "").trim() || undefined; + const sslMode = resolveTidbSslMode(parsed); + const sslCa = readStringParamFromMap(parsed.params, "ssl-ca", "sslCa", "ssl_ca"); + const sslCert = readStringParamFromMap(parsed.params, "ssl-cert", "sslCert", "ssl_cert"); + const sslKey = readStringParamFromMap(parsed.params, "ssl-key", "sslKey", "ssl_key"); + + const connectTimeoutRaw = readStringParamFromMap( + parsed.params, + "connect-timeout", + "connectTimeout", + "connect_timeout", + ); + const connectTimeout = connectTimeoutRaw ? Number.parseInt(connectTimeoutRaw, 10) : undefined; + + const argv: string[] = [ + command, + "--host", + parsed.host, + "--port", + String(parsed.port), + "--user", + parsed.user, + "--protocol", + "tcp", + "--batch", + "--raw", + ]; + if (database) argv.push("--database", database); + if (sslMode) argv.push("--ssl-mode", sslMode); + if (sslCa) argv.push("--ssl-ca", sslCa); + if (sslCert) argv.push("--ssl-cert", sslCert); + if (sslKey) argv.push("--ssl-key", sslKey); + if ( + typeof connectTimeout === "number" && + Number.isFinite(connectTimeout) && + connectTimeout > 0 + ) { + argv.push("--connect-timeout", String(connectTimeout)); + } + + const outputFormat = format ?? "rows"; + if (outputFormat === "raw") { + argv.push("--skip-column-names"); + } else { + argv.push("--column-names"); + } + + argv.push("--execute", sql); + + const timeoutSeconds = Math.max( + 1, + Math.min(600, timeoutSecondsOverride ?? resolveDefaultTimeoutSeconds(tidbCfg)), + ); + + const env: Record = { ...process.env } as Record; + if (parsed.password) { + env.MYSQL_PWD = parsed.password; + } + + const startedAt = Date.now(); + const child = spawn(argv[0], argv.slice(1), { stdio: ["ignore", "pipe", "pipe"], env }); + + let stdout = ""; + let stderr = ""; + let spawnError: string | null = null; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.once("error", (err) => { + spawnError = err instanceof Error ? err.message : String(err); + }); + + const timedOut = await new Promise((resolve) => { + let settled = false; + const finish = (value: boolean) => { + if (settled) return; + settled = true; + resolve(value); + }; + const timer = setTimeout(() => { + try { + child.kill("SIGKILL"); + } catch { + // ignore + } + finish(true); + }, timeoutSeconds * 1000); + const cleanup = () => { + clearTimeout(timer); + child.removeListener("exit", onExit); + child.removeListener("error", onError); + }; + const onExit = () => { + cleanup(); + finish(false); + }; + const onError = () => { + cleanup(); + finish(false); + }; + child.once("exit", onExit); + child.once("error", onError); + }); + + const exitCode = typeof child.exitCode === "number" ? child.exitCode : null; + const durationMs = Date.now() - startedAt; + + const stdoutTrimmed = truncate(stdout, maxOutputChars); + const stderrTrimmed = truncate(stderr, maxOutputChars); + + const base = { + ok: !timedOut && !spawnError && exitCode === 0, + timedOut, + spawnError, + exitCode, + durationMs, + connection: { + scheme: parsed.scheme, + host: parsed.host, + port: parsed.port, + user: parsed.user, + database, + sslMode: sslMode ?? null, + }, + command: argv[0], + sqlPreview: truncateSqlPreview(sql), + stdout: stdoutTrimmed, + stderr: stderrTrimmed, + } as const; + + if (outputFormat !== "rows") { + return jsonResult(base); + } + + const parsedRows = parseTsvToRows(stdoutTrimmed); + return jsonResult({ + ...base, + columns: parsedRows.columns, + rows: parsedRows.rows, + }); + }, + }; +} diff --git a/src/config/schema.ts b/src/config/schema.ts index 69846236e..d94093218 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -180,6 +180,11 @@ const FIELD_LABELS: Record = { "tools.exec.node": "Exec Node Binding", "tools.exec.pathPrepend": "Exec PATH Prepend", "tools.exec.safeBins": "Exec Safe Bins", + "tools.tidb.enabled": "Enable TiDB Tool", + "tools.tidb.url": "TiDB URL", + "tools.tidb.command": "TiDB mysql CLI Command", + "tools.tidb.timeoutSeconds": "TiDB Timeout (sec)", + "tools.tidb.maxOutputChars": "TiDB Max Output Chars", "tools.message.allowCrossContextSend": "Allow Cross-Context Messaging", "tools.message.crossContext.allowWithinProvider": "Allow Cross-Context (Same Provider)", "tools.message.crossContext.allowAcrossProviders": "Allow Cross-Context (Across Providers)", @@ -420,6 +425,14 @@ const FIELD_HELP: Record = { "tools.exec.pathPrepend": "Directories to prepend to PATH for exec runs (gateway/sandbox).", "tools.exec.safeBins": "Allow stdin-only safe binaries to run without explicit allowlist entries.", + "tools.tidb.enabled": + "Enable the tidb tool (mysql CLI-backed). Intended for saving/querying large structured datasets when it makes sense to persist results to a database.", + "tools.tidb.url": + 'TiDB/MySQL connection URL (supports "tidb://" and "mysql://"). Prefer using env substitution (e.g. "${TIDB_URL}") to avoid storing credentials in plaintext.', + "tools.tidb.command": 'mysql client binary name/path (default: "mysql").', + "tools.tidb.timeoutSeconds": "Default timeout in seconds for tidb tool mysql CLI runs.", + "tools.tidb.maxOutputChars": + "Max characters returned from mysql stdout/stderr in tidb tool results (default: 60000).", "tools.message.allowCrossContextSend": "Legacy override: allow cross-context sends across all providers.", "tools.message.crossContext.allowWithinProvider": diff --git a/src/config/types.tools.ts b/src/config/types.tools.ts index bb1d45bf0..f1d0bd575 100644 --- a/src/config/types.tools.ts +++ b/src/config/types.tools.ts @@ -126,6 +126,24 @@ export type LinkToolsConfig = { models?: LinkModelConfig[]; }; +export type TiDbToolConfig = { + /** Enable TiDB tool (default: false). */ + enabled?: boolean; + /** + * Connection URL for TiDB/MySQL protocol. + * + * Supports `tidb://` and `mysql://` schemes. + * Prefer using env substitution (e.g. "${TIDB_URL}") to avoid storing credentials in plaintext. + */ + url?: string; + /** mysql client binary name/path (default: "mysql"). */ + command?: string; + /** Default timeout in seconds for mysql CLI invocations. */ + timeoutSeconds?: number; + /** Max stdout/stderr characters returned in tool results (default: 60k). */ + maxOutputChars?: number; +}; + export type MediaToolsConfig = { /** Shared model list applied across image/audio/video. */ models?: MediaUnderstandingModelConfig[]; @@ -429,6 +447,8 @@ export type ToolsConfig = { }; /** Exec tool defaults. */ exec?: ExecToolConfig; + /** TiDB (MySQL protocol) query tool via mysql CLI. */ + tidb?: TiDbToolConfig; /** Sub-agent tool policy defaults (deny wins). */ subagents?: { /** Default model selection for spawned sub-agents (string or {primary,fallbacks}). */ diff --git a/src/config/zod-schema.agent-runtime.ts b/src/config/zod-schema.agent-runtime.ts index 7a63e307d..c2dc999db 100644 --- a/src/config/zod-schema.agent-runtime.ts +++ b/src/config/zod-schema.agent-runtime.ts @@ -521,6 +521,16 @@ export const ToolsSchema = z }) .strict() .optional(), + tidb: z + .object({ + enabled: z.boolean().optional(), + url: z.string().optional(), + command: z.string().optional(), + timeoutSeconds: z.number().int().positive().optional(), + maxOutputChars: z.number().int().positive().optional(), + }) + .strict() + .optional(), subagents: z .object({ tools: ToolPolicySchema,