diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index df5dfdd73..d64ce3860 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -314,7 +314,7 @@ importers: specifier: ^10.5.0 version: 10.5.0 devDependencies: - moltbot: + openclaw: specifier: workspace:* version: link:../.. @@ -322,7 +322,7 @@ importers: extensions/line: devDependencies: - moltbot: + openclaw: specifier: workspace:* version: link:../.. @@ -348,7 +348,7 @@ importers: specifier: ^4.3.6 version: 4.3.6 devDependencies: - moltbot: + openclaw: specifier: workspace:* version: link:../.. @@ -356,7 +356,7 @@ importers: extensions/memory-core: devDependencies: - moltbot: + openclaw: specifier: workspace:* version: link:../.. @@ -386,7 +386,7 @@ importers: express: specifier: ^5.2.1 version: 5.2.1 - moltbot: + openclaw: specifier: workspace:* version: link:../.. proper-lockfile: @@ -397,12 +397,12 @@ importers: extensions/nostr: dependencies: - moltbot: - specifier: workspace:* - version: link:../.. nostr-tools: specifier: ^2.20.0 version: 2.20.0(typescript@5.9.3) + openclaw: + specifier: workspace:* + version: link:../.. zod: specifier: ^4.3.6 version: 4.3.6 @@ -439,7 +439,7 @@ importers: specifier: ^4.3.5 version: 4.3.6 devDependencies: - moltbot: + openclaw: specifier: workspace:* version: link:../.. @@ -459,7 +459,7 @@ importers: extensions/zalo: dependencies: - moltbot: + openclaw: specifier: workspace:* version: link:../.. undici: @@ -471,13 +471,19 @@ importers: '@sinclair/typebox': specifier: 0.34.47 version: 0.34.47 - moltbot: + openclaw: specifier: workspace:* version: link:../.. packages/clawdbot: dependencies: - moltbot: + openclaw: + specifier: workspace:* + version: link:../.. + + packages/moltbot: + dependencies: + openclaw: specifier: workspace:* version: link:../.. @@ -495,10 +501,22 @@ importers: openai: specifier: ^4.77.0 version: 4.104.0(ws@8.19.0)(zod@3.25.76) + pdf-parse: + specifier: ^1.1.1 + version: 1.1.4 + pg: + specifier: ^8.11.3 + version: 8.17.2 + redis: + specifier: ^4.6.12 + version: 4.7.1 devDependencies: '@types/node': specifier: ^22.10.2 version: 22.19.7 + '@types/pg': + specifier: ^8.10.9 + version: 8.16.0 tsx: specifier: ^4.7.0 version: 4.21.0 @@ -2129,6 +2147,35 @@ packages: '@protobufjs/utf8@1.1.0': resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + '@redis/bloom@1.2.0': + resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/client@1.6.1': + resolution: {integrity: sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==} + engines: {node: '>=14'} + + '@redis/graph@1.1.1': + resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/json@1.0.7': + resolution: {integrity: sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/search@1.2.0': + resolution: {integrity: sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/time-series@1.1.0': + resolution: {integrity: sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==} + peerDependencies: + '@redis/client': ^1.0.0 + '@reflink/reflink-darwin-arm64@0.1.19': resolution: {integrity: sha512-ruy44Lpepdk1FqDz38vExBY/PVUsjxZA+chd9wozjUH9JjuDT/HEaQYA6wYN9mf041l0yLVar6BCZuWABJvHSA==} engines: {node: '>= 10'} @@ -2786,6 +2833,9 @@ packages: '@types/node@25.0.10': resolution: {integrity: sha512-zWW5KPngR/yvakJgGOmZ5vTBemDoSqF3AcV/LrO5u5wTWyEAVVh+IT39G4gtyAkh3CtTZs8aX/yRM82OfzHJRg==} + '@types/pg@8.16.0': + resolution: {integrity: sha512-RmhMd/wD+CF8Dfo+cVIy3RR5cl8CyfXQ0tGgW6XBL8L4LM/UTEbNXYRbLwU6w+CgrKBNbrQWt4FUtTfaU5jSYQ==} + '@types/proper-lockfile@4.1.4': resolution: {integrity: sha512-uo2ABllncSqg9F1D4nugVl9v93RmjxF6LJzQLMLDdPaXCUIDPeOJ21Gbqi43xNKzBi/WQ0Q0dICqufzQbMjipQ==} @@ -3282,6 +3332,10 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + cmake-js@7.4.0: resolution: {integrity: sha512-Lw0JxEHrmk+qNj1n9W9d4IvkDdYTBn7l2BW6XmtLj7WPpIo2shvxUy+YokfjMxAAOELNonQwX3stkPhM5xSC2Q==} engines: {node: '>= 14.15.0'} @@ -3739,6 +3793,10 @@ packages: resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} engines: {node: '>=18'} + generic-pool@3.9.0: + resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==} + engines: {node: '>= 4'} + get-caller-file@2.0.5: resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} engines: {node: 6.* || 8.* || >= 10.*} @@ -4474,6 +4532,9 @@ packages: resolution: {integrity: sha512-fvfW1dUgJdZAdTniC6MzLTMwnNUFKGKaUdRJ1OsveOYlfnPUETBU973CG89565txvbBowCQ4Czdeu3qSX8bNOg==} hasBin: true + node-ensure@0.0.0: + resolution: {integrity: sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw==} + node-fetch@2.7.0: resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} engines: {node: 4.x || >=6.0.0} @@ -4725,6 +4786,10 @@ packages: pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + pdf-parse@1.1.4: + resolution: {integrity: sha512-XRIRcLgk6ZnUbsHsYXExMw+krrPE81hJ6FQPLdBNhhBefqIQKXu/WeTgNBGSwPrfU0v+UCEwn7AoAUOsVKHFvQ==} + engines: {node: '>=6.8.1'} + pdfjs-dist@5.4.530: resolution: {integrity: sha512-r1hWsSIGGmyYUAHR26zSXkxYWLXLMd6AwqcaFYG9YUZ0GBf5GvcjJSeo512tabM4GYFhxhl5pMCmPr7Q72Rq2Q==} engines: {node: '>=20.16.0 || >=22.3.0'} @@ -4735,6 +4800,40 @@ packages: performance-now@2.1.0: resolution: {integrity: sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow==} + pg-cloudflare@1.3.0: + resolution: {integrity: sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==} + + pg-connection-string@2.10.1: + resolution: {integrity: sha512-iNzslsoeSH2/gmDDKiyMqF64DATUCWj3YJ0wP14kqcsf2TUklwimd+66yYojKwZCA7h2yRNLGug71hCBA2a4sw==} + + pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + + pg-pool@3.11.0: + resolution: {integrity: sha512-MJYfvHwtGp870aeusDh+hg9apvOe2zmpZJpyt+BMtzUWlVqbhFmMK6bOBXLBUPd7iRtIF9fZplDc7KrPN3PN7w==} + peerDependencies: + pg: '>=8.0' + + pg-protocol@1.11.0: + resolution: {integrity: sha512-pfsxk2M9M3BuGgDOfuy37VNRRX3jmKgMjcvAcWqNDpZSf4cUmv8HSOl5ViRQFsfARFn0KuUQTgLxVMbNq5NW3g==} + + pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + + pg@8.17.2: + resolution: {integrity: sha512-vjbKdiBJRqzcYw1fNU5KuHyYvdJ1qpcQg1CeBrHFqV1pWgHeVR6j/+kX0E1AAXfyuLUGY1ICrN2ELKA/z2HWzw==} + engines: {node: '>= 16.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -4786,6 +4885,22 @@ packages: resolution: {integrity: sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==} engines: {node: ^10 || ^12 || >=14} + postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + + postgres-bytea@1.0.1: + resolution: {integrity: sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==} + engines: {node: '>=0.10.0'} + + postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + + postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + postgres@3.4.8: resolution: {integrity: sha512-d+JFcLM17njZaOLkv6SCev7uoLaBtfK86vMUXhW1Z4glPWh4jozno9APvW/XKFJ3CCxVoC7OL38BqRydtu5nGg==} engines: {node: '>=12'} @@ -4929,6 +5044,9 @@ packages: resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} engines: {node: '>= 12.13.0'} + redis@4.7.1: + resolution: {integrity: sha512-S1bJDnqLftzHXHP8JsT5II/CtHWQrASX5K96REjWjlmWKrviSOLWmM7QnRLstAWsu1VBBV1ffV6DzCvxNP0UJQ==} + reflect-metadata@0.2.2: resolution: {integrity: sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q==} @@ -5601,6 +5719,10 @@ packages: utf-8-validate: optional: true + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} @@ -7828,6 +7950,32 @@ snapshots: '@protobufjs/utf8@1.1.0': {} + '@redis/bloom@1.2.0(@redis/client@1.6.1)': + dependencies: + '@redis/client': 1.6.1 + + '@redis/client@1.6.1': + dependencies: + cluster-key-slot: 1.1.2 + generic-pool: 3.9.0 + yallist: 4.0.0 + + '@redis/graph@1.1.1(@redis/client@1.6.1)': + dependencies: + '@redis/client': 1.6.1 + + '@redis/json@1.0.7(@redis/client@1.6.1)': + dependencies: + '@redis/client': 1.6.1 + + '@redis/search@1.2.0(@redis/client@1.6.1)': + dependencies: + '@redis/client': 1.6.1 + + '@redis/time-series@1.1.0(@redis/client@1.6.1)': + dependencies: + '@redis/client': 1.6.1 + '@reflink/reflink-darwin-arm64@0.1.19': optional: true @@ -8605,7 +8753,7 @@ snapshots: '@types/node-fetch@2.6.13': dependencies: - '@types/node': 22.19.7 + '@types/node': 25.0.10 form-data: 4.0.5 '@types/node@10.17.60': {} @@ -8630,6 +8778,12 @@ snapshots: dependencies: undici-types: 7.16.0 + '@types/pg@8.16.0': + dependencies: + '@types/node': 25.0.10 + pg-protocol: 1.11.0 + pg-types: 2.2.0 + '@types/proper-lockfile@4.1.4': dependencies: '@types/retry': 0.12.5 @@ -9235,6 +9389,8 @@ snapshots: clsx@2.1.1: {} + cluster-key-slot@1.1.2: {} + cmake-js@7.4.0: dependencies: axios: 1.13.2(debug@4.4.3) @@ -9767,6 +9923,8 @@ snapshots: transitivePeerDependencies: - supports-color + generic-pool@3.9.0: {} + get-caller-file@2.0.5: {} get-east-asian-width@1.4.0: {} @@ -10527,6 +10685,8 @@ snapshots: - supports-color - utf-8-validate + node-ensure@0.0.0: {} + node-fetch@2.7.0: dependencies: whatwg-url: 5.0.0 @@ -10827,6 +10987,10 @@ snapshots: pathe@2.0.3: {} + pdf-parse@1.1.4: + dependencies: + node-ensure: 0.0.0 + pdfjs-dist@5.4.530: optionalDependencies: '@napi-rs/canvas': 0.1.88 @@ -10835,6 +10999,41 @@ snapshots: performance-now@2.1.0: {} + pg-cloudflare@1.3.0: + optional: true + + pg-connection-string@2.10.1: {} + + pg-int8@1.0.1: {} + + pg-pool@3.11.0(pg@8.17.2): + dependencies: + pg: 8.17.2 + + pg-protocol@1.11.0: {} + + pg-types@2.2.0: + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.1 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + + pg@8.17.2: + dependencies: + pg-connection-string: 2.10.1 + pg-pool: 3.11.0(pg@8.17.2) + pg-protocol: 1.11.0 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.3.0 + + pgpass@1.0.5: + dependencies: + split2: 4.2.0 + picocolors@1.1.1: {} picomatch@2.3.1: {} @@ -10885,6 +11084,16 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + postgres-array@2.0.0: {} + + postgres-bytea@1.0.1: {} + + postgres-date@1.0.7: {} + + postgres-interval@1.2.0: + dependencies: + xtend: 4.0.2 + postgres@3.4.8: {} pretty-bytes@6.1.1: @@ -11073,6 +11282,15 @@ snapshots: real-require@0.2.0: {} + redis@4.7.1: + dependencies: + '@redis/bloom': 1.2.0(@redis/client@1.6.1) + '@redis/client': 1.6.1 + '@redis/graph': 1.1.1(@redis/client@1.6.1) + '@redis/json': 1.0.7(@redis/client@1.6.1) + '@redis/search': 1.2.0(@redis/client@1.6.1) + '@redis/time-series': 1.1.0(@redis/client@1.6.1) + reflect-metadata@0.2.2: {} request-promise-core@1.1.4(request@2.88.2): @@ -11814,6 +12032,8 @@ snapshots: ws@8.19.0: {} + xtend@4.0.2: {} + y18n@5.0.8: {} yallist@4.0.0: {} diff --git a/secure/README.md b/secure/README.md index 692066b87..b1020ce73 100644 --- a/secure/README.md +++ b/secure/README.md @@ -21,19 +21,24 @@ Your AI agent that runs on your infrastructure, answers only to you, and you can ``` ┌─────────────────────────────────────────────────────┐ │ TELEGRAM (your secure UI) │ -│ ├── Chat with AI (text, voice, images) │ +│ ├── Chat with AI (text, images, documents) │ │ ├── Forward anything → get analysis │ │ └── /commands for actions │ ├─────────────────────────────────────────────────────┤ +│ DOCUMENT ANALYSIS │ +│ ├── PDF extraction and summarization │ +│ ├── Code files, markdown, JSON, CSV │ +│ └── Up to 20MB per document │ +├─────────────────────────────────────────────────────┤ │ WEBHOOKS IN (authenticated) │ │ ├── GitHub → "PR merged, here's the summary" │ │ ├── Uptime → "Site down, checking why..." │ │ └── Anything → AI-summarized to Telegram │ ├─────────────────────────────────────────────────────┤ -│ SCHEDULED TASKS (cron) │ +│ SCHEDULED TASKS (persistent cron) │ │ ├── Morning briefing │ -│ ├── Monitor RSS/sites │ -│ └── Recurring research │ +│ ├── Stored in PostgreSQL (survives restarts) │ +│ └── Conversations cached in Redis │ ├─────────────────────────────────────────────────────┤ │ SANDBOX (isolated execution) │ │ ├── Docker container │ @@ -71,6 +76,10 @@ ANTHROPIC_API_KEY=sk-ant-... # Or OPENAI_API_KEY ### Optional ```bash +# Storage (Railway provides these automatically) +DATABASE_URL=postgresql://... # PostgreSQL for task persistence +REDIS_URL=redis://... # Redis for conversation caching + # Webhooks WEBHOOK_SECRET=random-32-chars # Auto-generated if missing WEBHOOK_BASE_PATH=/hooks # Default: /hooks @@ -178,9 +187,12 @@ All webhooks are: │ • Allowlist auth │ │ • Ephemeral │ └────────────────────┘ └────────────────────┘ │ - ▼ - [Anthropic/OpenAI] - (Direct API calls) + ┌────┴────┬─────────────┐ + ▼ ▼ ▼ +┌────────┐ ┌────────┐ ┌────────────────┐ +│ Pg │ │ Redis │ │ Anthropic/ │ +│ Tasks │ │ Cache │ │ OpenAI │ +└────────┘ └────────┘ └────────────────┘ ``` ## License diff --git a/secure/config.ts b/secure/config.ts index 6cb63edca..026fe0052 100644 --- a/secure/config.ts +++ b/secure/config.ts @@ -53,6 +53,12 @@ export type SecureConfig = { host: string; gatewayToken: string; }; + + // Storage (optional) + storage: { + postgresUrl?: string; + redisUrl?: string; + }; }; function required(name: string): string { @@ -179,6 +185,10 @@ export function loadSecureConfig(): SecureConfig { host: optional("HOST", "0.0.0.0"), gatewayToken: optional("ASSUREBOT_GATEWAY_TOKEN", generateSecureToken()), }, + storage: { + postgresUrl: process.env.DATABASE_URL || process.env.POSTGRES_URL, + redisUrl: process.env.REDIS_URL, + }, }; } @@ -231,5 +241,9 @@ export function redactConfig(config: SecureConfig): Record { host: config.server.host, gatewayToken: "[REDACTED]", }, + storage: { + postgresUrl: config.storage.postgresUrl ? "[CONFIGURED]" : undefined, + redisUrl: config.storage.redisUrl ? "[CONFIGURED]" : undefined, + }, }; } diff --git a/secure/documents.ts b/secure/documents.ts new file mode 100644 index 000000000..4d690816a --- /dev/null +++ b/secure/documents.ts @@ -0,0 +1,120 @@ +/** + * AssureBot - Document Analysis + * + * Extract text from various document formats for AI analysis. + */ + +export type DocumentResult = { + text: string; + pageCount?: number; + format: string; + truncated: boolean; +}; + +const MAX_TEXT_LENGTH = 50000; // ~12k tokens + +/** + * Extract text from a buffer based on mime type + */ +export async function extractText( + buffer: Buffer, + mimeType: string, + filename?: string +): Promise { + const ext = filename?.split(".").pop()?.toLowerCase(); + + // Plain text files + if ( + mimeType.startsWith("text/") || + ext === "txt" || + ext === "md" || + ext === "json" || + ext === "xml" || + ext === "csv" || + ext === "log" + ) { + return extractPlainText(buffer); + } + + // PDF + if (mimeType === "application/pdf" || ext === "pdf") { + return extractPdf(buffer); + } + + // Code files (treat as text) + const codeExtensions = [ + "js", "ts", "jsx", "tsx", "py", "rb", "go", "rs", "java", + "c", "cpp", "h", "hpp", "cs", "php", "swift", "kt", "scala", + "sh", "bash", "zsh", "yaml", "yml", "toml", "ini", "env", + "sql", "graphql", "html", "css", "scss", "less" + ]; + if (ext && codeExtensions.includes(ext)) { + return extractPlainText(buffer, ext); + } + + // Unsupported format + return { + text: `[Unsupported document format: ${mimeType}${ext ? ` (.${ext})` : ""}]`, + format: "unsupported", + truncated: false, + }; +} + +/** + * Extract plain text + */ +function extractPlainText(buffer: Buffer, format = "text"): DocumentResult { + let text = buffer.toString("utf-8"); + let truncated = false; + + if (text.length > MAX_TEXT_LENGTH) { + text = text.slice(0, MAX_TEXT_LENGTH) + "\n\n[... truncated ...]"; + truncated = true; + } + + return { text, format, truncated }; +} + +/** + * Extract text from PDF using pdf-parse + */ +async function extractPdf(buffer: Buffer): Promise { + try { + // Dynamic import to avoid bundling issues + const pdfParse = await import("pdf-parse").then(m => m.default); + const data = await pdfParse(buffer); + + let text = data.text; + let truncated = false; + + if (text.length > MAX_TEXT_LENGTH) { + text = text.slice(0, MAX_TEXT_LENGTH) + "\n\n[... truncated ...]"; + truncated = true; + } + + return { + text, + pageCount: data.numpages, + format: "pdf", + truncated, + }; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return { + text: `[Failed to parse PDF: ${msg}]`, + format: "pdf-error", + truncated: false, + }; + } +} + +/** + * Summarize document metadata for logging + */ +export function summarizeDocument(result: DocumentResult): string { + const parts = [result.format.toUpperCase()]; + if (result.pageCount) parts.push(`${result.pageCount} pages`); + parts.push(`${result.text.length} chars`); + if (result.truncated) parts.push("truncated"); + return parts.join(", "); +} diff --git a/secure/index.ts b/secure/index.ts index 3e11624ee..1e7ea7f97 100644 --- a/secure/index.ts +++ b/secure/index.ts @@ -15,6 +15,7 @@ import { createTelegramBot } from "./telegram.js"; import { createWebhookHandler } from "./webhooks.js"; import { createSandboxRunner } from "./sandbox.js"; import { createScheduler } from "./scheduler.js"; +import { createStorage, type Storage } from "./storage.js"; async function main() { console.log("=".repeat(50)); @@ -49,6 +50,15 @@ async function main() { }); audit.startup(); + // Create storage (PostgreSQL + Redis) + console.log("[init] Creating storage layer..."); + const storage = await createStorage({ + postgres: config.storage.postgresUrl ? { url: config.storage.postgresUrl } : undefined, + redis: config.storage.redisUrl ? { url: config.storage.redisUrl } : undefined, + }); + const storageHealthy = await storage.isHealthy(); + console.log(`[init] Storage healthy: ${storageHealthy}`); + // Create AI agent console.log(`[init] Creating AI agent (${config.ai.provider})...`); const agent = createAgent(config, audit); @@ -67,13 +77,14 @@ async function main() { const { Bot } = await import("grammy"); const bot = new Bot(config.telegram.botToken); - // Create scheduler (needs bot for notifications) + // Create scheduler (needs bot for notifications, storage for persistence) console.log("[init] Creating scheduler..."); const scheduler = createScheduler({ config, audit, agent, telegramBot: bot, + storage, }); // Create Telegram bot handler (with sandbox and scheduler) @@ -103,6 +114,7 @@ async function main() { // Health check if (url.pathname === "/health" || url.pathname === "/healthz") { + const isStorageHealthy = await storage.isHealthy(); res.statusCode = 200; res.setHeader("Content-Type", "application/json"); res.end(JSON.stringify({ @@ -111,6 +123,9 @@ async function main() { uptime: process.uptime(), telegram: "connected", sandbox: sandboxAvailable ? "available" : "unavailable", + storage: isStorageHealthy ? "healthy" : "degraded", + postgres: config.storage.postgresUrl ? "configured" : "none", + redis: config.storage.redisUrl ? "configured" : "none", })); return; } @@ -148,6 +163,7 @@ async function main() { try { scheduler.stop(); await telegram.stop(); + await storage.close(); await new Promise((resolve, reject) => { server.close((err) => { @@ -175,8 +191,8 @@ async function main() { console.log(`[start] HTTP server listening on ${config.server.host}:${config.server.port}`); }); - // Start scheduler - scheduler.start(); + // Start scheduler (loads tasks from storage) + await scheduler.start(); // Start Telegram bot (polling mode for simplicity) await telegram.start(); @@ -188,6 +204,7 @@ async function main() { console.log(` Telegram: Polling mode`); console.log(` Webhooks: http://localhost:${config.server.port}${config.webhooks.basePath}/*`); console.log(` Health: http://localhost:${config.server.port}/health`); + console.log(` Storage: ${config.storage.postgresUrl ? "PostgreSQL" : "memory"}${config.storage.redisUrl ? " + Redis" : ""}`); console.log(` Allowed: ${config.telegram.allowedUsers.length} users`); console.log(); console.log(" Press Ctrl+C to stop"); diff --git a/secure/package.json b/secure/package.json index 7e6a5e5ae..9c4b5bf03 100644 --- a/secure/package.json +++ b/secure/package.json @@ -13,10 +13,14 @@ "@anthropic-ai/sdk": "^0.39.0", "cron": "^3.1.7", "grammy": "^1.21.1", - "openai": "^4.77.0" + "openai": "^4.77.0", + "pdf-parse": "^1.1.1", + "pg": "^8.11.3", + "redis": "^4.6.12" }, "devDependencies": { "@types/node": "^22.10.2", + "@types/pg": "^8.10.9", "tsx": "^4.7.0", "typescript": "^5.3.3" }, diff --git a/secure/pdf-parse.d.ts b/secure/pdf-parse.d.ts new file mode 100644 index 000000000..992c512aa --- /dev/null +++ b/secure/pdf-parse.d.ts @@ -0,0 +1,13 @@ +declare module "pdf-parse" { + interface PDFData { + numpages: number; + numrender: number; + info: Record; + metadata: Record | null; + text: string; + version: string; + } + + function pdfParse(dataBuffer: Buffer, options?: Record): Promise; + export default pdfParse; +} diff --git a/secure/railway.toml b/secure/railway.toml new file mode 100644 index 000000000..31a65137a --- /dev/null +++ b/secure/railway.toml @@ -0,0 +1,10 @@ +[build] +builder = "dockerfile" +dockerfilePath = "Dockerfile" + +[deploy] +startCommand = "node dist/index.js" +healthcheckPath = "/health" +healthcheckTimeout = 30 +restartPolicyType = "ON_FAILURE" +restartPolicyMaxRetries = 3 diff --git a/secure/scheduler.ts b/secure/scheduler.ts index a3cb95dfc..ddd5132e0 100644 --- a/secure/scheduler.ts +++ b/secure/scheduler.ts @@ -11,6 +11,7 @@ import type { AuditLogger } from "./audit.js"; import type { AgentCore } from "./agent.js"; import type { Bot } from "grammy"; import { sendToUser } from "./telegram.js"; +import type { Storage } from "./storage.js"; export type ScheduledTask = { id: string; @@ -29,7 +30,7 @@ export type Scheduler = { enableTask: (id: string, enabled: boolean) => boolean; listTasks: () => ScheduledTask[]; runTask: (id: string) => Promise; - start: () => void; + start: () => Promise; stop: () => void; }; @@ -38,6 +39,7 @@ export type SchedulerDeps = { audit: AuditLogger; agent: AgentCore; telegramBot: Bot; + storage?: Storage; }; function generateId(): string { @@ -45,9 +47,44 @@ function generateId(): string { } export function createScheduler(deps: SchedulerDeps): Scheduler { - const { config, audit, agent, telegramBot } = deps; + const { config, audit, agent, telegramBot, storage } = deps; const tasks = new Map(); const cronJobs = new Map>(); + let initialized = false; + + // Save task to storage (if available) + async function persistTask(task: ScheduledTask): Promise { + if (storage) { + await storage.saveTask(task).catch((err) => { + console.error("[scheduler] Failed to persist task:", err); + }); + } + } + + // Delete task from storage (if available) + async function unpersistTask(id: string): Promise { + if (storage) { + await storage.deleteTask(id).catch((err) => { + console.error("[scheduler] Failed to delete persisted task:", err); + }); + } + } + + // Load tasks from storage + async function loadFromStorage(): Promise { + if (!storage || initialized) return; + initialized = true; + + try { + const storedTasks = await storage.getAllTasks(); + for (const task of storedTasks) { + tasks.set(task.id, task); + } + console.log(`[scheduler] Loaded ${storedTasks.length} tasks from storage`); + } catch (err) { + console.error("[scheduler] Failed to load tasks from storage:", err); + } + } async function executeTask(task: ScheduledTask): Promise { const startTime = Date.now(); @@ -67,6 +104,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler { task.lastRun = new Date(); task.lastStatus = "ok"; task.lastError = undefined; + await persistTask(task); audit.cron({ jobId: task.id, @@ -80,6 +118,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler { task.lastRun = new Date(); task.lastStatus = "error"; task.lastError = errorMsg; + await persistTask(task); audit.cron({ jobId: task.id, @@ -133,6 +172,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler { const task: ScheduledTask = { ...taskInput, id }; tasks.set(id, task); scheduleTask(task); + void persistTask(task); return id; }, @@ -147,6 +187,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler { } tasks.delete(id); + void unpersistTask(id); return true; }, @@ -156,6 +197,7 @@ export function createScheduler(deps: SchedulerDeps): Scheduler { task.enabled = enabled; scheduleTask(task); + void persistTask(task); return true; }, @@ -171,16 +213,21 @@ export function createScheduler(deps: SchedulerDeps): Scheduler { await executeTask(task); }, - start(): void { + async start(): Promise { if (!config.scheduler.enabled) { console.log("[scheduler] Scheduler is disabled"); return; } console.log("[scheduler] Starting scheduler..."); + + // Load tasks from persistent storage + await loadFromStorage(); + for (const task of tasks.values()) { scheduleTask(task); } + console.log(`[scheduler] ${tasks.size} tasks scheduled`); }, stop(): void { diff --git a/secure/storage.ts b/secure/storage.ts new file mode 100644 index 000000000..74d5891d8 --- /dev/null +++ b/secure/storage.ts @@ -0,0 +1,293 @@ +/** + * AssureBot - Storage Layer + * + * PostgreSQL for persistent data (tasks, audit) + * Redis for caching and sessions + */ + +import type { ScheduledTask } from "./scheduler.js"; + +export type StorageConfig = { + postgres?: { + url: string; + }; + redis?: { + url: string; + }; +}; + +export type Storage = { + // Tasks + saveTask: (task: ScheduledTask) => Promise; + getTask: (id: string) => Promise; + getAllTasks: () => Promise; + deleteTask: (id: string) => Promise; + + // Conversations (Redis cache) + getConversation: (userId: number) => Promise; + saveConversation: (userId: number, messages: ConversationMessage[]) => Promise; + clearConversation: (userId: number) => Promise; + + // Health + isHealthy: () => Promise; + close: () => Promise; +}; + +export type ConversationMessage = { + role: "user" | "assistant"; + content: string; + timestamp?: string; +}; + +/** + * In-memory storage (fallback when no DB configured) + */ +function createMemoryStorage(): Storage { + const tasks = new Map(); + const conversations = new Map(); + + return { + async saveTask(task) { + tasks.set(task.id, task); + }, + async getTask(id) { + return tasks.get(id) || null; + }, + async getAllTasks() { + return Array.from(tasks.values()); + }, + async deleteTask(id) { + return tasks.delete(id); + }, + async getConversation(userId) { + return conversations.get(userId) || []; + }, + async saveConversation(userId, messages) { + conversations.set(userId, messages); + }, + async clearConversation(userId) { + conversations.delete(userId); + }, + async isHealthy() { + return true; + }, + async close() { + // Nothing to close + }, + }; +} + +/** + * PostgreSQL storage for tasks + */ +async function createPostgresStorage(url: string): Promise<{ + saveTask: Storage["saveTask"]; + getTask: Storage["getTask"]; + getAllTasks: Storage["getAllTasks"]; + deleteTask: Storage["deleteTask"]; + isHealthy: () => Promise; + close: () => Promise; +}> { + const { default: pg } = await import("pg"); + const pool = new pg.Pool({ connectionString: url }); + + // Create tables if not exist + await pool.query(` + CREATE TABLE IF NOT EXISTS scheduled_tasks ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + schedule TEXT NOT NULL, + prompt TEXT NOT NULL, + enabled BOOLEAN DEFAULT true, + last_run TIMESTAMPTZ, + last_status TEXT, + last_error TEXT, + created_at TIMESTAMPTZ DEFAULT NOW(), + updated_at TIMESTAMPTZ DEFAULT NOW() + ) + `); + + console.log("[storage] PostgreSQL connected, tables ready"); + + return { + async saveTask(task) { + await pool.query( + `INSERT INTO scheduled_tasks (id, name, schedule, prompt, enabled, last_run, last_status, last_error, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, NOW()) + ON CONFLICT (id) DO UPDATE SET + name = $2, schedule = $3, prompt = $4, enabled = $5, + last_run = $6, last_status = $7, last_error = $8, updated_at = NOW()`, + [ + task.id, + task.name, + task.schedule, + task.prompt, + task.enabled, + task.lastRun || null, + task.lastStatus || null, + task.lastError || null, + ] + ); + }, + + async getTask(id) { + const result = await pool.query( + "SELECT * FROM scheduled_tasks WHERE id = $1", + [id] + ); + if (result.rows.length === 0) return null; + return rowToTask(result.rows[0]); + }, + + async getAllTasks() { + const result = await pool.query("SELECT * FROM scheduled_tasks ORDER BY created_at"); + return result.rows.map(rowToTask); + }, + + async deleteTask(id) { + const result = await pool.query( + "DELETE FROM scheduled_tasks WHERE id = $1", + [id] + ); + return (result.rowCount ?? 0) > 0; + }, + + async isHealthy() { + try { + await pool.query("SELECT 1"); + return true; + } catch { + return false; + } + }, + + async close() { + await pool.end(); + }, + }; +} + +function rowToTask(row: Record): ScheduledTask { + return { + id: row.id as string, + name: row.name as string, + schedule: row.schedule as string, + prompt: row.prompt as string, + enabled: row.enabled as boolean, + lastRun: row.last_run ? new Date(row.last_run as string) : undefined, + lastStatus: row.last_status as "ok" | "error" | undefined, + lastError: row.last_error as string | undefined, + }; +} + +/** + * Redis storage for conversations/cache + */ +async function createRedisStorage(url: string): Promise<{ + getConversation: Storage["getConversation"]; + saveConversation: Storage["saveConversation"]; + clearConversation: Storage["clearConversation"]; + isHealthy: () => Promise; + close: () => Promise; +}> { + const { createClient } = await import("redis"); + const client = createClient({ url }); + + client.on("error", (err) => console.error("[redis] Error:", err)); + await client.connect(); + + console.log("[storage] Redis connected"); + + const CONVERSATION_TTL = 60 * 60 * 24; // 24 hours + const MAX_MESSAGES = 50; + + return { + async getConversation(userId) { + const key = `conv:${userId}`; + const data = await client.get(key); + if (!data) return []; + try { + return JSON.parse(data) as ConversationMessage[]; + } catch { + return []; + } + }, + + async saveConversation(userId, messages) { + const key = `conv:${userId}`; + // Keep only last N messages + const trimmed = messages.slice(-MAX_MESSAGES); + await client.setEx(key, CONVERSATION_TTL, JSON.stringify(trimmed)); + }, + + async clearConversation(userId) { + const key = `conv:${userId}`; + await client.del(key); + }, + + async isHealthy() { + try { + await client.ping(); + return true; + } catch { + return false; + } + }, + + async close() { + await client.quit(); + }, + }; +} + +/** + * Create storage based on config + */ +export async function createStorage(config: StorageConfig): Promise { + const memory = createMemoryStorage(); + + let pgStorage: Awaited> | null = null; + let redisStorage: Awaited> | null = null; + + // Try PostgreSQL + if (config.postgres?.url) { + try { + pgStorage = await createPostgresStorage(config.postgres.url); + } catch (err) { + console.error("[storage] PostgreSQL connection failed, using memory:", err); + } + } + + // Try Redis + if (config.redis?.url) { + try { + redisStorage = await createRedisStorage(config.redis.url); + } catch (err) { + console.error("[storage] Redis connection failed, using memory:", err); + } + } + + return { + // Tasks: prefer PostgreSQL, fallback to memory + saveTask: pgStorage?.saveTask ?? memory.saveTask, + getTask: pgStorage?.getTask ?? memory.getTask, + getAllTasks: pgStorage?.getAllTasks ?? memory.getAllTasks, + deleteTask: pgStorage?.deleteTask ?? memory.deleteTask, + + // Conversations: prefer Redis, fallback to memory + getConversation: redisStorage?.getConversation ?? memory.getConversation, + saveConversation: redisStorage?.saveConversation ?? memory.saveConversation, + clearConversation: redisStorage?.clearConversation ?? memory.clearConversation, + + async isHealthy() { + const pgOk = pgStorage ? await pgStorage.isHealthy() : true; + const redisOk = redisStorage ? await redisStorage.isHealthy() : true; + return pgOk && redisOk; + }, + + async close() { + await pgStorage?.close(); + await redisStorage?.close(); + }, + }; +} diff --git a/secure/telegram.ts b/secure/telegram.ts index 4807ace6a..ab9df5e66 100644 --- a/secure/telegram.ts +++ b/secure/telegram.ts @@ -11,6 +11,7 @@ import type { AuditLogger } from "./audit.js"; import type { AgentCore, ConversationStore, ImageContent } from "./agent.js"; import type { SandboxRunner } from "./sandbox.js"; import type { Scheduler } from "./scheduler.js"; +import { extractText, summarizeDocument } from "./documents.js"; export type TelegramBot = { bot: Bot; @@ -501,13 +502,95 @@ Cron format: minute hour day month weekday // Handle documents bot.on("message:document", async (ctx) => { const userId = ctx.from?.id; + const username = formatUsername(ctx); + if (!userId || !isUserAllowed(userId, config.telegram.allowedUsers)) { + audit.messageBlocked({ + userId: userId || 0, + username, + reason: "User not in allowlist", + }); return; } - await ctx.reply( - "I received your document. Document analysis coming soon - for now, please copy/paste the text content." - ); + const doc = ctx.message?.document; + if (!doc) { + await ctx.reply("Could not process document."); + return; + } + + const startTime = Date.now(); + const caption = ctx.message?.caption || "Please analyze this document and summarize the key points."; + + try { + await ctx.replyWithChatAction("typing"); + + // Check file size (max 20MB) + if (doc.file_size && doc.file_size > 20 * 1024 * 1024) { + await ctx.reply("Document too large (max 20MB)."); + return; + } + + // Get file info + const file = await ctx.api.getFile(doc.file_id); + if (!file.file_path) { + await ctx.reply("Could not download document."); + return; + } + + // Download the file + const fileUrl = `https://api.telegram.org/file/bot${config.telegram.botToken}/${file.file_path}`; + const response = await fetch(fileUrl); + if (!response.ok) { + await ctx.reply("Failed to download document."); + return; + } + + const buffer = Buffer.from(await response.arrayBuffer()); + const mimeType = doc.mime_type || "application/octet-stream"; + + // Extract text + const extracted = await extractText(buffer, mimeType, doc.file_name); + + if (extracted.format === "unsupported") { + await ctx.reply( + `Unsupported document format: ${mimeType}\n\nSupported: PDF, TXT, MD, JSON, CSV, code files` + ); + return; + } + + if (extracted.format === "pdf-error") { + await ctx.reply(`Could not parse PDF: ${extracted.text}`); + return; + } + + // Analyze with AI + const result = await agent.chat([ + { + role: "user", + content: `${caption}\n\n--- Document Content (${summarizeDocument(extracted)}) ---\n\n${extracted.text}`, + }, + ]); + + await ctx.reply(result.text, { parse_mode: "Markdown" }).catch(async () => { + await ctx.reply(result.text); + }); + + audit.message({ + userId, + username, + text: `[DOCUMENT: ${doc.file_name || "unnamed"}] ${caption}`, + response: result.text, + durationMs: Date.now() - startTime, + }); + } catch (err) { + const errorMsg = err instanceof Error ? err.message : String(err); + audit.error({ + error: `Failed to analyze document: ${errorMsg}`, + metadata: { userId, username, filename: doc.file_name }, + }); + await ctx.reply("Sorry, I couldn't analyze that document. Please try again."); + } }); return {