From fdc7e118915436afcd3ec0853fd3f32186cc95d3 Mon Sep 17 00:00:00 2001 From: ronitchidara Date: Tue, 27 Jan 2026 19:00:30 +0530 Subject: [PATCH] mobile: Phase 4 polish - refactoring, app store prep, and infrastructure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tier 5 - Code Refactoring: - iOS: Split NodeAppModel.swift (988→493 LOC) into focused modules - NodeCommandHandlers.swift: command routing (372 LOC) - NodeGatewaySync.swift: gateway sync, branding (153 LOC) - Android: Split NodeRuntime.kt (1268→756 LOC, 40% reduction) - NodeCommandHandlers.kt: command routing (326 LOC) - NodeGatewaySync.kt: gateway sync, A2UI helpers (294 LOC) Tier 6 - App Store Preparation: - iOS: Add PrivacyInfo.xcprivacy (iOS 17+ privacy manifest) - Android: Add ProGuard rules and enable minification for release builds Infrastructure (from earlier tiers): - iOS: Add OfflineMessageQueue for disconnected message queueing - iOS: Add PushManager for VoIP push foundation - iOS: Add TTSVoiceSettingsView for voice selection - Android: Add ClawdbotMessagingService for FCM push - Android: Add SyncWorker for background chat sync - Both: Deep linking support (clawdbot:// URL scheme) - Both: Empty states and loading indicators in settings Testing: - iOS: Add OfflineMessageQueueTests (8 tests) - Android: Add NodeGatewaySyncTest (11 tests) Co-Authored-By: Claude Opus 4.5 --- apps/android/app/build.gradle.kts | 15 +- apps/android/app/proguard-rules.pro | 58 +++ apps/android/app/src/main/AndroidManifest.xml | 16 +- .../java/bot/molt/android/MainActivity.kt | 40 ++ .../java/bot/molt/android/MainViewModel.kt | 36 ++ .../src/main/java/bot/molt/android/NodeApp.kt | 5 + .../bot/molt/android/NodeCommandHandlers.kt | 326 +++++++++++++++ .../java/bot/molt/android/NodeGatewaySync.kt | 294 ++++++++++++++ .../bot/molt/android/chat/ChatController.kt | 26 +- .../molt/android/gateway/GatewaySession.kt | 9 +- .../android/push/ClawdbotMessagingService.kt | 161 ++++++++ .../java/bot/molt/android/sync/SyncWorker.kt | 93 +++++ .../java/bot/molt/android/ui/RootScreen.kt | 15 + .../java/bot/molt/android/ui/SettingsSheet.kt | 27 +- .../bot/molt/android/voice/TalkModeManager.kt | 27 +- .../bot/molt/android/NodeGatewaySyncTest.kt | 78 ++++ apps/android/build.gradle.kts | 1 + .../Sources/Chat/OfflineMessageQueue.swift | 92 +++++ apps/ios/Sources/ClawdbotApp.swift | 30 ++ apps/ios/Sources/Info.plist | 13 + .../Sources/Model/NodeCommandHandlers.swift | 371 ++++++++++++++++++ apps/ios/Sources/Model/NodeGatewaySync.swift | 152 +++++++ apps/ios/Sources/Push/PushManager.swift | 137 +++++++ .../Sources/Resources/PrivacyInfo.xcprivacy | 31 ++ apps/ios/Sources/RootTabs.swift | 9 + apps/ios/Sources/Settings/SettingsTab.swift | 15 +- .../Settings/TTSVoiceSettingsView.swift | 140 +++++++ apps/ios/Tests/OfflineMessageQueueTests.swift | 122 ++++++ apps/ios/project.yml | 6 + 29 files changed, 2317 insertions(+), 28 deletions(-) create mode 100644 apps/android/app/proguard-rules.pro create mode 100644 apps/android/app/src/main/java/bot/molt/android/NodeCommandHandlers.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/NodeGatewaySync.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/push/ClawdbotMessagingService.kt create mode 100644 apps/android/app/src/main/java/bot/molt/android/sync/SyncWorker.kt create mode 100644 apps/android/app/src/test/java/bot/molt/android/NodeGatewaySyncTest.kt create mode 100644 apps/ios/Sources/Chat/OfflineMessageQueue.swift create mode 100644 apps/ios/Sources/Model/NodeCommandHandlers.swift create mode 100644 apps/ios/Sources/Model/NodeGatewaySync.swift create mode 100644 apps/ios/Sources/Push/PushManager.swift create mode 100644 apps/ios/Sources/Resources/PrivacyInfo.xcprivacy create mode 100644 apps/ios/Sources/Settings/TTSVoiceSettingsView.swift create mode 100644 apps/ios/Tests/OfflineMessageQueueTests.swift diff --git a/apps/android/app/build.gradle.kts b/apps/android/app/build.gradle.kts index 3ddcb3b81..02e52953e 100644 --- a/apps/android/app/build.gradle.kts +++ b/apps/android/app/build.gradle.kts @@ -5,6 +5,7 @@ plugins { id("org.jetbrains.kotlin.android") id("org.jetbrains.kotlin.plugin.compose") id("org.jetbrains.kotlin.plugin.serialization") + id("com.google.gms.google-services") } android { @@ -27,7 +28,12 @@ android { buildTypes { release { - isMinifyEnabled = false + isMinifyEnabled = true + isShrinkResources = true + proguardFiles( + getDefaultProguardFile("proguard-android-optimize.txt"), + "proguard-rules.pro" + ) } } @@ -115,6 +121,13 @@ dependencies { // Unicast DNS-SD (Wide-Area Bonjour) for tailnet discovery domains. implementation("dnsjava:dnsjava:3.6.4") + // WorkManager for background sync + implementation("androidx.work:work-runtime-ktx:2.10.1") + + // Firebase Cloud Messaging for push notifications + implementation(platform("com.google.firebase:firebase-bom:33.7.0")) + implementation("com.google.firebase:firebase-messaging-ktx") + testImplementation("junit:junit:4.13.2") testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2") testImplementation("io.kotest:kotest-runner-junit5-jvm:6.0.7") diff --git a/apps/android/app/proguard-rules.pro b/apps/android/app/proguard-rules.pro new file mode 100644 index 000000000..d07528dc6 --- /dev/null +++ b/apps/android/app/proguard-rules.pro @@ -0,0 +1,58 @@ +# Clawdbot Android ProGuard Rules +# https://www.guardsquare.com/manual/configuration/usage + +# OkHttp +-dontwarn okhttp3.** +-dontwarn okio.** +-keep class okhttp3.** { *; } +-keep interface okhttp3.** { *; } + +# Kotlinx Serialization +-keepattributes *Annotation*, InnerClasses +-dontnote kotlinx.serialization.AnnotationsKt +-keepclassmembers class kotlinx.serialization.json.** { *; } +-keepclasseswithmembers class * { + @kotlinx.serialization.Serializable *; +} + +# Keep Serializable classes +-keep class com.clawdbot.android.** implements kotlinx.serialization.KSerializer { *; } +-keep @kotlinx.serialization.Serializable class * { *; } + +# Keep WebView JavaScript interface +-keepclassmembers class * { + @android.webkit.JavascriptInterface ; +} + +# Keep all enums +-keepclassmembers enum * { + public static **[] values(); + public static ** valueOf(java.lang.String); +} + +# Firebase Messaging +-keep class com.google.firebase.messaging.** { *; } + +# dnsjava (for DNS-SD discovery) +-dontwarn org.xbill.DNS.** +-keep class org.xbill.DNS.** { *; } + +# CameraX +-keep class androidx.camera.** { *; } + +# Keep R8 from stripping the companion objects used by serialization +-keepclassmembers class * { + *** Companion; +} +-keepclasseswithmembers class * { + kotlinx.serialization.KSerializer serializer(...); +} + +# Prevent stripping of coroutine internals +-dontwarn kotlinx.coroutines.** +-keep class kotlinx.coroutines.** { *; } + +# Keep data classes +-keep class com.clawdbot.android.chat.** { *; } +-keep class com.clawdbot.android.gateway.** { *; } +-keep class com.clawdbot.android.protocol.** { *; } diff --git a/apps/android/app/src/main/AndroidManifest.xml b/apps/android/app/src/main/AndroidManifest.xml index e0aab841e..e74fd47d2 100644 --- a/apps/android/app/src/main/AndroidManifest.xml +++ b/apps/android/app/src/main/AndroidManifest.xml @@ -37,13 +37,27 @@ android:name=".NodeForegroundService" android:exported="false" android:foregroundServiceType="dataSync|microphone|mediaProjection" /> + + + + + + android:exported="true" + android:launchMode="singleTask"> + + + + + + diff --git a/apps/android/app/src/main/java/bot/molt/android/MainActivity.kt b/apps/android/app/src/main/java/bot/molt/android/MainActivity.kt index 1e8707e72..94699e6df 100644 --- a/apps/android/app/src/main/java/bot/molt/android/MainActivity.kt +++ b/apps/android/app/src/main/java/bot/molt/android/MainActivity.kt @@ -1,8 +1,10 @@ package bot.molt.android import android.Manifest +import android.content.Intent import android.content.pm.ApplicationInfo import android.os.Bundle +import android.util.Log import android.os.Build import android.view.WindowManager import android.webkit.WebView @@ -27,6 +29,10 @@ class MainActivity : ComponentActivity() { private lateinit var permissionRequester: PermissionRequester private lateinit var screenCaptureRequester: ScreenCaptureRequester + companion object { + private const val TAG = "MainActivity" + } + override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) val isDebuggable = (applicationInfo.flags and ApplicationInfo.FLAG_DEBUGGABLE) != 0 @@ -62,6 +68,40 @@ class MainActivity : ComponentActivity() { } } } + + // Handle deep link from initial launch + handleDeepLink(intent) + } + + override fun onNewIntent(intent: Intent) { + super.onNewIntent(intent) + handleDeepLink(intent) + } + + private fun handleDeepLink(intent: Intent?) { + val uri = intent?.data ?: return + if (uri.scheme != "clawdbot") return + + val host = uri.host // e.g., "agent", "chat", "settings" + val path = uri.path?.trimStart('/') // e.g., "hello" from clawdbot://agent/hello + val sessionKey = uri.getQueryParameter("sessionKey") ?: "main" + + Log.d(TAG, "Deep link received: scheme=${uri.scheme}, host=$host, path=$path, sessionKey=$sessionKey") + + when (host) { + "agent", "chat" -> { + // Send message via chat if path contains text + if (!path.isNullOrBlank()) { + viewModel.handleDeepLinkMessage(path, sessionKey) + } + } + "settings" -> { + viewModel.showSettings() + } + else -> { + Log.w(TAG, "Unknown deep link host: $host") + } + } } override fun onResume() { diff --git a/apps/android/app/src/main/java/bot/molt/android/MainViewModel.kt b/apps/android/app/src/main/java/bot/molt/android/MainViewModel.kt index a4f5ee23e..3c8fd52d8 100644 --- a/apps/android/app/src/main/java/bot/molt/android/MainViewModel.kt +++ b/apps/android/app/src/main/java/bot/molt/android/MainViewModel.kt @@ -8,9 +8,13 @@ import bot.molt.android.node.CameraCaptureManager import bot.molt.android.node.CanvasController import bot.molt.android.node.ScreenRecordManager import bot.molt.android.node.SmsManager +import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow class MainViewModel(app: Application) : AndroidViewModel(app) { + private val _pendingDeepLinkSheet = MutableStateFlow(null) + val pendingDeepLinkSheet: StateFlow = _pendingDeepLinkSheet.asStateFlow() private val runtime: NodeRuntime = (app as NodeApp).runtime val canvas: CanvasController = runtime.canvas @@ -171,4 +175,36 @@ class MainViewModel(app: Application) : AndroidViewModel(app) { fun sendChat(message: String, thinking: String, attachments: List) { runtime.sendChat(message = message, thinking = thinking, attachments = attachments) } + + /** + * Handle a deep link message by switching to the specified session and sending the message. + */ + fun handleDeepLinkMessage(message: String, sessionKey: String) { + if (sessionKey != chatSessionKey.value) { + switchChatSession(sessionKey) + } + val thinking = chatThinkingLevel.value + sendChat(message = message, thinking = thinking, attachments = emptyList()) + // Signal the UI to open the chat sheet + _pendingDeepLinkSheet.value = DeepLinkSheet.Chat + } + + /** + * Request the settings sheet to be shown from a deep link. + */ + fun showSettings() { + _pendingDeepLinkSheet.value = DeepLinkSheet.Settings + } + + /** + * Clear the pending deep link sheet after the UI has consumed it. + */ + fun consumeDeepLinkSheet() { + _pendingDeepLinkSheet.value = null + } +} + +enum class DeepLinkSheet { + Chat, + Settings, } diff --git a/apps/android/app/src/main/java/bot/molt/android/NodeApp.kt b/apps/android/app/src/main/java/bot/molt/android/NodeApp.kt index 53b2c58f5..96a91eb92 100644 --- a/apps/android/app/src/main/java/bot/molt/android/NodeApp.kt +++ b/apps/android/app/src/main/java/bot/molt/android/NodeApp.kt @@ -2,12 +2,17 @@ package bot.molt.android import android.app.Application import android.os.StrictMode +import com.clawdbot.android.sync.SyncWorker class NodeApp : Application() { val runtime: NodeRuntime by lazy { NodeRuntime(this) } override fun onCreate() { super.onCreate() + + // Schedule periodic background sync for chat history + SyncWorker.schedule(this) + if (BuildConfig.DEBUG) { StrictMode.setThreadPolicy( StrictMode.ThreadPolicy.Builder() diff --git a/apps/android/app/src/main/java/bot/molt/android/NodeCommandHandlers.kt b/apps/android/app/src/main/java/bot/molt/android/NodeCommandHandlers.kt new file mode 100644 index 000000000..d2c42f272 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/NodeCommandHandlers.kt @@ -0,0 +1,326 @@ +package bot.molt.android + +import android.Manifest +import android.content.pm.PackageManager +import android.location.LocationManager +import android.util.Log +import androidx.core.content.ContextCompat +import bot.molt.android.gateway.GatewaySession +import bot.molt.android.node.CanvasController +import bot.molt.android.protocol.ClawdbotCameraCommand +import bot.molt.android.protocol.ClawdbotCanvasA2UICommand +import bot.molt.android.protocol.ClawdbotCanvasCommand +import bot.molt.android.protocol.ClawdbotLocationCommand +import bot.molt.android.protocol.ClawdbotScreenCommand +import bot.molt.android.protocol.ClawdbotSmsCommand +import kotlinx.coroutines.TimeoutCancellationException + +private const val TAG = "NodeCommandHandlers" + +// MARK: - Command Handlers Extension + +/** + * Extension for node.invoke command routing and handlers. + * Separated from NodeRuntime for maintainability. + */ +internal suspend fun NodeRuntime.routeInvoke( + command: String, + paramsJson: String? +): GatewaySession.InvokeResult { + // Background restrictions + if (command.startsWith(ClawdbotCanvasCommand.NamespacePrefix) || + command.startsWith(ClawdbotCanvasA2UICommand.NamespacePrefix) || + command.startsWith(ClawdbotCameraCommand.NamespacePrefix) || + command.startsWith(ClawdbotScreenCommand.NamespacePrefix) + ) { + if (!isForeground.value) { + return GatewaySession.InvokeResult.error( + code = "NODE_BACKGROUND_UNAVAILABLE", + message = "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground", + ) + } + } + + // Camera check + if (command.startsWith(ClawdbotCameraCommand.NamespacePrefix) && !cameraEnabled.value) { + return GatewaySession.InvokeResult.error( + code = "CAMERA_DISABLED", + message = "CAMERA_DISABLED: enable Camera in Settings", + ) + } + + // Location check + if (command.startsWith(ClawdbotLocationCommand.NamespacePrefix) && + locationMode.value == LocationMode.Off + ) { + return GatewaySession.InvokeResult.error( + code = "LOCATION_DISABLED", + message = "LOCATION_DISABLED: enable Location in Settings", + ) + } + + return when (command) { + ClawdbotCanvasCommand.Present.rawValue -> handleCanvasPresent(paramsJson) + ClawdbotCanvasCommand.Hide.rawValue -> GatewaySession.InvokeResult.ok(null) + ClawdbotCanvasCommand.Navigate.rawValue -> handleCanvasNavigate(paramsJson) + ClawdbotCanvasCommand.Eval.rawValue -> handleCanvasEval(paramsJson) + ClawdbotCanvasCommand.Snapshot.rawValue -> handleCanvasSnapshot(paramsJson) + ClawdbotCanvasA2UICommand.Reset.rawValue -> handleA2UIReset() + ClawdbotCanvasA2UICommand.Push.rawValue, + ClawdbotCanvasA2UICommand.PushJSONL.rawValue -> handleA2UIPush(command, paramsJson) + ClawdbotCameraCommand.Snap.rawValue -> handleCameraSnap(paramsJson) + ClawdbotCameraCommand.Clip.rawValue -> handleCameraClip(paramsJson) + ClawdbotLocationCommand.Get.rawValue -> handleLocationGet(paramsJson) + ClawdbotScreenCommand.Record.rawValue -> handleScreenRecord(paramsJson) + ClawdbotSmsCommand.Send.rawValue -> handleSmsSend(paramsJson) + else -> GatewaySession.InvokeResult.error( + code = "INVALID_REQUEST", + message = "INVALID_REQUEST: unknown command", + ) + } +} + +// MARK: - Canvas Handlers + +private fun NodeRuntime.handleCanvasPresent(paramsJson: String?): GatewaySession.InvokeResult { + val url = CanvasController.parseNavigateUrl(paramsJson) + canvas.navigate(url) + return GatewaySession.InvokeResult.ok(null) +} + +private fun NodeRuntime.handleCanvasNavigate(paramsJson: String?): GatewaySession.InvokeResult { + val url = CanvasController.parseNavigateUrl(paramsJson) + canvas.navigate(url) + return GatewaySession.InvokeResult.ok(null) +} + +private suspend fun NodeRuntime.handleCanvasEval(paramsJson: String?): GatewaySession.InvokeResult { + val js = CanvasController.parseEvalJs(paramsJson) + ?: return GatewaySession.InvokeResult.error( + code = "INVALID_REQUEST", + message = "INVALID_REQUEST: javaScript required", + ) + val result = try { + canvas.eval(js) + } catch (err: Throwable) { + return GatewaySession.InvokeResult.error( + code = "NODE_BACKGROUND_UNAVAILABLE", + message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable", + ) + } + return GatewaySession.InvokeResult.ok("""{"result":${result.toJsonStringEscaped()}}""") +} + +private suspend fun NodeRuntime.handleCanvasSnapshot(paramsJson: String?): GatewaySession.InvokeResult { + val snapshotParams = CanvasController.parseSnapshotParams(paramsJson) + val base64 = try { + canvas.snapshotBase64( + format = snapshotParams.format, + quality = snapshotParams.quality, + maxWidth = snapshotParams.maxWidth, + ) + } catch (err: Throwable) { + return GatewaySession.InvokeResult.error( + code = "NODE_BACKGROUND_UNAVAILABLE", + message = "NODE_BACKGROUND_UNAVAILABLE: canvas unavailable", + ) + } + return GatewaySession.InvokeResult.ok("""{"format":"${snapshotParams.format.rawValue}","base64":"$base64"}""") +} + +// MARK: - A2UI Handlers + +private suspend fun NodeRuntime.handleA2UIReset(): GatewaySession.InvokeResult { + val a2uiUrl = resolveA2uiHostUrl() + ?: return GatewaySession.InvokeResult.error( + code = "A2UI_HOST_NOT_CONFIGURED", + message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host", + ) + val ready = ensureA2uiReady(a2uiUrl) + if (!ready) { + return GatewaySession.InvokeResult.error( + code = "A2UI_HOST_UNAVAILABLE", + message = "A2UI host not reachable", + ) + } + val res = canvas.eval(A2UI_RESET_JS) + return GatewaySession.InvokeResult.ok(res) +} + +private suspend fun NodeRuntime.handleA2UIPush( + command: String, + paramsJson: String? +): GatewaySession.InvokeResult { + val messages = try { + decodeA2uiMessages(command, paramsJson) + } catch (err: Throwable) { + return GatewaySession.InvokeResult.error( + code = "INVALID_REQUEST", + message = err.message ?: "invalid A2UI payload" + ) + } + val a2uiUrl = resolveA2uiHostUrl() + ?: return GatewaySession.InvokeResult.error( + code = "A2UI_HOST_NOT_CONFIGURED", + message = "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host", + ) + val ready = ensureA2uiReady(a2uiUrl) + if (!ready) { + return GatewaySession.InvokeResult.error( + code = "A2UI_HOST_UNAVAILABLE", + message = "A2UI host not reachable", + ) + } + val js = a2uiApplyMessagesJS(messages) + val res = canvas.eval(js) + return GatewaySession.InvokeResult.ok(res) +} + +// MARK: - Camera Handlers + +private suspend fun NodeRuntime.handleCameraSnap(paramsJson: String?): GatewaySession.InvokeResult { + showCameraHud(message = "Taking photo…", kind = CameraHudKind.Photo) + triggerCameraFlash() + val res = try { + camera.snap(paramsJson) + } catch (err: Throwable) { + val (code, message) = invokeErrorFromThrowable(err) + showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2200) + return GatewaySession.InvokeResult.error(code = code, message = message) + } + showCameraHud(message = "Photo captured", kind = CameraHudKind.Success, autoHideMs = 1600) + return GatewaySession.InvokeResult.ok(res.payloadJson) +} + +private suspend fun NodeRuntime.handleCameraClip(paramsJson: String?): GatewaySession.InvokeResult { + val includeAudio = paramsJson?.contains("\"includeAudio\":true") != false + if (includeAudio) setExternalAudioCaptureActive(true) + try { + showCameraHud(message = "Recording…", kind = CameraHudKind.Recording) + val res = try { + camera.clip(paramsJson) + } catch (err: Throwable) { + val (code, message) = invokeErrorFromThrowable(err) + showCameraHud(message = message, kind = CameraHudKind.Error, autoHideMs = 2400) + return GatewaySession.InvokeResult.error(code = code, message = message) + } + showCameraHud(message = "Clip captured", kind = CameraHudKind.Success, autoHideMs = 1800) + return GatewaySession.InvokeResult.ok(res.payloadJson) + } finally { + if (includeAudio) setExternalAudioCaptureActive(false) + } +} + +// MARK: - Location Handler + +private suspend fun NodeRuntime.handleLocationGet(paramsJson: String?): GatewaySession.InvokeResult { + val mode = locationMode.value + if (!isForeground.value && mode != LocationMode.Always) { + return GatewaySession.InvokeResult.error( + code = "LOCATION_BACKGROUND_UNAVAILABLE", + message = "LOCATION_BACKGROUND_UNAVAILABLE: background location requires Always", + ) + } + if (!hasFineLocationPermission() && !hasCoarseLocationPermission()) { + return GatewaySession.InvokeResult.error( + code = "LOCATION_PERMISSION_REQUIRED", + message = "LOCATION_PERMISSION_REQUIRED: grant Location permission", + ) + } + if (!isForeground.value && mode == LocationMode.Always && !hasBackgroundLocationPermission()) { + return GatewaySession.InvokeResult.error( + code = "LOCATION_PERMISSION_REQUIRED", + message = "LOCATION_PERMISSION_REQUIRED: enable Always in system Settings", + ) + } + + val (maxAgeMs, timeoutMs, desiredAccuracy) = parseLocationParams(paramsJson) + val preciseEnabled = locationPreciseEnabled.value + val accuracy = when (desiredAccuracy) { + "precise" -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced" + "coarse" -> "coarse" + else -> if (preciseEnabled && hasFineLocationPermission()) "precise" else "balanced" + } + val providers = when (accuracy) { + "precise" -> listOf(LocationManager.GPS_PROVIDER, LocationManager.NETWORK_PROVIDER) + "coarse" -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER) + else -> listOf(LocationManager.NETWORK_PROVIDER, LocationManager.GPS_PROVIDER) + } + + return try { + val payload = location.getLocation( + desiredProviders = providers, + maxAgeMs = maxAgeMs, + timeoutMs = timeoutMs, + isPrecise = accuracy == "precise", + ) + GatewaySession.InvokeResult.ok(payload.payloadJson) + } catch (err: TimeoutCancellationException) { + GatewaySession.InvokeResult.error( + code = "LOCATION_TIMEOUT", + message = "LOCATION_TIMEOUT: no fix in time", + ) + } catch (err: Throwable) { + val message = err.message ?: "LOCATION_UNAVAILABLE: no fix" + GatewaySession.InvokeResult.error(code = "LOCATION_UNAVAILABLE", message = message) + } +} + +// MARK: - Screen Recording Handler + +private suspend fun NodeRuntime.handleScreenRecord(paramsJson: String?): GatewaySession.InvokeResult { + setScreenRecordActive(true) + try { + val res = try { + screenRecorder.record(paramsJson) + } catch (err: Throwable) { + val (code, message) = invokeErrorFromThrowable(err) + return GatewaySession.InvokeResult.error(code = code, message = message) + } + return GatewaySession.InvokeResult.ok(res.payloadJson) + } finally { + setScreenRecordActive(false) + } +} + +// MARK: - SMS Handler + +private fun NodeRuntime.handleSmsSend(paramsJson: String?): GatewaySession.InvokeResult { + val res = sms.send(paramsJson) + return if (res.ok) { + GatewaySession.InvokeResult.ok(res.payloadJson) + } else { + val error = res.error ?: "SMS_SEND_FAILED" + val idx = error.indexOf(':') + val code = if (idx > 0) error.substring(0, idx).trim() else "SMS_SEND_FAILED" + GatewaySession.InvokeResult.error(code = code, message = error) + } +} + +// MARK: - Permission Checks + +internal fun NodeRuntime.hasFineLocationPermission(): Boolean { + return ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_FINE_LOCATION) == + PackageManager.PERMISSION_GRANTED +} + +internal fun NodeRuntime.hasCoarseLocationPermission(): Boolean { + return ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_COARSE_LOCATION) == + PackageManager.PERMISSION_GRANTED +} + +internal fun NodeRuntime.hasBackgroundLocationPermission(): Boolean { + return ContextCompat.checkSelfPermission(appContext, Manifest.permission.ACCESS_BACKGROUND_LOCATION) == + PackageManager.PERMISSION_GRANTED +} + +// MARK: - Helper Extension + +private fun String.toJsonStringEscaped(): String { + val escaped = this + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + return "\"$escaped\"" +} diff --git a/apps/android/app/src/main/java/bot/molt/android/NodeGatewaySync.kt b/apps/android/app/src/main/java/bot/molt/android/NodeGatewaySync.kt new file mode 100644 index 000000000..e35394ce6 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/NodeGatewaySync.kt @@ -0,0 +1,294 @@ +package bot.molt.android + +import android.util.Log +import bot.molt.android.protocol.ClawdbotCanvasA2UICommand +import kotlinx.coroutines.delay +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.JsonPrimitive + +private const val TAG = "NodeGatewaySync" + +// MARK: - Gateway Sync Extension + +/** + * Extension for gateway synchronization, branding, and wake word sync. + * Separated from NodeRuntime for maintainability. + */ +internal suspend fun NodeRuntime.refreshBrandingFromGateway() { + if (!isConnected.value) return + try { + val res = operatorSession.request("config.get", "{}") + val root = jsonParser.parseToJsonElement(res).asObjectOrNull() + val config = root?.get("config").asObjectOrNull() + val ui = config?.get("ui").asObjectOrNull() + val raw = ui?.get("seamColor").asStringOrNull()?.trim() + val sessionCfg = config?.get("session").asObjectOrNull() + val mainKey = normalizeMainKey(sessionCfg?.get("mainKey").asStringOrNull()) + applyMainSessionKey(mainKey) + + val parsed = parseHexColorArgb(raw) + setSeamColorArgb(parsed ?: DEFAULT_SEAM_COLOR_ARGB) + } catch (e: Throwable) { + Log.d(TAG, "Failed to refresh branding from gateway", e) + } +} + +internal suspend fun NodeRuntime.refreshWakeWordsFromGateway() { + if (!isConnected.value) return + try { + val res = operatorSession.request("voicewake.get", "{}") + val payload = jsonParser.parseToJsonElement(res).asObjectOrNull() ?: return + val array = payload["triggers"] as? JsonArray ?: return + val triggers = array.mapNotNull { it.asStringOrNull() } + applyWakeWordsFromGateway(triggers) + } catch (e: Throwable) { + Log.d(TAG, "Failed to refresh wake words from gateway", e) + } +} + +internal fun NodeRuntime.handleGatewayEvent(event: String, payloadJson: String?) { + if (event == "voicewake.changed") { + if (payloadJson.isNullOrBlank()) return + try { + val payload = jsonParser.parseToJsonElement(payloadJson).asObjectOrNull() ?: return + val array = payload["triggers"] as? JsonArray ?: return + val triggers = array.mapNotNull { it.asStringOrNull() } + applyWakeWordsFromGateway(triggers) + } catch (e: Throwable) { + Log.w(TAG, "Failed to parse voicewake.changed event", e) + } + return + } + + talkModeManager.handleGatewayEvent(event, payloadJson) + chatController.handleGatewayEvent(event, payloadJson) +} + +internal suspend fun NodeRuntime.scheduleWakeWordsSyncIfNeeded() { + if (suppressWakeWordsSync) return + if (!isConnected.value) return + + val snapshot = prefs.wakeWords.value + wakeWordsSyncJob?.cancel() + wakeWordsSyncJob = scope.launch { + delay(650) + val jsonList = snapshot.joinToString(separator = ",") { it.toJsonStringInternal() } + val params = """{"triggers":[$jsonList]}""" + try { + operatorSession.request("voicewake.set", params) + } catch (e: Throwable) { + Log.d(TAG, "Failed to sync wake words to gateway", e) + } + } +} + +// MARK: - A2UI URL Resolution + +internal fun NodeRuntime.resolveA2uiHostUrl(): String? { + val nodeRaw = nodeSession.currentCanvasHostUrl()?.trim().orEmpty() + val operatorRaw = operatorSession.currentCanvasHostUrl()?.trim().orEmpty() + val raw = if (nodeRaw.isNotBlank()) nodeRaw else operatorRaw + if (raw.isBlank()) return null + val base = raw.trimEnd('/') + return "${base}/__clawdbot__/a2ui/?platform=android" +} + +internal suspend fun NodeRuntime.ensureA2uiReady(a2uiUrl: String): Boolean { + try { + val already = canvas.eval(A2UI_READY_CHECK_JS) + if (already == "true") return true + } catch (e: Throwable) { + Log.d(TAG, "A2UI ready check failed (will retry)", e) + } + + canvas.navigate(a2uiUrl) + repeat(50) { + try { + val ready = canvas.eval(A2UI_READY_CHECK_JS) + if (ready == "true") return true + } catch (e: Throwable) { + Log.v(TAG, "A2UI ready poll failed, retrying...", e) + } + delay(120) + } + return false +} + +internal fun NodeRuntime.maybeNavigateToA2uiOnConnect() { + val a2uiUrl = resolveA2uiHostUrl() ?: return + val current = canvas.currentUrl()?.trim().orEmpty() + if (current.isEmpty() || current == lastAutoA2uiUrl) { + lastAutoA2uiUrl = a2uiUrl + canvas.navigate(a2uiUrl) + } +} + +internal fun NodeRuntime.showLocalCanvasOnDisconnect() { + lastAutoA2uiUrl = null + canvas.navigate("") +} + +// MARK: - A2UI Message Decoding + +internal fun NodeRuntime.decodeA2uiMessages(command: String, paramsJson: String?): String { + val raw = paramsJson?.trim().orEmpty() + if (raw.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: paramsJSON required") + + val obj = jsonParser.parseToJsonElement(raw) as? JsonObject + ?: throw IllegalArgumentException("INVALID_REQUEST: expected object params") + + val jsonlField = (obj["jsonl"] as? JsonPrimitive)?.content?.trim().orEmpty() + val hasMessagesArray = obj["messages"] is JsonArray + + if (command == ClawdbotCanvasA2UICommand.PushJSONL.rawValue || (!hasMessagesArray && jsonlField.isNotBlank())) { + val jsonl = jsonlField + if (jsonl.isBlank()) throw IllegalArgumentException("INVALID_REQUEST: jsonl required") + val messages = jsonl + .lineSequence() + .map { it.trim() } + .filter { it.isNotBlank() } + .mapIndexed { idx, line -> + val el = jsonParser.parseToJsonElement(line) + val msg = el as? JsonObject + ?: throw IllegalArgumentException("A2UI JSONL line ${idx + 1}: expected a JSON object") + validateA2uiV0_8(msg, idx + 1) + msg + } + .toList() + return JsonArray(messages).toString() + } + + val arr = obj["messages"] as? JsonArray + ?: throw IllegalArgumentException("INVALID_REQUEST: messages[] required") + val out = arr.mapIndexed { idx, el -> + val msg = el as? JsonObject + ?: throw IllegalArgumentException("A2UI messages[${idx}]: expected a JSON object") + validateA2uiV0_8(msg, idx + 1) + msg + } + return JsonArray(out).toString() +} + +private fun validateA2uiV0_8(msg: JsonObject, lineNumber: Int) { + if (msg.containsKey("createSurface")) { + throw IllegalArgumentException( + "A2UI JSONL line $lineNumber: looks like A2UI v0.9 (`createSurface`). Canvas supports v0.8 messages only.", + ) + } + val allowed = setOf("beginRendering", "surfaceUpdate", "dataModelUpdate", "deleteSurface") + val matched = msg.keys.filter { allowed.contains(it) } + if (matched.size != 1) { + val found = msg.keys.sorted().joinToString(", ") + throw IllegalArgumentException( + "A2UI JSONL line $lineNumber: expected exactly one of ${allowed.sorted().joinToString(", ")}; found: $found", + ) + } +} + +// MARK: - Helper Methods + +internal fun NodeRuntime.invokeErrorFromThrowable(err: Throwable): Pair { + val raw = (err.message ?: "").trim() + if (raw.isEmpty()) return "UNAVAILABLE" to "UNAVAILABLE: camera error" + + val idx = raw.indexOf(':') + if (idx <= 0) return "UNAVAILABLE" to raw + val code = raw.substring(0, idx).trim().ifEmpty { "UNAVAILABLE" } + val message = raw.substring(idx + 1).trim().ifEmpty { raw } + return code to "$code: $message" +} + +internal fun NodeRuntime.parseLocationParams(paramsJson: String?): Triple { + if (paramsJson.isNullOrBlank()) { + return Triple(null, 10_000L, null) + } + val root = try { + jsonParser.parseToJsonElement(paramsJson).asObjectOrNull() + } catch (e: Throwable) { + Log.d(TAG, "Failed to parse location params JSON", e) + null + } + val maxAgeMs = (root?.get("maxAgeMs") as? JsonPrimitive)?.content?.toLongOrNull() + val timeoutMs = (root?.get("timeoutMs") as? JsonPrimitive)?.content?.toLongOrNull()?.coerceIn(1_000L, 60_000L) + ?: 10_000L + val desiredAccuracy = (root?.get("desiredAccuracy") as? JsonPrimitive)?.content?.trim()?.lowercase() + return Triple(maxAgeMs, timeoutMs, desiredAccuracy) +} + +// MARK: - A2UI JavaScript Constants + +internal const val A2UI_READY_CHECK_JS: String = """ +(() => { + try { + return !!globalThis.clawdbotA2UI && typeof globalThis.clawdbotA2UI.applyMessages === 'function'; + } catch (_) { + return false; + } +})() +""" + +internal const val A2UI_RESET_JS: String = """ +(() => { + try { + if (!globalThis.clawdbotA2UI) return { ok: false, error: "missing clawdbotA2UI" }; + return globalThis.clawdbotA2UI.reset(); + } catch (e) { + return { ok: false, error: String(e?.message ?? e) }; + } +})() +""" + +internal fun a2uiApplyMessagesJS(messagesJson: String): String { + return """ + (() => { + try { + if (!globalThis.clawdbotA2UI) return { ok: false, error: "missing clawdbotA2UI" }; + const messages = $messagesJson; + return globalThis.clawdbotA2UI.applyMessages(messages); + } catch (e) { + return { ok: false, error: String(e?.message ?? e) }; + } + })() + """.trimIndent() +} + +internal const val DEFAULT_SEAM_COLOR_ARGB: Long = 0xFF4F7A9A + +internal fun parseHexColorArgb(raw: String?): Long? { + val trimmed = raw?.trim().orEmpty() + if (trimmed.isEmpty()) return null + val hex = if (trimmed.startsWith("#")) trimmed.drop(1) else trimmed + if (hex.length != 6) return null + val rgb = hex.toLongOrNull(16) ?: return null + return 0xFF000000L or rgb +} + +internal fun normalizeMainKey(raw: String?): String { + val trimmed = raw?.trim().orEmpty() + return trimmed.ifEmpty { "main" } +} + +internal fun isCanonicalMainSessionKey(key: String): Boolean { + return key.trim().equals("main", ignoreCase = true) +} + +// MARK: - JSON Helpers + +internal fun kotlinx.serialization.json.JsonElement?.asObjectOrNull(): JsonObject? = this as? JsonObject + +internal fun kotlinx.serialization.json.JsonElement?.asStringOrNull(): String? = + when (this) { + is kotlinx.serialization.json.JsonNull -> null + is JsonPrimitive -> content + else -> null + } + +private fun String.toJsonStringInternal(): String { + val escaped = this + .replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", "\\n") + .replace("\r", "\\r") + return "\"$escaped\"" +} diff --git a/apps/android/app/src/main/java/bot/molt/android/chat/ChatController.kt b/apps/android/app/src/main/java/bot/molt/android/chat/ChatController.kt index eef66fece..8a925e268 100644 --- a/apps/android/app/src/main/java/bot/molt/android/chat/ChatController.kt +++ b/apps/android/app/src/main/java/bot/molt/android/chat/ChatController.kt @@ -1,5 +1,6 @@ package bot.molt.android.chat +import android.util.Log import bot.molt.android.gateway.GatewaySession import java.util.UUID import java.util.concurrent.ConcurrentHashMap @@ -24,6 +25,9 @@ class ChatController( private val json: Json, private val supportsChatSubscribe: Boolean, ) { + companion object { + private const val TAG = "ChatController" + } private val _sessionKey = MutableStateFlow("main") val sessionKey: StateFlow = _sessionKey.asStateFlow() @@ -218,8 +222,8 @@ class ChatController( put("runId", JsonPrimitive(runId)) } session.request("chat.abort", params.toString()) - } catch (_: Throwable) { - // best-effort + } catch (e: Throwable) { + Log.d(TAG, "Failed to abort chat (best-effort)", e) } } } @@ -263,8 +267,8 @@ class ChatController( if (supportsChatSubscribe) { try { session.sendNodeEvent("chat.subscribe", """{"sessionKey":"$key"}""") - } catch (_: Throwable) { - // best-effort + } catch (e: Throwable) { + Log.d(TAG, "Failed to subscribe to chat events (best-effort)", e) } } @@ -291,8 +295,8 @@ class ChatController( } val res = session.request("sessions.list", params.toString()) _sessions.value = parseSessions(res) - } catch (_: Throwable) { - // best-effort + } catch (e: Throwable) { + Log.d(TAG, "Failed to refresh chat sessions (best-effort)", e) } } @@ -304,7 +308,8 @@ class ChatController( try { session.request("health", null) _healthOk.value = true - } catch (_: Throwable) { + } catch (e: Throwable) { + Log.d(TAG, "Health check failed", e) _healthOk.value = false } } @@ -341,8 +346,8 @@ class ChatController( _messages.value = history.messages _sessionId.value = history.sessionId history.thinkingLevel?.trim()?.takeIf { it.isNotEmpty() }?.let { _thinkingLevel.value = it } - } catch (_: Throwable) { - // best-effort + } catch (e: Throwable) { + Log.d(TAG, "Failed to refresh chat history after complete event (best-effort)", e) } } } @@ -491,7 +496,8 @@ class ChatController( private fun parseRunId(resJson: String): String? { return try { json.parseToJsonElement(resJson).asObjectOrNull()?.get("runId").asStringOrNull() - } catch (_: Throwable) { + } catch (e: Throwable) { + Log.d(TAG, "Failed to parse runId from response", e) null } } diff --git a/apps/android/app/src/main/java/bot/molt/android/gateway/GatewaySession.kt b/apps/android/app/src/main/java/bot/molt/android/gateway/GatewaySession.kt index 13074b918..b18949db5 100644 --- a/apps/android/app/src/main/java/bot/molt/android/gateway/GatewaySession.kt +++ b/apps/android/app/src/main/java/bot/molt/android/gateway/GatewaySession.kt @@ -62,6 +62,9 @@ class GatewaySession( private val onInvoke: (suspend (InvokeRequest) -> InvokeResult)? = null, private val onTlsFingerprint: ((stableId: String, fingerprint: String) -> Unit)? = null, ) { + companion object { + private const val TAG = "GatewaySession" + } data class InvokeRequest( val id: String, val nodeId: String, @@ -458,7 +461,8 @@ class GatewaySession( if (isLoopbackHost(endpoint.host)) return null return try { withTimeout(2_000) { connectNonceDeferred.await() } - } catch (_: Throwable) { + } catch (e: Throwable) { + Log.d(TAG, "Timeout awaiting connect nonce", e) null } } @@ -473,7 +477,8 @@ class GatewaySession( val payload = try { json.parseToJsonElement(payloadJson).asObjectOrNull() - } catch (_: Throwable) { + } catch (e: Throwable) { + Log.w(TAG, "Failed to parse invoke event JSON", e) null } ?: return val id = payload["id"].asStringOrNull() ?: return diff --git a/apps/android/app/src/main/java/bot/molt/android/push/ClawdbotMessagingService.kt b/apps/android/app/src/main/java/bot/molt/android/push/ClawdbotMessagingService.kt new file mode 100644 index 000000000..fcf10bd7e --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/push/ClawdbotMessagingService.kt @@ -0,0 +1,161 @@ +package bot.molt.android.push + +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.content.Context +import android.content.Intent +import android.util.Log +import androidx.core.app.NotificationCompat +import bot.molt.android.MainActivity +import bot.molt.android.NodeApp +import com.google.firebase.messaging.FirebaseMessagingService +import com.google.firebase.messaging.RemoteMessage + +/** + * Firebase Cloud Messaging service for receiving push notifications. + * Handles both notification messages (shown automatically when app backgrounded) + * and data messages (always delivered to this handler). + */ +class ClawdbotMessagingService : FirebaseMessagingService() { + + override fun onCreate() { + super.onCreate() + createNotificationChannel() + } + + override fun onNewToken(token: String) { + super.onNewToken(token) + Log.d(TAG, "FCM Token refreshed: ${token.take(16)}...") + + // Send token to gateway for registration + val app = application as? NodeApp + app?.let { + // Store token for later registration with gateway + PushTokenStore.saveToken(this, token) + Log.d(TAG, "FCM token saved to local storage") + } + } + + override fun onMessageReceived(message: RemoteMessage) { + super.onMessageReceived(message) + Log.d(TAG, "FCM message received from: ${message.from}") + + // Check if message contains a notification payload (handled automatically if app is in background) + message.notification?.let { notification -> + Log.d(TAG, "FCM notification payload: title=${notification.title}, body=${notification.body}") + showNotification( + title = notification.title ?: "Clawdbot", + body = notification.body ?: "New message" + ) + } + + // Check if message contains a data payload (always handled here) + if (message.data.isNotEmpty()) { + Log.d(TAG, "FCM data payload: ${message.data.keys.joinToString()}") + handleDataMessage(message.data) + } + } + + private fun handleDataMessage(data: Map) { + val messageType = data["type"] + val title = data["title"] ?: "Clawdbot" + val body = data["body"] ?: "New message" + val sessionKey = data["sessionKey"] + + when (messageType) { + "chat" -> { + // Incoming chat message - show notification and optionally sync + Log.d(TAG, "Chat message received for session: $sessionKey") + showNotification(title, body, sessionKey) + } + "sync" -> { + // Silent sync request - trigger background sync + Log.d(TAG, "Sync request received") + // Could trigger SyncWorker.scheduleImmediately() here + } + else -> { + // Generic notification + showNotification(title, body) + } + } + } + + private fun createNotificationChannel() { + val channel = NotificationChannel( + CHANNEL_ID, + "Chat Messages", + NotificationManager.IMPORTANCE_HIGH + ).apply { + description = "Notifications for incoming chat messages" + enableLights(true) + enableVibration(true) + } + + val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + manager.createNotificationChannel(channel) + Log.d(TAG, "Notification channel created: $CHANNEL_ID") + } + + private fun showNotification(title: String, body: String, sessionKey: String? = null) { + // Create intent to open app when notification is tapped + val intent = Intent(this, MainActivity::class.java).apply { + flags = Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_CLEAR_TOP + sessionKey?.let { putExtra("sessionKey", it) } + } + + val pendingIntent = PendingIntent.getActivity( + this, + 0, + intent, + PendingIntent.FLAG_ONE_SHOT or PendingIntent.FLAG_IMMUTABLE + ) + + val notification = NotificationCompat.Builder(this, CHANNEL_ID) + .setSmallIcon(android.R.drawable.ic_dialog_info) // TODO: Replace with app icon + .setContentTitle(title) + .setContentText(body) + .setAutoCancel(true) + .setContentIntent(pendingIntent) + .setPriority(NotificationCompat.PRIORITY_HIGH) + .setDefaults(NotificationCompat.DEFAULT_ALL) + .build() + + val manager = getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + manager.notify(System.currentTimeMillis().toInt(), notification) + Log.d(TAG, "Notification shown: $title") + } + + companion object { + private const val TAG = "ClawdbotFCM" + private const val CHANNEL_ID = "clawdbot_chat" + } +} + +/** + * Simple storage for FCM push token. + * The token should be sent to the gateway when connection is established. + */ +object PushTokenStore { + private const val PREFS_NAME = "clawdbot_push" + private const val KEY_FCM_TOKEN = "fcm_token" + + fun saveToken(context: Context, token: String) { + context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + .edit() + .putString(KEY_FCM_TOKEN, token) + .apply() + } + + fun getToken(context: Context): String? { + return context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + .getString(KEY_FCM_TOKEN, null) + } + + fun clearToken(context: Context) { + context.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE) + .edit() + .remove(KEY_FCM_TOKEN) + .apply() + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/sync/SyncWorker.kt b/apps/android/app/src/main/java/bot/molt/android/sync/SyncWorker.kt new file mode 100644 index 000000000..cf081d553 --- /dev/null +++ b/apps/android/app/src/main/java/bot/molt/android/sync/SyncWorker.kt @@ -0,0 +1,93 @@ +package bot.molt.android.sync + +import android.content.Context +import android.util.Log +import androidx.work.BackoffPolicy +import androidx.work.Constraints +import androidx.work.CoroutineWorker +import androidx.work.ExistingPeriodicWorkPolicy +import androidx.work.NetworkType +import androidx.work.PeriodicWorkRequestBuilder +import androidx.work.WorkManager +import androidx.work.WorkerParameters +import bot.molt.android.NodeApp +import java.util.concurrent.TimeUnit + +/** + * Background sync worker that periodically syncs chat history when the app is backgrounded. + * Uses WorkManager for battery-efficient scheduling. + */ +class SyncWorker( + context: Context, + params: WorkerParameters +) : CoroutineWorker(context, params) { + + override suspend fun doWork(): Result { + Log.d(TAG, "Background sync starting") + + val app = applicationContext as? NodeApp + if (app == null) { + Log.w(TAG, "NodeApp not available") + return Result.failure() + } + + val runtime = app.runtime + + return try { + // Only sync if connected to gateway + if (runtime.isConnected.value) { + Log.d(TAG, "Gateway connected, refreshing chat") + runtime.refreshChat() + runtime.refreshChatSessions(limit = 50) + Log.d(TAG, "Background sync completed successfully") + Result.success() + } else { + Log.d(TAG, "Gateway not connected, skipping sync") + Result.success() + } + } catch (e: Exception) { + Log.e(TAG, "Background sync failed", e) + Result.retry() + } + } + + companion object { + private const val TAG = "SyncWorker" + private const val WORK_NAME = "clawdbot_background_sync" + + /** + * Schedule periodic background sync. + * Runs every 15 minutes (minimum interval) when network is available and battery isn't low. + */ + fun schedule(context: Context) { + val constraints = Constraints.Builder() + .setRequiredNetworkType(NetworkType.CONNECTED) + .setRequiresBatteryNotLow(true) + .build() + + val request = PeriodicWorkRequestBuilder( + 15, TimeUnit.MINUTES // Minimum interval allowed by WorkManager + ) + .setConstraints(constraints) + .setBackoffCriteria(BackoffPolicy.EXPONENTIAL, 1, TimeUnit.MINUTES) + .build() + + WorkManager.getInstance(context) + .enqueueUniquePeriodicWork( + WORK_NAME, + ExistingPeriodicWorkPolicy.KEEP, + request + ) + + Log.d(TAG, "Background sync scheduled") + } + + /** + * Cancel scheduled background sync. + */ + fun cancel(context: Context) { + WorkManager.getInstance(context).cancelUniqueWork(WORK_NAME) + Log.d(TAG, "Background sync cancelled") + } + } +} diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/RootScreen.kt b/apps/android/app/src/main/java/bot/molt/android/ui/RootScreen.kt index 67d76b82f..720e2e70a 100644 --- a/apps/android/app/src/main/java/bot/molt/android/ui/RootScreen.kt +++ b/apps/android/app/src/main/java/bot/molt/android/ui/RootScreen.kt @@ -50,11 +50,13 @@ import androidx.compose.material.icons.filled.Refresh import androidx.compose.material.icons.filled.Report import androidx.compose.material.icons.filled.Settings import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue +import com.clawdbot.android.DeepLinkSheet import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color as ComposeColor @@ -73,6 +75,19 @@ import bot.molt.android.MainViewModel fun RootScreen(viewModel: MainViewModel) { var sheet by remember { mutableStateOf(null) } val sheetState = rememberModalBottomSheetState(skipPartiallyExpanded = true) + val pendingDeepLinkSheet by viewModel.pendingDeepLinkSheet.collectAsState() + + // Handle deep link navigation requests + LaunchedEffect(pendingDeepLinkSheet) { + when (pendingDeepLinkSheet) { + DeepLinkSheet.Chat -> sheet = Sheet.Chat + DeepLinkSheet.Settings -> sheet = Sheet.Settings + null -> {} + } + if (pendingDeepLinkSheet != null) { + viewModel.consumeDeepLinkSheet() + } + } val safeOverlayInsets = WindowInsets.safeDrawing.only(WindowInsetsSides.Top + WindowInsetsSides.Horizontal) val context = LocalContext.current val serverName by viewModel.serverName.collectAsState() diff --git a/apps/android/app/src/main/java/bot/molt/android/ui/SettingsSheet.kt b/apps/android/app/src/main/java/bot/molt/android/ui/SettingsSheet.kt index f96731acf..0cde941d6 100644 --- a/apps/android/app/src/main/java/bot/molt/android/ui/SettingsSheet.kt +++ b/apps/android/app/src/main/java/bot/molt/android/ui/SettingsSheet.kt @@ -33,6 +33,7 @@ import androidx.compose.foundation.text.KeyboardOptions import androidx.compose.material.icons.Icons import androidx.compose.material.icons.filled.ExpandLess import androidx.compose.material.icons.filled.ExpandMore +import androidx.compose.material.icons.filled.WifiOff import androidx.compose.material3.Button import androidx.compose.material3.HorizontalDivider import androidx.compose.material3.Icon @@ -49,6 +50,7 @@ import androidx.compose.runtime.getValue import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.alpha import androidx.compose.ui.focus.onFocusChanged @@ -315,7 +317,30 @@ fun SettingsSheet(viewModel: MainViewModel) { ) } if (!isConnected && visibleGateways.isEmpty()) { - item { Text("No gateways found yet.", color = MaterialTheme.colorScheme.onSurfaceVariant) } + item { + Column( + modifier = Modifier.fillMaxWidth().padding(vertical = 24.dp), + horizontalAlignment = Alignment.CenterHorizontally, + verticalArrangement = Arrangement.spacedBy(8.dp), + ) { + Icon( + imageVector = Icons.Default.WifiOff, + contentDescription = null, + modifier = Modifier.size(48.dp), + tint = MaterialTheme.colorScheme.onSurfaceVariant, + ) + Text( + "No Gateways Found", + style = MaterialTheme.typography.titleMedium, + ) + Text( + "Make sure your Clawdbot gateway is running on the same network.", + style = MaterialTheme.typography.bodyMedium, + color = MaterialTheme.colorScheme.onSurfaceVariant, + textAlign = TextAlign.Center, + ) + } + } } else { items(items = visibleGateways, key = { it.stableId }) { gateway -> val detailLines = diff --git a/apps/android/app/src/main/java/bot/molt/android/voice/TalkModeManager.kt b/apps/android/app/src/main/java/bot/molt/android/voice/TalkModeManager.kt index f050f8bd2..4efbe2e50 100644 --- a/apps/android/app/src/main/java/bot/molt/android/voice/TalkModeManager.kt +++ b/apps/android/app/src/main/java/bot/molt/android/voice/TalkModeManager.kt @@ -140,7 +140,8 @@ class TalkModeManager( val obj = try { json.parseToJsonElement(payloadJson).asObjectOrNull() - } catch (_: Throwable) { + } catch (e: Throwable) { + Log.d(tag, "Failed to parse gateway event JSON", e) null } ?: return val runId = obj["runId"].asStringOrNull() ?: return @@ -245,8 +246,8 @@ class TalkModeManager( val shouldInterrupt = _isSpeaking.value && interruptOnSpeech if (!shouldListen && !shouldInterrupt) return@post startListeningInternal(markListening = shouldListen) - } catch (_: Throwable) { - // handled by onError + } catch (e: Throwable) { + Log.d(tag, "Failed to restart listening after end-of-speech", e) } } } @@ -395,7 +396,8 @@ class TalkModeManager( withContext(Dispatchers.IO) { try { kotlinx.coroutines.withTimeout(120_000) { deferred.await() } - } catch (_: Throwable) { + } catch (e: Throwable) { + Log.d(tag, "Timeout waiting for agent response", e) false } } @@ -702,7 +704,8 @@ class TalkModeManager( TextToSpeech(context) { status -> deferred.complete(status == TextToSpeech.SUCCESS) } - } catch (_: Throwable) { + } catch (e: Throwable) { + Log.w(tag, "Failed to initialize system TTS", e) deferred.complete(false) null } @@ -743,7 +746,8 @@ class TalkModeManager( val ok = try { deferred.await() - } catch (_: Throwable) { + } catch (e: Throwable) { + Log.d(tag, "TTS initialization deferred await failed", e) false } if (ok) { @@ -793,8 +797,8 @@ class TalkModeManager( track.pause() track.flush() track.stop() - } catch (_: Throwable) { - // ignore cleanup errors + } catch (e: Throwable) { + Log.d(tag, "PCM track cleanup error (ignored)", e) } finally { track.release() } @@ -842,7 +846,8 @@ class TalkModeManager( defaultOutputFormat = outputFormat ?: defaultOutputFormatFallback apiKey = key ?: envKey?.takeIf { it.isNotEmpty() } if (interrupt != null) interruptOnSpeech = interrupt - } catch (_: Throwable) { + } catch (e: Throwable) { + Log.w(tag, "Failed to parse talk config from gateway, using defaults", e) defaultVoiceId = envVoice?.takeIf { it.isNotEmpty() } ?: sagVoice?.takeIf { it.isNotEmpty() } defaultModelId = defaultModelIdFallback if (!modelOverrideActive) currentModelId = defaultModelId @@ -1099,8 +1104,8 @@ class TalkModeManager( } recognizer?.cancel() startListeningInternal(markListening = false) - } catch (_: Throwable) { - // ignore + } catch (e: Throwable) { + Log.d(tag, "Failed to start interrupt listener", e) } } } diff --git a/apps/android/app/src/test/java/bot/molt/android/NodeGatewaySyncTest.kt b/apps/android/app/src/test/java/bot/molt/android/NodeGatewaySyncTest.kt new file mode 100644 index 000000000..ba6381b12 --- /dev/null +++ b/apps/android/app/src/test/java/bot/molt/android/NodeGatewaySyncTest.kt @@ -0,0 +1,78 @@ +package bot.molt.android + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class NodeGatewaySyncTest { + @Test + fun parseHexColorArgbParsesValidHex() { + val result = parseHexColorArgb("#4F7A9A") + assertEquals(0xFF4F7A9AL, result) + } + + @Test + fun parseHexColorArgbParsesWithoutHash() { + val result = parseHexColorArgb("4F7A9A") + assertEquals(0xFF4F7A9AL, result) + } + + @Test + fun parseHexColorArgbReturnsNullForInvalidLength() { + assertNull(parseHexColorArgb("#FFF")) + assertNull(parseHexColorArgb("#FFFFFFFF")) + assertNull(parseHexColorArgb("12345")) + } + + @Test + fun parseHexColorArgbReturnsNullForEmpty() { + assertNull(parseHexColorArgb(null)) + assertNull(parseHexColorArgb("")) + assertNull(parseHexColorArgb(" ")) + } + + @Test + fun parseHexColorArgbReturnsNullForInvalidHex() { + assertNull(parseHexColorArgb("#GGGGGG")) + assertNull(parseHexColorArgb("#ZZZZZZ")) + } + + @Test + fun parseHexColorArgbTrimsWhitespace() { + val result = parseHexColorArgb(" #FFFFFF ") + assertEquals(0xFFFFFFFFL, result) + } + + @Test + fun normalizeMainKeyReturnsMainForNull() { + assertEquals("main", normalizeMainKey(null)) + } + + @Test + fun normalizeMainKeyReturnsMainForEmpty() { + assertEquals("main", normalizeMainKey("")) + assertEquals("main", normalizeMainKey(" ")) + } + + @Test + fun normalizeMainKeyReturnsTrimmedValue() { + assertEquals("custom-key", normalizeMainKey(" custom-key ")) + } + + @Test + fun isCanonicalMainSessionKeyReturnsTrueForMain() { + assertTrue(isCanonicalMainSessionKey("main")) + assertTrue(isCanonicalMainSessionKey("Main")) + assertTrue(isCanonicalMainSessionKey("MAIN")) + assertTrue(isCanonicalMainSessionKey(" main ")) + } + + @Test + fun isCanonicalMainSessionKeyReturnsFalseForOther() { + assertFalse(isCanonicalMainSessionKey("custom")) + assertFalse(isCanonicalMainSessionKey("session-1")) + assertFalse(isCanonicalMainSessionKey("")) + } +} diff --git a/apps/android/build.gradle.kts b/apps/android/build.gradle.kts index f79902d56..421386f4b 100644 --- a/apps/android/build.gradle.kts +++ b/apps/android/build.gradle.kts @@ -3,4 +3,5 @@ plugins { id("org.jetbrains.kotlin.android") version "2.2.21" apply false id("org.jetbrains.kotlin.plugin.compose") version "2.2.21" apply false id("org.jetbrains.kotlin.plugin.serialization") version "2.2.21" apply false + id("com.google.gms.google-services") version "4.4.2" apply false } diff --git a/apps/ios/Sources/Chat/OfflineMessageQueue.swift b/apps/ios/Sources/Chat/OfflineMessageQueue.swift new file mode 100644 index 000000000..2b7b4636e --- /dev/null +++ b/apps/ios/Sources/Chat/OfflineMessageQueue.swift @@ -0,0 +1,92 @@ +import Foundation +import OSLog + +/// Queues chat messages when the gateway is disconnected and delivers them on reconnect. +@MainActor +@Observable +final class OfflineMessageQueue { + struct PendingMessage: Codable, Identifiable, Sendable { + let id: UUID + let text: String + let sessionKey: String + let thinking: String + let createdAt: Date + var retryCount: Int + } + + private let logger = Logger(subsystem: "com.clawdbot", category: "offline-queue") + private let fileURL: URL + private let maxRetries = 3 + + private(set) var pending: [PendingMessage] = [] + + init() { + let docs = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0] + self.fileURL = docs.appendingPathComponent("offline-queue.json") + self.load() + } + + var count: Int { pending.count } + var isEmpty: Bool { pending.isEmpty } + + /// Add a message to the queue when gateway is disconnected. + func enqueue(text: String, sessionKey: String, thinking: String = "off") { + let msg = PendingMessage( + id: UUID(), + text: text, + sessionKey: sessionKey, + thinking: thinking, + createdAt: Date(), + retryCount: 0 + ) + pending.append(msg) + save() + logger.info("Queued message for session \(sessionKey, privacy: .public): \(text.prefix(50), privacy: .public)...") + } + + /// Remove a message after successful delivery. + func dequeue(_ id: UUID) { + pending.removeAll { $0.id == id } + save() + logger.info("Dequeued message \(id.uuidString, privacy: .public)") + } + + /// Mark a message as failed. Removes it if max retries exceeded. + func markFailed(_ id: UUID) { + guard let idx = pending.firstIndex(where: { $0.id == id }) else { return } + pending[idx].retryCount += 1 + if pending[idx].retryCount >= maxRetries { + logger.warning("Message \(id.uuidString, privacy: .public) exceeded max retries, removing") + pending.remove(at: idx) + } + save() + } + + /// Clear all pending messages. + func clear() { + pending.removeAll() + save() + logger.info("Cleared all pending messages") + } + + private func load() { + guard FileManager.default.fileExists(atPath: fileURL.path) else { return } + do { + let data = try Data(contentsOf: fileURL) + let decoded = try JSONDecoder().decode([PendingMessage].self, from: data) + pending = decoded + logger.info("Loaded \(decoded.count) pending messages from disk") + } catch { + logger.error("Failed to load offline queue: \(error.localizedDescription)") + } + } + + private func save() { + do { + let data = try JSONEncoder().encode(pending) + try data.write(to: fileURL, options: .atomic) + } catch { + logger.error("Failed to save offline queue: \(error.localizedDescription)") + } + } +} diff --git a/apps/ios/Sources/ClawdbotApp.swift b/apps/ios/Sources/ClawdbotApp.swift index d189e4964..a27bcb689 100644 --- a/apps/ios/Sources/ClawdbotApp.swift +++ b/apps/ios/Sources/ClawdbotApp.swift @@ -1,7 +1,9 @@ import SwiftUI +import UIKit @main struct MoltbotApp: App { + @UIApplicationDelegateAdaptor(AppDelegate.self) var appDelegate @State private var appModel: NodeAppModel @State private var gatewayController: GatewayConnectionController @Environment(\.scenePhase) private var scenePhase @@ -11,6 +13,12 @@ struct MoltbotApp: App { let appModel = NodeAppModel() _appModel = State(initialValue: appModel) _gatewayController = State(initialValue: GatewayConnectionController(appModel: appModel)) + + // Register for push notifications + PushManager.shared.registerForVoIPPush() + Task { + await PushManager.shared.registerForRemoteNotifications() + } } var body: some Scene { @@ -29,3 +37,25 @@ struct MoltbotApp: App { } } } + +// MARK: - AppDelegate for Push Notification Token Handling + +class AppDelegate: NSObject, UIApplicationDelegate { + func application( + _ application: UIApplication, + didRegisterForRemoteNotificationsWithDeviceToken deviceToken: Data + ) { + Task { @MainActor in + PushManager.shared.didRegisterForRemoteNotifications(deviceToken: deviceToken) + } + } + + func application( + _ application: UIApplication, + didFailToRegisterForRemoteNotificationsWithError error: Error + ) { + Task { @MainActor in + PushManager.shared.didFailToRegisterForRemoteNotifications(error: error) + } + } +} diff --git a/apps/ios/Sources/Info.plist b/apps/ios/Sources/Info.plist index d3e398ab4..e59d6f084 100644 --- a/apps/ios/Sources/Info.plist +++ b/apps/ios/Sources/Info.plist @@ -51,6 +51,19 @@ UIBackgroundModes audio + voip + remote-notification + + CFBundleURLTypes + + + CFBundleURLName + com.clawdbot.ios + CFBundleURLSchemes + + clawdbot + + UILaunchScreen diff --git a/apps/ios/Sources/Model/NodeCommandHandlers.swift b/apps/ios/Sources/Model/NodeCommandHandlers.swift new file mode 100644 index 000000000..f1bd65468 --- /dev/null +++ b/apps/ios/Sources/Model/NodeCommandHandlers.swift @@ -0,0 +1,371 @@ +import ClawdbotKit +import Foundation +import UIKit + +// MARK: - Command Handlers + +/// Extension containing all node.invoke command handlers. +/// Separated from NodeAppModel for maintainability. +extension NodeAppModel { + func routeInvoke(_ req: BridgeInvokeRequest) async -> BridgeInvokeResponse { + let command = req.command + + if self.isBackgrounded, self.isBackgroundRestricted(command) { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: ClawdbotNodeError( + code: .backgroundUnavailable, + message: "NODE_BACKGROUND_UNAVAILABLE: canvas/camera/screen commands require foreground")) + } + + if command.hasPrefix("camera."), !self.isCameraEnabled() { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: ClawdbotNodeError( + code: .unavailable, + message: "CAMERA_DISABLED: enable Camera in iOS Settings → Camera → Allow Camera")) + } + + do { + switch command { + case ClawdbotLocationCommand.get.rawValue: + return try await self.handleLocationInvoke(req) + case ClawdbotCanvasCommand.present.rawValue, + ClawdbotCanvasCommand.hide.rawValue, + ClawdbotCanvasCommand.navigate.rawValue, + ClawdbotCanvasCommand.evalJS.rawValue, + ClawdbotCanvasCommand.snapshot.rawValue: + return try await self.handleCanvasInvoke(req) + case ClawdbotCanvasA2UICommand.reset.rawValue, + ClawdbotCanvasA2UICommand.push.rawValue, + ClawdbotCanvasA2UICommand.pushJSONL.rawValue: + return try await self.handleCanvasA2UIInvoke(req) + case ClawdbotCameraCommand.list.rawValue, + ClawdbotCameraCommand.snap.rawValue, + ClawdbotCameraCommand.clip.rawValue: + return try await self.handleCameraInvoke(req) + case ClawdbotScreenCommand.record.rawValue: + return try await self.handleScreenRecordInvoke(req) + default: + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: ClawdbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command")) + } + } catch { + if command.hasPrefix("camera.") { + let text = (error as? LocalizedError)?.errorDescription ?? error.localizedDescription + self.showCameraHUD(text: text, kind: .error, autoHideSeconds: 2.2) + } + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: ClawdbotNodeError(code: .unavailable, message: error.localizedDescription)) + } + } + + func isBackgroundRestricted(_ command: String) -> Bool { + command.hasPrefix("canvas.") || command.hasPrefix("camera.") || command.hasPrefix("screen.") + } + + // MARK: - Location + + func handleLocationInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + let mode = self.locationMode() + guard mode != .off else { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: ClawdbotNodeError( + code: .unavailable, + message: "LOCATION_DISABLED: enable Location in Settings")) + } + if self.isBackgrounded, mode != .always { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: ClawdbotNodeError( + code: .backgroundUnavailable, + message: "LOCATION_BACKGROUND_UNAVAILABLE: background location requires Always")) + } + let params = (try? Self.decodeParams(ClawdbotLocationGetParams.self, from: req.paramsJSON)) ?? + ClawdbotLocationGetParams() + let desired = params.desiredAccuracy ?? + (self.isLocationPreciseEnabled() ? .precise : .balanced) + let status = self.locationService.authorizationStatus() + if status != .authorizedAlways, status != .authorizedWhenInUse { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: ClawdbotNodeError( + code: .unavailable, + message: "LOCATION_PERMISSION_REQUIRED: grant Location permission")) + } + if self.isBackgrounded, status != .authorizedAlways { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: ClawdbotNodeError( + code: .unavailable, + message: "LOCATION_PERMISSION_REQUIRED: enable Always for background access")) + } + let location = try await self.locationService.currentLocation( + params: params, + desiredAccuracy: desired, + maxAgeMs: params.maxAgeMs, + timeoutMs: params.timeoutMs) + let isPrecise = self.locationService.accuracyAuthorization() == .fullAccuracy + let payload = ClawdbotLocationPayload( + lat: location.coordinate.latitude, + lon: location.coordinate.longitude, + accuracyMeters: location.horizontalAccuracy, + altitudeMeters: location.verticalAccuracy >= 0 ? location.altitude : nil, + speedMps: location.speed >= 0 ? location.speed : nil, + headingDeg: location.course >= 0 ? location.course : nil, + timestamp: ISO8601DateFormatter().string(from: location.timestamp), + isPrecise: isPrecise, + source: nil) + let json = try Self.encodePayload(payload) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json) + } + + // MARK: - Canvas + + func handleCanvasInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + switch req.command { + case ClawdbotCanvasCommand.present.rawValue: + let params = (try? Self.decodeParams(ClawdbotCanvasPresentParams.self, from: req.paramsJSON)) ?? + ClawdbotCanvasPresentParams() + let url = params.url?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + if url.isEmpty { + self.screen.showDefaultCanvas() + } else { + self.screen.navigate(to: url) + } + return BridgeInvokeResponse(id: req.id, ok: true) + case ClawdbotCanvasCommand.hide.rawValue: + return BridgeInvokeResponse(id: req.id, ok: true) + case ClawdbotCanvasCommand.navigate.rawValue: + let params = try Self.decodeParams(ClawdbotCanvasNavigateParams.self, from: req.paramsJSON) + self.screen.navigate(to: params.url) + return BridgeInvokeResponse(id: req.id, ok: true) + case ClawdbotCanvasCommand.evalJS.rawValue: + let params = try Self.decodeParams(ClawdbotCanvasEvalParams.self, from: req.paramsJSON) + let result = try await self.screen.eval(javaScript: params.javaScript) + let payload = try Self.encodePayload(["result": result]) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + case ClawdbotCanvasCommand.snapshot.rawValue: + let params = try? Self.decodeParams(ClawdbotCanvasSnapshotParams.self, from: req.paramsJSON) + let format = params?.format ?? .jpeg + let maxWidth: CGFloat? = { + if let raw = params?.maxWidth, raw > 0 { return CGFloat(raw) } + return switch format { + case .png: 900 + case .jpeg: 1600 + } + }() + let base64 = try await self.screen.snapshotBase64( + maxWidth: maxWidth, + format: format, + quality: params?.quality) + let payload = try Self.encodePayload([ + "format": format == .jpeg ? "jpeg" : "png", + "base64": base64, + ]) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + default: + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: ClawdbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command")) + } + } + + // MARK: - Canvas A2UI + + func handleCanvasA2UIInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + let command = req.command + switch command { + case ClawdbotCanvasA2UICommand.reset.rawValue: + guard let a2uiUrl = await self.resolveA2UIHostURL() else { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: ClawdbotNodeError( + code: .unavailable, + message: "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host")) + } + self.screen.navigate(to: a2uiUrl) + if await !self.screen.waitForA2UIReady(timeoutMs: 5000) { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: ClawdbotNodeError( + code: .unavailable, + message: "A2UI_HOST_UNAVAILABLE: A2UI host not reachable")) + } + + let json = try await self.screen.eval(javaScript: """ + (() => { + if (!globalThis.clawdbotA2UI) return JSON.stringify({ ok: false, error: "missing clawdbotA2UI" }); + return JSON.stringify(globalThis.clawdbotA2UI.reset()); + })() + """) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: json) + case ClawdbotCanvasA2UICommand.push.rawValue, ClawdbotCanvasA2UICommand.pushJSONL.rawValue: + let messages: [AnyCodable] + if command == ClawdbotCanvasA2UICommand.pushJSONL.rawValue { + let params = try Self.decodeParams(ClawdbotCanvasA2UIPushJSONLParams.self, from: req.paramsJSON) + messages = try ClawdbotCanvasA2UIJSONL.decodeMessagesFromJSONL(params.jsonl) + } else { + do { + let params = try Self.decodeParams(ClawdbotCanvasA2UIPushParams.self, from: req.paramsJSON) + messages = params.messages + } catch { + let params = try Self.decodeParams(ClawdbotCanvasA2UIPushJSONLParams.self, from: req.paramsJSON) + messages = try ClawdbotCanvasA2UIJSONL.decodeMessagesFromJSONL(params.jsonl) + } + } + + guard let a2uiUrl = await self.resolveA2UIHostURL() else { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: ClawdbotNodeError( + code: .unavailable, + message: "A2UI_HOST_NOT_CONFIGURED: gateway did not advertise canvas host")) + } + self.screen.navigate(to: a2uiUrl) + if await !self.screen.waitForA2UIReady(timeoutMs: 5000) { + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: ClawdbotNodeError( + code: .unavailable, + message: "A2UI_HOST_UNAVAILABLE: A2UI host not reachable")) + } + + let messagesJSON = try ClawdbotCanvasA2UIJSONL.encodeMessagesJSONArray(messages) + let js = """ + (() => { + try { + if (!globalThis.clawdbotA2UI) return JSON.stringify({ ok: false, error: "missing clawdbotA2UI" }); + const messages = \(messagesJSON); + return JSON.stringify(globalThis.clawdbotA2UI.applyMessages(messages)); + } catch (e) { + return JSON.stringify({ ok: false, error: String(e?.message ?? e) }); + } + })() + """ + let resultJSON = try await self.screen.eval(javaScript: js) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: resultJSON) + default: + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: ClawdbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command")) + } + } + + // MARK: - Camera + + func handleCameraInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + switch req.command { + case ClawdbotCameraCommand.list.rawValue: + let devices = await self.camera.listDevices() + struct Payload: Codable { + var devices: [CameraController.CameraDeviceInfo] + } + let payload = try Self.encodePayload(Payload(devices: devices)) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + case ClawdbotCameraCommand.snap.rawValue: + self.showCameraHUD(text: "Taking photo…", kind: .photo) + self.triggerCameraFlash() + let params = (try? Self.decodeParams(ClawdbotCameraSnapParams.self, from: req.paramsJSON)) ?? + ClawdbotCameraSnapParams() + let res = try await self.camera.snap(params: params) + + struct Payload: Codable { + var format: String + var base64: String + var width: Int + var height: Int + } + let payload = try Self.encodePayload(Payload( + format: res.format, + base64: res.base64, + width: res.width, + height: res.height)) + self.showCameraHUD(text: "Photo captured", kind: .success, autoHideSeconds: 1.6) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + case ClawdbotCameraCommand.clip.rawValue: + let params = (try? Self.decodeParams(ClawdbotCameraClipParams.self, from: req.paramsJSON)) ?? + ClawdbotCameraClipParams() + + let suspended = (params.includeAudio ?? true) ? self.voiceWake.suspendForExternalAudioCapture() : false + defer { self.voiceWake.resumeAfterExternalAudioCapture(wasSuspended: suspended) } + + self.showCameraHUD(text: "Recording…", kind: .recording) + let res = try await self.camera.clip(params: params) + + struct Payload: Codable { + var format: String + var base64: String + var durationMs: Int + var hasAudio: Bool + } + let payload = try Self.encodePayload(Payload( + format: res.format, + base64: res.base64, + durationMs: res.durationMs, + hasAudio: res.hasAudio)) + self.showCameraHUD(text: "Clip captured", kind: .success, autoHideSeconds: 1.8) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + default: + return BridgeInvokeResponse( + id: req.id, + ok: false, + error: ClawdbotNodeError(code: .invalidRequest, message: "INVALID_REQUEST: unknown command")) + } + } + + // MARK: - Screen Recording + + func handleScreenRecordInvoke(_ req: BridgeInvokeRequest) async throws -> BridgeInvokeResponse { + let params = (try? Self.decodeParams(ClawdbotScreenRecordParams.self, from: req.paramsJSON)) ?? + ClawdbotScreenRecordParams() + if let format = params.format, format.lowercased() != "mp4" { + throw NSError(domain: "Screen", code: 30, userInfo: [ + NSLocalizedDescriptionKey: "INVALID_REQUEST: screen format must be mp4", + ]) + } + self.screenRecordActive = true + defer { self.screenRecordActive = false } + let path = try await self.screenRecorder.record( + screenIndex: params.screenIndex, + durationMs: params.durationMs, + fps: params.fps, + includeAudio: params.includeAudio, + outPath: nil) + defer { try? FileManager().removeItem(atPath: path) } + let data = try Data(contentsOf: URL(fileURLWithPath: path)) + struct Payload: Codable { + var format: String + var base64: String + var durationMs: Int? + var fps: Double? + var screenIndex: Int? + var hasAudio: Bool + } + let payload = try Self.encodePayload(Payload( + format: "mp4", + base64: data.base64EncodedString(), + durationMs: params.durationMs, + fps: params.fps, + screenIndex: params.screenIndex, + hasAudio: params.includeAudio ?? true)) + return BridgeInvokeResponse(id: req.id, ok: true, payloadJSON: payload) + } +} diff --git a/apps/ios/Sources/Model/NodeGatewaySync.swift b/apps/ios/Sources/Model/NodeGatewaySync.swift new file mode 100644 index 000000000..a82210e41 --- /dev/null +++ b/apps/ios/Sources/Model/NodeGatewaySync.swift @@ -0,0 +1,152 @@ +import ClawdbotKit +import Foundation +import SwiftUI + +// MARK: - Gateway Sync & Branding + +/// Extension for gateway synchronization, branding, and wake word sync. +extension NodeAppModel { + func applyMainSessionKey(_ key: String?) { + let trimmed = (key ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + let current = self.mainSessionKey.trimmingCharacters(in: .whitespacesAndNewlines) + if SessionKey.isCanonicalMainSessionKey(current) { return } + if trimmed == current { return } + self.mainSessionKey = trimmed + self.talkMode.updateMainSessionKey(trimmed) + } + + func refreshBrandingFromGateway() async { + do { + let res = try await self.gateway.request(method: "config.get", paramsJSON: "{}", timeoutSeconds: 8) + guard let json = try JSONSerialization.jsonObject(with: res) as? [String: Any] else { return } + guard let config = json["config"] as? [String: Any] else { return } + let ui = config["ui"] as? [String: Any] + let raw = (ui?["seamColor"] as? String)?.trimmingCharacters(in: .whitespacesAndNewlines) ?? "" + let session = config["session"] as? [String: Any] + let mainKey = SessionKey.normalizeMainKey(session?["mainKey"] as? String) + await MainActor.run { + self.seamColorHex = raw.isEmpty ? nil : raw + if !SessionKey.isCanonicalMainSessionKey(self.mainSessionKey) { + self.mainSessionKey = mainKey + self.talkMode.updateMainSessionKey(mainKey) + } + } + } catch { + // ignore + } + } + + func setGlobalWakeWords(_ words: [String]) async { + let sanitized = VoiceWakePreferences.sanitizeTriggerWords(words) + + struct Payload: Codable { + var triggers: [String] + } + let payload = Payload(triggers: sanitized) + guard let data = try? JSONEncoder().encode(payload), + let json = String(data: data, encoding: .utf8) + else { return } + + do { + _ = try await self.gateway.request(method: "voicewake.set", paramsJSON: json, timeoutSeconds: 12) + } catch { + // Best-effort only. + } + } + + func startVoiceWakeSync() async { + self.voiceWakeSyncTask?.cancel() + self.voiceWakeSyncTask = Task { [weak self] in + guard let self else { return } + + await self.refreshWakeWordsFromGateway() + + let stream = await self.gateway.subscribeServerEvents(bufferingNewest: 200) + for await evt in stream { + if Task.isCancelled { return } + guard evt.event == "voicewake.changed" else { continue } + guard let payload = evt.payload else { continue } + struct Payload: Decodable { var triggers: [String] } + guard let decoded = try? GatewayPayloadDecoding.decode(payload, as: Payload.self) else { continue } + let triggers = VoiceWakePreferences.sanitizeTriggerWords(decoded.triggers) + VoiceWakePreferences.saveTriggerWords(triggers) + } + } + } + + func refreshWakeWordsFromGateway() async { + do { + let data = try await self.gateway.request(method: "voicewake.get", paramsJSON: "{}", timeoutSeconds: 8) + guard let triggers = VoiceWakePreferences.decodeGatewayTriggers(from: data) else { return } + VoiceWakePreferences.saveTriggerWords(triggers) + } catch { + // Best-effort only. + } + } + + // MARK: - Color Utilities + + var seamColor: Color { + Self.color(fromHex: self.seamColorHex) ?? Self.defaultSeamColor + } + + static let defaultSeamColor = Color(red: 79 / 255.0, green: 122 / 255.0, blue: 154 / 255.0) + + static func color(fromHex raw: String?) -> Color? { + let trimmed = (raw ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + let hex = trimmed.hasPrefix("#") ? String(trimmed.dropFirst()) : trimmed + guard hex.count == 6, let value = Int(hex, radix: 16) else { return nil } + let r = Double((value >> 16) & 0xFF) / 255.0 + let g = Double((value >> 8) & 0xFF) / 255.0 + let b = Double(value & 0xFF) / 255.0 + return Color(red: r, green: g, blue: b) + } + + // MARK: - A2UI URL Resolution + + func resolveA2UIHostURL() async -> String? { + guard let raw = await self.gateway.currentCanvasHostUrl() else { return nil } + let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty, let base = URL(string: trimmed) else { return nil } + return base.appendingPathComponent("__clawdbot__/a2ui/").absoluteString + "?platform=ios" + } + + func showA2UIOnConnectIfNeeded() async { + guard let a2uiUrl = await self.resolveA2UIHostURL() else { return } + let current = self.screen.urlString.trimmingCharacters(in: .whitespacesAndNewlines) + if current.isEmpty || current == self.lastAutoA2uiURL { + self.screen.navigate(to: a2uiUrl) + self.lastAutoA2uiURL = a2uiUrl + } + } + + func showLocalCanvasOnDisconnect() { + self.lastAutoA2uiURL = nil + self.screen.showDefaultCanvas() + } + + // MARK: - Offline Queue + + func flushOfflineQueue() async { + let pending = await MainActor.run { self.offlineQueue.pending } + guard !pending.isEmpty else { return } + + for msg in pending { + do { + struct Payload: Codable { + var text: String + var sessionKey: String? + } + let payload = Payload(text: msg.text, sessionKey: msg.sessionKey) + let data = try JSONEncoder().encode(payload) + guard let json = String(bytes: data, encoding: .utf8) else { continue } + await self.gateway.sendEvent(event: "voice.transcript", payloadJSON: json) + await MainActor.run { self.offlineQueue.dequeue(msg.id) } + } catch { + await MainActor.run { self.offlineQueue.markFailed(msg.id) } + } + } + } +} diff --git a/apps/ios/Sources/Push/PushManager.swift b/apps/ios/Sources/Push/PushManager.swift new file mode 100644 index 000000000..1a8d0c289 --- /dev/null +++ b/apps/ios/Sources/Push/PushManager.swift @@ -0,0 +1,137 @@ +import Foundation +import OSLog +import PushKit +import UIKit +import UserNotifications + +/// Manages push notification registration for VoIP (PushKit) and remote notifications. +/// VoIP push allows the app to wake from background for incoming calls/messages. +@MainActor +final class PushManager: NSObject, Sendable { + static let shared = PushManager() + + private let logger = Logger(subsystem: "com.clawdbot", category: "push") + private var voipRegistry: PKPushRegistry? + private(set) var voipToken: Data? + private(set) var remoteToken: Data? + + /// Called when a VoIP push is received. The gateway can use this to wake the app. + var onIncomingVoIPPush: (@Sendable (PKPushPayload) async -> Void)? + + /// Called when the VoIP token is updated. Send this to the gateway for registration. + var onVoIPTokenUpdated: (@Sendable (String) async -> Void)? + + /// Called when the remote notification token is updated. + var onRemoteTokenUpdated: (@Sendable (String) async -> Void)? + + private override init() { + super.init() + } + + // MARK: - VoIP Push (PushKit) + + /// Register for VoIP push notifications. + /// This enables the app to receive pushes that can wake it from suspended state. + func registerForVoIPPush() { + logger.info("Registering for VoIP push") + voipRegistry = PKPushRegistry(queue: .main) + voipRegistry?.delegate = self + voipRegistry?.desiredPushTypes = [.voIP] + } + + // MARK: - Remote Notifications + + /// Request permission and register for standard remote notifications. + func registerForRemoteNotifications() async { + let center = UNUserNotificationCenter.current() + + do { + let granted = try await center.requestAuthorization(options: [.alert, .sound, .badge]) + if granted { + logger.info("Remote notification permission granted") + await MainActor.run { + UIApplication.shared.registerForRemoteNotifications() + } + } else { + logger.info("Remote notification permission denied") + } + } catch { + logger.error("Failed to request notification permission: \(error.localizedDescription)") + } + } + + /// Called by AppDelegate when remote notification registration succeeds. + func didRegisterForRemoteNotifications(deviceToken: Data) { + remoteToken = deviceToken + let tokenString = deviceToken.map { String(format: "%02x", $0) }.joined() + logger.info("Remote notification token: \(tokenString.prefix(16))...") + + Task { + await onRemoteTokenUpdated?(tokenString) + } + } + + /// Called by AppDelegate when remote notification registration fails. + func didFailToRegisterForRemoteNotifications(error: Error) { + logger.error("Failed to register for remote notifications: \(error.localizedDescription)") + } + + // MARK: - Token Helpers + + /// Returns the VoIP token as a hex string, or nil if not registered. + var voipTokenString: String? { + voipToken.map { $0.map { String(format: "%02x", $0) }.joined() } + } + + /// Returns the remote notification token as a hex string, or nil if not registered. + var remoteTokenString: String? { + remoteToken.map { $0.map { String(format: "%02x", $0) }.joined() } + } +} + +// MARK: - PKPushRegistryDelegate + +extension PushManager: PKPushRegistryDelegate { + nonisolated func pushRegistry( + _ registry: PKPushRegistry, + didUpdate pushCredentials: PKPushCredentials, + for type: PKPushType + ) { + Task { @MainActor in + self.voipToken = pushCredentials.token + let tokenString = pushCredentials.token.map { String(format: "%02x", $0) }.joined() + self.logger.info("VoIP push token: \(tokenString.prefix(16))...") + + await self.onVoIPTokenUpdated?(tokenString) + } + } + + nonisolated func pushRegistry( + _ registry: PKPushRegistry, + didReceiveIncomingPushWith payload: PKPushPayload, + for type: PKPushType, + completion: @escaping () -> Void + ) { + Task { @MainActor in + self.logger.info("Received VoIP push: \(payload.dictionaryPayload.keys.joined(separator: ", "))") + + // Note: For VoIP pushes, iOS requires that you report a new incoming call + // using CallKit within a few seconds, otherwise the app will be terminated. + // For non-call use cases (like chat wakeup), consider using regular push + // notifications with background mode instead. + + await self.onIncomingVoIPPush?(payload) + completion() + } + } + + nonisolated func pushRegistry( + _ registry: PKPushRegistry, + didInvalidatePushTokenFor type: PKPushType + ) { + Task { @MainActor in + self.logger.info("VoIP push token invalidated") + self.voipToken = nil + } + } +} diff --git a/apps/ios/Sources/Resources/PrivacyInfo.xcprivacy b/apps/ios/Sources/Resources/PrivacyInfo.xcprivacy new file mode 100644 index 000000000..9c3364e2a --- /dev/null +++ b/apps/ios/Sources/Resources/PrivacyInfo.xcprivacy @@ -0,0 +1,31 @@ + + + + + NSPrivacyTracking + + NSPrivacyTrackingDomains + + NSPrivacyCollectedDataTypes + + NSPrivacyAccessedAPITypes + + + NSPrivacyAccessedAPIType + NSPrivacyAccessedAPICategoryUserDefaults + NSPrivacyAccessedAPITypeReasons + + CA92.1 + + + + NSPrivacyAccessedAPIType + NSPrivacyAccessedAPICategorySystemBootTime + NSPrivacyAccessedAPITypeReasons + + 35F9.1 + + + + + diff --git a/apps/ios/Sources/RootTabs.swift b/apps/ios/Sources/RootTabs.swift index f7b3fd822..b2973e34e 100644 --- a/apps/ios/Sources/RootTabs.swift +++ b/apps/ios/Sources/RootTabs.swift @@ -138,6 +138,15 @@ struct RootTabs: View { } } + // Show pending offline messages indicator + let pendingCount = self.appModel.offlineQueue.count + if pendingCount > 0 { + return StatusPill.Activity( + title: "\(pendingCount) pending", + systemImage: "clock.arrow.circlepath", + tint: .orange) + } + return nil } } diff --git a/apps/ios/Sources/Settings/SettingsTab.swift b/apps/ios/Sources/Settings/SettingsTab.swift index 8aaf39264..5179bc068 100644 --- a/apps/ios/Sources/Settings/SettingsTab.swift +++ b/apps/ios/Sources/Settings/SettingsTab.swift @@ -186,6 +186,12 @@ struct SettingsTab: View { "Wake Words", value: VoiceWakePreferences.displayString(for: self.voiceWake.triggerWords)) } + + NavigationLink { + TTSVoiceSettingsView() + } label: { + Text("TTS Voice Settings") + } } Section("Camera") { @@ -278,8 +284,13 @@ struct SettingsTab: View { @ViewBuilder private func gatewayList(showing: GatewayListMode) -> some View { if self.gatewayController.gateways.isEmpty { - Text("No gateways found yet.") - .foregroundStyle(.secondary) + ContentUnavailableView { + Label("No Gateways Found", systemImage: "network.slash") + } description: { + Text("Make sure your Clawdbot gateway is running on the same network.") + } + .frame(maxWidth: .infinity) + .padding(.vertical, 8) } else { let connectedID = self.appModel.connectedGatewayID let rows = self.gatewayController.gateways.filter { gateway in diff --git a/apps/ios/Sources/Settings/TTSVoiceSettingsView.swift b/apps/ios/Sources/Settings/TTSVoiceSettingsView.swift new file mode 100644 index 000000000..52ef60be4 --- /dev/null +++ b/apps/ios/Sources/Settings/TTSVoiceSettingsView.swift @@ -0,0 +1,140 @@ +import AVFoundation +import SwiftUI + +/// Settings view for configuring Text-to-Speech voice parameters. +struct TTSVoiceSettingsView: View { + @AppStorage("tts.voiceIdentifier") private var selectedVoiceId = "" + @AppStorage("tts.rate") private var speechRate = 0.5 + @AppStorage("tts.pitch") private var speechPitch = 1.0 + @State private var availableVoices: [AVSpeechSynthesisVoice] = [] + @State private var isTestPlaying = false + private let synthesizer = AVSpeechSynthesizer() + + var body: some View { + Form { + Section { + Picker("Voice", selection: self.$selectedVoiceId) { + Text("System Default").tag("") + ForEach(self.availableVoices, id: \.identifier) { voice in + Text(self.voiceDisplayName(voice)) + .tag(voice.identifier) + } + } + + VStack(alignment: .leading, spacing: 8) { + HStack { + Text("Speech Rate") + Spacer() + Text(self.rateLabel) + .foregroundStyle(.secondary) + } + Slider(value: self.$speechRate, in: 0.0 ... 1.0, step: 0.1) + } + + VStack(alignment: .leading, spacing: 8) { + HStack { + Text("Pitch") + Spacer() + Text(String(format: "%.1f", self.speechPitch)) + .foregroundStyle(.secondary) + } + Slider(value: self.$speechPitch, in: 0.5 ... 2.0, step: 0.1) + } + + Button { + self.testVoice() + } label: { + HStack { + if self.isTestPlaying { + ProgressView() + .progressViewStyle(.circular) + .scaleEffect(0.8) + } + Text(self.isTestPlaying ? "Playing..." : "Test Voice") + } + } + .disabled(self.isTestPlaying) + } header: { + Text("Text-to-Speech") + } footer: { + Text("These settings apply to system TTS fallback when cloud voices are unavailable.") + } + + Section { + Button("Reset to Defaults") { + self.resetDefaults() + } + } + } + .navigationTitle("TTS Voice") + .onAppear { + self.loadVoices() + } + } + + private var rateLabel: String { + switch self.speechRate { + case 0.0 ..< 0.3: + "Slow" + case 0.3 ..< 0.6: + "Normal" + case 0.6 ..< 0.8: + "Fast" + default: + "Very Fast" + } + } + + private func voiceDisplayName(_ voice: AVSpeechSynthesisVoice) -> String { + let language = Locale.current.localizedString(forIdentifier: voice.language) ?? voice.language + return "\(voice.name) (\(language))" + } + + private func loadVoices() { + let voices = AVSpeechSynthesisVoice.speechVoices() + // Filter to show only enhanced/premium voices and common languages + let filtered = voices.filter { voice in + voice.quality == .enhanced || voice.quality == .premium + } + // Sort by language then name + self.availableVoices = filtered.sorted { a, b in + if a.language != b.language { + return a.language < b.language + } + return a.name < b.name + } + } + + private func testVoice() { + self.isTestPlaying = true + let utterance = AVSpeechUtterance(string: "Hello, I'm Clawdbot. How can I help you today?") + + // Apply selected voice + if !self.selectedVoiceId.isEmpty, + let voice = AVSpeechSynthesisVoice(identifier: self.selectedVoiceId) + { + utterance.voice = voice + } + + // Apply rate (AVSpeechUtterance rate is 0.0-1.0 but default speaking rate is around 0.5) + utterance.rate = Float(self.speechRate) * AVSpeechUtteranceMaximumSpeechRate + utterance.pitchMultiplier = Float(self.speechPitch) + + self.synthesizer.speak(utterance) + + // Reset playing state when done + Task { + // Simple delay approximation since we can't easily track utterance completion here + try? await Task.sleep(nanoseconds: 3_000_000_000) + await MainActor.run { + self.isTestPlaying = false + } + } + } + + private func resetDefaults() { + self.selectedVoiceId = "" + self.speechRate = 0.5 + self.speechPitch = 1.0 + } +} diff --git a/apps/ios/Tests/OfflineMessageQueueTests.swift b/apps/ios/Tests/OfflineMessageQueueTests.swift new file mode 100644 index 000000000..7adac1f5f --- /dev/null +++ b/apps/ios/Tests/OfflineMessageQueueTests.swift @@ -0,0 +1,122 @@ +import Foundation +import Testing +@testable import Clawdbot + +@Suite @MainActor struct OfflineMessageQueueTests { + @Test func enqueueAddsMessage() async { + let queue = OfflineMessageQueue() + queue.clear() + + queue.enqueue(text: "Hello", sessionKey: "test-session") + + #expect(queue.count == 1) + #expect(queue.pending.first?.text == "Hello") + #expect(queue.pending.first?.sessionKey == "test-session") + #expect(queue.pending.first?.retryCount == 0) + + queue.clear() + } + + @Test func dequeueRemovesMessage() async { + let queue = OfflineMessageQueue() + queue.clear() + + queue.enqueue(text: "Message 1", sessionKey: "session") + queue.enqueue(text: "Message 2", sessionKey: "session") + + let firstId = queue.pending.first!.id + queue.dequeue(firstId) + + #expect(queue.count == 1) + #expect(queue.pending.first?.text == "Message 2") + + queue.clear() + } + + @Test func markFailedIncrementsRetryCount() async { + let queue = OfflineMessageQueue() + queue.clear() + + queue.enqueue(text: "Retry test", sessionKey: "session") + let id = queue.pending.first!.id + + queue.markFailed(id) + #expect(queue.pending.first?.retryCount == 1) + + queue.markFailed(id) + #expect(queue.pending.first?.retryCount == 2) + + queue.clear() + } + + @Test func markFailedRemovesAfterMaxRetries() async { + let queue = OfflineMessageQueue() + queue.clear() + + queue.enqueue(text: "Max retry test", sessionKey: "session") + let id = queue.pending.first!.id + + // Max retries is 3 + queue.markFailed(id) // retry 1 + queue.markFailed(id) // retry 2 + queue.markFailed(id) // retry 3 - should remove + + #expect(queue.isEmpty) + + queue.clear() + } + + @Test func clearRemovesAllMessages() async { + let queue = OfflineMessageQueue() + queue.clear() + + queue.enqueue(text: "Message 1", sessionKey: "s1") + queue.enqueue(text: "Message 2", sessionKey: "s2") + queue.enqueue(text: "Message 3", sessionKey: "s3") + + #expect(queue.count == 3) + + queue.clear() + + #expect(queue.isEmpty) + #expect(queue.count == 0) + } + + @Test func isEmptyReturnsCorrectValue() async { + let queue = OfflineMessageQueue() + queue.clear() + + #expect(queue.isEmpty) + + queue.enqueue(text: "Test", sessionKey: "session") + #expect(!queue.isEmpty) + + queue.clear() + #expect(queue.isEmpty) + } + + @Test func enqueueWithThinkingLevel() async { + let queue = OfflineMessageQueue() + queue.clear() + + queue.enqueue(text: "Deep thought", sessionKey: "session", thinking: "high") + + #expect(queue.pending.first?.thinking == "high") + + queue.clear() + } + + @Test func dequeueNonExistentIdDoesNothing() async { + let queue = OfflineMessageQueue() + queue.clear() + + queue.enqueue(text: "Test", sessionKey: "session") + let originalCount = queue.count + + queue.dequeue(UUID()) // Random ID that doesn't exist + + #expect(queue.count == originalCount) + + queue.clear() + } +} diff --git a/apps/ios/project.yml b/apps/ios/project.yml index a6728cd98..c626aa340 100644 --- a/apps/ios/project.yml +++ b/apps/ios/project.yml @@ -88,6 +88,12 @@ targets: UIApplicationSupportsMultipleScenes: false UIBackgroundModes: - audio + - voip + - remote-notification + CFBundleURLTypes: + - CFBundleURLName: bot.molt.ios + CFBundleURLSchemes: + - moltbot NSLocalNetworkUsageDescription: Moltbot discovers and connects to your Moltbot gateway on the local network. NSAppTransportSecurity: NSAllowsArbitraryLoadsInWebContent: true