diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/DeveloperSettingsPresenterImpl.kt b/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/DeveloperSettingsPresenterImpl.kt index e98f815ca34..936f2c51e54 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/DeveloperSettingsPresenterImpl.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/settings/developer/DeveloperSettingsPresenterImpl.kt @@ -46,6 +46,7 @@ class DeveloperSettingsPresenterImpl @Inject constructor( override fun getBoolean(key: String?, defValue: Boolean): Boolean = runBlocking { return@runBlocking when (key) { "webview_debug" -> prefsRepository.isWebViewDebugEnabled() + "webrtc_native_player" -> prefsRepository.isNativeWebRtcPlayerEnabled() else -> throw IllegalArgumentException("No boolean found by this key: $key") } } @@ -57,6 +58,7 @@ class DeveloperSettingsPresenterImpl @Inject constructor( prefsRepository.setWebViewDebugEnabled(value) WebView.setWebContentsDebuggingEnabled(BuildConfig.DEBUG || value) } + "webrtc_native_player" -> prefsRepository.setNativeWebRtcPlayerEnabled(value) else -> throw IllegalArgumentException("No boolean found by this key: $key") } } diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewActivity.kt b/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewActivity.kt index 58cbf43e5bc..813a9d82a25 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewActivity.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewActivity.kt @@ -156,6 +156,11 @@ import io.homeassistant.companion.android.util.hasSameOrigin import io.homeassistant.companion.android.util.isStarted import io.homeassistant.companion.android.util.sensitive import io.homeassistant.companion.android.util.toRelativeUrl +import io.homeassistant.companion.android.webrtc.core.PlayerFailure +import io.homeassistant.companion.android.webrtc.core.PlayerState +import io.homeassistant.companion.android.webrtc.core.session.WebRtcSession +import io.homeassistant.companion.android.webrtc.core.session.libwebrtc.LibWebRtcPeerConnectionControllerFactory +import io.homeassistant.companion.android.webrtc.signaling.HaSignalingClient import io.homeassistant.companion.android.websocket.WebsocketManager import io.homeassistant.companion.android.webview.WebView.ErrorType import io.homeassistant.companion.android.webview.externalbus.EntityAddToActionsResponse @@ -164,6 +169,7 @@ import io.homeassistant.companion.android.webview.externalbus.ExternalConfigResp import io.homeassistant.companion.android.webview.externalbus.ExternalEntityAddToAction import io.homeassistant.companion.android.webview.externalbus.NavigateTo import io.homeassistant.companion.android.webview.externalbus.ShowSidebar +import io.homeassistant.companion.android.webview.externalbus.WebRtcErrorEvent import io.homeassistant.companion.android.webview.insecure.BlockInsecureFragment import javax.inject.Inject import kotlinx.coroutines.CancellationException @@ -178,6 +184,8 @@ import kotlinx.serialization.json.JsonObject import org.json.JSONObject import timber.log.Timber +private const val WEBRTC_DEFAULT_VIDEO_HEIGHT_RATIO = 9.0 / 16.0 + @AndroidEntryPoint class WebViewActivity : BaseActivity(), @@ -289,6 +297,9 @@ class WebViewActivity : @Inject lateinit var dataSourceFactoryProvider: SuspendProvider + @Inject + lateinit var peerConnectionControllerFactory: LibWebRtcPeerConnectionControllerFactory + private lateinit var webView: WebView private var loadedUrl: Uri? = null private lateinit var decor: FrameLayout @@ -314,6 +325,11 @@ class WebViewActivity : private var playerSize = mutableStateOf(null) private var playerTop = mutableStateOf(0.dp) private var playerLeft = mutableStateOf(0.dp) + private var webRtcSession = mutableStateOf(null) + private var webRtcPlayerSize = mutableStateOf(null) + private var webRtcPlayerTop = mutableStateOf(0.dp) + private var webRtcPlayerLeft = mutableStateOf(0.dp) + private var webRtcStateJob: Job? = null private var statusBarColor = mutableStateOf(null) private var backgroundColor = mutableStateOf(null) @@ -412,6 +428,10 @@ class WebViewActivity : val playerSize by remember { playerSize } val playerTop by remember { playerTop } val playerLeft by remember { playerLeft } + val webRtcPlayer by remember { webRtcSession } + val webRtcPlayerSize by remember { webRtcPlayerSize } + val webRtcPlayerTop by remember { webRtcPlayerTop } + val webRtcPlayerLeft by remember { webRtcPlayerLeft } val currentAppLocked by remember { appLocked } val customViewFromWebView by remember { customViewFromWebView } val statusBarColor by remember { statusBarColor } @@ -450,6 +470,15 @@ class WebViewActivity : playerSize = playerSize, playerTop = playerTop, playerLeft = playerLeft, + webRtcPlayer = webRtcPlayer, + webRtcEglContext = if (webRtcPlayer != null) { + peerConnectionControllerFactory.eglBase.eglBaseContext + } else { + null + }, + webRtcPlayerSize = webRtcPlayerSize, + webRtcPlayerTop = webRtcPlayerTop, + webRtcPlayerLeft = webRtcPlayerLeft, currentAppLocked, customViewFromWebView, shouldAskNotificationPermission = shouldAskNotificationPermission, @@ -1075,16 +1104,19 @@ class WebViewActivity : } else { 0 } - sendExternalBusMessage( - ExternalConfigResponse( - id = messageId, - hasNfc = hasNfc, - canCommissionMatter = canCommissionMatter, - canExportThread = canExportThread, - hasBarCodeScanner = hasBarCodeScanner, - appVersion = appVersionProvider(), - ), - ) + lifecycleScope.launch { + sendExternalBusMessage( + ExternalConfigResponse( + id = messageId, + hasNfc = hasNfc, + canCommissionMatter = canCommissionMatter, + canExportThread = canExportThread, + hasBarCodeScanner = hasBarCodeScanner, + hasNativeWebRtc = presenter.isNativeWebRtcPlayerEnabled(), + appVersion = appVersionProvider(), + ), + ) + } // TODO This feature is deprecated and should be removed after 2022.6 getAndSetStatusBarNavigationBarColors() @@ -1178,6 +1210,9 @@ class WebViewActivity : "exoplayer/play_hls" -> exoPlayHls(json) "exoplayer/stop" -> exoStopHls() "exoplayer/resize" -> exoResizeHls(json) + "webrtc/play_camera" -> webRtcPlayCamera(json) + "webrtc/stop" -> webRtcStop() + "webrtc/resize" -> webRtcResize(json) "haptic" -> processHaptic( json["payload"]?.jsonObjectOrNull()?.getStringOrNull("hapticType") ?: "", ) @@ -1526,6 +1561,92 @@ class WebViewActivity : playerSize.value = DpSize((right - left).dp, (bottom - top).dp) } + private fun webRtcPlayCamera(json: JsonObject) { + val payload = json["payload"]?.jsonObjectOrNull() + val entityId = payload?.getStringOrNull("entity_id") ?: return + val muted = payload.getBooleanOrElse("muted", false) + lifecycleScope.launch { + releaseWebRtcSession() + val signalingClient = HaSignalingClient( + serverManager.webSocketRepository(presenter.getActiveServer()), + ) + val session = WebRtcSession( + entityId = entityId, + signalingClient = signalingClient, + controllerFactory = peerConnectionControllerFactory, + ) + session.setAudioEnabled(!muted) + session.start() + webRtcSession.value = session + webRtcStateJob = lifecycleScope.launch { + session.state.collect { state -> + if (state is PlayerState.Failed) { + onWebRtcSessionFailed(entityId, state.failure) + } + } + } + sendExternalBusMessage( + ExternalBusMessage( + id = json["id"], + type = "result", + success = true, + callback = { + Timber.d("Callback $it") + }, + ), + ) + } + } + + private fun onWebRtcSessionFailed(entityId: String, failure: PlayerFailure) { + Timber.w("Native WebRTC session failed: $failure") + // Let the frontend fall back to its own in-WebView playback + sendExternalBusMessage( + WebRtcErrorEvent( + entityId = entityId, + code = (failure as? PlayerFailure.Signaling)?.code, + message = when (failure) { + is PlayerFailure.Signaling -> failure.message + is PlayerFailure.Internal -> failure.message + PlayerFailure.ConnectionLost -> null + }, + ), + ) + webRtcStop() + } + + private fun webRtcStop() { + runOnUiThread { + releaseWebRtcSession() + webRtcPlayerSize.value = null + webRtcPlayerTop.value = 0.dp + webRtcPlayerLeft.value = 0.dp + } + } + + private fun releaseWebRtcSession() { + webRtcStateJob?.cancel() + webRtcStateJob = null + webRtcSession.value?.release() + webRtcSession.value = null + } + + private fun webRtcResize(json: JsonObject) { + val payload = json["payload"]?.jsonObjectOrNull() ?: return + // Payload is https://developer.mozilla.org/en-US/docs/Web/API/Element/getBoundingClientRect + // like in exoResizeHls. The video size is not known before the first frame, so when the + // frontend does not constrain the height a 16:9 ratio is used until it does. + val left = payload.getDoubleOrElse("left", 0.0) + val top = payload.getDoubleOrElse("top", 0.0) + val right = payload.getDoubleOrElse("right", 0.0) + val bottom = payload.getDoubleOrNull("bottom")?.takeIf { it > 0 } + ?: (top + (right - left) * WEBRTC_DEFAULT_VIDEO_HEIGHT_RATIO) + + webRtcPlayerTop.value = top.dp + webRtcPlayerLeft.value = left.dp + webRtcPlayerSize.value = DpSize((right - left).dp, (bottom - top).dp) + } + fun processHaptic(hapticType: String) { Timber.d("Processing haptic tag for $hapticType") HapticFeedbackPerformer.perform( @@ -1815,6 +1936,7 @@ class WebViewActivity : WebViewCompat.removeWebMessageListener(webView, EXTERNAL_APP_V2_LISTENER) } + releaseWebRtcSession() presenter.onFinish() super.onDestroy() } diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewContentScreen.kt b/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewContentScreen.kt index 2af00584113..39ce2453aec 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewContentScreen.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewContentScreen.kt @@ -50,7 +50,10 @@ import io.homeassistant.companion.android.common.util.SdkVersion import io.homeassistant.companion.android.frontend.permissions.NotificationPermissionPrompt import io.homeassistant.companion.android.util.compose.media.player.HAMediaPlayer import io.homeassistant.companion.android.util.compose.webview.HAWebView +import io.homeassistant.companion.android.webrtc.compose.WebRtcVideo +import io.homeassistant.companion.android.webrtc.core.CameraPlayer import kotlinx.coroutines.launch +import livekit.org.webrtc.EglBase import timber.log.Timber @SuppressLint("UnusedMaterial3ScaffoldPaddingParameter") @@ -62,6 +65,11 @@ internal fun WebViewContentScreen( playerSize: DpSize?, playerTop: Dp, playerLeft: Dp, + webRtcPlayer: CameraPlayer?, + webRtcEglContext: EglBase.Context?, + webRtcPlayerSize: DpSize?, + webRtcPlayerTop: Dp, + webRtcPlayerLeft: Dp, currentAppLocked: Boolean, customViewFromWebView: View?, shouldAskNotificationPermission: Boolean, @@ -115,6 +123,18 @@ internal fun WebViewContentScreen( ) } } + webRtcPlayer?.let { webRtcPlayer -> + webRtcPlayerSize?.let { webRtcPlayerSize -> + WebRtcVideo( + player = webRtcPlayer, + eglContext = webRtcEglContext, + zOrderMediaOverlay = true, + modifier = Modifier + .offset(webRtcPlayerLeft, webRtcPlayerTop) + .size(webRtcPlayerSize), + ) + } + } customViewFromWebView?.let { customViewFromWebView -> AndroidView( factory = { @@ -233,6 +253,11 @@ private fun WebViewContentScreenPreview() { playerSize = null, playerTop = 0.dp, playerLeft = 0.dp, + webRtcPlayer = null, + webRtcEglContext = null, + webRtcPlayerSize = null, + webRtcPlayerTop = 0.dp, + webRtcPlayerLeft = 0.dp, currentAppLocked = false, shouldAskNotificationPermission = false, webViewInitialized = true, diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewPresenter.kt b/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewPresenter.kt index 62f00f39547..0c27e6441fb 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewPresenter.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewPresenter.kt @@ -72,6 +72,9 @@ interface WebViewPresenter { suspend fun parseWebViewColor(webViewColor: String): Int fun appCanCommissionMatterDevice(): Boolean + + /** Whether the beta native WebRTC camera player is enabled in the developer settings. */ + suspend fun isNativeWebRtcPlayerEnabled(): Boolean fun startCommissioningMatterDevice() /** @return `true` if the app can send this device's preferred Thread credential to the server */ diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewPresenterImpl.kt b/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewPresenterImpl.kt index 5bd5658b0e6..b665a266ed8 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewPresenterImpl.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/webview/WebViewPresenterImpl.kt @@ -529,6 +529,8 @@ class WebViewPresenterImpl @Inject constructor( override fun appCanCommissionMatterDevice(): Boolean = matterManager.appSupportsCommissioning() + override suspend fun isNativeWebRtcPlayerEnabled(): Boolean = prefsRepository.isNativeWebRtcPlayerEnabled() + override fun startCommissioningMatterDevice() { if (mutableMatterThreadStep.value != MatterThreadStep.REQUESTED) { mutableMatterThreadStep.tryEmit(MatterThreadStep.REQUESTED) diff --git a/app/src/main/kotlin/io/homeassistant/companion/android/webview/externalbus/ExternalBusMessage.kt b/app/src/main/kotlin/io/homeassistant/companion/android/webview/externalbus/ExternalBusMessage.kt index aebefd60b71..cd6ee925630 100644 --- a/app/src/main/kotlin/io/homeassistant/companion/android/webview/externalbus/ExternalBusMessage.kt +++ b/app/src/main/kotlin/io/homeassistant/companion/android/webview/externalbus/ExternalBusMessage.kt @@ -59,6 +59,7 @@ class ExternalConfigResponse( canCommissionMatter: Boolean, canExportThread: Boolean, hasBarCodeScanner: Int, + hasNativeWebRtc: Boolean, appVersion: AppVersion, ) : ExternalBusMessage( id = id, @@ -78,12 +79,29 @@ class ExternalConfigResponse( "appVersion" to appVersion.value, "hasEntityAddTo" to true, "hasAssistSettings" to true, + "hasNativeWebRTC" to hasNativeWebRtc, ).toJsonObject(), callback = { Timber.d("Callback from external config (id=$id): $it") }, ) +/** + * Notifies the frontend that the native WebRTC session of a camera failed, so it can fall back + * to its own in-WebView playback. + */ +class WebRtcErrorEvent(entityId: String, code: String?, message: String?) : + ExternalBusMessage( + id = -1, + type = "command", + command = "webrtc/error", + payload = buildMap { + put("entity_id", entityId) + code?.let { put("code", it) } + message?.let { put("message", it) } + }, + ) + @Serializable data class ExternalEntityAddToAction( @SerialName("app_payload") val appPayload: String, diff --git a/app/src/main/res/xml/preferences_developer.xml b/app/src/main/res/xml/preferences_developer.xml index a5b06415206..cb4efe5e5a3 100644 --- a/app/src/main/res/xml/preferences_developer.xml +++ b/app/src/main/res/xml/preferences_developer.xml @@ -25,6 +25,11 @@ android:icon="@drawable/ic_play" android:title="@string/webrtc_debug" android:summary="@string/webrtc_debug_summary"/> + Start Stop Test the native WebRTC player against a camera entity of the active server + Native WebRTC player (beta) + Let dashboards play camera WebRTC streams with the native player instead of the WebView Connected to Home Assistant Unable to open connection to Home Assistant Update settings to fix or disable diff --git a/webrtc-compose/src/main/kotlin/io/homeassistant/companion/android/webrtc/compose/WebRtcVideo.kt b/webrtc-compose/src/main/kotlin/io/homeassistant/companion/android/webrtc/compose/WebRtcVideo.kt index 5b5d8a1615d..a39866e4920 100644 --- a/webrtc-compose/src/main/kotlin/io/homeassistant/companion/android/webrtc/compose/WebRtcVideo.kt +++ b/webrtc-compose/src/main/kotlin/io/homeassistant/companion/android/webrtc/compose/WebRtcVideo.kt @@ -40,6 +40,8 @@ import livekit.org.webrtc.VideoSink * decoder and this renderer use the same context * @param modifier the modifier for the video container * @param scalingType how the video is scaled inside the container + * @param zOrderMediaOverlay places the video surface on top of other surface views, for example + * when the video overlays a WebView * @param connectingContent overlay shown until the first frame is rendered, defaults to a * progress indicator * @param failedContent overlay shown when the player reaches [PlayerState.Failed], for example to @@ -51,6 +53,7 @@ fun WebRtcVideo( eglContext: EglBase.Context?, modifier: Modifier = Modifier, scalingType: RendererCommon.ScalingType = RendererCommon.ScalingType.SCALE_ASPECT_FIT, + zOrderMediaOverlay: Boolean = false, connectingContent: @Composable BoxScope.() -> Unit = { DefaultConnectingContent() }, failedContent: @Composable BoxScope.(PlayerFailure) -> Unit = {}, ) { @@ -71,6 +74,7 @@ fun WebRtcVideo( ) setScalingType(scalingType) setEnableHardwareScaler(true) + setZOrderMediaOverlay(zOrderMediaOverlay) } }, releaseRenderer = SurfaceViewRenderer::release,