Add native WebRTC session foundation (core engine + HA signaling)

common/src/main/kotlin/io/homeassistant/companion/android/common/data/websocket/WebSocketRepository.kt

@@ -8,6 +8,9 @@ import io.homeassistant.companion.android.common.data.websocket.impl.entities.Ar
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineEvent
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineListResponse
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineResponse
+import io.homeassistant.companion.android.common.data.websocket.impl.entities.CameraCapabilitiesResponse
+import io.homeassistant.companion.android.common.data.websocket.impl.entities.CameraStreamTypes
+import io.homeassistant.companion.android.common.data.websocket.impl.entities.CameraWebRtcClientConfigResponse
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.CompressedStateChangedEvent
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.ConversationResponse
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.CurrentUserResponse
@@ -24,6 +27,8 @@ import io.homeassistant.companion.android.common.data.websocket.impl.entities.Te
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.ThreadDatasetResponse
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.ThreadDatasetTlvResponse
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.TriggerEvent
+import io.homeassistant.companion.android.common.data.websocket.impl.entities.WebRtcCandidate
+import io.homeassistant.companion.android.common.data.websocket.impl.entities.WebRtcEvent
 import javax.inject.Inject
 import javax.inject.Provider
 import kotlinx.coroutines.flow.Flow
@@ -134,6 +139,55 @@ interface WebSocketRepository {
      * @return `true`/`false` indicating if it was enqueued, or `null` on unexpected failures
      */
     suspend fun sendVoiceData(binaryHandlerId: Int, data: ByteArray): Boolean
+
+    /**
+     * Get the stream types the frontend can use for a camera entity. Consumers should only start
+     * a WebRTC session when [CameraStreamTypes.WEB_RTC] is reported and fall back to HLS
+     * otherwise.
+     *
+     * Requires Home Assistant Core 2024.11 or later.
+     *
+     * @return [CameraCapabilitiesResponse] for the entity, or `null` if the server did not return
+     * a successful response.
+     */
+    suspend fun getCameraCapabilities(entityId: String): CameraCapabilitiesResponse?
+
+    /**
+     * Get the WebRTC client configuration (STUN/TURN servers and optional data channel label) to
+     * use when creating a peer connection for a camera entity.
+     *
+     * Requires Home Assistant Core 2024.11 or later.
+     *
+     * @return [CameraWebRtcClientConfigResponse] for the entity, or `null` if the server did not
+     * return a successful response, for example when the camera does not support WebRTC
+     * (`webrtc_get_client_config_failed`).
+     */
+    suspend fun getCameraWebRtcClientConfig(entityId: String): CameraWebRtcClientConfigResponse?
+
+    /**
+     * Start a WebRTC session for a camera entity by sending the SDP offer, and subscribe to the
+     * signaling events for this session.
+     *
+     * The subscription lifetime is the session lifetime: when the returned Flow is no longer
+     * collected the subscription is cancelled with `unsubscribe_events`, which closes the WebRTC
+     * session on the server (there is no dedicated close command).
+     *
+     * Requires Home Assistant Core 2024.11 or later (2024.12 or later for trickle ICE with
+     * `RTCIceCandidateInit` dictionaries).
+     *
+     * @return a Flow that will emit all [WebRtcEvent]s for the session, or `null` if the
+     * subscription could not be started.
+     */
+    suspend fun startCameraWebRtcSession(entityId: String, offerSdp: String): Flow<WebRtcEvent>?
+
+    /**
+     * Send a local ICE candidate for a WebRTC session previously started with
+     * [startCameraWebRtcSession].
+     *
+     * @param sessionId the session identifier received in [WebRtcEvent.Session]
+     * @return `true` if the server accepted the candidate
+     */
+    suspend fun sendCameraWebRtcCandidate(entityId: String, sessionId: String, candidate: WebRtcCandidate): Boolean
 }
 
 internal class WebSocketRepositoryFactory @Inject internal constructor(

common/src/main/kotlin/io/homeassistant/companion/android/common/data/websocket/impl/WebSocketConstants.kt

@@ -8,6 +8,7 @@ internal object WebSocketConstants {
     const val EVENT_ENTITY_REGISTRY_UPDATED = "entity_registry_updated"
 
     const val SUBSCRIBE_TYPE_ASSIST_PIPELINE_RUN = "assist_pipeline/run"
+    const val SUBSCRIBE_TYPE_CAMERA_WEBRTC_OFFER = "camera/webrtc/offer"
     const val SUBSCRIBE_TYPE_SUBSCRIBE_EVENTS = "subscribe_events"
     const val SUBSCRIBE_TYPE_SUBSCRIBE_ENTITIES = "subscribe_entities"
     const val SUBSCRIBE_TYPE_SUBSCRIBE_TRIGGER = "subscribe_trigger"

common/src/main/kotlin/io/homeassistant/companion/android/common/data/websocket/impl/WebSocketCoreImpl.kt

@@ -17,6 +17,7 @@ import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketCo
 import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketConstants.EVENT_ENTITY_REGISTRY_UPDATED
 import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketConstants.EVENT_STATE_CHANGED
 import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketConstants.SUBSCRIBE_TYPE_ASSIST_PIPELINE_RUN
+import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketConstants.SUBSCRIBE_TYPE_CAMERA_WEBRTC_OFFER
 import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketConstants.SUBSCRIBE_TYPE_RENDER_TEMPLATE
 import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketConstants.SUBSCRIBE_TYPE_SUBSCRIBE_ENTITIES
 import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketConstants.SUBSCRIBE_TYPE_SUBSCRIBE_TRIGGER
@@ -46,6 +47,8 @@ import io.homeassistant.companion.android.common.data.websocket.impl.entities.St
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.TemplateUpdatedEvent
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.TriggerEvent
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.UnknownTypeSocketResponse
+import io.homeassistant.companion.android.common.data.websocket.impl.entities.WebRtcEvent
+import io.homeassistant.companion.android.common.data.websocket.impl.entities.webRtcJsonMapper
 import io.homeassistant.companion.android.common.util.FailFast
 import io.homeassistant.companion.android.common.util.MapAnySerializer
 import io.homeassistant.companion.android.common.util.kotlinJsonMapper
@@ -921,6 +924,20 @@ internal class WebSocketCoreImpl(
                     Timber.w("Received Assist pipeline event without type, skipping")
                     return
                 }
+            } else if (subscriptionType == SUBSCRIBE_TYPE_CAMERA_WEBRTC_OFFER) {
+                if (response.event != null) {
+                    try {
+                        webRtcJsonMapper.decodeFromJsonElement<WebRtcEvent>(response.event)
+                    } catch (e: IllegalArgumentException) {
+                        // Covers SerializationException too, a malformed event must not kill the
+                        // whole message handling
+                        Timber.w(e, "Received malformed WebRTC event, skipping")
+                        return
+                    }
+                } else {
+                    Timber.w("Received no event for WebRTC subscription, skipping")
+                    return
+                }
             } else if (eventResponseType != null && (eventResponseType as? JsonPrimitive)?.isString == true) {
                 when (eventResponseType.content) {
                     EVENT_STATE_CHANGED -> {

common/src/main/kotlin/io/homeassistant/companion/android/common/data/websocket/impl/WebSocketRepositoryImpl.kt

@@ -13,6 +13,7 @@ import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketCo
 import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketConstants.EVENT_ENTITY_REGISTRY_UPDATED
 import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketConstants.EVENT_STATE_CHANGED
 import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketConstants.SUBSCRIBE_TYPE_ASSIST_PIPELINE_RUN
+import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketConstants.SUBSCRIBE_TYPE_CAMERA_WEBRTC_OFFER
 import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketConstants.SUBSCRIBE_TYPE_PUSH_NOTIFICATION_CHANNEL
 import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketConstants.SUBSCRIBE_TYPE_RENDER_TEMPLATE
 import io.homeassistant.companion.android.common.data.websocket.impl.WebSocketConstants.SUBSCRIBE_TYPE_SUBSCRIBE_ENTITIES
@@ -23,6 +24,8 @@ import io.homeassistant.companion.android.common.data.websocket.impl.entities.Ar
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineEvent
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineListResponse
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.AssistPipelineResponse
+import io.homeassistant.companion.android.common.data.websocket.impl.entities.CameraCapabilitiesResponse
+import io.homeassistant.companion.android.common.data.websocket.impl.entities.CameraWebRtcClientConfigResponse
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.CompressedStateChangedEvent
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.ConversationResponse
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.CurrentUserResponse
@@ -41,6 +44,9 @@ import io.homeassistant.companion.android.common.data.websocket.impl.entities.Te
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.ThreadDatasetResponse
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.ThreadDatasetTlvResponse
 import io.homeassistant.companion.android.common.data.websocket.impl.entities.TriggerEvent
+import io.homeassistant.companion.android.common.data.websocket.impl.entities.WebRtcCandidate
+import io.homeassistant.companion.android.common.data.websocket.impl.entities.WebRtcEvent
+import io.homeassistant.companion.android.common.data.websocket.impl.entities.webRtcJsonMapper
 import io.homeassistant.companion.android.common.util.kotlinJsonMapper
 import io.homeassistant.companion.android.common.util.toHexString
 import io.homeassistant.companion.android.database.server.ServerUserInfo
@@ -50,6 +56,7 @@ import kotlinx.coroutines.flow.Flow
 import kotlinx.serialization.json.JsonObject
 import kotlinx.serialization.json.JsonPrimitive
 import kotlinx.serialization.json.decodeFromJsonElement
+import kotlinx.serialization.json.encodeToJsonElement
 import kotlinx.serialization.json.intOrNull
 import okhttp3.WebSocketListener
 
@@ -420,6 +427,60 @@ class WebSocketRepositoryImpl internal constructor(
         return response?.success == true
     }
 
+    override suspend fun getCameraCapabilities(entityId: String): CameraCapabilitiesResponse? {
+        val socketResponse = webSocketCore.sendMessage(
+            mapOf(
+                "type" to "camera/capabilities",
+                "entity_id" to entityId,
+            ),
+        )
+
+        return mapResponse(socketResponse)
+    }
+
+    override suspend fun getCameraWebRtcClientConfig(entityId: String): CameraWebRtcClientConfigResponse? {
+        val socketResponse = webSocketCore.sendMessage(
+            mapOf(
+                "type" to "camera/webrtc/get_client_config",
+                "entity_id" to entityId,
+            ),
+        )
+
+        // The response follows the camelCase W3C dictionaries so it cannot be decoded with the
+        // shared snake_case mapper used by mapResponse. The success check avoids decoding an
+        // error payload when the camera does not support WebRTC.
+        return socketResponse?.takeIf { it.success == true }?.result?.let {
+            webRtcJsonMapper.decodeFromJsonElement(it)
+        }
+    }
+
+    override suspend fun startCameraWebRtcSession(entityId: String, offerSdp: String): Flow<WebRtcEvent>? =
+        webSocketCore.subscribeTo(
+            SUBSCRIBE_TYPE_CAMERA_WEBRTC_OFFER,
+            mapOf(
+                "entity_id" to entityId,
+                "offer" to offerSdp,
+            ),
+        )
+
+    override suspend fun sendCameraWebRtcCandidate(
+        entityId: String,
+        sessionId: String,
+        candidate: WebRtcCandidate,
+    ): Boolean {
+        val response = webSocketCore.sendMessage(
+            mapOf(
+                "type" to "camera/webrtc/candidate",
+                "entity_id" to entityId,
+                "session_id" to sessionId,
+                // Pre-encoded with the WebRTC mapper to keep the camelCase keys, the message
+                // serializer passes JsonElement values through untouched
+                "candidate" to webRtcJsonMapper.encodeToJsonElement(candidate),
+            ),
+        )
+        return response?.success == true
+    }
+
     /**
      * Update server entry in [serverManager] with information from a [CurrentUserResponse] like user
      * name and admin status.

common/src/main/kotlin/io/homeassistant/companion/android/common/data/websocket/impl/entities/CameraCapabilitiesResponse.kt

@@ -0,0 +1,22 @@
+package io.homeassistant.companion.android.common.data.websocket.impl.entities
+
+import kotlinx.serialization.Serializable
+
+/**
+ * Response for the `camera/capabilities` WebSocket command.
+ *
+ * The server reports which stream types the frontend can use for a camera entity. Consumers
+ * should check for [CameraStreamTypes.WEB_RTC] before starting a WebRTC session and fall back to
+ * HLS otherwise.
+ */
+@Serializable
+data class CameraCapabilitiesResponse(val frontendStreamTypes: List<String> = emptyList())
+
+/**
+ * Known values of [CameraCapabilitiesResponse.frontendStreamTypes], matching the
+ * `StreamType` enum of Home Assistant Core.
+ */
+object CameraStreamTypes {
+    const val HLS = "hls"
+    const val WEB_RTC = "web_rtc"
+}

common/src/main/kotlin/io/homeassistant/companion/android/common/data/websocket/impl/entities/CameraWebRtcClientConfigResponse.kt

@@ -0,0 +1,52 @@
+package io.homeassistant.companion.android.common.data.websocket.impl.entities
+
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.builtins.ListSerializer
+import kotlinx.serialization.builtins.serializer
+import kotlinx.serialization.json.JsonArray
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.JsonTransformingSerializer
+
+/**
+ * Response for the `camera/webrtc/get_client_config` WebSocket command.
+ *
+ * The keys are camelCase on the wire (they mirror the W3C `RTCConfiguration` dictionary), so this
+ * class must be deserialized with [webRtcJsonMapper] and not the shared snake_case mapper.
+ *
+ * @property configuration the `RTCConfiguration` to create the peer connection with
+ * @property dataChannel label of a data channel the client should open, used by some WebRTC
+ * providers (like go2rtc) to negotiate additional features. `null` when the provider does not use
+ * a data channel.
+ */
+@Serializable
+data class CameraWebRtcClientConfigResponse(
+    val configuration: WebRtcConfiguration = WebRtcConfiguration(),
+    val dataChannel: String? = null,
+)
+
+/**
+ * The subset of the W3C `RTCConfiguration` dictionary sent by Home Assistant Core.
+ */
+@Serializable
+data class WebRtcConfiguration(val iceServers: List<WebRtcIceServer> = emptyList())
+
+/**
+ * A single `RTCIceServer` entry (STUN or TURN server) of an `RTCConfiguration`.
+ */
+@Serializable
+data class WebRtcIceServer(
+    @Serializable(with = StringOrStringListSerializer::class)
+    val urls: List<String> = emptyList(),
+    val username: String? = null,
+    val credential: String? = null,
+)
+
+/**
+ * The `urls` member of `RTCIceServer` is allowed to be either a single string or a list of
+ * strings. This serializer normalizes both shapes to a list.
+ */
+private object StringOrStringListSerializer :
+    JsonTransformingSerializer<List<String>>(ListSerializer(String.serializer())) {
+    override fun transformDeserialize(element: JsonElement): JsonElement =
+        element as? JsonArray ?: JsonArray(listOf(element))
+}

common/src/main/kotlin/io/homeassistant/companion/android/common/data/websocket/impl/entities/WebRtcEvent.kt

@@ -0,0 +1,87 @@
+package io.homeassistant.companion.android.common.data.websocket.impl.entities
+
+import io.homeassistant.companion.android.common.util.UnknownJsonContent
+import io.homeassistant.companion.android.common.util.UnknownJsonContentBuilder
+import io.homeassistant.companion.android.common.util.UnknownJsonContentDeserializer
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.modules.SerializersModule
+
+/**
+ * JSON mapper dedicated to the `camera/webrtc` WebSocket commands.
+ *
+ * These payloads cannot be handled by the shared [io.homeassistant.companion.android.common.util.kotlinJsonMapper]:
+ * its global snake_case naming strategy would also rewrite explicit `@SerialName` values, while
+ * the WebRTC API mixes snake_case keys (`session_id`) with the camelCase keys of the standard
+ * `RTCIceCandidateInit`/`RTCConfiguration` dictionaries (`sdpMid`, `iceServers`, ...).
+ */
+internal val webRtcJsonMapper = Json {
+    ignoreUnknownKeys = true
+    // Omit null values so optional candidate fields are sent the same way as the frontend,
+    // which leaves undefined values out of the JSON payload
+    explicitNulls = false
+    serializersModule = SerializersModule {
+        polymorphicDefaultDeserializer(WebRtcEvent::class) { className ->
+            object : UnknownJsonContentDeserializer<WebRtcEvent.Unknown>() {
+                override val builder = UnknownJsonContentBuilder { content ->
+                    WebRtcEvent.Unknown(className, content)
+                }
+            }
+        }
+    }
+}
+
+/**
+ * Event received on a `camera/webrtc/offer` subscription.
+ *
+ * The server pushes these events while a WebRTC session is being negotiated: first a [Session]
+ * with the identifier needed to send candidates back, then an [Answer], then zero or more
+ * [Candidate]s (trickle ICE). An [Error] can arrive at any time and ends the negotiation.
+ */
+@Serializable
+sealed interface WebRtcEvent {
+
+    /** The server created a session and assigned it an identifier. */
+    @Serializable
+    @SerialName("session")
+    data class Session(@SerialName("session_id") val sessionId: String) : WebRtcEvent
+
+    /** The SDP answer from the camera or its WebRTC provider. */
+    @Serializable
+    @SerialName("answer")
+    data class Answer(val answer: String) : WebRtcEvent
+
+    /** A remote ICE candidate discovered by the camera or its WebRTC provider. */
+    @Serializable
+    @SerialName("candidate")
+    data class Candidate(val candidate: WebRtcCandidate) : WebRtcEvent
+
+    /** Negotiation failed, for example `webrtc_offer_failed`. */
+    @Serializable
+    @SerialName("error")
+    data class Error(val code: String, val message: String? = null) : WebRtcEvent
+
+    /**
+     * Fallback for event types this version of the app does not know, so that a server-side
+     * addition never breaks an ongoing subscription.
+     */
+    data class Unknown(override val discriminator: String?, override val content: JsonElement) :
+        WebRtcEvent,
+        UnknownJsonContent
+}
+
+/**
+ * A standard `RTCIceCandidateInit` dictionary, exchanged in both directions during trickle ICE.
+ *
+ * The keys are camelCase on the wire (like in the W3C WebRTC specification), which is why this
+ * class must be serialized with [webRtcJsonMapper] and not the shared snake_case mapper.
+ */
+@Serializable
+data class WebRtcCandidate(
+    val candidate: String,
+    val sdpMid: String? = null,
+    val sdpMLineIndex: Int? = null,
+    val usernameFragment: String? = null,
+)