Release Android plugin resources when detached from Flutter engine

CHANGELOG.md

@@ -15,7 +15,12 @@ All user visible changes to this project will be documented in this file. This p
 - Upgraded [libwebrtc] to [136.0.7103.113] version. ([9613c018])
 - Upgraded [`flutter_rust_bridge`] crate to [2.10.0][frb-2.10.0] version. ([#201])
 
+### Fixed
+
+- Resources cleanup when `medea_flutter_webrtc` Flutter plugin is detached on Android. ([#202])
+
 [#201]: https://github.com/instrumentisto/medea-flutter-webrtc/pull/201
+[#202]: https://github.com/instrumentisto/medea-flutter-webrtc/pull/202
 [9613c018]: https://github.com/instrumentisto/medea-flutter-webrtc/commit/9613c018f9c08739f1121366f6049f23d7d1b51c
 [136.0.7103.113]: https://github.com/instrumentisto/libwebrtc-bin/releases/tag/136.0.7103.113
 [frb-2.10.0]: https://github.com/fzyzcjy/flutter_rust_bridge/releases/tag/v2.10.0

android/src/main/AndroidManifest.xml

@@ -1,14 +1,27 @@
 <manifest xmlns:android="http://schemas.android.com/apk/res/android"
   package="com.instrumentisto.medea_flutter_webrtc">
-    <uses-feature android:name="android.hardware.camera" />
-    <uses-feature android:name="android.hardware.camera.autofocus" />
+    <uses-feature android:name="android.hardware.camera" android:required="false" />
+    <uses-feature android:name="android.hardware.camera.autofocus" android:required="false" />
+    <uses-feature android:name="android.hardware.microphone" android:required="false"/>
     <uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
     <uses-permission android:name="android.permission.BLUETOOTH" android:maxSdkVersion="30" />
     <uses-permission android:name="android.permission.BLUETOOTH_ADMIN" android:maxSdkVersion="30" />
     <uses-permission android:name="android.permission.BLUETOOTH_CONNECT" />
     <uses-permission android:name="android.permission.CAMERA" />
     <uses-permission android:name="android.permission.CHANGE_NETWORK_STATE" />
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE_CAMERA" />
+    <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
     <uses-permission android:name="android.permission.INTERNET" />
     <uses-permission android:name="android.permission.MODIFY_AUDIO_SETTINGS" />
     <uses-permission android:name="android.permission.RECORD_AUDIO" />
+
+    <application>
+        <service
+                android:name="com.instrumentisto.medea_flutter_webrtc.ForegroundCallService"
+                android:exported="false"
+                android:stopWithTask="true"
+                android:foregroundServiceType="microphone|camera" />
+    </application>
+
 </manifest>

android/src/main/kotlin/com/instrumentisto/medea_flutter_webrtc/ForegroundCallService.kt

@@ -0,0 +1,104 @@
+package com.instrumentisto.medea_flutter_webrtc
+
+import android.Manifest
+import android.app.Notification
+import android.app.NotificationChannel
+import android.app.NotificationManager
+import android.app.Service
+import android.content.Context
+import android.content.Intent
+import android.content.pm.PackageManager
+import android.content.pm.ServiceInfo
+import android.os.Build
+import android.os.IBinder
+import android.util.Log
+import androidx.core.app.ServiceCompat
+import androidx.core.content.ContextCompat
+
+private val TAG = ForegroundCallService::class.java.simpleName
+
+class ForegroundCallService : Service() {
+
+  companion object {
+    fun start(context: Context) {
+      Log.v(TAG, "ForegroundCallService::start")
+
+      if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
+        // Foreground services not required before SDK 28
+        return
+      }
+
+      val intent = Intent(context, ForegroundCallService::class.java)
+      intent.putExtra("inputExtra", "Foreground Service Example in Android")
+
+      // TODO: block if no permission?
+      ContextCompat.startForegroundService(context, intent)
+    }
+
+    fun stop(context: Context) {
+      Log.v(TAG, "ForegroundCallService::stop")
+
+      val serviceIntent = Intent(context, ForegroundCallService::class.java)
+      context.stopService(serviceIntent)
+    }
+  }
+
+  override fun onCreate() {
+    Log.d(TAG, "onCreate")
+    super.onCreate()
+  }
+
+  override fun onStartCommand(intent: Intent, flags: Int, startId: Int): Int {
+    Log.d(TAG, "Started")
+    createNotificationChannel()
+
+    val notification: Notification =
+        Notification.Builder(this, "ForegroundServiceChannel")
+            .setContentTitle("AZAZAZAZAZAZAZAZAZAZAZAZ")
+            .setContentText(
+                "AZAZAZAZAZAZAZAZAZAZAZAZAZ") //            .setContentIntent(pendingIntent)
+            .build()
+
+    var serviceType = 0
+    if (this.checkSelfPermission(Manifest.permission.RECORD_AUDIO) ==
+        PackageManager.PERMISSION_GRANTED) {
+      serviceType = serviceType or ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
+    }
+    if (this.checkSelfPermission(Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED) {
+      serviceType = serviceType or ServiceInfo.FOREGROUND_SERVICE_TYPE_CAMERA
+    }
+
+    ServiceCompat.startForeground(this, 1863424652, notification, serviceType)
+
+    return START_NOT_STICKY
+  }
+
+  override fun onDestroy() {
+    Log.d(TAG, "Destroyed")
+    super.onDestroy()
+    ServiceCompat.stopForeground(this, ServiceCompat.STOP_FOREGROUND_REMOVE)
+  }
+
+  override fun onBind(intent: Intent?): IBinder? {
+    return null
+  }
+
+  override fun onTaskRemoved(rootIntent: Intent?) {
+    Log.e(TAG, "onTaskRemoved")
+  }
+
+  private fun createNotificationChannel() {
+    Log.e(TAG, "createNotificationChannel")
+    if (Build.VERSION.SDK_INT >= 26) {
+      val serviceChannel =
+          NotificationChannel(
+              "ForegroundServiceChannel",
+              "Foreground Service Channel",
+              NotificationManager.IMPORTANCE_DEFAULT)
+
+      val manager: NotificationManager =
+          getSystemService<NotificationManager>(NotificationManager::class.java)
+      manager.createNotificationChannel(serviceChannel)
+    }
+  }
+}

android/src/main/kotlin/com/instrumentisto/medea_flutter_webrtc/MedeaFlutterWebrtcPlugin.kt

@@ -1,5 +1,7 @@
 package com.instrumentisto.medea_flutter_webrtc
 
+import android.util.Log
+import com.instrumentisto.medea_flutter_webrtc.controller.ControllerRegistry
 import com.instrumentisto.medea_flutter_webrtc.controller.MediaDevicesController
 import com.instrumentisto.medea_flutter_webrtc.controller.PeerConnectionFactoryController
 import com.instrumentisto.medea_flutter_webrtc.controller.VideoRendererFactoryController
@@ -9,26 +11,48 @@ import io.flutter.embedding.engine.plugins.activity.ActivityPluginBinding
 import io.flutter.plugin.common.BinaryMessenger
 import io.flutter.view.TextureRegistry
 
+private val TAG = MedeaFlutterWebrtcPlugin::class.java.simpleName
+
 class MedeaFlutterWebrtcPlugin : FlutterPlugin, ActivityAware {
   private var peerConnectionFactory: PeerConnectionFactoryController? = null
   private var mediaDevices: MediaDevicesController? = null
   private var videoRendererFactory: VideoRendererFactoryController? = null
   private var messenger: BinaryMessenger? = null
   private var state: State? = null
   private var textureRegistry: TextureRegistry? = null
+  private var permissions: Permissions? = null
+  private var activityPluginBinding: ActivityPluginBinding? = null
 
   override fun onAttachedToEngine(registrar: FlutterPlugin.FlutterPluginBinding) {
+    Log.i(TAG, "Attached to engine")
+
     messenger = registrar.binaryMessenger
     state = State(registrar.applicationContext)
     textureRegistry = registrar.textureRegistry
   }
 
-  override fun onDetachedFromEngine(registrar: FlutterPlugin.FlutterPluginBinding) {}
+  override fun onDetachedFromEngine(registrar: FlutterPlugin.FlutterPluginBinding) {
+    Log.i(TAG, "Detached from engine")
+
+    ControllerRegistry.disposeAll()
+
+    messenger = null
+    state = null
+    textureRegistry = null
+    activityPluginBinding = null
+    permissions = null
+    mediaDevices = null
+    peerConnectionFactory = null
+    videoRendererFactory = null
+  }
 
   override fun onAttachedToActivity(registrar: ActivityPluginBinding) {
-    val permissions = Permissions(registrar.activity)
-    registrar.addRequestPermissionsResultListener(permissions)
-    mediaDevices = MediaDevicesController(messenger!!, state!!, permissions)
+    Log.i(TAG, "Attached to activity")
+
+    activityPluginBinding = registrar
+    permissions = Permissions(activityPluginBinding!!.activity)
+    activityPluginBinding!!.addRequestPermissionsResultListener(permissions!!)
+    mediaDevices = MediaDevicesController(messenger!!, state!!, permissions!!)
     peerConnectionFactory = PeerConnectionFactoryController(messenger!!, state!!)
     videoRendererFactory = VideoRendererFactoryController(messenger!!, textureRegistry!!)
   }
@@ -37,5 +61,10 @@ class MedeaFlutterWebrtcPlugin : FlutterPlugin, ActivityAware {
 
   override fun onReattachedToActivityForConfigChanges(binding: ActivityPluginBinding) {}
 
-  override fun onDetachedFromActivity() {}
+  override fun onDetachedFromActivity() {
+    Log.i(TAG, "Detached from activity")
+
+    activityPluginBinding!!.removeRequestPermissionsResultListener(permissions!!)
+    activityPluginBinding = null
+  }
 }

android/src/main/kotlin/com/instrumentisto/medea_flutter_webrtc/MediaDevices.kt

@@ -77,7 +77,7 @@ class MediaDevices(val state: State, private val permissions: Permissions) : Bro
    * Enumerator for the camera devices, based on which new video [MediaStreamTrackProxy]s will be
    * created.
    */
-  private val cameraEnumerator: CameraEnumerator = getCameraEnumerator(state.getAppContext())
+  private val cameraEnumerator: CameraEnumerator = getCameraEnumerator(state.context)
 
   /** List of [EventObserver]s of these [MediaDevices]. */
   private var eventObservers: HashSet<EventObserver> = HashSet()
@@ -103,6 +103,14 @@ class MediaDevices(val state: State, private val permissions: Permissions) : Bro
   /** Indicator whether bluetooth SCO is connected. */
   private var scoAudioStateConnected: Boolean = false
 
+  /**
+   * [AudioDeviceCallback] provided to [AudioManager.registerAudioDeviceCallback] which fires once
+   * new audio device is connected.
+   *
+   * [isBluetoothHeadsetConnected] will be updated based on this subscription.
+   */
+  private var audioDeviceCallback: AudioDeviceCallback? = null
+
   companion object {
     /** Observer of [MediaDevices] events. */
     interface EventObserver {
@@ -134,21 +142,7 @@ class MediaDevices(val state: State, private val permissions: Permissions) : Bro
   }
 
   init {
-    state
-        .getAppContext()
-        .registerReceiver(this, IntentFilter(AudioManager.ACTION_SCO_AUDIO_STATE_UPDATED))
-    synchronizeHeadsetState()
-    registerHeadsetStateReceiver()
-  }
-
-  /**
-   * Subscribes to the [AudioManager.registerAudioDeviceCallback] which fires once new audio device
-   * is connected.
-   *
-   * [isBluetoothHeadsetConnected] will be updated based on this subscription.
-   */
-  private fun registerHeadsetStateReceiver() {
-    audioManager.registerAudioDeviceCallback(
+    audioDeviceCallback =
         object : AudioDeviceCallback() {
           override fun onAudioDevicesAdded(addedDevices: Array<AudioDeviceInfo>) {
             if (addedDevices.any { isBluetoothDevice(it) }) {
@@ -161,8 +155,11 @@ class MediaDevices(val state: State, private val permissions: Permissions) : Bro
               synchronizeHeadsetState()
             }
           }
-        },
-        null)
+        }
+
+    state.context.registerReceiver(this, IntentFilter(AudioManager.ACTION_SCO_AUDIO_STATE_UPDATED))
+    synchronizeHeadsetState()
+    audioManager.registerAudioDeviceCallback(audioDeviceCallback, null)
   }
 
   /** Actualizes Bluetooth headset state based on the [AudioManager.getDevices]. */
@@ -405,19 +402,8 @@ class MediaDevices(val state: State, private val permissions: Permissions) : Bro
 
     val surfaceTextureRenderer =
         SurfaceTextureHelper.create(Thread.currentThread().name, EglUtils.rootEglBaseContext)
-    val videoCapturer =
-        cameraEnumerator.createCapturer(
-            deviceId,
-            object : CameraVideoCapturer.CameraEventsHandler {
-              override fun onCameraError(p0: String?) {}
-              override fun onCameraDisconnected() {}
-              override fun onCameraFreezed(p0: String?) {}
-              override fun onCameraOpening(p0: String?) {}
-              override fun onFirstFrameAvailable() {}
-              override fun onCameraClosed() {}
-            })
-    videoCapturer.initialize(
-        surfaceTextureRenderer, state.getAppContext(), videoSource.capturerObserver)
+    val videoCapturer = cameraEnumerator.createCapturer(deviceId, null)
+    videoCapturer.initialize(surfaceTextureRenderer, state.context, videoSource.capturerObserver)
     videoCapturer.startCapture(width, height, fps)
 
     val facingMode =
@@ -491,4 +477,11 @@ class MediaDevices(val state: State, private val permissions: Permissions) : Bro
       }
     }
   }
+
+  /** Releases allocated resources. */
+  fun dispose() {
+    state.context.unregisterReceiver(this)
+    audioManager.unregisterAudioDeviceCallback(audioDeviceCallback)
+    audioDeviceCallback = null
+  }
 }

android/src/main/kotlin/com/instrumentisto/medea_flutter_webrtc/State.kt

@@ -7,6 +7,7 @@ import org.webrtc.PeerConnectionFactory
 import org.webrtc.VideoDecoderFactory
 import org.webrtc.VideoEncoderFactory
 import org.webrtc.audio.JavaAudioDeviceModule
+import org.webrtc.audio.JavaAudioDeviceModule.AudioRecordStateCallback
 
 /**
  * Global context of the `flutter_webrtc` library.
@@ -15,10 +16,7 @@ import org.webrtc.audio.JavaAudioDeviceModule
  *
  * @property context Android [Context] used, for example, for `getUserMedia` requests.
  */
-class State(private val context: Context) {
-  /** Module for the controlling audio devices in context of `libwebrtc`. */
-  private var audioDeviceModule: JavaAudioDeviceModule? = null
-
+class State(val context: Context) {
   /** [VideoEncoderFactory] used by the [PeerConnectionFactory]. */
   var encoder: WebrtcVideoEncoderFactory
 
@@ -50,10 +48,20 @@ class State(private val context: Context) {
    * @return Current [PeerConnectionFactory] of this [State].
    */
   fun getPeerConnectionFactory(): PeerConnectionFactory {
-    if (factory == null) {
-      audioDeviceModule =
+    if (factory == null || factory!!.nativeOwnedFactoryAndThreads == 0L) {
+      var audioDeviceModule =
           JavaAudioDeviceModule.builder(context)
               .setUseHardwareAcousticEchoCanceler(true)
+              .setAudioRecordStateCallback(
+                  object : AudioRecordStateCallback {
+                    override fun onWebRtcAudioRecordStart() {
+                      //                      ForegroundCallService.start(context)
+                    }
+
+                    override fun onWebRtcAudioRecordStop() {
+                      //                      ForegroundCallService.stop(context)
+                    }
+                  })
               .setUseHardwareNoiseSuppressor(true)
               .createAudioDeviceModule()
 
@@ -64,18 +72,12 @@ class State(private val context: Context) {
               .setVideoDecoderFactory(decoder)
               .setAudioDeviceModule(audioDeviceModule)
               .createPeerConnectionFactory()
-
-      audioDeviceModule!!.setSpeakerMute(false)
+      audioDeviceModule.release()
     }
 
     return factory!!
   }
 
-  /** @return Android SDK [Context]. */
-  fun getAppContext(): Context {
-    return context
-  }
-
   /** @return [AudioManager] system service. */
   fun getAudioManager(): AudioManager {
     return context.getSystemService(Context.AUDIO_SERVICE) as AudioManager