Add English and French dialog to the sample application

Author: Joss STUART

FaceMaskDetection/src/main/java/com/softbankrobotics/facemaskdetection/FaceMaskDetection.kt

@@ -28,7 +28,7 @@ class FaceMaskDetection(private val detector: FaceMaskDetector, private val came
 
     private val detectionScope = CoroutineScope(
         ThreadPoolExecutor(1, 1, 30, TimeUnit.SECONDS,
-            ArrayBlockingQueue(1), ThreadPoolExecutor.DiscardOldestPolicy()).asCoroutineDispatcher())
+        ArrayBlockingQueue(1), ThreadPoolExecutor.DiscardOldestPolicy()).asCoroutineDispatcher())
 
     sealed class Message {
         class FaceMaskDetect(

app/build.gradle

@@ -7,11 +7,11 @@ android {
     buildToolsVersion "29.0.3"
 
     defaultConfig {
-        applicationId "com.softbankrobotics.peppermaskdetection"
+        applicationId "com.softbankrobotics.dx.peppermaskdetection"
         minSdkVersion 23
         targetSdkVersion 29
-        versionCode 6
-        versionName "1.3.2"
+        versionCode 9
+        versionName "2.3"
 
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
     }
@@ -61,4 +61,12 @@ dependencies {
     // OpenCV
     implementation project(":OpenCV")
     implementation project(":FaceMaskDetection")
+
+    implementation 'com.aldebaran:qisdk-conversationalcontent-greetings:0.19.1-experimental-05'
+    implementation 'com.aldebaran:qisdk-conversationalcontent-robotabilities:0.19.1-experimental-05'
+    implementation 'com.aldebaran:qisdk-conversationalcontent-volumecontrol:0.19.1-experimental-05'
+    implementation 'com.aldebaran:qisdk-conversationalcontent-farewell:0.19.1-experimental-05'
+    implementation 'com.aldebaran:qisdk-conversationalcontent-datetime:0.19.1-experimental-05'
+    implementation 'com.aldebaran:qisdk-conversationalcontent-askrobotname:0.19.1-experimental-05'
+    implementation 'com.aldebaran:qisdk-conversationalcontent-conversationbasics:0.19.1-experimental-05'
 }

app/src/main/java/com/softbankrobotics/peppermaskdetection/MainActivity.kt

@@ -14,24 +14,37 @@ import com.aldebaran.qi.Future
 import com.aldebaran.qi.sdk.QiContext
 import com.aldebaran.qi.sdk.QiSDK
 import com.aldebaran.qi.sdk.RobotLifecycleCallbacks
+import com.aldebaran.qi.sdk.`object`.conversation.*
+import com.aldebaran.qi.sdk.builder.QiChatbotBuilder
+import com.aldebaran.qi.sdk.builder.TopicBuilder
+import com.aldebaran.qi.sdk.conversationalcontentlibrary.askrobotname.AskRobotNameConversationalContent
+import com.aldebaran.qi.sdk.conversationalcontentlibrary.base.AbstractConversationalContent
+import com.aldebaran.qi.sdk.conversationalcontentlibrary.base.ConversationalContentChatBuilder
+import com.aldebaran.qi.sdk.conversationalcontentlibrary.datetime.DateTimeConversationalContent
+import com.aldebaran.qi.sdk.conversationalcontentlibrary.farewell.FarewellConversationalContent
+import com.aldebaran.qi.sdk.conversationalcontentlibrary.greetings.GreetingsConversationalContent
+import com.aldebaran.qi.sdk.conversationalcontentlibrary.robotabilities.RobotAbilitiesConversationalContent
+import com.aldebaran.qi.sdk.conversationalcontentlibrary.volumecontrol.VolumeControlConversationalContent
 import com.aldebaran.qi.sdk.design.activity.RobotActivity
-import com.aldebaran.qi.sdk.design.activity.conversationstatus.SpeechBarDisplayStrategy
 import com.softbankrobotics.facemaskdetection.FaceMaskDetection
 import com.softbankrobotics.facemaskdetection.capturer.BottomCameraCapturer
-import com.softbankrobotics.facemaskdetection.detector.AizooFaceMaskDetector
 import com.softbankrobotics.facemaskdetection.capturer.TopCameraCapturer
+import com.softbankrobotics.facemaskdetection.detector.AizooFaceMaskDetector
 import com.softbankrobotics.facemaskdetection.detector.FaceMaskDetector
 import com.softbankrobotics.facemaskdetection.utils.OpenCVUtils
 import com.softbankrobotics.facemaskdetection.utils.TAG
 import kotlinx.android.synthetic.main.activity_main.*
 
 class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
 
-    private val useTopCamera = false
+    private var qiChatbot: QiChatbot? = null
+    private var chatFuture: Future<Void>? = null
+    private val useTopCamera = true
+    private var shouldBeRecognizing = false
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
-        setSpeechBarDisplayStrategy(SpeechBarDisplayStrategy.IMMERSIVE)
+        //setSpeechBarDisplayStrategy(SpeechBarDisplayStrategy.IMMERSIVE)
         setContentView(R.layout.activity_main)
         clearFaces()
         if (useTopCamera || cameraPermissionAlreadyGranted()) {
@@ -45,6 +58,8 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
 
     public override fun onPause() {
         super.onPause()
+        Log.i(TAG, "onPause")
+        shouldBeRecognizing = false
         detectionFuture?.requestCancellation()
         detectionFuture = null
     }
@@ -53,6 +68,7 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
         super.onResume()
         OpenCVUtils.loadOpenCV(this)
         clearFaces()
+        shouldBeRecognizing = true
         startDetecting()
     }
 
@@ -73,14 +89,17 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
 
     private fun requestPermissionForCamera() {
         if (ActivityCompat.shouldShowRequestPermissionRationale(this, Manifest.permission.CAMERA)) {
-            Toast.makeText(this,
+            Toast.makeText(
+                this,
                 R.string.permissions_needed,
-                Toast.LENGTH_LONG).show()
+                Toast.LENGTH_LONG
+            ).show()
         } else {
             ActivityCompat.requestPermissions(
                 this,
                 arrayOf(Manifest.permission.CAMERA),
-                CAMERA_PERMISSION_REQUEST_CODE)
+                CAMERA_PERMISSION_REQUEST_CODE
+            )
         }
     }
 
@@ -89,9 +108,11 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
         return resultCamera == PackageManager.PERMISSION_GRANTED
     }
 
-    override fun onRequestPermissionsResult(requestCode: Int,
-                                            permissions: Array<String>,
-                                            grantResults: IntArray) {
+    override fun onRequestPermissionsResult(
+        requestCode: Int,
+        permissions: Array<String>,
+        grantResults: IntArray
+    ) {
         if (requestCode == CAMERA_PERMISSION_REQUEST_CODE) {
             var cameraPermissionGranted = true
 
@@ -102,9 +123,11 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
             if (cameraPermissionGranted) {
                 QiSDK.register(this, this)
             } else {
-                Toast.makeText(this,
+                Toast.makeText(
+                    this,
                     R.string.permissions_needed,
-                    Toast.LENGTH_LONG).show()
+                    Toast.LENGTH_LONG
+                ).show()
             }
         }
     }
@@ -116,17 +139,18 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
     inner class FacesForDisplay(rawFaces: List<FaceMaskDetector.DetectedFace>) {
         // Choose the "main" focused faced, which is either the biggest or, when there are a lot of
         // people, the one in the middle.
-        val mainFace : FaceMaskDetector.DetectedFace? = when {
+        val mainFace: FaceMaskDetector.DetectedFace? = when {
             rawFaces.size >= 5 -> rawFaces[2]
             rawFaces.size == 4 -> rawFaces.subList(1, 3).maxBy { it.size() }
             else -> rawFaces.maxBy { it.size() }
         }
         private val mainFaceIndex = rawFaces.indexOf(mainFace)
+
         // Set the other faces relatively
-        val leftFarFace : FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex - 2)
-        val leftNearFace : FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex - 1)
-        val rightNearFace : FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex + 1)
-        val rightFarFace : FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex + 2)
+        val leftFarFace: FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex - 2)
+        val leftNearFace: FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex - 1)
+        val rightNearFace: FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex + 1)
+        val rightFarFace: FaceMaskDetector.DetectedFace? = rawFaces.getOrNull(mainFaceIndex + 2)
     }
 
     private fun setFaces(faces: List<FaceMaskDetector.DetectedFace>) {
@@ -150,7 +174,11 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
         }
     }
 
-    private fun setFace(card: View, face: FaceMaskDetector.DetectedFace?, hideIfEmpty : Boolean = true) {
+    private fun setFace(
+        card: View,
+        face: FaceMaskDetector.DetectedFace?,
+        hideIfEmpty: Boolean = true
+    ) {
         if (hideIfEmpty && face == null) {
             card.visibility = View.INVISIBLE
         } else {
@@ -167,7 +195,8 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
                     resources.getColor(R.color.colorNoMaskDetected, null)
                 }
                 circle.setColorFilter(color)
-                label.text = resources.getString(if(face.hasMask) R.string.mask else R.string.no_mask)
+                label.text =
+                    resources.getString(if (face.hasMask) R.string.mask else R.string.no_mask)
             } else {
                 photo.visibility = View.INVISIBLE
                 circle.setColorFilter(resources.getColor(R.color.colorNobody, null))
@@ -176,35 +205,189 @@ class MainActivity : RobotActivity(), RobotLifecycleCallbacks {
         }
     }
 
+
+    /**********************
+     * Greetings
+     **********************/
+
+    var engaged = false
+    var sawNobodyCount = 0
+
+    private fun jumpToBookmark(bookmark: Bookmark?) {
+        bookmark?.let {
+            qiChatbot?.async()?.goToBookmark(
+                it,
+                AutonomousReactionImportance.HIGH,
+                AutonomousReactionValidity.DELAYABLE
+            )
+        }
+    }
+
+    var lastSawWithoutMask = false
+    var annoyance = 0
+
+    var lastMentionedPeopleNum = 0
+    var worthMentioningPeopleCounter = 0
+
+    private fun updateState(seesWithMask: Boolean, seesWithoutMask: Boolean, numPeople: Int) {
+        val seesSomebody = seesWithMask || seesWithoutMask
+        Log.w(
+            TAG,
+            "DBG updating state wMask=${seesWithMask} noMask=${seesWithoutMask} engaged=${engaged}"
+        )
+
+        if (engaged) {
+            // Update status when already engaged
+            if (seesSomebody) {
+                sawNobodyCount = 0 // Stay engaged
+                // See if they put on a mask
+                var saidSomething = false
+
+                // See if it's worth mentioning people putting on masks or taking them off
+                if (seesWithoutMask == lastSawWithoutMask) {
+                    annoyance = 0
+                } else {
+                    // Something changed !
+                    annoyance += 1
+                    if (annoyance >= 2) {
+                        annoyance = 0
+                        lastSawWithoutMask = seesWithoutMask
+                        saidSomething = true
+                        if (seesWithoutMask) {
+                            if (seesWithMask) {
+                                jumpToBookmark(newWithoutMaskBookmark)
+                            } else {
+                                jumpToBookmark(tookOffMaskBookmark)
+                            }
+                        } else {
+                            jumpToBookmark(putOnMaskBookmark)
+                        }
+                        lastMentionedPeopleNum = numPeople
+                        worthMentioningPeopleCounter = 0
+                    }
+                }
+
+                // See if it's worth mentioning a lot of people
+                if (numPeople > lastMentionedPeopleNum) {
+                    if (worthMentioningPeopleCounter > 2) {
+                        lastMentionedPeopleNum = numPeople
+                        worthMentioningPeopleCounter = 0
+                        jumpToBookmark(manyPeopleBookmark)
+                    } else {
+                        worthMentioningPeopleCounter += 1
+                    }
+                } else {
+                    lastMentionedPeopleNum = numPeople
+                    worthMentioningPeopleCounter = 0
+                }
+
+            } else {
+                sawNobodyCount += 1
+                if (sawNobodyCount > 2) {
+                    engaged = false
+                    //chat?.removeAllOnStartedListeners()
+                    //chatFuture?.cancel(false)
+                    chatFuture = null
+                    lastSawWithoutMask = false
+                    annoyance = 0
+                }
+            }
+        } else if (seesSomebody) {
+            engaged = true
+            chat?.let { chat ->
+                if (seesWithoutMask) {
+                    jumpToBookmark(noMaskBookmark)
+                } else {
+                    jumpToBookmark(maskBookmark)
+                }
+                lastSawWithoutMask = seesWithoutMask
+                annoyance = 0
+            }
+        }
+    }
+
+
     /**********************
      * Robot Lifecycle
      **********************/
 
+
     private fun startDetecting() {
         detectionFuture = detection?.start { faces ->
-            // Filter and sort the faces so that they're left to right, with no uncertain or
-            // non-unique results
+            // Filter and sort the faces so that they're left to right and certain enough
             val sortedFaces = faces
-                .filter { (it.confidence > 0.5)}
+                .filter { (it.confidence > 0.5) }
                 .sortedBy { -it.bb.left }
-            Log.v(TAG, "Filtered faces ${faces.size}, ->  ${sortedFaces.size}")
+            Log.i(TAG, "Filtered faces ${faces.size}, ->  ${sortedFaces.size}")
             setFaces(sortedFaces)
+            // Now update the logic
+            val seesWithMask = sortedFaces.any { it.hasMask }
+            val seesWithoutMask = sortedFaces.any { !it.hasMask }
+            val numPeople = sortedFaces.size
+            updateState(seesWithMask, seesWithoutMask, numPeople)
         }
         detectionFuture?.thenConsume {
-            Log.i(TAG, "Detection future has finished: success=${it.isSuccess}, cancelled=${it.isCancelled}")
+            Log.i(
+                TAG,
+                "Detection future has finished: success=${it.isSuccess}, cancelled=${it.isCancelled}"
+            )
+            if (shouldBeRecognizing) {
+                Log.w(TAG, "Stopped, but it shouldn't have - starting it again")
+                startDetecting()
+            }
         }
     }
 
+    var chat: Chat? = null
+    var maskBookmark: Bookmark? = null
+    var noMaskBookmark: Bookmark? = null
+    var tookOffMaskBookmark: Bookmark? = null
+    var putOnMaskBookmark: Bookmark? = null
+    var newWithoutMaskBookmark: Bookmark? = null
+    var manyPeopleBookmark: Bookmark? = null
+
     override fun onRobotFocusGained(qiContext: QiContext) {
         Log.i(TAG, "onRobotFocusGained")
+        if (chat == null) {
+            val topic = TopicBuilder.with(qiContext).withResource(R.raw.chat).build()
+            val qiChatbot = QiChatbotBuilder.with(qiContext).withTopic(topic).build()
+            this.qiChatbot = qiChatbot
+            maskBookmark = topic.bookmarks["GREETING_MASK"]
+            noMaskBookmark = topic.bookmarks["GREETING_NO_MASK"]
+            tookOffMaskBookmark = topic.bookmarks["TOOK_OFF_MASK"]
+            putOnMaskBookmark = topic.bookmarks["PUT_ON_MASK"]
+            newWithoutMaskBookmark = topic.bookmarks["NEW_WITHOUT_MASK"]
+            manyPeopleBookmark = topic.bookmarks["MANY_PEOPLE"]
+
+            val conversationalContents: List<AbstractConversationalContent> = listOf(
+                GreetingsConversationalContent(),
+                FarewellConversationalContent(),
+                AskRobotNameConversationalContent(),
+                DateTimeConversationalContent(),
+                RobotAbilitiesConversationalContent(),
+                VolumeControlConversationalContent()
+            )
+
+            chat = ConversationalContentChatBuilder.with(qiContext)
+                .withChatbot(qiChatbot)
+                .withConversationalContents(conversationalContents)
+                .build()
+            chat?.listeningBodyLanguage = BodyLanguageOption.DISABLED
+        }
+
+        Log.i(TAG, "Initialised chat")
+
         val capturer = if (useTopCamera) {
             TopCameraCapturer(qiContext)
         } else {
             BottomCameraCapturer(this, this)
         }
         val detector = AizooFaceMaskDetector(this)
         detection = FaceMaskDetection(detector, capturer)
+        shouldBeRecognizing = true
         startDetecting()
+        Log.i(TAG, "Starting chat")
+        chatFuture = chat?.async()?.run()
     }
 
     override fun onRobotFocusLost() {