resolves #1271 case insensitive model

Signed-off-by: Julien Buret <jburet@voyages-sncf.com>
theopenconversationkit · Aug 28, 2021 · 55cd46c · 55cd46c
1 parent 3eccd0c
commit 55cd46c
Show file tree

Hide file tree

Showing 22 changed files with 325 additions and 101 deletions.
diff --git a/bot/admin/web/src/app/configuration/bot/new-bot.component.ts b/bot/admin/web/src/app/configuration/bot/new-bot.component.ts
@@ -65,7 +65,7 @@ export class NewBotComponent implements OnInit {
   validate() {
     const locale = this.firstFormGroup.value.firstCtrl;
     const channel = this.channel.value;
-    const newApp = new Application("new_assistant", "new_assistant", this.state.user.organization, [], [locale], StateService.DEFAULT_ENGINE, true, true, false, 0.0);
+    const newApp = new Application("new_assistant", "new_assistant", this.state.user.organization, [], [locale], StateService.DEFAULT_ENGINE, true, true, false, 0.0, false);
     this.applicationService.saveApplication(newApp)
       .subscribe(app => {
         this.applicationService.refreshCurrentApplication(app);

diff --git a/nlp/admin/server/src/main/kotlin/AdminVerticle.kt b/nlp/admin/server/src/main/kotlin/AdminVerticle.kt
@@ -307,12 +307,13 @@ open class AdminVerticle : WebVerticle() {
                 if (existingApp != null && existingApp.name != application.name) {
                     badRequest("Application name cannot be changed")
                 }
-                val newApp = saveApplication(
+                    val newApp = saveApplication(
                     existingApp,
                     application.toApplication().copy(name = application.name.lowercase())
                 )
                 // trigger a full rebuild if nlp engine change
-                if (appWithSameName?.nlpEngineType != newApp.nlpEngineType) {
+                if (appWithSameName?.nlpEngineType != newApp.nlpEngineType
+                    || appWithSameName.caseInsensitive != newApp.caseInsensitive) {
                     front.triggerBuild(ModelBuildTrigger(newApp._id, true))
                 }
                 ApplicationWithIntents(newApp, front.getIntentsByApplicationId(newApp._id))

diff --git a/nlp/admin/server/src/main/kotlin/model/ApplicationWithIntents.kt b/nlp/admin/server/src/main/kotlin/model/ApplicationWithIntents.kt
@@ -67,6 +67,10 @@ data class ApplicationWithIntents(
      * Unknown intent threshold level.
      */
     val unknownIntentThreshold: Double = 0.0,
+    /**
+     * Case insensitive model - sentences are persisted with lower case.
+     */
+    val caseInsensitive: Boolean = false,
     /**
      * The id of the app.
      */
@@ -85,6 +89,7 @@ data class ApplicationWithIntents(
             application.useEntityModels,
             application.supportSubEntities,
             application.unknownIntentThreshold,
+            application.caseInsensitive,
             application._id
         )
 
@@ -101,6 +106,7 @@ data class ApplicationWithIntents(
             useEntityModels,
             supportSubEntities,
             unknownIntentThreshold,
+            caseInsensitive,
             _id ?: newId()
         )
     }

diff --git a/nlp/admin/web/src/app/applications/application/application.component.html b/nlp/admin/web/src/app/applications/application/application.component.html
@@ -64,6 +64,10 @@
           </mat-form-field>
         </p>
         <p>
+          <nb-checkbox class="separator" [(ngModel)]="application.caseInsensitive"
+                       nbTooltip="If selected, this option uses case insensitive models">
+            Case insensitive models
+          </nb-checkbox>
           <nb-checkbox class="separator" [(ngModel)]="application.mergeEngineTypes"
                        nbTooltip="If selected, this option uses built-in entity models (like dates) in conjunction with standard NER">
             Use entity models

diff --git a/nlp/admin/web/src/app/applications/application/application.component.ts b/nlp/admin/web/src/app/applications/application/application.component.ts
@@ -60,7 +60,7 @@ export class ApplicationComponent implements OnInit {
           }
         } else {
           this.newApplication = true;
-          this.application = new Application("", "", this.state.user.organization, [], [], StateService.DEFAULT_ENGINE, true, true, false, 0.0);
+          this.application = new Application("", "", this.state.user.organization, [], [], StateService.DEFAULT_ENGINE, true, true, false, 0.0, false);
         }
         this.nlpEngineType = this.application.nlpEngineType.name;
         if (this.application) {

diff --git a/nlp/admin/web/src/app/model/application.ts b/nlp/admin/web/src/app/model/application.ts
@@ -29,6 +29,7 @@ export class Application {
               public useEntityModels: boolean,
               public supportSubEntities: boolean,
               public unknownIntentThreshold: number,
+              public caseInsensitive: boolean,
               public _id?: string) {
   }
 
@@ -44,6 +45,7 @@ export class Application {
       this.useEntityModels,
       this.supportSubEntities,
       this.unknownIntentThreshold,
+      this.caseInsensitive,
       this._id)
   }
 

diff --git a/nlp/core/service/src/main/kotlin/ModelCoreService.kt b/nlp/core/service/src/main/kotlin/ModelCoreService.kt
@@ -31,6 +31,7 @@ import ai.tock.nlp.core.quality.TestContext
 import ai.tock.nlp.core.quality.TestModelReport
 import ai.tock.nlp.core.sample.SampleEntity
 import ai.tock.nlp.core.sample.SampleExpression
+import ai.tock.nlp.core.service.ModelCoreService.formatExpressions
 import ai.tock.nlp.model.EntityBuildContext
 import ai.tock.nlp.model.EntityBuildContextForIntent
 import ai.tock.nlp.model.EntityBuildContextForSubEntities
@@ -72,7 +73,7 @@ internal object ModelCoreService : ModelCore {
     override fun updateIntentModel(context: BuildContext, expressions: List<SampleExpression>) {
         val nlpContext = IntentContext(context)
         if (!context.onlyIfNotExists || !nlpClassifier.isIntentModelExist(nlpContext)) {
-            nlpClassifier.buildAndSaveIntentModel(nlpContext, expressions)
+            nlpClassifier.buildAndSaveIntentModel(nlpContext, context.formatExpressions(expressions))
         }
     }
 
@@ -102,7 +103,7 @@ internal object ModelCoreService : ModelCore {
         if (!context.onlyIfNotExists ||
             !nlpClassifier.isEntityModelExist(nlpContext)
         ) {
-            nlpClassifier.buildAndSaveEntityModel(nlpContext, expressions)
+            nlpClassifier.buildAndSaveEntityModel(nlpContext, context.formatExpressions(expressions))
         }
     }
 
@@ -123,16 +124,15 @@ internal object ModelCoreService : ModelCore {
         val startDate = Instant.now()
 
         val intentContext = IntentContext(context)
-        val intentModel = nlpClassifier.buildIntentModel(intentContext, modelExpressions)
+        val intentModel = nlpClassifier.buildIntentModel(intentContext, context.formatExpressions(modelExpressions))
         val entityModels = modelExpressions
-            .asSequence()
             .groupBy { it.intent }
             .mapNotNull { (intent, expressions)
                 ->
                 try {
                     intent to nlpClassifier.buildEntityModel(
                         EntityBuildContextForIntent(context, intent),
-                        expressions
+                        context.formatExpressions(expressions)
                     )
                 } catch (e: Exception) {
                     logger.error { "entity model build fail for $intent " }
@@ -185,13 +185,13 @@ internal object ModelCoreService : ModelCore {
                 )
             }
         } ||
-            entities.any {
-                expectedEntities.none { e ->
-                    it.role == e.definition.role && it.entityType == e.definition.entityType && it.isSameRange(
-                        e
-                    )
+                entities.any {
+                    expectedEntities.none { e ->
+                        it.role == e.definition.role && it.entityType == e.definition.entityType && it.isSameRange(
+                            e
+                        )
+                    }
                 }
-            }
     }
 
     override fun getCurrentModelConfiguration(
@@ -205,4 +205,12 @@ internal object ModelCoreService : ModelCore {
         engineType: NlpEngineType,
         configuration: NlpApplicationConfiguration
     ) = nlpClassifier.updateModelConfiguration(applicationName, engineType, configuration)
+
+    private fun BuildContext.formatExpressions(expressions: List<SampleExpression>): List<SampleExpression> =
+        if (application.caseInsensitive) expressions.map { e -> e.copy(text = e.text.lowercase(language)) }
+        else expressions
+
+    private fun TestContext.formatExpressions(expressions: List<SampleExpression>): List<SampleExpression> =
+        if (callContext.application.caseInsensitive) expressions.map { e -> e.copy(text = e.text.lowercase(callContext.language)) }
+        else expressions
 }
diff --git a/nlp/core/service/src/main/kotlin/NlpCoreService.kt b/nlp/core/service/src/main/kotlin/NlpCoreService.kt
@@ -62,7 +62,7 @@ internal object NlpCoreService : NlpCore {
         text: String,
         intentSelector: IntentSelector
     ): ParsingResult {
-        val t = checkMaxLengthAllowed(text)
+        val t = context.prepareText(checkMaxLengthAllowed(text))
         return parse(
             context,
             t,
@@ -83,16 +83,17 @@ internal object NlpCoreService : NlpCore {
         intentModelHolder: ModelHolder,
         entityModelHolders: Map<Intent, ModelHolder?>
     ): ParsingResult {
+        val t = context.prepareText(text)
         return parse(
             context.callContext,
-            text,
-            { nlpClassifier.classifyIntent(IntentContext(context), intentModelHolder, text) },
+            t,
+            { nlpClassifier.classifyIntent(IntentContext(context), intentModelHolder, t) },
             { intent ->
                 entityModelHolders[intent]?.let { entityModel ->
                     nlpClassifier.classifyEntities(
                         EntityCallContextForIntent(context, intent),
                         entityModel,
-                        text
+                        t
                     )
                 } ?: emptyList()
             },
@@ -109,7 +110,7 @@ internal object NlpCoreService : NlpCore {
     ): ParsingResult {
         try {
             val intents = intentClassifier.invoke()
-            val (intent, probability) = intentSelector.selectIntent(intents) ?: null to null
+            val (intent, probability) = intentSelector.selectIntent(intents) ?: (null to null)
 
             if (intent == null || probability == null) {
                 return unknownResult
@@ -158,14 +159,20 @@ internal object NlpCoreService : NlpCore {
                 if (classifiedEntityTypes.isNotEmpty()) {
                     if (context.evaluationContext.mergeEntityTypes) {
                         val result =
-                            entityMerge.mergeEntityTypes(context, text, intent, evaluatedEntities, classifiedEntityTypes)
+                            entityMerge.mergeEntityTypes(
+                                context,
+                                text,
+                                intent,
+                                evaluatedEntities,
+                                classifiedEntityTypes
+                            )
                         result to
-                            (evaluatedEntities + classifiedEntityTypes.map { it.toEntityRecognition(it.entityType.name) })
-                                .subtract(result).toList()
+                                (evaluatedEntities + classifiedEntityTypes.map { it.toEntityRecognition(it.entityType.name) })
+                                    .subtract(result).toList()
                     } else {
                         evaluatedEntities to
-                            classifiedEntityTypes.map { it.toEntityRecognition(it.entityType.name) }
-                                .subtract(evaluatedEntities).toList()
+                                classifiedEntityTypes.map { it.toEntityRecognition(it.entityType.name) }
+                                    .subtract(evaluatedEntities).toList()
                     }
                 } else {
                     evaluatedEntities to emptyList()
@@ -204,4 +211,9 @@ internal object NlpCoreService : NlpCore {
     override fun healthcheck(): Boolean {
         return entityCore.healthcheck()
     }
+
+    private fun CallContext.prepareText(text: String): String =
+        if (application.caseInsensitive) text.lowercase(language) else text
+
+    private fun TestContext.prepareText(text: String): String = callContext.prepareText(text)
 }
diff --git a/nlp/core/shared/src/main/kotlin/Application.kt b/nlp/core/shared/src/main/kotlin/Application.kt
@@ -25,7 +25,8 @@ import java.util.Locale
 data class Application(
     val name: String,
     val intents: List<Intent>,
-    val supportedLocales: Set<Locale>
+    val supportedLocales: Set<Locale>,
+    val caseInsensitive: Boolean = false,
 ) {
 
     companion object {

diff --git a/nlp/front/service/src/main/kotlin/ApplicationConfigurationService.kt b/nlp/front/service/src/main/kotlin/ApplicationConfigurationService.kt
@@ -60,6 +60,13 @@ object ApplicationConfigurationService :
     UserNamespaceDAO by userNamespaceDAO,
     ApplicationConfiguration {
 
+    override fun save(application: ApplicationDefinition): ApplicationDefinition {
+        if(application.caseInsensitive) {
+            sentenceDAO.updateCaseInsensitiveSentences(application._id)
+        }
+        return applicationDAO.save(application)
+    }
+
     override fun save(sentence: ClassifiedSentence, user: UserLogin?) {
         sentenceDAO.save(sentence.copy(qualifier = user))
     }

diff --git a/nlp/front/service/src/main/kotlin/ConfigurationRepository.kt b/nlp/front/service/src/main/kotlin/ConfigurationRepository.kt
@@ -155,7 +155,12 @@ internal object ConfigurationRepository {
                 it.entitiesRegexp.mapValues { r -> LinkedHashSet(r.value.map { v -> EntitiesRegexp(v.regexp) }) }
             )
         }
-        return Application(applicationDefinition.qualifiedName, intents, applicationDefinition.supportedLocales)
+        return Application(
+            applicationDefinition.qualifiedName,
+            intents,
+            applicationDefinition.supportedLocales,
+            applicationDefinition.caseInsensitive
+        )
     }
 
     private fun toEntityWithEntityTypesTree(e: EntityDefinition): Entity? {

diff --git a/nlp/front/service/src/main/kotlin/ParserService.kt b/nlp/front/service/src/main/kotlin/ParserService.kt
@@ -197,7 +197,8 @@ object ParserService : Parser {
                         language,
                         search = q,
                         status = setOf(validated, model),
-                        onlyExactMatch = true
+                        onlyExactMatch = true,
+                        caseInsensitiveExactMatch = application.caseInsensitive
                     )
                 )
                 .sentences

diff --git a/nlp/front/service/src/main/kotlin/storage/ClassifiedSentenceDAO.kt b/nlp/front/service/src/main/kotlin/storage/ClassifiedSentenceDAO.kt
@@ -31,6 +31,8 @@ import java.util.Locale
  */
 interface ClassifiedSentenceDAO {
 
+    fun updateCaseInsensitiveSentences(applicationId: Id<ApplicationDefinition>)
+
     fun getSentences(
         intents: Set<Id<IntentDefinition>>?,
         language: Locale?,

diff --git a/nlp/front/shared/src/main/kotlin/config/ApplicationDefinition.kt b/nlp/front/shared/src/main/kotlin/config/ApplicationDefinition.kt
@@ -70,6 +70,10 @@ data class ApplicationDefinition(
      * Unknown intent threshold level.
      */
     val unknownIntentThreshold: Double = 0.0,
+    /**
+     * Case insensitive model - sentences are persisted with lower case.
+     */
+    val caseInsensitive: Boolean = false,
     /**
      * The id of the app.
      */

diff --git a/nlp/front/shared/src/main/kotlin/config/SentencesQuery.kt b/nlp/front/shared/src/main/kotlin/config/SentencesQuery.kt
@@ -61,5 +61,6 @@ data class SentencesQuery(
      */
     val allButUser: String? = null,
     val maxIntentProbability: Float = 1f,
-    val minIntentProbability: Float = 0f
+    val minIntentProbability: Float = 0f,
+    val caseInsensitiveExactMatch: Boolean = false,
 )