From 55cd46c2e56d33413d4abc54e6f922803d1c7310 Mon Sep 17 00:00:00 2001 From: Julien Buret Date: Sat, 28 Aug 2021 23:05:54 +0200 Subject: [PATCH] resolves #1271 case insensitive model Signed-off-by: Julien Buret --- .../configuration/bot/new-bot.component.ts | 2 +- .../server/src/main/kotlin/AdminVerticle.kt | 5 +- .../kotlin/model/ApplicationWithIntents.kt | 6 + .../application/application.component.html | 4 + .../application/application.component.ts | 2 +- nlp/admin/web/src/app/model/application.ts | 2 + .../src/main/kotlin/ModelCoreService.kt | 30 ++-- .../service/src/main/kotlin/NlpCoreService.kt | 32 ++-- .../shared/src/main/kotlin/Application.kt | 3 +- .../kotlin/ApplicationConfigurationService.kt | 7 + .../main/kotlin/ConfigurationRepository.kt | 7 +- .../service/src/main/kotlin/ParserService.kt | 3 +- .../kotlin/storage/ClassifiedSentenceDAO.kt | 2 + .../kotlin/config/ApplicationDefinition.kt | 4 + .../src/main/kotlin/config/SentencesQuery.kt | 3 +- .../main/kotlin/ClassifiedSentenceMongoDAO.kt | 139 ++++++++++-------- .../shared/config/ApplicationDefinition_.kt | 29 ++++ .../ApplicationDefinition_Deserializer.kt | 35 ++++- .../ApplicationDefinition_Serializer.kt | 19 +++ .../storage/mongo/ClassifiedSentenceCol_.kt | 29 ++++ .../ClassifiedSentenceCol_Deserializer.kt | 44 ++++-- .../mongo/ClassifiedSentenceCol_Serializer.kt | 19 +++ 22 files changed, 325 insertions(+), 101 deletions(-) diff --git a/bot/admin/web/src/app/configuration/bot/new-bot.component.ts b/bot/admin/web/src/app/configuration/bot/new-bot.component.ts index 1629143589..8f046def29 100644 --- a/bot/admin/web/src/app/configuration/bot/new-bot.component.ts +++ b/bot/admin/web/src/app/configuration/bot/new-bot.component.ts @@ -65,7 +65,7 @@ export class NewBotComponent implements OnInit { validate() { const locale = this.firstFormGroup.value.firstCtrl; const channel = this.channel.value; - const newApp = new Application("new_assistant", "new_assistant", this.state.user.organization, [], [locale], StateService.DEFAULT_ENGINE, true, true, false, 0.0); + const newApp = new Application("new_assistant", "new_assistant", this.state.user.organization, [], [locale], StateService.DEFAULT_ENGINE, true, true, false, 0.0, false); this.applicationService.saveApplication(newApp) .subscribe(app => { this.applicationService.refreshCurrentApplication(app); diff --git a/nlp/admin/server/src/main/kotlin/AdminVerticle.kt b/nlp/admin/server/src/main/kotlin/AdminVerticle.kt index f2cde3fd07..e059f2e04a 100644 --- a/nlp/admin/server/src/main/kotlin/AdminVerticle.kt +++ b/nlp/admin/server/src/main/kotlin/AdminVerticle.kt @@ -307,12 +307,13 @@ open class AdminVerticle : WebVerticle() { if (existingApp != null && existingApp.name != application.name) { badRequest("Application name cannot be changed") } - val newApp = saveApplication( + val newApp = saveApplication( existingApp, application.toApplication().copy(name = application.name.lowercase()) ) // trigger a full rebuild if nlp engine change - if (appWithSameName?.nlpEngineType != newApp.nlpEngineType) { + if (appWithSameName?.nlpEngineType != newApp.nlpEngineType + || appWithSameName.caseInsensitive != newApp.caseInsensitive) { front.triggerBuild(ModelBuildTrigger(newApp._id, true)) } ApplicationWithIntents(newApp, front.getIntentsByApplicationId(newApp._id)) diff --git a/nlp/admin/server/src/main/kotlin/model/ApplicationWithIntents.kt b/nlp/admin/server/src/main/kotlin/model/ApplicationWithIntents.kt index c8a6cfaf93..97bde09226 100644 --- a/nlp/admin/server/src/main/kotlin/model/ApplicationWithIntents.kt +++ b/nlp/admin/server/src/main/kotlin/model/ApplicationWithIntents.kt @@ -67,6 +67,10 @@ data class ApplicationWithIntents( * Unknown intent threshold level. */ val unknownIntentThreshold: Double = 0.0, + /** + * Case insensitive model - sentences are persisted with lower case. + */ + val caseInsensitive: Boolean = false, /** * The id of the app. */ @@ -85,6 +89,7 @@ data class ApplicationWithIntents( application.useEntityModels, application.supportSubEntities, application.unknownIntentThreshold, + application.caseInsensitive, application._id ) @@ -101,6 +106,7 @@ data class ApplicationWithIntents( useEntityModels, supportSubEntities, unknownIntentThreshold, + caseInsensitive, _id ?: newId() ) } diff --git a/nlp/admin/web/src/app/applications/application/application.component.html b/nlp/admin/web/src/app/applications/application/application.component.html index ce2a99c10d..775c29254b 100644 --- a/nlp/admin/web/src/app/applications/application/application.component.html +++ b/nlp/admin/web/src/app/applications/application/application.component.html @@ -64,6 +64,10 @@

+ + Case insensitive models + Use entity models diff --git a/nlp/admin/web/src/app/applications/application/application.component.ts b/nlp/admin/web/src/app/applications/application/application.component.ts index 8cee7bd79f..e66358ada1 100644 --- a/nlp/admin/web/src/app/applications/application/application.component.ts +++ b/nlp/admin/web/src/app/applications/application/application.component.ts @@ -60,7 +60,7 @@ export class ApplicationComponent implements OnInit { } } else { this.newApplication = true; - this.application = new Application("", "", this.state.user.organization, [], [], StateService.DEFAULT_ENGINE, true, true, false, 0.0); + this.application = new Application("", "", this.state.user.organization, [], [], StateService.DEFAULT_ENGINE, true, true, false, 0.0, false); } this.nlpEngineType = this.application.nlpEngineType.name; if (this.application) { diff --git a/nlp/admin/web/src/app/model/application.ts b/nlp/admin/web/src/app/model/application.ts index fca46d55ea..a804f5d121 100644 --- a/nlp/admin/web/src/app/model/application.ts +++ b/nlp/admin/web/src/app/model/application.ts @@ -29,6 +29,7 @@ export class Application { public useEntityModels: boolean, public supportSubEntities: boolean, public unknownIntentThreshold: number, + public caseInsensitive: boolean, public _id?: string) { } @@ -44,6 +45,7 @@ export class Application { this.useEntityModels, this.supportSubEntities, this.unknownIntentThreshold, + this.caseInsensitive, this._id) } diff --git a/nlp/core/service/src/main/kotlin/ModelCoreService.kt b/nlp/core/service/src/main/kotlin/ModelCoreService.kt index 1849aa638c..5de7a1c070 100644 --- a/nlp/core/service/src/main/kotlin/ModelCoreService.kt +++ b/nlp/core/service/src/main/kotlin/ModelCoreService.kt @@ -31,6 +31,7 @@ import ai.tock.nlp.core.quality.TestContext import ai.tock.nlp.core.quality.TestModelReport import ai.tock.nlp.core.sample.SampleEntity import ai.tock.nlp.core.sample.SampleExpression +import ai.tock.nlp.core.service.ModelCoreService.formatExpressions import ai.tock.nlp.model.EntityBuildContext import ai.tock.nlp.model.EntityBuildContextForIntent import ai.tock.nlp.model.EntityBuildContextForSubEntities @@ -72,7 +73,7 @@ internal object ModelCoreService : ModelCore { override fun updateIntentModel(context: BuildContext, expressions: List) { val nlpContext = IntentContext(context) if (!context.onlyIfNotExists || !nlpClassifier.isIntentModelExist(nlpContext)) { - nlpClassifier.buildAndSaveIntentModel(nlpContext, expressions) + nlpClassifier.buildAndSaveIntentModel(nlpContext, context.formatExpressions(expressions)) } } @@ -102,7 +103,7 @@ internal object ModelCoreService : ModelCore { if (!context.onlyIfNotExists || !nlpClassifier.isEntityModelExist(nlpContext) ) { - nlpClassifier.buildAndSaveEntityModel(nlpContext, expressions) + nlpClassifier.buildAndSaveEntityModel(nlpContext, context.formatExpressions(expressions)) } } @@ -123,16 +124,15 @@ internal object ModelCoreService : ModelCore { val startDate = Instant.now() val intentContext = IntentContext(context) - val intentModel = nlpClassifier.buildIntentModel(intentContext, modelExpressions) + val intentModel = nlpClassifier.buildIntentModel(intentContext, context.formatExpressions(modelExpressions)) val entityModels = modelExpressions - .asSequence() .groupBy { it.intent } .mapNotNull { (intent, expressions) -> try { intent to nlpClassifier.buildEntityModel( EntityBuildContextForIntent(context, intent), - expressions + context.formatExpressions(expressions) ) } catch (e: Exception) { logger.error { "entity model build fail for $intent " } @@ -185,13 +185,13 @@ internal object ModelCoreService : ModelCore { ) } } || - entities.any { - expectedEntities.none { e -> - it.role == e.definition.role && it.entityType == e.definition.entityType && it.isSameRange( - e - ) + entities.any { + expectedEntities.none { e -> + it.role == e.definition.role && it.entityType == e.definition.entityType && it.isSameRange( + e + ) + } } - } } override fun getCurrentModelConfiguration( @@ -205,4 +205,12 @@ internal object ModelCoreService : ModelCore { engineType: NlpEngineType, configuration: NlpApplicationConfiguration ) = nlpClassifier.updateModelConfiguration(applicationName, engineType, configuration) + + private fun BuildContext.formatExpressions(expressions: List): List = + if (application.caseInsensitive) expressions.map { e -> e.copy(text = e.text.lowercase(language)) } + else expressions + + private fun TestContext.formatExpressions(expressions: List): List = + if (callContext.application.caseInsensitive) expressions.map { e -> e.copy(text = e.text.lowercase(callContext.language)) } + else expressions } diff --git a/nlp/core/service/src/main/kotlin/NlpCoreService.kt b/nlp/core/service/src/main/kotlin/NlpCoreService.kt index 79d1d59914..e8e8bbd89f 100644 --- a/nlp/core/service/src/main/kotlin/NlpCoreService.kt +++ b/nlp/core/service/src/main/kotlin/NlpCoreService.kt @@ -62,7 +62,7 @@ internal object NlpCoreService : NlpCore { text: String, intentSelector: IntentSelector ): ParsingResult { - val t = checkMaxLengthAllowed(text) + val t = context.prepareText(checkMaxLengthAllowed(text)) return parse( context, t, @@ -83,16 +83,17 @@ internal object NlpCoreService : NlpCore { intentModelHolder: ModelHolder, entityModelHolders: Map ): ParsingResult { + val t = context.prepareText(text) return parse( context.callContext, - text, - { nlpClassifier.classifyIntent(IntentContext(context), intentModelHolder, text) }, + t, + { nlpClassifier.classifyIntent(IntentContext(context), intentModelHolder, t) }, { intent -> entityModelHolders[intent]?.let { entityModel -> nlpClassifier.classifyEntities( EntityCallContextForIntent(context, intent), entityModel, - text + t ) } ?: emptyList() }, @@ -109,7 +110,7 @@ internal object NlpCoreService : NlpCore { ): ParsingResult { try { val intents = intentClassifier.invoke() - val (intent, probability) = intentSelector.selectIntent(intents) ?: null to null + val (intent, probability) = intentSelector.selectIntent(intents) ?: (null to null) if (intent == null || probability == null) { return unknownResult @@ -158,14 +159,20 @@ internal object NlpCoreService : NlpCore { if (classifiedEntityTypes.isNotEmpty()) { if (context.evaluationContext.mergeEntityTypes) { val result = - entityMerge.mergeEntityTypes(context, text, intent, evaluatedEntities, classifiedEntityTypes) + entityMerge.mergeEntityTypes( + context, + text, + intent, + evaluatedEntities, + classifiedEntityTypes + ) result to - (evaluatedEntities + classifiedEntityTypes.map { it.toEntityRecognition(it.entityType.name) }) - .subtract(result).toList() + (evaluatedEntities + classifiedEntityTypes.map { it.toEntityRecognition(it.entityType.name) }) + .subtract(result).toList() } else { evaluatedEntities to - classifiedEntityTypes.map { it.toEntityRecognition(it.entityType.name) } - .subtract(evaluatedEntities).toList() + classifiedEntityTypes.map { it.toEntityRecognition(it.entityType.name) } + .subtract(evaluatedEntities).toList() } } else { evaluatedEntities to emptyList() @@ -204,4 +211,9 @@ internal object NlpCoreService : NlpCore { override fun healthcheck(): Boolean { return entityCore.healthcheck() } + + private fun CallContext.prepareText(text: String): String = + if (application.caseInsensitive) text.lowercase(language) else text + + private fun TestContext.prepareText(text: String): String = callContext.prepareText(text) } diff --git a/nlp/core/shared/src/main/kotlin/Application.kt b/nlp/core/shared/src/main/kotlin/Application.kt index f883799001..3bd49a633c 100644 --- a/nlp/core/shared/src/main/kotlin/Application.kt +++ b/nlp/core/shared/src/main/kotlin/Application.kt @@ -25,7 +25,8 @@ import java.util.Locale data class Application( val name: String, val intents: List, - val supportedLocales: Set + val supportedLocales: Set, + val caseInsensitive: Boolean = false, ) { companion object { diff --git a/nlp/front/service/src/main/kotlin/ApplicationConfigurationService.kt b/nlp/front/service/src/main/kotlin/ApplicationConfigurationService.kt index 43f0fbee4c..23b872810c 100644 --- a/nlp/front/service/src/main/kotlin/ApplicationConfigurationService.kt +++ b/nlp/front/service/src/main/kotlin/ApplicationConfigurationService.kt @@ -60,6 +60,13 @@ object ApplicationConfigurationService : UserNamespaceDAO by userNamespaceDAO, ApplicationConfiguration { + override fun save(application: ApplicationDefinition): ApplicationDefinition { + if(application.caseInsensitive) { + sentenceDAO.updateCaseInsensitiveSentences(application._id) + } + return applicationDAO.save(application) + } + override fun save(sentence: ClassifiedSentence, user: UserLogin?) { sentenceDAO.save(sentence.copy(qualifier = user)) } diff --git a/nlp/front/service/src/main/kotlin/ConfigurationRepository.kt b/nlp/front/service/src/main/kotlin/ConfigurationRepository.kt index e826b06e59..595466e022 100644 --- a/nlp/front/service/src/main/kotlin/ConfigurationRepository.kt +++ b/nlp/front/service/src/main/kotlin/ConfigurationRepository.kt @@ -155,7 +155,12 @@ internal object ConfigurationRepository { it.entitiesRegexp.mapValues { r -> LinkedHashSet(r.value.map { v -> EntitiesRegexp(v.regexp) }) } ) } - return Application(applicationDefinition.qualifiedName, intents, applicationDefinition.supportedLocales) + return Application( + applicationDefinition.qualifiedName, + intents, + applicationDefinition.supportedLocales, + applicationDefinition.caseInsensitive + ) } private fun toEntityWithEntityTypesTree(e: EntityDefinition): Entity? { diff --git a/nlp/front/service/src/main/kotlin/ParserService.kt b/nlp/front/service/src/main/kotlin/ParserService.kt index 312e6ef43e..96185760ae 100644 --- a/nlp/front/service/src/main/kotlin/ParserService.kt +++ b/nlp/front/service/src/main/kotlin/ParserService.kt @@ -197,7 +197,8 @@ object ParserService : Parser { language, search = q, status = setOf(validated, model), - onlyExactMatch = true + onlyExactMatch = true, + caseInsensitiveExactMatch = application.caseInsensitive ) ) .sentences diff --git a/nlp/front/service/src/main/kotlin/storage/ClassifiedSentenceDAO.kt b/nlp/front/service/src/main/kotlin/storage/ClassifiedSentenceDAO.kt index 9d510902fc..b6b10156a7 100644 --- a/nlp/front/service/src/main/kotlin/storage/ClassifiedSentenceDAO.kt +++ b/nlp/front/service/src/main/kotlin/storage/ClassifiedSentenceDAO.kt @@ -31,6 +31,8 @@ import java.util.Locale */ interface ClassifiedSentenceDAO { + fun updateCaseInsensitiveSentences(applicationId: Id) + fun getSentences( intents: Set>?, language: Locale?, diff --git a/nlp/front/shared/src/main/kotlin/config/ApplicationDefinition.kt b/nlp/front/shared/src/main/kotlin/config/ApplicationDefinition.kt index 1d01f050ef..4e6ba4e019 100644 --- a/nlp/front/shared/src/main/kotlin/config/ApplicationDefinition.kt +++ b/nlp/front/shared/src/main/kotlin/config/ApplicationDefinition.kt @@ -70,6 +70,10 @@ data class ApplicationDefinition( * Unknown intent threshold level. */ val unknownIntentThreshold: Double = 0.0, + /** + * Case insensitive model - sentences are persisted with lower case. + */ + val caseInsensitive: Boolean = false, /** * The id of the app. */ diff --git a/nlp/front/shared/src/main/kotlin/config/SentencesQuery.kt b/nlp/front/shared/src/main/kotlin/config/SentencesQuery.kt index ef46c9244f..a074ba7f07 100644 --- a/nlp/front/shared/src/main/kotlin/config/SentencesQuery.kt +++ b/nlp/front/shared/src/main/kotlin/config/SentencesQuery.kt @@ -61,5 +61,6 @@ data class SentencesQuery( */ val allButUser: String? = null, val maxIntentProbability: Float = 1f, - val minIntentProbability: Float = 0f + val minIntentProbability: Float = 0f, + val caseInsensitiveExactMatch: Boolean = false, ) diff --git a/nlp/front/storage-mongo/src/main/kotlin/ClassifiedSentenceMongoDAO.kt b/nlp/front/storage-mongo/src/main/kotlin/ClassifiedSentenceMongoDAO.kt index 1da491ea75..034cc89326 100644 --- a/nlp/front/storage-mongo/src/main/kotlin/ClassifiedSentenceMongoDAO.kt +++ b/nlp/front/storage-mongo/src/main/kotlin/ClassifiedSentenceMongoDAO.kt @@ -40,6 +40,7 @@ import ai.tock.nlp.front.storage.mongo.ClassifiedSentenceCol_.Companion.Language import ai.tock.nlp.front.storage.mongo.ClassifiedSentenceCol_.Companion.LastEntityProbability import ai.tock.nlp.front.storage.mongo.ClassifiedSentenceCol_.Companion.LastIntentProbability import ai.tock.nlp.front.storage.mongo.ClassifiedSentenceCol_.Companion.LastUsage +import ai.tock.nlp.front.storage.mongo.ClassifiedSentenceCol_.Companion.LowerCaseText import ai.tock.nlp.front.storage.mongo.ClassifiedSentenceCol_.Companion.Status import ai.tock.nlp.front.storage.mongo.ClassifiedSentenceCol_.Companion.Text import ai.tock.nlp.front.storage.mongo.ClassifiedSentenceCol_.Companion.UnknownCount @@ -77,6 +78,7 @@ import org.litote.kmongo.combine import org.litote.kmongo.descendingSort import org.litote.kmongo.distinct import org.litote.kmongo.eq +import org.litote.kmongo.exists import org.litote.kmongo.find import org.litote.kmongo.getCollection import org.litote.kmongo.gt @@ -91,9 +93,11 @@ import org.litote.kmongo.orderBy import org.litote.kmongo.pullByFilter import org.litote.kmongo.regex import org.litote.kmongo.replaceOneWithFilter +import org.litote.kmongo.save import org.litote.kmongo.setTo import org.litote.kmongo.setValue import org.litote.kmongo.updateMany +import org.litote.kmongo.updateOneById import java.time.Duration import java.time.Instant import java.time.Instant.now @@ -115,6 +119,7 @@ internal object ClassifiedSentenceMongoDAO : ClassifiedSentenceDAO { @JacksonData(internal = true) data class ClassifiedSentenceCol( val text: String, + val lowerCaseText: String = text.lowercase(), val fullText: String = text, val language: Locale, val applicationId: Id, @@ -134,25 +139,26 @@ internal object ClassifiedSentenceMongoDAO : ClassifiedSentenceDAO { ) { constructor(sentence: ClassifiedSentence) : - this( - textKey(sentence.text), - sentence.text, - sentence.language, - sentence.applicationId, - sentence.creationDate, - now(), - sentence.status, - sentence.classification, - sentence.lastIntentProbability, - sentence.lastEntityProbability, - sentence.lastUsage, - sentence.usageCount, - sentence.unknownCount, - sentence.forReview, - sentence.reviewComment, - sentence.qualifier, - sentence.otherIntentsProbabilities - ) + this( + textKey(sentence.text), + sentence.text.lowercase(sentence.language), + sentence.text, + sentence.language, + sentence.applicationId, + sentence.creationDate, + now(), + sentence.status, + sentence.classification, + sentence.lastIntentProbability, + sentence.lastEntityProbability, + sentence.lastUsage, + sentence.usageCount, + sentence.unknownCount, + sentence.forReview, + sentence.reviewComment, + sentence.qualifier, + sentence.otherIntentsProbabilities + ) fun toSentence(): ClassifiedSentence = ClassifiedSentence( @@ -179,6 +185,7 @@ internal object ClassifiedSentenceMongoDAO : ClassifiedSentenceDAO { val c = database.getCollection("classified_sentence") try { c.ensureUniqueIndex(Text, Language, ApplicationId) + c.ensureIndex(LowerCaseText, Language, ApplicationId) c.ensureIndex(Language, ApplicationId, Status) c.ensureIndex(Status) c.ensureIndex(orderBy(mapOf(ApplicationId to true, Language to true, UpdateDate to false))) @@ -224,6 +231,14 @@ internal object ClassifiedSentenceMongoDAO : ClassifiedSentenceDAO { c } + override fun updateCaseInsensitiveSentences(applicationId: Id) { + logger.debug { "start updating case insensitive sentences" } + col.find(LowerCaseText exists false, ApplicationId eq applicationId).forEach { + save(it.copy(lowerCaseText = it.text.lowercase(it.language))) + } + logger.debug { "end updating case insensitive sentences" } + } + override fun getSentences( intents: Set>?, language: Locale?, @@ -258,13 +273,17 @@ internal object ClassifiedSentenceMongoDAO : ClassifiedSentenceDAO { } override fun save(sentence: ClassifiedSentence) { + save(ClassifiedSentenceCol(sentence)) + } + + private fun save(s: ClassifiedSentenceCol) { col.replaceOneWithFilter( and( - Text eq textKey(sentence.text), - Language eq sentence.language, - ApplicationId eq sentence.applicationId + Text eq textKey(s.text), + Language eq s.language, + ApplicationId eq s.applicationId ), - ClassifiedSentenceCol(sentence), + s, ReplaceOptions().upsert(true) ) } @@ -354,10 +373,10 @@ internal object ClassifiedSentenceMongoDAO : ClassifiedSentenceDAO { private fun SentencesQuery.filterApplication() = if (wholeNamespace) ApplicationId `in` - ( - getApplicationById(applicationId)?.namespace?.let { n -> getApplicationsByNamespace(n).map { it._id } } - ?: emptyList() - ) + ( + getApplicationById(applicationId)?.namespace?.let { n -> getApplicationsByNamespace(n).map { it._id } } + ?: emptyList() + ) else ApplicationId eq applicationId private fun SentencesQuery.filterReviewOnly() = if (onlyToReview) ForReview eq true else null @@ -400,7 +419,9 @@ internal object ClassifiedSentenceMongoDAO : ClassifiedSentenceDAO { private fun SentencesQuery.filterText(): Bson? = when { search.isNullOrBlank() -> null - onlyExactMatch -> Text eq search + onlyExactMatch -> if (caseInsensitiveExactMatch) LowerCaseText eq search?.lowercase( + language ?: defaultLocale + ) else Text eq search else -> FullText.regex(search!!.trim(), "i") } @@ -501,38 +522,38 @@ internal object ClassifiedSentenceMongoDAO : ClassifiedSentenceDAO { s.classification.entities.filter { e -> e.role == oldEntity.role && e.type == oldEntity.entityTypeName } val newEntities = s.classification.entities.filterNot { e -> e.role == oldEntity.role && e.type == oldEntity.entityTypeName } + - selectedEntities.map { e -> - // select already existing roles and change type - val subEntitiesWithExistingRole = e.subEntities - .filter { subEntity -> newEntityType.subEntities.any { it.role == subEntity.role } } - .map { sub -> - sub.copy(type = newEntityType.subEntities.first { it.role == sub.role }.entityTypeName) - } - - val subEntitiesWithNotExistingRole = - if (newEntityType.name.namespace() == allowedNamespace) { - // for non existing roles, add the new sub entity to entity - e.subEntities - .filterNot { subEntity -> newEntityType.subEntities.any { it.role == subEntity.role } } - .apply { - forEach { sub -> - oldEntityType.subEntities.find { sub.role == it.role } - ?.let { newSubEntity -> - newSubEntityDefinitions.add(newSubEntity) - } + selectedEntities.map { e -> + // select already existing roles and change type + val subEntitiesWithExistingRole = e.subEntities + .filter { subEntity -> newEntityType.subEntities.any { it.role == subEntity.role } } + .map { sub -> + sub.copy(type = newEntityType.subEntities.first { it.role == sub.role }.entityTypeName) + } + + val subEntitiesWithNotExistingRole = + if (newEntityType.name.namespace() == allowedNamespace) { + // for non existing roles, add the new sub entity to entity + e.subEntities + .filterNot { subEntity -> newEntityType.subEntities.any { it.role == subEntity.role } } + .apply { + forEach { sub -> + oldEntityType.subEntities.find { sub.role == it.role } + ?.let { newSubEntity -> + newSubEntityDefinitions.add(newSubEntity) + } + } } - } - } else { - emptyList() - } - val newSubEntities = subEntitiesWithExistingRole + subEntitiesWithNotExistingRole - - e.copy( - type = newEntity.entityTypeName, - role = newEntity.role, - subEntities = newSubEntities.sorted() - ) - } + } else { + emptyList() + } + val newSubEntities = subEntitiesWithExistingRole + subEntitiesWithNotExistingRole + + e.copy( + type = newEntity.entityTypeName, + role = newEntity.role, + subEntities = newSubEntities.sorted() + ) + } save( s.copy( classification = s.classification.copy( diff --git a/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/shared/config/ApplicationDefinition_.kt b/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/shared/config/ApplicationDefinition_.kt index e2ff43a97d..0df368cf2a 100644 --- a/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/shared/config/ApplicationDefinition_.kt +++ b/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/shared/config/ApplicationDefinition_.kt @@ -1,3 +1,19 @@ +/* + * Copyright (C) 2017/2021 e-voyageurs technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package ai.tock.nlp.front.shared.config import ai.tock.nlp.core.NlpEngineType @@ -40,6 +56,8 @@ private val __SupportSubEntities: KProperty1 get() = ApplicationDefinition::supportSubEntities private val __UnknownIntentThreshold: KProperty1 get() = ApplicationDefinition::unknownIntentThreshold +private val __CaseInsensitive: KProperty1 + get() = ApplicationDefinition::caseInsensitive private val ___id: KProperty1?> get() = ApplicationDefinition::_id class ApplicationDefinition_(previous: KPropertyPath?, property: KProperty1<*, @@ -77,6 +95,9 @@ class ApplicationDefinition_(previous: KPropertyPath?, property: KPrope val unknownIntentThreshold: KPropertyPath get() = KPropertyPath(this,__UnknownIntentThreshold) + val caseInsensitive: KPropertyPath + get() = KPropertyPath(this,__CaseInsensitive) + val _id: KPropertyPath?> get() = KPropertyPath(this,___id) @@ -104,6 +125,8 @@ class ApplicationDefinition_(previous: KPropertyPath?, property: KPrope get() = __SupportSubEntities val UnknownIntentThreshold: KProperty1 get() = __UnknownIntentThreshold + val CaseInsensitive: KProperty1 + get() = __CaseInsensitive val _id: KProperty1?> get() = ___id} } @@ -144,6 +167,9 @@ class ApplicationDefinition_Col(previous: KPropertyPath?, property: KPr val unknownIntentThreshold: KPropertyPath get() = KPropertyPath(this,__UnknownIntentThreshold) + val caseInsensitive: KPropertyPath + get() = KPropertyPath(this,__CaseInsensitive) + val _id: KPropertyPath?> get() = KPropertyPath(this,___id) @@ -187,6 +213,9 @@ class ApplicationDefinition_Map(previous: KPropertyPath?, property: val unknownIntentThreshold: KPropertyPath get() = KPropertyPath(this,__UnknownIntentThreshold) + val caseInsensitive: KPropertyPath + get() = KPropertyPath(this,__CaseInsensitive) + val _id: KPropertyPath?> get() = KPropertyPath(this,___id) diff --git a/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/shared/config/ApplicationDefinition_Deserializer.kt b/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/shared/config/ApplicationDefinition_Deserializer.kt index 0aa67afba7..584ab854f5 100644 --- a/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/shared/config/ApplicationDefinition_Deserializer.kt +++ b/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/shared/config/ApplicationDefinition_Deserializer.kt @@ -1,3 +1,19 @@ +/* + * Copyright (C) 2017/2021 e-voyageurs technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package ai.tock.nlp.front.shared.config import ai.tock.nlp.core.NlpEngineType @@ -50,6 +66,8 @@ internal class ApplicationDefinition_Deserializer : JsonDeserializer? = null var __id_set : Boolean = false var _token_ : JsonToken? = currentToken @@ -117,6 +135,11 @@ internal class ApplicationDefinition_Deserializer : JsonDeserializer { + _caseInsensitive_ = if(_token_ == JsonToken.VALUE_NULL) null + else p.booleanValue; + _caseInsensitive_set = true + } "_id" -> { __id_ = if(_token_ == JsonToken.VALUE_NULL) null else p.readValueAs(__id__reference); @@ -133,14 +156,15 @@ internal class ApplicationDefinition_Deserializer : JsonDeserializer() if(_name_set) @@ -165,6 +189,8 @@ internal class ApplicationDefinition_Deserializer : JsonDeserializer>> = object : TypeReference>>() {} diff --git a/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/shared/config/ApplicationDefinition_Serializer.kt b/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/shared/config/ApplicationDefinition_Serializer.kt index 2fa318c9c7..29affaff6d 100644 --- a/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/shared/config/ApplicationDefinition_Serializer.kt +++ b/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/shared/config/ApplicationDefinition_Serializer.kt @@ -1,3 +1,19 @@ +/* + * Copyright (C) 2017/2021 e-voyageurs technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package ai.tock.nlp.front.shared.config import com.fasterxml.jackson.core.JsonGenerator @@ -78,6 +94,9 @@ internal class ApplicationDefinition_Serializer : gen.writeFieldName("unknownIntentThreshold") val _unknownIntentThreshold_ = value.unknownIntentThreshold gen.writeNumber(_unknownIntentThreshold_) + gen.writeFieldName("caseInsensitive") + val _caseInsensitive_ = value.caseInsensitive + gen.writeBoolean(_caseInsensitive_) gen.writeFieldName("_id") val __id_ = value._id serializers.defaultSerializeValue(__id_, gen) diff --git a/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/storage/mongo/ClassifiedSentenceCol_.kt b/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/storage/mongo/ClassifiedSentenceCol_.kt index da073dc834..c933f7d045 100644 --- a/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/storage/mongo/ClassifiedSentenceCol_.kt +++ b/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/storage/mongo/ClassifiedSentenceCol_.kt @@ -1,3 +1,19 @@ +/* + * Copyright (C) 2017/2021 e-voyageurs technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package ai.tock.nlp.front.storage.mongo import ai.tock.nlp.front.shared.config.ApplicationDefinition @@ -22,6 +38,8 @@ import org.litote.kmongo.property.KPropertyPath private val __Text: KProperty1 get() = ClassifiedSentenceMongoDAO.ClassifiedSentenceCol::text +private val __LowerCaseText: KProperty1 + get() = ClassifiedSentenceMongoDAO.ClassifiedSentenceCol::lowerCaseText private val __FullText: KProperty1 get() = ClassifiedSentenceMongoDAO.ClassifiedSentenceCol::fullText private val __Language: KProperty1 @@ -66,6 +84,9 @@ internal class ClassifiedSentenceCol_(previous: KPropertyPath?, propert val text: KPropertyPath get() = KPropertyPath(this,__Text) + val lowerCaseText: KPropertyPath + get() = KPropertyPath(this,__LowerCaseText) + val fullText: KPropertyPath get() = KPropertyPath(this,__FullText) @@ -119,6 +140,8 @@ internal class ClassifiedSentenceCol_(previous: KPropertyPath?, propert companion object { val Text: KProperty1 get() = __Text + val LowerCaseText: KProperty1 + get() = __LowerCaseText val FullText: KProperty1 get() = __FullText val Language: KProperty1 @@ -166,6 +189,9 @@ internal class ClassifiedSentenceCol_Col(previous: KPropertyPath?, prop val text: KPropertyPath get() = KPropertyPath(this,__Text) + val lowerCaseText: KPropertyPath + get() = KPropertyPath(this,__LowerCaseText) + val fullText: KPropertyPath get() = KPropertyPath(this,__FullText) @@ -227,6 +253,9 @@ internal class ClassifiedSentenceCol_Map(previous: KPropertyPath?, p val text: KPropertyPath get() = KPropertyPath(this,__Text) + val lowerCaseText: KPropertyPath + get() = KPropertyPath(this,__LowerCaseText) + val fullText: KPropertyPath get() = KPropertyPath(this,__FullText) diff --git a/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/storage/mongo/ClassifiedSentenceCol_Deserializer.kt b/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/storage/mongo/ClassifiedSentenceCol_Deserializer.kt index 99a79c6d05..ab609f2bc3 100644 --- a/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/storage/mongo/ClassifiedSentenceCol_Deserializer.kt +++ b/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/storage/mongo/ClassifiedSentenceCol_Deserializer.kt @@ -1,3 +1,19 @@ +/* + * Copyright (C) 2017/2021 e-voyageurs technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package ai.tock.nlp.front.storage.mongo import ai.tock.nlp.front.shared.config.ApplicationDefinition @@ -36,6 +52,8 @@ internal class ClassifiedSentenceCol_Deserializer : with(p) { var _text_: String? = null var _text_set : Boolean = false + var _lowerCaseText_: String? = null + var _lowerCaseText_set : Boolean = false var _fullText_: String? = null var _fullText_set : Boolean = false var _language_: Locale? = null @@ -83,6 +101,11 @@ internal class ClassifiedSentenceCol_Deserializer : else p.text; _text_set = true } + "lowerCaseText" -> { + _lowerCaseText_ = if(_token_ == JsonToken.VALUE_NULL) null + else p.text; + _lowerCaseText_set = true + } "fullText" -> { _fullText_ = if(_token_ == JsonToken.VALUE_NULL) null else p.text; @@ -171,15 +194,15 @@ internal class ClassifiedSentenceCol_Deserializer : } _token_ = currentToken } - return if(_text_set && _fullText_set && _language_set && _applicationId_set && - _creationDate_set && _updateDate_set && _status_set && _classification_set && - _lastIntentProbability_set && _lastEntityProbability_set && _lastUsage_set && - _usageCount_set && _unknownCount_set && _forReview_set && _reviewComment_set && - _classifier_set && _otherIntentsProbabilities_set) - ClassifiedSentenceMongoDAO.ClassifiedSentenceCol(text = _text_!!, fullText = - _fullText_!!, language = _language_!!, applicationId = - _applicationId_!!, creationDate = _creationDate_!!, updateDate = - _updateDate_!!, status = _status_!!, classification = + return if(_text_set && _lowerCaseText_set && _fullText_set && _language_set && + _applicationId_set && _creationDate_set && _updateDate_set && _status_set && + _classification_set && _lastIntentProbability_set && _lastEntityProbability_set + && _lastUsage_set && _usageCount_set && _unknownCount_set && _forReview_set && + _reviewComment_set && _classifier_set && _otherIntentsProbabilities_set) + ClassifiedSentenceMongoDAO.ClassifiedSentenceCol(text = _text_!!, lowerCaseText + = _lowerCaseText_!!, fullText = _fullText_!!, language = _language_!!, + applicationId = _applicationId_!!, creationDate = _creationDate_!!, + updateDate = _updateDate_!!, status = _status_!!, classification = _classification_!!, lastIntentProbability = _lastIntentProbability_, lastEntityProbability = _lastEntityProbability_, lastUsage = _lastUsage_, usageCount = _usageCount_, unknownCount = _unknownCount_, @@ -189,6 +212,8 @@ internal class ClassifiedSentenceCol_Deserializer : val map = mutableMapOf() if(_text_set) map[parameters.getValue("text")] = _text_ + if(_lowerCaseText_set) + map[parameters.getValue("lowerCaseText")] = _lowerCaseText_ if(_fullText_set) map[parameters.getValue("fullText")] = _fullText_ if(_language_set) @@ -234,6 +259,7 @@ internal class ClassifiedSentenceCol_Deserializer : private val parameters: Map by lazy(LazyThreadSafetyMode.PUBLICATION) { kotlin.collections.mapOf("text" to primaryConstructor.findParameterByName("text")!!, + "lowerCaseText" to primaryConstructor.findParameterByName("lowerCaseText")!!, "fullText" to primaryConstructor.findParameterByName("fullText")!!, "language" to primaryConstructor.findParameterByName("language")!!, "applicationId" to primaryConstructor.findParameterByName("applicationId")!!, "creationDate" to diff --git a/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/storage/mongo/ClassifiedSentenceCol_Serializer.kt b/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/storage/mongo/ClassifiedSentenceCol_Serializer.kt index 172a2efde7..2c82de5302 100644 --- a/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/storage/mongo/ClassifiedSentenceCol_Serializer.kt +++ b/nlp/front/storage-mongo/target/generated-sources/kapt/compile/ai/tock/nlp/front/storage/mongo/ClassifiedSentenceCol_Serializer.kt @@ -1,3 +1,19 @@ +/* + * Copyright (C) 2017/2021 e-voyageurs technologies + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package ai.tock.nlp.front.storage.mongo import com.fasterxml.jackson.core.JsonGenerator @@ -22,6 +38,9 @@ internal class ClassifiedSentenceCol_Serializer : gen.writeFieldName("text") val _text_ = value.text gen.writeString(_text_) + gen.writeFieldName("lowerCaseText") + val _lowerCaseText_ = value.lowerCaseText + gen.writeString(_lowerCaseText_) gen.writeFieldName("fullText") val _fullText_ = value.fullText gen.writeString(_fullText_)