diff --git a/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/AiCacheBuildItem.java b/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/AiCacheBuildItem.java new file mode 100644 index 000000000..46a78368e --- /dev/null +++ b/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/AiCacheBuildItem.java @@ -0,0 +1,16 @@ +package io.quarkiverse.langchain4j.deployment; + +import io.quarkus.builder.item.SimpleBuildItem; + +public final class AiCacheBuildItem extends SimpleBuildItem { + + private boolean enable; + + public AiCacheBuildItem(boolean enable) { + this.enable = enable; + } + + public boolean isEnable() { + return enable; + } +} diff --git a/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/AiCacheProcessor.java b/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/AiCacheProcessor.java new file mode 100644 index 000000000..0948ff3e6 --- /dev/null +++ b/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/AiCacheProcessor.java @@ -0,0 +1,65 @@ +package io.quarkiverse.langchain4j.deployment; + +import jakarta.enterprise.context.ApplicationScoped; + +import org.jboss.jandex.AnnotationInstance; +import org.jboss.jandex.AnnotationTarget; +import org.jboss.jandex.ClassInfo; +import org.jboss.jandex.ClassType; +import org.jboss.jandex.IndexView; + +import io.quarkiverse.langchain4j.runtime.AiCacheRecorder; +import io.quarkiverse.langchain4j.runtime.cache.AiCacheConfig; +import io.quarkiverse.langchain4j.runtime.cache.AiCacheProvider; +import io.quarkiverse.langchain4j.runtime.cache.AiCacheStore; +import io.quarkus.arc.deployment.SyntheticBeanBuildItem; +import io.quarkus.arc.deployment.UnremovableBeanBuildItem; +import io.quarkus.deployment.annotations.BuildProducer; +import io.quarkus.deployment.annotations.BuildStep; +import io.quarkus.deployment.annotations.ExecutionTime; +import io.quarkus.deployment.annotations.Record; +import io.quarkus.deployment.builditem.CombinedIndexBuildItem; + +public class AiCacheProcessor { + + @BuildStep + @Record(ExecutionTime.RUNTIME_INIT) + void setupBeans(ChatMemoryBuildConfig buildConfig, AiCacheConfig cacheConfig, + AiCacheRecorder recorder, + CombinedIndexBuildItem indexBuildItem, + BuildProducer aiCacheBuildItemProducer, + BuildProducer unremovableProducer, + BuildProducer syntheticBeanProducer) { + + IndexView index = indexBuildItem.getIndex(); + boolean enableCache = false; + + for (AnnotationInstance instance : index.getAnnotations(LangChain4jDotNames.REGISTER_AI_SERVICES)) { + if (instance.target().kind() != AnnotationTarget.Kind.CLASS) { + continue; + } + + ClassInfo declarativeAiServiceClassInfo = instance.target().asClass(); + + if (declarativeAiServiceClassInfo.hasAnnotation(LangChain4jDotNames.CACHE_RESULT)) { + enableCache = true; + break; + } + } + + aiCacheBuildItemProducer.produce(new AiCacheBuildItem(enableCache)); + + if (enableCache) { + var configurator = SyntheticBeanBuildItem + .configure(AiCacheProvider.class) + .setRuntimeInit() + .addInjectionPoint(ClassType.create(AiCacheStore.class)) + .scope(ApplicationScoped.class) + .createWith(recorder.messageWindow(cacheConfig)) + .defaultBean(); + + syntheticBeanProducer.produce(configurator.done()); + unremovableProducer.produce(UnremovableBeanBuildItem.beanTypes(AiCacheStore.class)); + } + } +} diff --git a/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/AiServicesProcessor.java b/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/AiServicesProcessor.java index a12da68bf..0aa75caee 100644 --- a/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/AiServicesProcessor.java +++ b/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/AiServicesProcessor.java @@ -51,7 +51,6 @@ import org.objectweb.asm.tree.analysis.AnalyzerException; import dev.langchain4j.exception.IllegalConfigurationException; -import dev.langchain4j.service.Moderate; import io.quarkiverse.langchain4j.ModelName; import io.quarkiverse.langchain4j.ToolBox; import io.quarkiverse.langchain4j.deployment.items.SelectedChatModelProviderBuildItem; @@ -185,6 +184,7 @@ public void findDeclarativeServices(CombinedIndexBuildItem indexBuildItem, Set chatModelNames = new HashSet<>(); Set moderationModelNames = new HashSet<>(); + for (AnnotationInstance instance : index.getAnnotations(LangChain4jDotNames.REGISTER_AI_SERVICES)) { if (instance.target().kind() != AnnotationTarget.Kind.CLASS) { continue; // should never happen @@ -206,14 +206,12 @@ public void findDeclarativeServices(CombinedIndexBuildItem indexBuildItem, } String chatModelName = NamedConfigUtil.DEFAULT_NAME; + String moderationModelName = NamedConfigUtil.DEFAULT_NAME; + String embeddingModelName = getModelName(instance.value("modelName")); + if (chatLanguageModelSupplierClassDotName == null) { AnnotationValue modelNameValue = instance.value("modelName"); - if (modelNameValue != null) { - String modelNameValueStr = modelNameValue.asString(); - if ((modelNameValueStr != null) && !modelNameValueStr.isEmpty()) { - chatModelName = modelNameValueStr; - } - } + chatModelName = getModelName(modelNameValue); chatModelNames.add(chatModelName); } @@ -239,6 +237,18 @@ public void findDeclarativeServices(CombinedIndexBuildItem indexBuildItem, } } + // the default value depends on whether tools exists or not - if they do, then we require a AiCacheProvider bean + DotName aiCacheProviderSupplierClassDotName = LangChain4jDotNames.BEAN_AI_CACHE_PROVIDER_SUPPLIER; + AnnotationValue aiCacheProviderSupplierValue = instance.value("cacheProviderSupplier"); + if (aiCacheProviderSupplierValue != null) { + aiCacheProviderSupplierClassDotName = aiCacheProviderSupplierValue.asClass().name(); + if (!aiCacheProviderSupplierClassDotName + .equals(LangChain4jDotNames.BEAN_AI_CACHE_PROVIDER_SUPPLIER)) { + validateSupplierAndRegisterForReflection(aiCacheProviderSupplierClassDotName, index, + reflectiveClassProducer); + } + } + DotName retrieverClassDotName = null; AnnotationValue retrieverValue = instance.value("retriever"); if (retrieverValue != null) { @@ -292,17 +302,11 @@ public void findDeclarativeServices(CombinedIndexBuildItem indexBuildItem, } // determine whether the method is annotated with @Moderate - String moderationModelName = NamedConfigUtil.DEFAULT_NAME; for (MethodInfo method : declarativeAiServiceClassInfo.methods()) { if (method.hasAnnotation(LangChain4jDotNames.MODERATE)) { if (moderationModelSupplierClassName.equals(LangChain4jDotNames.BEAN_IF_EXISTS_MODERATION_MODEL_SUPPLIER)) { AnnotationValue modelNameValue = instance.value("modelName"); - if (modelNameValue != null) { - String modelNameValueStr = modelNameValue.asString(); - if ((modelNameValueStr != null) && !modelNameValueStr.isEmpty()) { - moderationModelName = modelNameValueStr; - } - } + moderationModelName = getModelName(modelNameValue); moderationModelNames.add(moderationModelName); } break; @@ -321,13 +325,16 @@ public void findDeclarativeServices(CombinedIndexBuildItem indexBuildItem, chatLanguageModelSupplierClassDotName, toolDotNames, chatMemoryProviderSupplierClassDotName, + aiCacheProviderSupplierClassDotName, retrieverClassDotName, retrievalAugmentorSupplierClassName, customRetrievalAugmentorSupplierClassIsABean, auditServiceSupplierClassName, moderationModelSupplierClassName, cdiScope, - chatModelName, moderationModelName)); + chatModelName, + moderationModelName, + embeddingModelName)); } for (String chatModelName : chatModelNames) { @@ -361,7 +368,8 @@ public void handleDeclarativeServices(AiServicesRecorder recorder, List declarativeAiServiceItems, List selectedChatModelProvider, BuildProducer syntheticBeanProducer, - BuildProducer unremoveableProducer) { + BuildProducer unremoveableProducer, + AiCacheBuildItem aiCacheBuildItem) { boolean needsChatModelBean = false; boolean needsStreamingChatModelBean = false; @@ -370,6 +378,8 @@ public void handleDeclarativeServices(AiServicesRecorder recorder, boolean needsRetrievalAugmentorBean = false; boolean needsAuditServiceBean = false; boolean needsModerationModelBean = false; + boolean needsAiCacheProvider = false; + Set allToolNames = new HashSet<>(); for (DeclarativeAiServiceBuildItem bi : declarativeAiServiceItems) { @@ -386,6 +396,10 @@ public void handleDeclarativeServices(AiServicesRecorder recorder, ? bi.getChatMemoryProviderSupplierClassDotName().toString() : null; + String aiCacheProviderSupplierClassName = bi.getAiCacheProviderSupplierClassDotName() != null + ? bi.getAiCacheProviderSupplierClassDotName().toString() + : null; + String retrieverClassName = bi.getRetrieverClassDotName() != null ? bi.getRetrieverClassDotName().toString() : null; @@ -403,7 +417,7 @@ public void handleDeclarativeServices(AiServicesRecorder recorder, : null); // determine whether the method returns Multi - boolean injectStreamingChatModelBean = false; + boolean needsStreamingChatModel = false; for (MethodInfo method : declarativeAiServiceClassInfo.methods()) { if (!LangChain4jDotNames.MULTI.equals(method.returnType().name())) { continue; @@ -419,29 +433,36 @@ public void handleDeclarativeServices(AiServicesRecorder recorder, throw illegalConfiguration("Only Multi is supported as a Multi return type. Offending method is '" + method.declaringClass().name().toString() + "#" + method.name() + "'"); } - injectStreamingChatModelBean = true; + needsStreamingChatModel = true; } - boolean injectModerationModelBean = false; + boolean needsModerationModel = false; for (MethodInfo method : declarativeAiServiceClassInfo.methods()) { - if (method.hasAnnotation(Moderate.class)) { - injectModerationModelBean = true; + if (method.hasAnnotation(LangChain4jDotNames.MODERATE)) { + needsModerationModel = true; break; } } String chatModelName = bi.getChatModelName(); String moderationModelName = bi.getModerationModelName(); + String embeddingModelName = bi.getEmbeddingModelName(); + boolean enableCache = aiCacheBuildItem.isEnable(); + SyntheticBeanBuildItem.ExtendedBeanConfigurator configurator = SyntheticBeanBuildItem .configure(QuarkusAiServiceContext.class) .forceApplicationClass() .createWith(recorder.createDeclarativeAiService( new DeclarativeAiServiceCreateInfo(serviceClassName, chatLanguageModelSupplierClassName, - toolClassNames, chatMemoryProviderSupplierClassName, retrieverClassName, + toolClassNames, chatMemoryProviderSupplierClassName, aiCacheProviderSupplierClassName, + retrieverClassName, retrievalAugmentorSupplierClassName, auditServiceClassSupplierName, moderationModelSupplierClassName, chatModelName, moderationModelName, - injectStreamingChatModelBean, injectModerationModelBean))) + embeddingModelName, + needsStreamingChatModel, + needsModerationModel, + enableCache))) .setRuntimeInit() .addQualifier() .annotation(LangChain4jDotNames.QUARKUS_AI_SERVICE_CONTEXT_QUALIFIER).addValue("value", serviceClassName) @@ -451,7 +472,7 @@ public void handleDeclarativeServices(AiServicesRecorder recorder, if ((chatLanguageModelSupplierClassName == null) && !selectedChatModelProvider.isEmpty()) { if (NamedConfigUtil.isDefault(chatModelName)) { configurator.addInjectionPoint(ClassType.create(LangChain4jDotNames.CHAT_MODEL)); - if (injectStreamingChatModelBean) { + if (needsStreamingChatModel) { configurator.addInjectionPoint(ClassType.create(LangChain4jDotNames.STREAMING_CHAT_MODEL)); needsStreamingChatModelBean = true; } @@ -459,7 +480,7 @@ public void handleDeclarativeServices(AiServicesRecorder recorder, configurator.addInjectionPoint(ClassType.create(LangChain4jDotNames.CHAT_MODEL), AnnotationInstance.builder(ModelName.class).add("value", chatModelName).build()); - if (injectStreamingChatModelBean) { + if (needsStreamingChatModel) { configurator.addInjectionPoint(ClassType.create(LangChain4jDotNames.STREAMING_CHAT_MODEL), AnnotationInstance.builder(ModelName.class).add("value", chatModelName).build()); needsStreamingChatModelBean = true; @@ -515,7 +536,7 @@ public void handleDeclarativeServices(AiServicesRecorder recorder, } if (LangChain4jDotNames.BEAN_IF_EXISTS_MODERATION_MODEL_SUPPLIER.toString() - .equals(moderationModelSupplierClassName) && injectModerationModelBean) { + .equals(moderationModelSupplierClassName) && needsModerationModel) { if (NamedConfigUtil.isDefault(moderationModelName)) { configurator.addInjectionPoint(ClassType.create(LangChain4jDotNames.MODERATION_MODEL)); @@ -527,6 +548,15 @@ public void handleDeclarativeServices(AiServicesRecorder recorder, needsModerationModelBean = true; } + if (enableCache) { + if (LangChain4jDotNames.BEAN_AI_CACHE_PROVIDER_SUPPLIER.toString().equals(aiCacheProviderSupplierClassName)) { + configurator.addInjectionPoint(ClassType.create(LangChain4jDotNames.AI_CACHE_PROVIDER)); + } + configurator.addInjectionPoint(ClassType.create(LangChain4jDotNames.AI_CACHE_PROVIDER)); + configurator.addInjectionPoint(ClassType.create(LangChain4jDotNames.EMBEDDING_MODEL)); + needsAiCacheProvider = true; + } + syntheticBeanProducer.produce(configurator.done()); } @@ -551,6 +581,10 @@ public void handleDeclarativeServices(AiServicesRecorder recorder, if (needsModerationModelBean) { unremoveableProducer.produce(UnremovableBeanBuildItem.beanTypes(LangChain4jDotNames.MODERATION_MODEL)); } + if (needsAiCacheProvider) { + unremoveableProducer.produce(UnremovableBeanBuildItem.beanTypes(LangChain4jDotNames.AI_CACHE_PROVIDER)); + unremoveableProducer.produce(UnremovableBeanBuildItem.beanTypes(LangChain4jDotNames.EMBEDDING_MODEL)); + } if (!allToolNames.isEmpty()) { unremoveableProducer.produce(UnremovableBeanBuildItem.beanTypes(allToolNames)); } @@ -870,6 +904,8 @@ private AiServiceMethodCreateInfo gatherMethodMetadata(MethodInfo method, boolea } boolean requiresModeration = method.hasAnnotation(LangChain4jDotNames.MODERATE); + boolean requiresCache = method.declaringClass().hasDeclaredAnnotation(LangChain4jDotNames.CACHE_RESULT) + || method.hasDeclaredAnnotation(LangChain4jDotNames.CACHE_RESULT); List params = method.parameters(); @@ -887,7 +923,7 @@ private AiServiceMethodCreateInfo gatherMethodMetadata(MethodInfo method, boolea List methodToolClassNames = gatherMethodToolClassNames(method); return new AiServiceMethodCreateInfo(method.declaringClass().name().toString(), method.name(), systemMessageInfo, - userMessageInfo, memoryIdParamPosition, requiresModeration, + userMessageInfo, memoryIdParamPosition, requiresModeration, requiresCache, returnType, metricsTimedInfo, metricsCountedInfo, spanInfo, methodToolClassNames); } @@ -1222,6 +1258,16 @@ static Map toNameToArgsPositionMap(List } } + private String getModelName(AnnotationValue value) { + if (value != null) { + String modelNameValueStr = value.asString(); + if ((modelNameValueStr != null) && !modelNameValueStr.isEmpty()) { + return modelNameValueStr; + } + } + return NamedConfigUtil.DEFAULT_NAME; + } + public static final class AiServicesMethodBuildItem extends MultiBuildItem { private final MethodInfo methodInfo; diff --git a/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/BeansProcessor.java b/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/BeansProcessor.java index 9b3beaa00..1828852cb 100644 --- a/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/BeansProcessor.java +++ b/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/BeansProcessor.java @@ -64,7 +64,9 @@ void indexDependencies(BuildProducer producer) { } @BuildStep - public void handleProviders(BeanDiscoveryFinishedBuildItem beanDiscoveryFinished, + public void handleProviders( + AiCacheBuildItem aiCacheBuildItem, + BeanDiscoveryFinishedBuildItem beanDiscoveryFinished, List chatCandidateItems, List embeddingCandidateItems, List moderationCandidateItems, @@ -165,7 +167,8 @@ public void handleProviders(BeanDiscoveryFinishedBuildItem beanDiscoveryFinished } } // If the Easy RAG extension requested to automatically generate an embedding model... - if (requestEmbeddingModels.isEmpty() && autoCreateEmbeddingModelBuildItem.isPresent()) { + if (requestEmbeddingModels.isEmpty() + && (aiCacheBuildItem.isEnable() || autoCreateEmbeddingModelBuildItem.isPresent())) { String provider = selectEmbeddingModelProvider(inProcessEmbeddingBuildItems, embeddingCandidateItems, beanDiscoveryFinished.beanStream().withBeanType(EmbeddingModel.class), Optional.empty(), "EmbeddingModel", "embedding-model"); diff --git a/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/DeclarativeAiServiceBuildItem.java b/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/DeclarativeAiServiceBuildItem.java index f37e67ee5..bd1c6e4d5 100644 --- a/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/DeclarativeAiServiceBuildItem.java +++ b/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/DeclarativeAiServiceBuildItem.java @@ -17,6 +17,7 @@ public final class DeclarativeAiServiceBuildItem extends MultiBuildItem { private final List toolDotNames; private final DotName chatMemoryProviderSupplierClassDotName; + private final DotName aiCacheProviderSupplierClassDotName; private final DotName retrieverClassDotName; private final DotName retrievalAugmentorSupplierClassDotName; private final boolean customRetrievalAugmentorSupplierClassIsABean; @@ -25,10 +26,12 @@ public final class DeclarativeAiServiceBuildItem extends MultiBuildItem { private final DotName cdiScope; private final String chatModelName; private final String moderationModelName; + private final String embeddingModelName; public DeclarativeAiServiceBuildItem(ClassInfo serviceClassInfo, DotName languageModelSupplierClassDotName, List toolDotNames, DotName chatMemoryProviderSupplierClassDotName, + DotName aiCacheProviderSupplierClassDotName, DotName retrieverClassDotName, DotName retrievalAugmentorSupplierClassDotName, boolean customRetrievalAugmentorSupplierClassIsABean, @@ -36,11 +39,13 @@ public DeclarativeAiServiceBuildItem(ClassInfo serviceClassInfo, DotName languag DotName moderationModelSupplierDotName, DotName cdiScope, String chatModelName, - String moderationModelName) { + String moderationModelName, + String embeddingModelName) { this.serviceClassInfo = serviceClassInfo; this.languageModelSupplierClassDotName = languageModelSupplierClassDotName; this.toolDotNames = toolDotNames; this.chatMemoryProviderSupplierClassDotName = chatMemoryProviderSupplierClassDotName; + this.aiCacheProviderSupplierClassDotName = aiCacheProviderSupplierClassDotName; this.retrieverClassDotName = retrieverClassDotName; this.retrievalAugmentorSupplierClassDotName = retrievalAugmentorSupplierClassDotName; this.customRetrievalAugmentorSupplierClassIsABean = customRetrievalAugmentorSupplierClassIsABean; @@ -49,6 +54,7 @@ public DeclarativeAiServiceBuildItem(ClassInfo serviceClassInfo, DotName languag this.cdiScope = cdiScope; this.chatModelName = chatModelName; this.moderationModelName = moderationModelName; + this.embeddingModelName = embeddingModelName; } public ClassInfo getServiceClassInfo() { @@ -67,6 +73,10 @@ public DotName getChatMemoryProviderSupplierClassDotName() { return chatMemoryProviderSupplierClassDotName; } + public DotName getAiCacheProviderSupplierClassDotName() { + return aiCacheProviderSupplierClassDotName; + } + public DotName getRetrieverClassDotName() { return retrieverClassDotName; } @@ -98,4 +108,8 @@ public String getChatModelName() { public String getModerationModelName() { return moderationModelName; } + + public String getEmbeddingModelName() { + return embeddingModelName; + } } diff --git a/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/LangChain4jDotNames.java b/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/LangChain4jDotNames.java index 670b8d14c..ef25165b0 100644 --- a/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/LangChain4jDotNames.java +++ b/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/LangChain4jDotNames.java @@ -19,14 +19,15 @@ import dev.langchain4j.service.SystemMessage; import dev.langchain4j.service.UserMessage; import dev.langchain4j.service.UserName; -import dev.langchain4j.service.V; import dev.langchain4j.web.search.WebSearchEngine; import dev.langchain4j.web.search.WebSearchTool; +import io.quarkiverse.langchain4j.CacheResult; import io.quarkiverse.langchain4j.CreatedAware; import io.quarkiverse.langchain4j.ModelName; import io.quarkiverse.langchain4j.RegisterAiService; import io.quarkiverse.langchain4j.audit.AuditService; import io.quarkiverse.langchain4j.runtime.aiservice.QuarkusAiServiceContextQualifier; +import io.quarkiverse.langchain4j.runtime.cache.AiCacheProvider; import io.smallrye.mutiny.Multi; public class LangChain4jDotNames { @@ -41,6 +42,7 @@ public class LangChain4jDotNames { static final DotName USER_MESSAGE = DotName.createSimple(UserMessage.class); static final DotName USER_NAME = DotName.createSimple(UserName.class); static final DotName MODERATE = DotName.createSimple(Moderate.class); + static final DotName CACHE_RESULT = DotName.createSimple(CacheResult.class); static final DotName MEMORY_ID = DotName.createSimple(MemoryId.class); static final DotName DESCRIPTION = DotName.createSimple(Description.class); static final DotName STRUCTURED_PROMPT = DotName.createSimple(StructuredPrompt.class); @@ -54,10 +56,14 @@ public class LangChain4jDotNames { RegisterAiService.BeanChatLanguageModelSupplier.class); static final DotName CHAT_MEMORY_PROVIDER = DotName.createSimple(ChatMemoryProvider.class); + static final DotName AI_CACHE_PROVIDER = DotName.createSimple(AiCacheProvider.class); static final DotName BEAN_CHAT_MEMORY_PROVIDER_SUPPLIER = DotName.createSimple( RegisterAiService.BeanChatMemoryProviderSupplier.class); + static final DotName BEAN_AI_CACHE_PROVIDER_SUPPLIER = DotName.createSimple( + RegisterAiService.BeanAiCacheProviderSupplier.class); + static final DotName NO_CHAT_MEMORY_PROVIDER_SUPPLIER = DotName.createSimple( RegisterAiService.NoChatMemoryProviderSupplier.class); diff --git a/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/items/AutoCreateEmbeddingModelBuildItem.java b/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/items/AutoCreateEmbeddingModelBuildItem.java index 5dfc0faa5..b1aba4f63 100644 --- a/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/items/AutoCreateEmbeddingModelBuildItem.java +++ b/core/deployment/src/main/java/io/quarkiverse/langchain4j/deployment/items/AutoCreateEmbeddingModelBuildItem.java @@ -4,8 +4,8 @@ /** * Request to generate an embedding model even if there are no - * non-synthetic injection points for it. This is used by the Easy RAG - * extension to have an embedding model created automatically. + * non-synthetic injection points for it. This is used by the Easy RAG and CacheResult + * to have an embedding model created automatically. */ public final class AutoCreateEmbeddingModelBuildItem extends SimpleBuildItem { diff --git a/core/runtime/src/main/java/io/quarkiverse/langchain4j/CacheResult.java b/core/runtime/src/main/java/io/quarkiverse/langchain4j/CacheResult.java new file mode 100644 index 000000000..cfc8fd3ba --- /dev/null +++ b/core/runtime/src/main/java/io/quarkiverse/langchain4j/CacheResult.java @@ -0,0 +1,11 @@ +package io.quarkiverse.langchain4j; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Retention(RetentionPolicy.RUNTIME) +@Target(value = { ElementType.TYPE, ElementType.METHOD }) +public @interface CacheResult { +} diff --git a/core/runtime/src/main/java/io/quarkiverse/langchain4j/RegisterAiService.java b/core/runtime/src/main/java/io/quarkiverse/langchain4j/RegisterAiService.java index 6e9df6744..9306bed03 100644 --- a/core/runtime/src/main/java/io/quarkiverse/langchain4j/RegisterAiService.java +++ b/core/runtime/src/main/java/io/quarkiverse/langchain4j/RegisterAiService.java @@ -20,45 +20,51 @@ import dev.langchain4j.store.memory.chat.ChatMemoryStore; import dev.langchain4j.store.memory.chat.InMemoryChatMemoryStore; import io.quarkiverse.langchain4j.audit.AuditService; +import io.quarkiverse.langchain4j.runtime.cache.AiCache; +import io.quarkiverse.langchain4j.runtime.cache.AiCacheProvider; +import io.quarkiverse.langchain4j.runtime.cache.AiCacheStore; +import io.quarkiverse.langchain4j.runtime.cache.InMemoryAiCacheStore; +import io.quarkiverse.langchain4j.runtime.cache.MessageWindowAiCache; /** - * Used to create LangChain4j's {@link AiServices} in a declarative manner that the application can then use simply by - * using the class as a CDI bean. - * Under the hood LangChain4j's {@link AiServices#builder(Class)} is called - * while also providing the builder with the proper {@link ChatLanguageModel} bean (mandatory), {@code tools} bean (optional), - * {@link ChatMemoryProvider} and {@link Retriever} beans (which by default are configured if such beans exist). + * Used to create LangChain4j's {@link AiServices} in a declarative manner that the application can then use simply by using the + * class as a CDI bean. Under the hood LangChain4j's {@link AiServices#builder(Class)} is called while also providing the + * builder + * with the proper {@link ChatLanguageModel} bean (mandatory), {@code tools} bean (optional), {@link ChatMemoryProvider} and + * {@link Retriever} beans (which by default are configured if such beans exist). *

* NOTE: The resulting CDI bean is {@link jakarta.enterprise.context.RequestScoped} by default. If you need to change the scope, - * simply annotate the class with a CDI scope. - * CAUTION: When using anything other than the request scope, you need to be very careful with the chat memory implementation. + * simply annotate the class with a CDI scope. CAUTION: When using anything other than the request scope, you need to be very + * careful with the chat memory implementation. *

- * NOTE: When the application also contains the {@code quarkus-micrometer} extension, metrics are automatically generated - * for the method invocations. + * NOTE: When the application also contains the {@code quarkus-micrometer} extension, metrics are automatically generated for + * the + * method invocations. */ @Retention(RUNTIME) @Target(ElementType.TYPE) public @interface RegisterAiService { /** - * Configures the way to obtain the {@link ChatLanguageModel} to use. - * If not configured, the default CDI bean implementing the model is looked up. - * Such a bean provided automatically by extensions such as {@code quarkus-langchain4j-openai}, - * {@code quarkus-langchain4j-azure-openai} or - * {@code quarkus-langchain4j-hugging-face} + * Configures the way to obtain the {@link ChatLanguageModel} to use. If not configured, the default CDI bean implementing + * the + * model is looked up. Such a bean provided automatically by extensions such as {@code quarkus-langchain4j-openai}, + * {@code quarkus-langchain4j-azure-openai} or {@code quarkus-langchain4j-hugging-face} */ Class> chatLanguageModelSupplier() default BeanChatLanguageModelSupplier.class; /** - * When {@code chatLanguageModelSupplier} is set to {@code BeanChatLanguageModelSupplier.class} (which is the default) - * this allows the selection of the {@link ChatLanguageModel} CDI bean to use. + * When {@code chatLanguageModelSupplier} is set to {@code BeanChatLanguageModelSupplier.class} (which is the default) this + * allows + * the selection of the {@link ChatLanguageModel} CDI bean to use. *

- * If not set, the default model (i.e. the one configured without setting the model name) is used. - * An example of the default model configuration is the following: - * {@code quarkus.langchain4j.openai.chat-model.model-name=gpt-4-turbo-preview} + * If not set, the default model (i.e. the one configured without setting the model name) is used. An example of the default + * model + * configuration is the following: {@code quarkus.langchain4j.openai.chat-model.model-name=gpt-4-turbo-preview} * - * If set, it uses the model configured by name. For example if this is set to {@code somename} - * an example configuration value for that named model could be: - * {@code quarkus.langchain4j.somename.openai.chat-model.model-name=gpt-4-turbo-preview} + * If set, it uses the model configured by name. For example if this is set to {@code somename} an example configuration + * value for + * that named model could be: {@code quarkus.langchain4j.somename.openai.chat-model.model-name=gpt-4-turbo-preview} */ String modelName() default ""; @@ -70,27 +76,49 @@ /** * Configures the way to obtain the {@link ChatMemoryProvider}. *

- * Be default, Quarkus configures a {@link ChatMemoryProvider} bean that uses a {@link InMemoryChatMemoryStore} bean - * as the backing store. The default type for the actual {@link ChatMemory} is {@link MessageWindowChatMemory} - * and it is configured with the value of the {@code quarkus.langchain4j.chat-memory.memory-window.max-messages} - * configuration property (which default to 10) as a way of limiting the number of messages in each chat. + * Be default, Quarkus configures a {@link ChatMemoryProvider} bean that uses a {@link InMemoryChatMemoryStore} bean as the + * backing store. The default type for the actual {@link ChatMemory} is {@link MessageWindowChatMemory} and it is configured + * with + * the value of the {@code quarkus.langchain4j.chat-memory.memory-window.max-messages} configuration property (which default + * to + * 10) as a way of limiting the number of messages in each chat. *

* If the application provides its own {@link ChatMemoryProvider} bean, that takes precedence over what Quarkus provides as - * the default. + * the + * default. *

* If the application provides an implementation of {@link ChatMemoryStore}, then that is used instead of the default * {@link InMemoryChatMemoryStore}. *

* In the most advances case, an arbitrary {@link ChatMemoryProvider} can be used by having a custom - * {@code Supplier} configured in this property. - * {@link Supplier} needs to be provided. + * {@code Supplier} configured in this property. {@link Supplier} needs to be + * provided. *

*/ Class> chatMemoryProviderSupplier() default BeanChatMemoryProviderSupplier.class; /** - * Configures the way to obtain the {@link Retriever} to use (when using RAG). - * By default, no retriever is used. + * Configures the way to obtain the {@link AiCacheProvider}. + *

+ * Be default, Quarkus configures a {@link AiCacheProvider} bean that uses a {@link InMemoryAiCacheStore} bean as the + * backing store. The default type for the actual {@link AiCache} is {@link MessageWindowAiCache} and it is configured with + * the value of the {@code quarkus.langchain4j.cache.max-size} configuration property (which default to + * 1) as a way of limiting the number of messages in each cache. + *

+ * If the application provides its own {@link AiCacheProvider} bean, that takes precedence over what Quarkus provides as the + * default. + *

+ * If the application provides an implementation of {@link AiCacheStore}, then that is used instead of the default + * {@link InMemoryAiCacheStore}. + *

+ * In the most advances case, an arbitrary {@link AiCacheProvider} can be used by having a custom + * {@code Supplier} configured in this property. {@link Supplier} needs to be provided. + *

+ */ + Class> cacheProviderSupplier() default BeanAiCacheProviderSupplier.class; + + /** + * Configures the way to obtain the {@link Retriever} to use (when using RAG). By default, no retriever is used. * * @deprecated Use retrievalAugmentor instead */ @@ -98,38 +126,39 @@ Class> retriever() default NoRetriever.class; /** - * Configures the way to obtain the {@link RetrievalAugmentor} to use - * (when using RAG). The Supplier may or may not be a CDI bean (but most - * typically it will, so consider adding a bean-defining annotation to - * it). If it is not a CDI bean, Quarkus will create an instance - * by calling its no-arg constructor. + * Configures the way to obtain the {@link RetrievalAugmentor} to use (when using RAG). The Supplier may or may not be a CDI + * bean + * (but most typically it will, so consider adding a bean-defining annotation to it). If it is not a CDI bean, Quarkus will + * create + * an instance by calling its no-arg constructor. * - * If unspecified, Quarkus will attempt to locate a CDI bean that - * implements {@link RetrievalAugmentor} and use it if one exists. + * If unspecified, Quarkus will attempt to locate a CDI bean that implements {@link RetrievalAugmentor} and use it if one + * exists. */ Class> retrievalAugmentor() default BeanIfExistsRetrievalAugmentorSupplier.class; /** - * Configures the way to obtain the {@link AuditService} to use. - * By default, Quarkus will look for a CDI bean that implements {@link AuditService}, but will fall back to not using - * any memory if no such bean exists. - * If an arbitrary {@link AuditService} instance is needed, a custom implementation of - * {@link Supplier} needs to be provided. + * Configures the way to obtain the {@link AuditService} to use. By default, Quarkus will look for a CDI bean that + * implements + * {@link AuditService}, but will fall back to not using any memory if no such bean exists. If an arbitrary + * {@link AuditService} + * instance is needed, a custom implementation of {@link Supplier} needs to be provided. */ Class> auditServiceSupplier() default BeanIfExistsAuditServiceSupplier.class; /** - * Configures the way to obtain the {@link ModerationModel} to use. - * By default, Quarkus will look for a CDI bean that implements {@link ModerationModel} if at least one method is annotated - * with @Moderate. - * If an arbitrary {@link ModerationModel} instance is needed, a custom implementation of {@link Supplier} - * needs to be provided. + * Configures the way to obtain the {@link ModerationModel} to use. By default, Quarkus will look for a CDI bean that + * implements + * {@link ModerationModel} if at least one method is annotated with @Moderate. If an arbitrary {@link ModerationModel} + * instance is + * needed, a custom implementation of {@link Supplier} needs to be provided. */ Class> moderationModelSupplier() default BeanIfExistsModerationModelSupplier.class; /** - * Marker that is used to tell Quarkus to use the {@link ChatLanguageModel} that has been configured as a CDI bean by - * any of the extensions providing such capability (such as {@code quarkus-langchain4j-openai} and + * Marker that is used to tell Quarkus to use the {@link ChatLanguageModel} that has been configured as a CDI bean by any of + * the + * extensions providing such capability (such as {@code quarkus-langchain4j-openai} and * {@code quarkus-langchain4j-hugging-face}). */ final class BeanChatLanguageModelSupplier implements Supplier { @@ -141,11 +170,12 @@ public ChatLanguageModel get() { } /** - * Marker that is used to tell Quarkus to use the retriever that the user has configured as a CDI bean. - * Be default, Quarkus configures an {@link ChatMemoryProvider} by using an {@link InMemoryChatMemoryStore} - * as the backing store while using {@link MessageWindowChatMemory} with the value of - * configuration property {@code quarkus.langchain4j.chat-memory.memory-window.max-messages} (which default to 10) - * as a way of limiting the number of messages in each chat. + * Marker that is used to tell Quarkus to use the retriever that the user has configured as a CDI bean. Be default, Quarkus + * configures an {@link ChatMemoryProvider} by using an {@link InMemoryChatMemoryStore} as the backing store while using + * {@link MessageWindowChatMemory} with the value of configuration property + * {@code quarkus.langchain4j.chat-memory.memory-window.max-messages} (which default to 10) as a way of limiting the number + * of + * messages in each chat. */ final class BeanChatMemoryProviderSupplier implements Supplier { @@ -155,6 +185,21 @@ public ChatMemoryProvider get() { } } + /** + * Marker that is used to tell Quarkus to use the retriever that the user has configured as a CDI bean. Be default, Quarkus + * configures an {@link AiCacheProvider} by using an {@link InMemoryAiCacheStore} as the backing store while using + * {@link MessageWindowAiCacheMemory} with the value of configuration property + * {@code quarkus.langchain4j.cache.max-size} (which default to 1) as a way of limiting the number of + * messages in each cache. + */ + final class BeanAiCacheProviderSupplier implements Supplier { + + @Override + public AiCacheProvider get() { + throw new UnsupportedOperationException("should never be called"); + } + } + /** * Marker that is used when the user does not want any memory configured for the AiService */ @@ -178,8 +223,9 @@ public List findRelevant(String text) { } /** - * Marker that is used to tell Quarkus to use the {@link RetrievalAugmentor} that the user has configured as a CDI bean. - * If no such bean exists, then no retrieval augmentor will be used. + * Marker that is used to tell Quarkus to use the {@link RetrievalAugmentor} that the user has configured as a CDI bean. If + * no + * such bean exists, then no retrieval augmentor will be used. */ final class BeanIfExistsRetrievalAugmentorSupplier implements Supplier { @@ -190,8 +236,9 @@ public RetrievalAugmentor get() { } /** - * Marker that is used to tell Quarkus to not use any retrieval augmentor even if a CDI bean implementing - * the `RetrievalAugmentor` interface exists. + * Marker that is used to tell Quarkus to not use any retrieval augmentor even if a CDI bean implementing the + * `RetrievalAugmentor` + * interface exists. */ final class NoRetrievalAugmentorSupplier implements Supplier { @@ -202,8 +249,9 @@ public RetrievalAugmentor get() { } /** - * Marker that is used to tell Quarkus to use the {@link AuditService} that the user has configured as a CDI bean. - * If no such bean exists, then no audit service will be used. + * Marker that is used to tell Quarkus to use the {@link AuditService} that the user has configured as a CDI bean. If no + * such bean + * exists, then no audit service will be used. */ final class BeanIfExistsAuditServiceSupplier implements Supplier { @@ -214,8 +262,9 @@ public AuditService get() { } /** - * Marker that is used to tell Quarkus to use the {@link ModerationModel} that the user has configured as a CDI bean. - * If no such bean exists, then no audit service will be used. + * Marker that is used to tell Quarkus to use the {@link ModerationModel} that the user has configured as a CDI bean. If no + * such + * bean exists, then no moderation model will be used. */ final class BeanIfExistsModerationModelSupplier implements Supplier { diff --git a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/AiCacheRecorder.java b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/AiCacheRecorder.java new file mode 100644 index 000000000..8df459b80 --- /dev/null +++ b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/AiCacheRecorder.java @@ -0,0 +1,42 @@ +package io.quarkiverse.langchain4j.runtime; + +import java.time.Duration; +import java.util.function.Function; + +import io.quarkiverse.langchain4j.runtime.cache.AiCache; +import io.quarkiverse.langchain4j.runtime.cache.AiCacheConfig; +import io.quarkiverse.langchain4j.runtime.cache.AiCacheProvider; +import io.quarkiverse.langchain4j.runtime.cache.AiCacheStore; +import io.quarkiverse.langchain4j.runtime.cache.MessageWindowAiCache; +import io.quarkus.arc.SyntheticCreationalContext; +import io.quarkus.runtime.annotations.Recorder; + +@Recorder +public class AiCacheRecorder { + + public Function, AiCacheProvider> messageWindow(AiCacheConfig config) { + return new Function<>() { + @Override + public AiCacheProvider apply(SyntheticCreationalContext context) { + + AiCacheStore aiCacheStore = context.getInjectedReference(AiCacheStore.class); + double threshold = config.threshold(); + int maxSize = config.maxSize(); + Duration ttl = config.ttl().orElse(null); + + return new AiCacheProvider() { + @Override + public AiCache get(Object memoryId) { + return MessageWindowAiCache.Builder + .create(memoryId) + .ttl(ttl) + .maxSize(maxSize) + .threshold(threshold) + .store(aiCacheStore) + .build(); + } + }; + } + }; + } +} diff --git a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/AiServicesRecorder.java b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/AiServicesRecorder.java index 000790b6b..3a92f9cd0 100644 --- a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/AiServicesRecorder.java +++ b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/AiServicesRecorder.java @@ -7,6 +7,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; import java.util.function.Supplier; @@ -17,6 +18,7 @@ import dev.langchain4j.memory.chat.ChatMemoryProvider; import dev.langchain4j.model.chat.ChatLanguageModel; import dev.langchain4j.model.chat.StreamingChatLanguageModel; +import dev.langchain4j.model.embedding.EmbeddingModel; import dev.langchain4j.model.moderation.ModerationModel; import dev.langchain4j.rag.RetrievalAugmentor; import dev.langchain4j.retriever.Retriever; @@ -27,6 +29,7 @@ import io.quarkiverse.langchain4j.runtime.aiservice.AiServiceMethodCreateInfo; import io.quarkiverse.langchain4j.runtime.aiservice.DeclarativeAiServiceCreateInfo; import io.quarkiverse.langchain4j.runtime.aiservice.QuarkusAiServiceContext; +import io.quarkiverse.langchain4j.runtime.cache.AiCacheProvider; import io.quarkus.arc.Arc; import io.quarkus.arc.SyntheticCreationalContext; import io.quarkus.runtime.annotations.Recorder; @@ -234,6 +237,26 @@ public T apply(SyntheticCreationalContext creationalContext) { } } + if (info.enableCache()) { + + if (info.aiCacheProviderSupplierClassName() != null) { + if (RegisterAiService.BeanAiCacheProviderSupplier.class.getName() + .equals(info.aiCacheProviderSupplierClassName())) { + aiServiceContext.aiCacheProvider = creationalContext.getInjectedReference( + AiCacheProvider.class); + } else { + Supplier supplier = (Supplier) Thread + .currentThread().getContextClassLoader() + .loadClass(info.aiCacheProviderSupplierClassName()) + .getConstructor().newInstance(); + aiServiceContext.aiCacheProvider = supplier.get(); + } + } + + aiServiceContext.embeddingModel = creationalContext.getInjectedReference(EmbeddingModel.class); + aiServiceContext.aiCaches = new ConcurrentHashMap<>(); + } + return (T) aiServiceContext; } catch (ClassNotFoundException e) { throw new IllegalStateException(e); diff --git a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/AiServiceMethodCreateInfo.java b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/AiServiceMethodCreateInfo.java index 22124dabe..21e68d750 100644 --- a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/AiServiceMethodCreateInfo.java +++ b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/AiServiceMethodCreateInfo.java @@ -18,6 +18,7 @@ public final class AiServiceMethodCreateInfo { private final UserMessageInfo userMessageInfo; private final Optional memoryIdParamPosition; private final boolean requiresModeration; + private final boolean requiresCache; private final Class returnType; private final Optional metricsTimedInfo; private final Optional metricsCountedInfo; @@ -34,8 +35,7 @@ public AiServiceMethodCreateInfo(String interfaceName, String methodName, Optional systemMessageInfo, UserMessageInfo userMessageInfo, Optional memoryIdParamPosition, - boolean requiresModeration, - Class returnType, + boolean requiresModeration, boolean requiresCache, Class returnType, Optional metricsTimedInfo, Optional metricsCountedInfo, Optional spanInfo, @@ -46,6 +46,7 @@ public AiServiceMethodCreateInfo(String interfaceName, String methodName, this.userMessageInfo = userMessageInfo; this.memoryIdParamPosition = memoryIdParamPosition; this.requiresModeration = requiresModeration; + this.requiresCache = requiresCache; this.returnType = returnType; this.metricsTimedInfo = metricsTimedInfo; this.metricsCountedInfo = metricsCountedInfo; @@ -77,6 +78,10 @@ public boolean isRequiresModeration() { return requiresModeration; } + public boolean isRequiresCache() { + return requiresCache; + } + public Class getReturnType() { return returnType; } diff --git a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/AiServiceMethodImplementationSupport.java b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/AiServiceMethodImplementationSupport.java index 66f11fff1..b15683136 100644 --- a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/AiServiceMethodImplementationSupport.java +++ b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/AiServiceMethodImplementationSupport.java @@ -16,6 +16,7 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.Callable; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.function.Consumer; @@ -26,6 +27,7 @@ import dev.langchain4j.agent.tool.ToolExecutionRequest; import dev.langchain4j.agent.tool.ToolExecutor; import dev.langchain4j.agent.tool.ToolSpecification; +import dev.langchain4j.data.embedding.Embedding; import dev.langchain4j.data.message.AiMessage; import dev.langchain4j.data.message.ChatMessage; import dev.langchain4j.data.message.SystemMessage; @@ -46,6 +48,7 @@ import dev.langchain4j.spi.ServiceHelper; import io.quarkiverse.langchain4j.audit.Audit; import io.quarkiverse.langchain4j.audit.AuditService; +import io.quarkiverse.langchain4j.runtime.cache.AiCache; import io.quarkiverse.langchain4j.spi.DefaultMemoryIdProvider; import io.smallrye.mutiny.Multi; import io.smallrye.mutiny.infrastructure.Infrastructure; @@ -57,10 +60,9 @@ public class AiServiceMethodImplementationSupport { private static final Logger log = Logger.getLogger(AiServiceMethodImplementationSupport.class); - private static final int MAX_SEQUENTIAL_TOOL_EXECUTIONS = 10; - private static final List DEFAULT_MEMORY_ID_PROVIDERS; + private static final Map CACHE = new ConcurrentHashMap<>(); static { var defaultMemoryIdProviders = ServiceHelper.loadFactories( @@ -121,7 +123,16 @@ private static Object doImplement(AiServiceMethodCreateInfo methodCreateInfo, Ob } Object memoryId = memoryId(methodCreateInfo, methodArgs, context.chatMemoryProvider != null); + AiCache cache = null; + + // TODO: REMOVE THIS COMMENT BEFORE MERGING THE PR. + // - Understand how to implement the concept of cache for the stream responses. + // - What do we have to do when we have the tools? + if (methodCreateInfo.isRequiresCache()) { + Object cacheId = cacheId(methodCreateInfo, methodArgs); + cache = context.aiCacheProvider.get(cacheId); + } if (context.retrievalAugmentor != null) { // TODO extract method/class List chatMemory = context.hasChatMemory() ? context.chatMemory(memoryId).messages() @@ -175,9 +186,6 @@ public void accept(Response message) { } Future moderationFuture = triggerModerationIfNeeded(context, methodCreateInfo, messages); - - log.debug("Attempting to obtain AI response"); - List toolSpecifications = context.toolSpecifications; Map toolExecutors = context.toolExecutors; // override with method specific info @@ -186,16 +194,31 @@ public void accept(Response message) { toolExecutors = methodCreateInfo.getToolExecutors(); } - Response response = toolSpecifications == null - ? context.chatModel.generate(messages) - : context.chatModel.generate(messages, toolSpecifications); - log.debug("AI response obtained"); + Response response; + + if (cache != null) { + log.debug("Attempting to obtain AI response from cache"); + + Embedding query = context.embeddingModel.embed(userMessage.text()).content(); + var cacheResponse = cache.search(query); + + if (cacheResponse.isPresent()) { + log.debug("Return cached response"); + response = Response.from(cacheResponse.get()); + } else { + response = executeLLMCall(context, messages, moderationFuture, toolSpecifications); + cache.add(query, response.content()); + } + + } else { + response = executeLLMCall(context, messages, moderationFuture, toolSpecifications); + } + if (audit != null) { audit.addLLMToApplicationMessage(response); } - TokenUsage tokenUsageAccumulator = response.tokenUsage(); - verifyModerationIfNeeded(moderationFuture); + TokenUsage tokenUsageAccumulator = response.tokenUsage(); int executionsLeft = MAX_SEQUENTIAL_TOOL_EXECUTIONS; while (true) { @@ -249,6 +272,17 @@ public void accept(Response message) { return parse(response, returnType); } + private static Response executeLLMCall(QuarkusAiServiceContext context, List messages, + Future moderationFuture, List toolSpecifications) { + log.debug("Attempting to obtain AI response"); + var response = context.toolSpecifications == null + ? context.chatModel.generate(messages) + : context.chatModel.generate(messages, toolSpecifications); + log.debug("AI response obtained"); + verifyModerationIfNeeded(moderationFuture); + return response; + } + private static Future triggerModerationIfNeeded(AiServiceContext context, AiServiceMethodCreateInfo createInfo, List messages) { @@ -366,7 +400,20 @@ private static Object memoryId(AiServiceMethodCreateInfo createInfo, Object[] me return "default"; } - //TODO: share these methods with LangChain4j + private static Object cacheId(AiServiceMethodCreateInfo createInfo, Object[] methodArgs) { + for (DefaultMemoryIdProvider provider : DEFAULT_MEMORY_ID_PROVIDERS) { + Object memoryId = provider.getMemoryId(); + if (memoryId != null) { + String perServiceSuffix = "#" + createInfo.getInterfaceName() + "." + createInfo.getMethodName(); + return memoryId + perServiceSuffix; + } + } + + // fallback to the default since there is nothing else we can really use here + return "default"; + } + + // TODO: share these methods with LangChain4j private static String toString(Object arg) { if (arg.getClass().isArray()) { diff --git a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/DeclarativeAiServiceCreateInfo.java b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/DeclarativeAiServiceCreateInfo.java index 0b8615079..65ea1a3c7 100644 --- a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/DeclarativeAiServiceCreateInfo.java +++ b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/DeclarativeAiServiceCreateInfo.java @@ -6,12 +6,15 @@ public record DeclarativeAiServiceCreateInfo(String serviceClassName, String languageModelSupplierClassName, List toolsClassNames, String chatMemoryProviderSupplierClassName, + String aiCacheProviderSupplierClassName, String retrieverClassName, String retrievalAugmentorSupplierClassName, String auditServiceClassSupplierName, String moderationModelSupplierClassName, String chatModelName, String moderationModelName, + String embeddingModelName, boolean needsStreamingChatModel, - boolean needsModerationModel) { + boolean needsModerationModel, + boolean enableCache) { } diff --git a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/InMemoryAiCacheStoreProducer.java b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/InMemoryAiCacheStoreProducer.java new file mode 100644 index 000000000..2f768373d --- /dev/null +++ b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/InMemoryAiCacheStoreProducer.java @@ -0,0 +1,21 @@ +package io.quarkiverse.langchain4j.runtime.aiservice; + +import jakarta.enterprise.inject.Produces; +import jakarta.inject.Singleton; + +import io.quarkiverse.langchain4j.RegisterAiService; +import io.quarkiverse.langchain4j.runtime.cache.InMemoryAiCacheStore; +import io.quarkus.arc.DefaultBean; + +/** + * Creates the default {@link InMemoryAiCacheStoreProducer} store to be used by classes annotated with {@link RegisterAiService} + */ +public class InMemoryAiCacheStoreProducer { + + @Produces + @Singleton + @DefaultBean + public InMemoryAiCacheStore chatMemoryStore() { + return new InMemoryAiCacheStore(); + } +} diff --git a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/QuarkusAiServiceContext.java b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/QuarkusAiServiceContext.java index d9edf9f91..cdf7155e5 100644 --- a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/QuarkusAiServiceContext.java +++ b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/aiservice/QuarkusAiServiceContext.java @@ -1,15 +1,22 @@ package io.quarkiverse.langchain4j.runtime.aiservice; +import java.util.Map; import java.util.function.BiConsumer; import dev.langchain4j.memory.ChatMemory; +import dev.langchain4j.model.embedding.EmbeddingModel; import dev.langchain4j.service.AiServiceContext; import io.quarkiverse.langchain4j.RegisterAiService; import io.quarkiverse.langchain4j.audit.AuditService; +import io.quarkiverse.langchain4j.runtime.cache.AiCache; +import io.quarkiverse.langchain4j.runtime.cache.AiCacheProvider; public class QuarkusAiServiceContext extends AiServiceContext { public AuditService auditService; + public Map aiCaches; + public AiCacheProvider aiCacheProvider; + public EmbeddingModel embeddingModel; // needed by Arc public QuarkusAiServiceContext() { @@ -20,12 +27,21 @@ public QuarkusAiServiceContext(Class aiServiceClass) { super(aiServiceClass); } + public boolean hasCache() { + return aiCaches != null; + } + + public AiCache cache(Object memoryId) { + return aiCaches.computeIfAbsent(memoryId, ignored -> aiCacheProvider.get(memoryId)); + } + /** * This is called by the {@code close} method of AiServices registered with {@link RegisterAiService} * when the bean's scope is closed */ public void close() { clearChatMemory(); + clearAiCache(); } private void clearChatMemory() { @@ -40,6 +56,18 @@ public void accept(Object memoryId, ChatMemory chatMemory) { } } + private void clearAiCache() { + if (aiCaches != null) { + aiCaches.forEach(new BiConsumer<>() { + @Override + public void accept(Object memoryId, AiCache aiCache) { + aiCache.clear(); + } + }); + aiCaches = null; + } + } + /** * This is called by the {@code remove(Object... ids)} method of AiServices when a user manually requests removal of chat * memories diff --git a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/AiCache.java b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/AiCache.java new file mode 100644 index 000000000..ef50b053e --- /dev/null +++ b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/AiCache.java @@ -0,0 +1,40 @@ +package io.quarkiverse.langchain4j.runtime.cache; + +import java.util.Optional; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.message.AiMessage; + +/** + * Represents the cache of a AI. It can be used to reduces response time for similar queries. + */ +public interface AiCache { + + /** + * The ID of the {@link AiCache}. + * + * @return The ID of the {@link AiCache}. + */ + Object id(); + + /** + * Cache a new message. + * + * @param query Embedded value to add to the cache. + * @param response Response returned by the AI to add to the cache. + */ + void add(Embedding query, AiMessage response); + + /** + * Check to see if there is a response in the cache that is semantically close to a cached query. + * + * @param query + * @return + */ + Optional search(Embedding query); + + /** + * Clears the cache. + */ + void clear(); +} diff --git a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/AiCacheConfig.java b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/AiCacheConfig.java new file mode 100644 index 000000000..57a50590a --- /dev/null +++ b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/AiCacheConfig.java @@ -0,0 +1,32 @@ +package io.quarkiverse.langchain4j.runtime.cache; + +import static io.quarkus.runtime.annotations.ConfigPhase.RUN_TIME; + +import java.time.Duration; +import java.util.Optional; + +import io.quarkus.runtime.annotations.ConfigRoot; +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithDefault; + +@ConfigRoot(phase = RUN_TIME) +@ConfigMapping(prefix = "quarkus.langchain4j.cache") +public interface AiCacheConfig { + + /** + * Threshold used during semantic search to validate whether a cache result should be returned. + */ + @WithDefault("1") + double threshold(); + + /** + * Maximum number of messages to cache. + */ + @WithDefault("1") + int maxSize(); + + /** + * Time to live for messages stored in the cache. + */ + Optional ttl(); +} diff --git a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/AiCacheProvider.java b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/AiCacheProvider.java new file mode 100644 index 000000000..d2e8279a1 --- /dev/null +++ b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/AiCacheProvider.java @@ -0,0 +1,19 @@ +package io.quarkiverse.langchain4j.runtime.cache; + +import io.quarkiverse.langchain4j.RegisterAiService; + +/** + * Provides instances of {@link AiCache}. + * Intended to be used with {@link RegisterAiService} + */ +@FunctionalInterface +public interface AiCacheProvider { + + /** + * Provides an instance of {@link AiCache}. + * + * @param id The ID of the cache. + * @return A {@link AiCache} instance. + */ + AiCache get(Object id); +} diff --git a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/AiCacheStore.java b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/AiCacheStore.java new file mode 100644 index 000000000..abaee7e36 --- /dev/null +++ b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/AiCacheStore.java @@ -0,0 +1,43 @@ +package io.quarkiverse.langchain4j.runtime.cache; + +import java.time.Instant; +import java.util.List; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.message.AiMessage; + +/** + * Represents a store for the {@link AiCache} state. + * Allows for flexibility in terms of where and how cache is stored. + */ +public interface AiCacheStore { + + public record CacheRecord(Embedding embedded, AiMessage response, Instant creation) { + public static CacheRecord of(Embedding embedded, AiMessage response) { + return new CacheRecord(embedded, response, Instant.now()); + } + }; + + /** + * Get all items stored in the cache. + * + * @param id Unique identifier for the cache + * @return {@link List} of {@link CacheRecord} + */ + public List getAll(Object id); + + /** + * Delete all items stored in the cache. + * + * @param id Unique identifier for the cache + */ + public void deleteCache(Object id); + + /** + * Update all items stored in the cache. + * + * @param id Unique identifier for the cache + * @param items Items to update + */ + public void updateCache(Object id, List items); +} diff --git a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/InMemoryAiCacheStore.java b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/InMemoryAiCacheStore.java new file mode 100644 index 000000000..9f3e21771 --- /dev/null +++ b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/InMemoryAiCacheStore.java @@ -0,0 +1,34 @@ +package io.quarkiverse.langchain4j.runtime.cache; + +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Implementation of {@link AiCacheStore}. + *

+ * This storage mechanism is transient and does not persist data across application restarts. + */ +public class InMemoryAiCacheStore implements AiCacheStore { + + private final Map> store = new ConcurrentHashMap<>(); + + @Override + public List getAll(Object memoryId) { + var elements = store.get(memoryId); + if (elements == null) + return new LinkedList<>(); + return elements; + } + + @Override + public void deleteCache(Object memoryId) { + store.remove(memoryId); + } + + @Override + public void updateCache(Object memoryId, List elements) { + store.put(memoryId, elements); + } +} diff --git a/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/MessageWindowAiCache.java b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/MessageWindowAiCache.java new file mode 100644 index 000000000..29e58be1a --- /dev/null +++ b/core/runtime/src/main/java/io/quarkiverse/langchain4j/runtime/cache/MessageWindowAiCache.java @@ -0,0 +1,155 @@ +package io.quarkiverse.langchain4j.runtime.cache; + +import java.time.Duration; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.locks.ReentrantLock; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.store.embedding.CosineSimilarity; +import io.quarkiverse.langchain4j.runtime.cache.AiCacheStore.CacheRecord; + +/** + * This {@link AiCache} implementation operates as a sliding window of messages. + */ +public class MessageWindowAiCache implements AiCache { + + private final Object id; + private final Integer maxMessages; + private final AiCacheStore store; + private final Double threshold; + private final Duration ttl; + private final ReentrantLock lock; + + public MessageWindowAiCache(Builder builder) { + this.id = builder.id; + this.maxMessages = builder.maxSize; + this.store = builder.store; + this.ttl = builder.ttl; + this.threshold = builder.threshold; + this.lock = new ReentrantLock(); + } + + @Override + public Object id() { + return id; + } + + @Override + public void add(Embedding query, AiMessage response) { + + if (Objects.isNull(query) || Objects.isNull(response)) { + return; + } + + try { + + lock.lock(); + + List elements = store.getAll(id); + if (elements.size() == maxMessages) { + elements.remove(0); + } + + List update = new LinkedList<>(); + for (int i = 0; i < elements.size(); i++) { + + var expiredTime = Date.from(elements.get(i).creation().plus(ttl)); + var currentTime = new Date(); + + if (currentTime.after(expiredTime)) + continue; + + update.add(elements.get(i)); + } + + update.add(CacheRecord.of(query, response)); + store.updateCache(id, update); + + } finally { + lock.unlock(); + } + } + + @Override + public Optional search(Embedding query) { + + if (Objects.isNull(query)) + return Optional.empty(); + + var elements = store.getAll(id); + double maxScore = 0; + AiMessage result = null; + + for (var cacheRecord : elements) { + + if (ttl != null) { + var expiredTime = Date.from(cacheRecord.creation().plus(ttl)); + var currentTime = new Date(); + + if (currentTime.after(expiredTime)) + continue; + } + + var relevanceScore = CosineSimilarity.between(query, cacheRecord.embedded()); + var score = (float) CosineSimilarity.fromRelevanceScore(relevanceScore); + + if (score >= threshold.doubleValue() && score >= maxScore) { + maxScore = score; + result = cacheRecord.response(); + } + } + + return Optional.ofNullable(result); + } + + @Override + public void clear() { + store.deleteCache(id); + } + + public static class Builder { + + Object id; + Integer maxSize; + AiCacheStore store; + Double threshold; + Duration ttl; + + private Builder(Object id) { + this.id = id; + } + + public static Builder create(Object id) { + return new Builder(id); + } + + public Builder maxSize(Integer maxSize) { + this.maxSize = maxSize; + return this; + } + + public Builder store(AiCacheStore store) { + this.store = store; + return this; + } + + public Builder threshold(Double threshold) { + this.threshold = threshold; + return this; + } + + public Builder ttl(Duration ttl) { + this.ttl = ttl; + return this; + } + + public AiCache build() { + return new MessageWindowAiCache(this); + } + } +} diff --git a/integration-tests/cache/pom.xml b/integration-tests/cache/pom.xml new file mode 100644 index 000000000..d2bfa18df --- /dev/null +++ b/integration-tests/cache/pom.xml @@ -0,0 +1,129 @@ + + + 4.0.0 + + io.quarkiverse.langchain4j + quarkus-langchain4j-integration-tests-parent + 999-SNAPSHOT + + quarkus-langchain4j-integration-tests-cache + Quarkus LangChain4j - Integration Tests - Cache + + true + + + + io.quarkus + quarkus-resteasy-reactive-jackson + + + io.quarkiverse.langchain4j + quarkus-langchain4j-bam + ${project.version} + + + dev.langchain4j + langchain4j-embeddings-all-minilm-l6-v2 + ${langchain4j-embeddings.version} + + + dev.langchain4j + langchain4j-core + + + + + io.quarkus + quarkus-junit5 + test + + + io.rest-assured + rest-assured + test + + + org.assertj + assertj-core + ${assertj.version} + test + + + io.quarkus + quarkus-devtools-testing + test + + + + + io.quarkiverse.langchain4j + quarkus-langchain4j-azure-openai-deployment + ${project.version} + pom + test + + + * + * + + + + + + + + io.quarkus + quarkus-maven-plugin + + + + build + + + + + + maven-failsafe-plugin + + + + integration-test + verify + + + + ${project.build.directory}/${project.build.finalName}-runner + org.jboss.logmanager.LogManager + ${maven.home} + + + + + + + + + + native-image + + + native + + + + + + maven-surefire-plugin + + ${native.surefire.skip} + + + + + + false + native + + + + diff --git a/integration-tests/cache/src/main/java/org/acme/example/cache/AiService.java b/integration-tests/cache/src/main/java/org/acme/example/cache/AiService.java new file mode 100644 index 000000000..26b2995e6 --- /dev/null +++ b/integration-tests/cache/src/main/java/org/acme/example/cache/AiService.java @@ -0,0 +1,9 @@ +package org.acme.example.cache; + +import dev.langchain4j.service.UserMessage; +import io.quarkiverse.langchain4j.RegisterAiService; + +@RegisterAiService +public interface AiService { + public String poem(@UserMessage("Write a poem about {topic}") String topic); +} diff --git a/integration-tests/cache/src/main/resources/application.properties b/integration-tests/cache/src/main/resources/application.properties new file mode 100644 index 000000000..32b1c5577 --- /dev/null +++ b/integration-tests/cache/src/main/resources/application.properties @@ -0,0 +1,3 @@ +quarkus.langchain4j.bam.api-key=test +quarkus.langchain4j.bam.url=http://dummy +quarkus.langchain4j.embedding-model.provider=dev.langchain4j.model.embedding.AllMiniLmL6V2EmbeddingModel diff --git a/integration-tests/cache/src/test/java/org/acme/example/cache/MultipleEmbeddingTest.java b/integration-tests/cache/src/test/java/org/acme/example/cache/MultipleEmbeddingTest.java new file mode 100644 index 000000000..bbbd92034 --- /dev/null +++ b/integration-tests/cache/src/test/java/org/acme/example/cache/MultipleEmbeddingTest.java @@ -0,0 +1,21 @@ +package org.acme.example.cache; + +import jakarta.inject.Inject; + +import io.quarkus.test.junit.QuarkusTest; + +@QuarkusTest +public class MultipleEmbeddingTest { + + @Inject + AiService aiService; + + //@Test + void defaultModel() { + + // TODO: This test doesn't work. Look at the BeansProcessor class to understand why. + // Line 170! + aiService.poem("dog"); + aiService.poem("dog"); + } +} diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index b7b9f4165..b885d6ecf 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -20,6 +20,7 @@ azure-openai multiple-providers mistralai + cache devui devui-multiple-embedding-models embed-all-minilm-l6-v2-q diff --git a/model-providers/bam/deployment/src/test/java/io/quarkiverse/langchain4j/bam/deployment/CacheClassTest.java b/model-providers/bam/deployment/src/test/java/io/quarkiverse/langchain4j/bam/deployment/CacheClassTest.java new file mode 100644 index 000000000..e39b35ff4 --- /dev/null +++ b/model-providers/bam/deployment/src/test/java/io/quarkiverse/langchain4j/bam/deployment/CacheClassTest.java @@ -0,0 +1,1141 @@ +package io.quarkiverse.langchain4j.bam.deployment; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; + +import jakarta.enterprise.context.control.ActivateRequestContext; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; + +import org.jboss.shrinkwrap.api.ShrinkWrap; +import org.jboss.shrinkwrap.api.spec.JavaArchive; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.chat.ChatLanguageModel; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.output.Response; +import dev.langchain4j.service.SystemMessage; +import dev.langchain4j.service.UserMessage; +import io.quarkiverse.langchain4j.CacheResult; +import io.quarkiverse.langchain4j.RegisterAiService; +import io.quarkiverse.langchain4j.runtime.cache.AiCacheStore; +import io.quarkus.arc.Arc; +import io.quarkus.arc.ArcContainer; +import io.quarkus.arc.ManagedContext; +import io.quarkus.test.QuarkusUnitTest; + +public class CacheClassTest { + + @RegisterExtension + static QuarkusUnitTest unitTest = new QuarkusUnitTest() + .overrideRuntimeConfigKey("quarkus.langchain4j.bam.base-url", WireMockUtil.URL) + .overrideRuntimeConfigKey("quarkus.langchain4j.bam.api-key", WireMockUtil.API_KEY) + .setArchiveProducer( + () -> ShrinkWrap.create(JavaArchive.class).addClass(WireMockUtil.class)); + + @RegisterAiService + @Singleton + @CacheResult + interface LLMService { + + @SystemMessage("This is a systemMessage") + @UserMessage("This is a userMessage {text}") + String chat(String text); + + @SystemMessage("This is a systemMessage") + @UserMessage("This is a userMessage {text}") + String chat2(String text); + + @Singleton + public class CustomChatModel implements ChatLanguageModel { + @Override + public Response generate(List messages) { + return Response.from(AiMessage.from("result")); + } + } + + @Singleton + public class CustomEmbedding implements EmbeddingModel { + + @Override + public Response> embedAll(List textSegments) { + return Response.from(List.of(Embedding.from(es))); + } + } + } + + @Inject + LLMService service; + + @Inject + AiCacheStore aiCacheStore; + + @Test + void cache_test() { + + String cacheId = "default"; + + assertEquals(0, aiCacheStore.getAll(cacheId).size()); + service.chat("chat"); + assertEquals(1, aiCacheStore.getAll(cacheId).size()); + + service.chat2("chat2"); + assertEquals(1, aiCacheStore.getAll(cacheId).size()); + assertEquals("result", aiCacheStore.getAll(cacheId).get(0).response().text()); + assertEquals(es, aiCacheStore.getAll(cacheId).get(0).embedded().vector()); + } + + @Test + @ActivateRequestContext + void cache_test_with_request_context() { + + ArcContainer container = Arc.container(); + ManagedContext requestContext = container.requestContext(); + String chatCacheId = requestContext.getState() + "#" + LLMService.class.getName() + ".chat"; + String chat2CacheId = requestContext.getState() + "#" + LLMService.class.getName() + ".chat2"; + + assertEquals(0, aiCacheStore.getAll(chatCacheId).size()); + service.chat("chat"); + assertEquals(1, aiCacheStore.getAll(chatCacheId).size()); + assertEquals("result", aiCacheStore.getAll(chatCacheId).get(0).response().text()); + assertEquals(es, aiCacheStore.getAll(chatCacheId).get(0).embedded().vector()); + + service.chat2("chat2"); + assertEquals(1, aiCacheStore.getAll(chat2CacheId).size()); + assertEquals("result", aiCacheStore.getAll(chat2CacheId).get(0).response().text()); + assertEquals(es, aiCacheStore.getAll(chat2CacheId).get(0).embedded().vector()); + } + + static float[] es = { + 0.039016734808683395f, + 0.010098248720169067f, + -0.02687959559261799f, + -0.0656861960887909f, + 0.026491783559322357f, + -0.034191329032182693f, + -0.01605447754263878f, + 0.05859170854091644f, + 0.05299557000398636f, + -0.037048257887363434f, + 0.033948030322790146f, + 0.021606875583529472f, + -0.02515285648405552f, + -0.026005540043115616f, + -0.03924171254038811f, + -0.018618367612361908f, + 0.019430793821811676f, + 0.018351968377828598f, + 0.023682260885834694f, + 0.009436847642064095f, + 0.02903357706964016f, + -0.0030121582094579935f, + -0.014792232774198055f, + -0.03732290863990784f, + -0.015421096235513687f, + -0.012878560461103916f, + -0.03516802936792374f, + -0.028987467288970947f, + 0.0025460650213062763f, + -0.023227907717227936f, + 0.016212934628129005f, + 0.04003201052546501f, + -0.0716291293501854f, + -0.024192918092012405f, + -0.022297894582152367f, + 0.04087153077125549f, + 0.018859276548027992f, + -0.0024456740356981754f, + -0.02141951397061348f, + 0.013890198431909084f, + -0.024088209494948387f, + 0.03852872923016548f, + -0.01587112806737423f, + -0.0343363992869854f, + -0.004860903602093458f, + 0.011112665757536888f, + 0.0342666395008564f, + 0.003387244651094079f, + -0.011313678696751595f, + 0.009971032850444317f, + 0.038495250046253204f, + 0.025818120688199997f, + 0.0005494438228197396f, + -0.018540427088737488f, + -0.04263493791222572f, + 0.04554981738328934f, + -0.004352741874754429f, + -0.008359842002391815f, + -0.06736010313034058f, + 0.02220807410776615f, + -0.039974700659513474f, + 0.013279488310217857f, + 0.03903115540742874f, + -0.008271003141999245f, + -0.0400996208190918f, + 0.0388149656355381f, + 0.01530807837843895f, + 0.000992060056887567f, + -0.05355864390730858f, + 0.03955431655049324f, + -0.012402032501995564f, + 0.025374174118041992f, + -0.04465978592634201f, + -0.0018460184801369905f, + -0.025601740926504135f, + -0.010232556611299515f, + 0.04121308773756027f, + -0.014194999821484089f, + 0.0003599336196202785f, + -0.031029604375362396f, + 0.07545553147792816f, + -0.023090437054634094f, + 0.011858539655804634f, + -0.0059988973662257195f, + 0.040219854563474655f, + 0.008092803880572319f, + 0.013441123999655247f, + 0.07714559137821198f, + 0.027036814019083977f, + 0.010920023545622826f, + -0.014534782618284225f, + 0.04086093232035637f, + 0.023914894089102745f, + 0.012763736769557f, + -0.04254480451345444f, + 0.011305714957416058f, + 0.014120567589998245f, + 0.0196896493434906f, + -0.017100676894187927f, + -0.0013453649589791894f, + -0.007299546152353287f, + -0.0016225805738940835f, + 0.02249031327664852f, + -0.035537295043468475f, + -0.03191295638680458f, + 0.04402138665318489f, + 0.045873962342739105f, + 0.033952441066503525f, + 0.004062877967953682f, + -0.016052523627877235f, + 0.033559925854206085f, + 0.040684811770915985f, + 0.0027443845756351948f, + -0.036945641040802f, + -0.03932597115635872f, + 0.014583739452064037f, + 0.009190680459141731f, + -0.01264051254838705f, + -0.005108111072331667f, + 0.04355351999402046f, + 0.03141055628657341f, + 0.04495703801512718f, + -0.009294744580984116f, + -0.035847608000040054f, + 0.019878702238202095f, + -0.02355729602277279f, + -0.014673866331577301f, + 0.021822089329361916f, + -0.0028103822842240334f, + 0.028539221733808517f, + 0.0329912044107914f, + 0.04047539830207825f, + -0.014253382571041584f, + -0.00660439720377326f, + -0.0639570951461792f, + -0.03520612418651581f, + 0.011432294733822346f, + 0.018043745309114456f, + -0.014185379259288311f, + 0.006499403156340122f, + 0.012511038221418858f, + 0.03146800398826599f, + 0.02470427192747593f, + -0.027075596153736115f, + -0.023012971505522728f, + -0.03645343333482742f, + -0.03147031366825104f, + 0.03326469659805298f, + -0.02293357253074646f, + 0.00007493131852243096f, + 0.027926260605454445f, + 0.0010919274063780904f, + -0.03810613974928856f, + -0.0009479214786551893f, + 0.01032050047069788f, + -0.02284139208495617f, + 0.022822747007012367f, + 0.013322929851710796f, + 0.01755167357623577f, + 0.006281423382461071f, + -0.05161968991160393f, + -0.03454432263970375f, + -0.015171839855611324f, + -0.013086988590657711f, + 0.009599068202078342f, + -0.023829564452171326f, + 0.03170377388596535f, + 0.025455616414546967f, + 0.006359482184052467f, + -0.006833407562226057f, + -0.02959762141108513f, + -0.04312501475214958f, + 0.020601991564035416f, + -0.04397542029619217f, + -0.011911196634173393f, + 0.030813507735729218f, + 0.04656203091144562f, + 0.043676819652318954f, + 0.06412668526172638f, + -0.02376391552388668f, + 0.020222248509526253f, + 0.048504240810871124f, + 0.04187004268169403f, + -0.03268779441714287f, + -0.013356110081076622f, + 0.003990362398326397f, + 0.03669435530900955f, + 0.03567170351743698f, + 0.02201189659535885f, + 0.007053814362734556f, + 0.02911461517214775f, + -0.016084877774119377f, + -0.013361542485654354f, + -0.0025507789105176926f, + -0.017344053834676743f, + 0.03305065259337425f, + 0.03494741767644882f, + 0.02428996004164219f, + -0.024909833446145058f, + -0.03706767037510872f, + -0.014623625203967094f, + 0.025253746658563614f, + -0.04022902622818947f, + 0.013688928447663784f, + 0.020935285836458206f, + -0.03215555474162102f, + -0.01734493114054203f, + -0.016057977452874184f, + 0.0332186259329319f, + -0.062192827463150024f, + -0.06338762491941452f, + -0.013384060002863407f, + 0.0031143389642238617f, + 0.007829300127923489f, + -0.05590514838695526f, + -0.028918664902448654f, + -0.018310556188225746f, + -0.02395123988389969f, + 0.0227365605533123f, + -0.037892408668994904f, + 0.01014055497944355f, + 0.056368228048086166f, + -0.000996291171759367f, + 0.05579328536987305f, + 0.018668418750166893f, + 0.027242204174399376f, + 0.05181897431612015f, + 0.02047695778310299f, + 0.037248481065034866f, + 0.048321932554244995f, + 0.0017545935697853565f, + 0.010634876787662506f, + -0.030261456966400146f, + -0.026554463431239128f, + 0.0051453569903969765f, + -0.013943018391728401f, + -0.04623245820403099f, + 0.0003986647061537951f, + 0.03525836020708084f, + 0.049685895442962646f, + -0.03695899248123169f, + 0.02713131159543991f, + 0.010237785056233406f, + -0.03279371187090874f, + -0.02466234378516674f, + 0.009988408535718918f, + 0.018033919855952263f, + -0.0013647808227688074f, + 0.03687773644924164f, + -0.0029583261348307133f, + 0.0067709567956626415f, + 0.03685348480939865f, + -0.031049998477101326f, + -0.001257769064977765f, + 0.04537429288029671f, + 0.010919678024947643f, + 0.04005173593759537f, + -0.013691303320229053f, + -0.002520656678825617f, + -0.01983926258981228f, + 0.02462667040526867f, + -0.01412965264171362f, + -0.01593015529215336f, + 0.056297656148672104f, + 0.03142832964658737f, + 0.009161954745650291f, + -0.02579585276544094f, + -0.031644586473703384f, + -0.01380220428109169f, + -0.04127073660492897f, + -0.04070030152797699f, + -0.048543404787778854f, + -0.05622323229908943f, + -0.0021230385173112154f, + -0.03724157437682152f, + -0.017728988081216812f, + -0.049515388906002045f, + -0.013393918052315712f, + 0.00653441995382309f, + -0.008645026944577694f, + -0.03468145802617073f, + -0.008628074079751968f, + 0.035358525812625885f, + -0.030219580978155136f, + 0.017552250996232033f, + 0.017619483172893524f, + 0.00875946693122387f, + -0.019581124186515808f, + 0.003319726325571537f, + -0.0051649208180606365f, + 0.03903667628765106f, + -0.0009742019465193152f, + 0.05749356374144554f, + 0.011132277548313141f, + -0.016711536794900894f, + 0.04648770019412041f, + -0.021648593246936798f, + -0.10148133337497711f, + -0.03178001940250397f, + 0.01529928483068943f, + -0.032578423619270325f, + -0.05097171664237976f, + 0.01338517852127552f, + 0.012569229118525982f, + -0.006618269253522158f, + -0.03592976555228233f, + 0.02533770725131035f, + 0.03580918163061142f, + -0.06759073585271835f, + 0.027629327028989792f, + 0.04527544975280762f, + 0.029171645641326904f, + 0.02645142935216427f, + -0.01734251342713833f, + -0.06954346597194672f, + 0.013422731310129166f, + -0.03871454671025276f, + 0.018256206065416336f, + -0.024370983242988586f, + 0.05545681715011597f, + -0.039528828114271164f, + -0.013660108670592308f, + -0.059706974774599075f, + 0.016853436827659607f, + 0.031912483274936676f, + 0.015487623400986195f, + 0.011767021380364895f, + -0.05857180431485176f, + -0.005341519135981798f, + -0.04033682122826576f, + 0.025871943682432175f, + -0.02402888983488083f, + 0.02561469003558159f, + 0.030010027810931206f, + -0.03678053244948387f, + 0.03396481275558472f, + -0.07291311025619507f, + 0.02396109327673912f, + 0.025626540184020996f, + 0.00955990795046091f, + 0.013724202290177345f, + -0.014750544913113117f, + -0.032967016100883484f, + 0.02058098278939724f, + 0.0033076023682951927f, + -0.029527032747864723f, + -0.0009443678427487612f, + -0.02781200222671032f, + 0.02402203157544136f, + 0.014563879929482937f, + -0.011585783213376999f, + -0.026427732780575752f, + 0.019133128225803375f, + 0.021863577887415886f, + 0.006209978833794594f, + 0.07440700381994247f, + 0.008599634282290936f, + -0.04177406057715416f, + 0.02020418830215931f, + -0.004323258996009827f, + 0.005790286231786013f, + -0.025323275476694107f, + 0.058458585292100906f, + -0.03278389200568199f, + -0.011032545939087868f, + -0.0036148857325315475f, + -0.0004926437395624816f, + -0.022332727909088135f, + -0.04997304454445839f, + -0.0007595937349833548f, + 0.03481508418917656f, + 0.020037902519106865f, + -0.021608928218483925f, + 0.01581759564578533f, + -0.011867176741361618f, + -0.00947039108723402f, + -0.07732773572206497f, + -0.011982443742454052f, + 0.023730168119072914f, + -0.04481136426329613f, + -0.011057991534471512f, + -0.04485539346933365f, + 0.01307082362473011f, + -0.03840833529829979f, + -0.005540383048355579f, + -0.03217718377709389f, + 0.19306251406669617f, + 0.01557007897645235f, + 0.03433694690465927f, + -0.015490150079131126f, + 0.014035001397132874f, + 0.008068409748375416f, + 0.029123784974217415f, + 0.010411231778562069f, + 0.03180669620633125f, + 0.016272393986582756f, + -0.03405269607901573f, + 0.008909656666219234f, + 0.026794541627168655f, + -0.007454108912497759f, + 0.016099030151963234f, + 0.05538764223456383f, + 0.008271883241832256f, + 0.023237742483615875f, + 0.034973062574863434f, + -0.0081742312759161f, + 0.04468708857893944f, + -0.03461684659123421f, + 0.03345729038119316f, + 0.0005239890888333321f, + -0.056889843195676804f, + -0.042187951505184174f, + -0.007005332037806511f, + 0.031517598778009415f, + -0.057308562099933624f, + 0.03141198307275772f, + 0.0007111141458153725f, + 0.028522223234176636f, + -0.04995636269450188f, + -0.022562386468052864f, + 0.039314135909080505f, + 0.015615178272128105f, + 0.02232501097023487f, + -0.03433450683951378f, + 0.012382658198475838f, + -0.02625468373298645f, + -0.0083118611946702f, + 0.017250170931220055f, + -0.02019653283059597f, + 0.03392040356993675f, + 0.0013320763828232884f, + 0.04296816885471344f, + 0.05378912016749382f, + -0.018141886219382286f, + -0.02347896434366703f, + 0.03130969777703285f, + 0.021074768155813217f, + 0.011518022045493126f, + -0.004174298606812954f, + -0.03582172095775604f, + -0.015318387188017368f, + -0.007444401737302542f, + -0.03830752149224281f, + -0.06777879595756531f, + -0.006020883098244667f, + 0.008447406813502312f, + 0.062179744243621826f, + 0.01918203756213188f, + -0.016178149729967117f, + -0.023090755566954613f, + 0.028789259493350983f, + 0.040439531207084656f, + -0.02672472968697548f, + -0.04095323011279106f, + -0.05732221528887749f, + -0.0020660210866481066f, + -0.055687226355075836f, + -0.008511627092957497f, + -0.039763759821653366f, + 0.03596954047679901f, + 0.033216822892427444f, + -0.005118542350828648f, + -0.03271672502160072f, + 0.018839454278349876f, + 0.016132114455103874f, + 0.028364259749650955f, + 0.006845584139227867f, + -0.0012978515587747097f, + -0.018664628267288208f, + 0.03485922887921333f, + 0.04039550945162773f, + -0.031199388206005096f, + 0.01957610994577408f, + 0.03947604075074196f, + -0.04612560197710991f, + 0.022226179018616676f, + 0.03061433508992195f, + 0.007180668413639069f, + -0.033031608909368515f, + 0.05610867962241173f, + 0.0284646637737751f, + -0.05204550921916962f, + 0.017941653728485107f, + 0.022062553092837334f, + -0.04683050885796547f, + -0.023560887202620506f, + -0.0276334211230278f, + 0.0041246069595217705f, + -0.032277919352054596f, + -0.01940121315419674f, + 0.0013466235250234604f, + -0.024347569793462753f, + 0.01447154488414526f, + 0.03692981228232384f, + 0.02556624449789524f, + -0.013659082353115082f, + 0.030253291130065918f, + 0.004965346772223711f, + 0.023664172738790512f, + -0.01723453216254711f, + 0.05455326661467552f, + 0.02362307533621788f, + 0.022859739139676094f, + -0.054154444485902786f, + 0.008361054584383965f, + 0.009449029341340065f, + -0.014727693982422352f, + 0.010806793347001076f, + -0.04166347533464432f, + 0.07135552912950516f, + -0.007770725060254335f, + 0.030580557882785797f, + -0.04844602942466736f, + 0.0008229002705775201f, + 0.026535481214523315f, + 0.021868199110031128f, + 0.004678168799728155f, + -0.03931296244263649f, + -0.04464258253574371f, + 0.03712749108672142f, + 0.014044600538909435f, + -0.0023832779843360186f, + 0.08839696645736694f, + -0.011742178350687027f, + -0.02589561603963375f, + 0.04511994495987892f, + -0.011089873500168324f, + 0.04424068331718445f, + -0.031209222972393036f, + 0.018096735700964928f, + 0.0276931282132864f, + 0.032378826290369034f, + -0.031537868082523346f, + -0.027904240414500237f, + -0.015190032310783863f, + -0.004481968469917774f, + 0.04267747700214386f, + -0.028845272958278656f, + 0.019086001440882683f, + 0.009203453548252583f, + -0.039411865174770355f, + -0.06638588011264801f, + 0.019709186628460884f, + 0.03591395914554596f, + 0.05008409172296524f, + -0.011427966877818108f, + -0.04971498250961304f, + -0.004772470332682133f, + 0.01432930026203394f, + 0.1095755398273468f, + 0.02603503316640854f, + 0.019698046147823334f, + -0.032637741416692734f, + 0.033184606581926346f, + 0.02692507766187191f, + 0.010238883085548878f, + -0.010183892212808132f, + -0.029479920864105225f, + -0.019146805629134178f, + 0.02746221050620079f, + 0.021641163155436516f, + -0.024658724665641785f, + -0.01630176603794098f, + 0.02958461083471775f, + -0.04108947888016701f, + 0.032371439039707184f, + -0.027287479490041733f, + 0.020171169191598892f, + 0.03383186459541321f, + -0.015322552062571049f, + -0.010307767428457737f, + 0.013115078210830688f, + 0.005613786168396473f, + -0.02910318784415722f, + -0.021252978593111038f, + 0.005560233723372221f, + -0.04231780767440796f, + 0.08192117512226105f, + -0.029788488522171974f, + -0.0016719227423891425f, + -0.03858398273587227f, + -0.027913816273212433f, + 0.012290316633880138f, + 0.015825187787413597f, + -0.026750769466161728f, + -0.039809394627809525f, + 0.010853637009859085f, + -0.04720485955476761f, + -0.010609184391796589f, + -0.01278828363865614f, + -0.03623141720890999f, + -0.020634643733501434f, + 0.030923841521143913f, + 0.03285284712910652f, + 0.01452653668820858f, + 0.03416169807314873f, + 0.010913162492215633f, + -0.024978244677186012f, + 0.03296514227986336f, + 0.02161172777414322f, + 0.030312156304717064f, + 0.03071526065468788f, + 0.04639757424592972f, + 0.031048797070980072f, + 0.03407664597034454f, + -0.041457146406173706f, + 0.011761465109884739f, + 0.012112592346966267f, + 0.019076142460107803f, + -0.059248920530080795f, + -0.04760892689228058f, + -0.0314653106033802f, + 0.016123836860060692f, + 0.006126987282186747f, + -0.031283795833587646f, + -0.0035350737161934376f, + 0.012261965312063694f, + -0.024649467319250107f, + -0.009447114542126656f, + -0.08359000086784363f, + 0.005851270165294409f, + -0.030149782076478004f, + 0.027616258710622787f, + 0.03182379528880119f, + 0.010887391865253448f, + -0.027072472497820854f, + 0.020552828907966614f, + 0.017932042479515076f, + 0.04406345263123512f, + -0.028712112456560135f, + -0.008050181902945042f, + 0.04549504071474075f, + -0.06913027167320251f, + -0.004572774283587933f, + 0.00011219691077712923f, + -0.006029843352735043f, + -0.012317972257733345f, + -0.04016095772385597f, + 0.030960263684391975f, + 0.039273906499147415f, + -0.015845293179154396f, + -0.011702924966812134f, + 0.011656280606985092f, + -0.03036012314260006f, + -0.08103150129318237f, + -0.023546282202005386f, + 0.016684504225850105f, + 0.0006793781649321318f, + -0.030003944411873817f, + 0.017032481729984283f, + -0.023390373215079308f, + 0.018340930342674255f, + 0.024219363927841187f, + -0.03645186498761177f, + -0.018286602571606636f, + -0.048507433384656906f, + 0.046701669692993164f, + -0.020781058818101883f, + -0.01605192758142948f, + -0.0903584435582161f, + 0.06904299557209015f, + 0.015601576305925846f, + -0.018518496304750443f, + -0.038458120077848434f, + -0.019991688430309296f, + -0.024300318211317062f, + -0.012263032607734203f, + 0.024427302181720734f, + -0.024809060618281364f, + 0.019496100023388863f, + 0.01758641004562378f, + 0.006886046379804611f, + -0.03860282525420189f, + 0.009409677237272263f, + -0.013964204117655754f, + -0.04017601162195206f, + -0.016977611929178238f, + 0.02575852908194065f, + -0.013815070502460003f, + 0.006740113254636526f, + -0.04932718351483345f, + -0.05205217003822327f, + -0.04415235295891762f, + -0.01197398267686367f, + 0.011467654258012772f, + -0.03154896944761276f, + 0.0028590233996510506f, + -0.03314562886953354f, + -0.038118354976177216f, + -0.03914812579751015f, + -0.0005633080145344138f, + 0.005159502383321524f, + -0.009560227394104004f, + -0.025696488097310066f, + 0.03951350972056389f, + 0.04184769093990326f, + -0.010419424623250961f, + -0.03487123176455498f, + 0.001081227557733655f, + 0.031209642067551613f, + 0.01224304549396038f, + -0.012603535316884518f, + 0.029348352923989296f, + -0.026243818923830986f, + -0.023612577468156815f, + 0.04481798782944679f, + -0.04039709270000458f, + -0.01403393130749464f, + -0.022715067490935326f, + 0.008543731644749641f, + 0.04796065390110016f, + 0.00940101407468319f, + 0.03649118170142174f, + 0.016881216317415237f, + 0.013232183642685413f, + 0.014445126056671143f, + 0.010655111633241177f, + -0.015311007387936115f, + -0.03107623942196369f, + -0.012181959114968777f, + -0.030173713341355324f, + -0.03136519715189934f, + -0.00968134868890047f, + -0.004136156756430864f, + 0.0005978004774078727f, + -0.05459468439221382f, + -0.015934104099869728f, + 0.053266167640686035f, + 0.03784262016415596f, + -0.010738855227828026f, + 0.03190033510327339f, + -0.001496695913374424f, + -0.024366231635212898f, + 0.03414757177233696f, + -0.0077161891385912895f, + -0.044079799205064774f, + -0.015896553173661232f, + 0.03637973219156265f, + -0.02161337621510029f, + 0.006903046742081642f, + -0.006962891202419996f, + 0.05574609339237213f, + -0.029158806428313255f, + 0.03076884150505066f, + -0.009158081375062466f, + -0.04707910493016243f, + -0.024784710258245468f, + 0.015435277484357357f, + 0.007692893035709858f, + 0.0006725761923007667f, + -0.038610704243183136f, + 0.009134447202086449f, + 0.009849862195551395f, + 0.024519752711057663f, + -0.03863726183772087f, + 0.007874547503888607f, + -0.020069731399416924f, + 0.007399733178317547f, + -0.05068531632423401f, + -0.009004154242575169f, + -0.04068838059902191f, + 0.040781937539577484f, + -0.02937493473291397f, + 0.016979916021227837f, + 0.037655118852853775f, + -0.01305563747882843f, + 0.013344315811991692f, + -0.05640937015414238f, + -0.020350581035017967f, + -0.02501261793076992f, + 0.038834795355796814f, + -0.04604274779558182f, + 0.0004385723441373557f, + 0.023745035752654076f, + 0.003656078828498721f, + 0.039705485105514526f, + -0.011241079308092594f, + 0.044178713113069534f, + -0.011368539184331894f, + 0.012771987356245518f, + -0.08492542803287506f, + -0.009417480789124966f, + -0.010677497833967209f, + -0.03395388647913933f, + -0.011490026488900185f, + -0.006386809982359409f, + -0.014871163293719292f, + 0.004204850643873215f, + 0.029643211513757706f, + -0.04543495550751686f, + 0.02437104657292366f, + 0.015322508290410042f, + 0.004863101989030838f, + -0.002539763692766428f, + 0.015903716906905174f, + 0.016624106094241142f, + 0.023341473191976547f, + -0.030201323330402374f, + 0.007918000221252441f, + 0.0009107357473112643f, + 0.03916248679161072f, + -0.010109283030033112f, + 0.018451586365699768f, + 0.0062872618436813354f, + 0.023154068738222122f, + 0.028111852705478668f, + -0.014428956434130669f, + 0.036793746054172516f, + -0.03575288504362106f, + -0.057246819138526917f, + -0.021768808364868164f, + -0.023564910516142845f, + 0.02397366799414158f, + 0.020718568935990334f, + -0.015275984071195126f, + 0.03703267127275467f, + -0.010308920405805111f, + -0.04211723431944847f, + -0.01663627289235592f, + 0.02286018803715706f, + -0.03881242126226425f, + 0.03525247052311897f, + -0.042307011783123016f, + -0.003567407839000225f, + 0.001457449048757553f, + 0.05936582013964653f, + 0.002895828103646636f, + -0.025598157197237015f, + 0.01283274032175541f, + -0.05007452890276909f, + -0.03235140070319176f, + -0.011465230956673622f, + 0.026079369708895683f, + 0.058149781078100204f, + 0.024806367233395576f, + 0.01770002953708172f, + 0.0005065941368229687f, + -0.026064587756991386f, + 0.015047438442707062f, + 0.03801387920975685f, + 0.01752178929746151f, + -0.02259741723537445f, + -0.03879963234066963f, + -0.02445252798497677f, + -0.029194211587309837f, + -0.04118459299206734f, + -0.07230067998170853f, + -0.009569251909852028f, + -0.01829957403242588f, + 0.03917815536260605f, + -0.019530808553099632f, + -0.03650574013590813f, + 0.028397979214787483f, + -0.026918869465589523f, + 0.023724351078271866f, + -0.011466676369309425f, + -0.040093325078487396f, + 0.010017112828791142f, + 0.012075957842171192f, + 0.010774342343211174f, + -0.016421642154455185f, + -0.04688034579157829f, + 0.02431156113743782f, + -0.003046917961910367f, + -0.0010734435636550188f, + 0.022572355344891548f, + 0.011197290383279324f, + -0.025137169286608696f, + -0.049277711659669876f, + 0.04823236167430878f, + -0.04765353724360466f, + -0.018735116347670555f, + 0.0003122256603091955f, + -0.008810674771666527f, + -0.017892984673380852f, + 0.00021120262681506574f, + -0.031895555555820465f, + -0.023480713367462158f, + 0.010128460824489594f, + 0.016516825184226036f, + -0.019813979044556618f, + -0.0044202678836882114f, + -0.00702977180480957f, + -0.002187698846682906f, + 0.013112161308526993f, + 0.008308488875627518f, + 0.03817952051758766f, + -0.020533859729766846f, + 0.028264036402106285f, + -0.04136377573013306f, + -0.05692916363477707f, + -0.021915340796113014f, + -0.019034316763281822f, + -0.015118561685085297f, + -0.0319315642118454f, + 0.03268538787961006f, + -0.0010479907505214214f, + 0.00024182928609661758f, + -0.0012983399210497737f, + -0.0035084697883576155f, + -0.01018751785159111f, + -0.06536149233579636f, + 0.03195962309837341f, + -0.004426909144967794f, + 0.0029187293257564306f, + -0.0256631039083004f, + 0.036579765379428864f, + -0.06405307352542877f, + 0.013227395713329315f, + -0.03624671325087547f, + 0.037127405405044556f, + -0.06633934378623962f, + 0.023713771253824234f, + 0.001657120301388204f, + -0.030938217416405678f, + -0.005628079175949097f, + -0.053462110459804535f, + 0.07506952434778214f, + -0.007784669753164053f, + 0.028080616146326065f, + -0.01574862003326416f, + -0.04546362906694412f, + -0.05016990005970001f, + -0.05019865557551384f, + -0.039244186133146286f, + 0.030151229351758957f, + -0.05171814560890198f, + -0.03881967440247536f, + -0.009908814914524555f, + -0.016938813030719757f, + 0.011704900301992893f, + -0.014907843433320522f, + -0.03325396403670311f, + 0.047350019216537476f, + 0.0045817457139492035f, + 0.028241490945219994f, + -0.003765946952626109f, + 0.01571817509829998f, + 0.03741828724741936f, + 0.0156949982047081f, + -0.01970336213707924f, + 0.009402992203831673f, + -0.03331432864069939f, + 0.025959284976124763f, + -0.0041138906963169575f, + -0.040316756814718246f, + 0.0012001245049759746f, + 0.009792031720280647f, + -0.047768693417310715f, + -0.02163655124604702f, + 0.018602421507239342f, + 0.000602972402703017f, + -0.03311364725232124f, + 0.026526229456067085f, + 0.04202568158507347f, + 0.008449001237750053f, + 0.06475643813610077f, + 0.0035848424304276705f, + 0.0488557443022728f, + 0.020431524142622948f, + 0.010165947489440441f, + 0.00012580514885485172f, + 0.011490631848573685f, + 0.049949027597904205f, + 0.03915407881140709f, + 0.059800226241350174f, + 0.024663494899868965f, + 0.02410218119621277f, + -0.014562365598976612f, + 0.02789120003581047f, + -0.03704062104225159f, + 0.02282542549073696f, + -0.047931917011737823f, + 0.05969615653157234f, + 0.038760412484407425f, + -0.007845507934689522f, + -0.0011439616791903973f, + 0.014877899549901485f, + -0.033575110137462616f, + -0.021374600008130074f, + 0.014881900511682034f, + 0.01631966605782509f, + 0.04771406203508377f, + -0.023484190925955772f, + -0.03400464728474617f, + -0.018462782725691795f, + -0.056461531668901443f, + 0.02125520072877407f, + -0.04083051532506943f, + 0.04490841552615166f, + 0.05638275668025017f, + -0.03922900930047035f, + 0.015876512974500656f, + 0.02240809239447117f, + -0.00000876183094078442f, + 0.041717130690813065f, + 0.041573211550712585f, + -0.014516608789563179f, + -0.020156629383563995f, + 0.03390810638666153f, + 0.04974587634205818f, + 0.011070906184613705f, + -0.0019445940852165222f, + 0.014449193142354488f, + -0.02405550517141819f, + 0.021492162719368935f, + -0.05518610030412674f, + 0.01799919828772545f, + -0.039734020829200745f, + -0.0037850206717848778f, + 0.007841100916266441f, + 0.014386530965566635f, + 0.02055184915661812f, + -0.0069200219586491585f, + -0.036611758172512054f, + -0.044122129678726196f, + 0.045821838080883026f, + 0.012124829925596714f, + -0.00812555756419897f, + 0.02017771080136299f, + -0.0007992296013981104f, + 0.003344304393976927f, + -0.007832828909158707f, + 0.0202498622238636f + }; +} diff --git a/model-providers/bam/deployment/src/test/java/io/quarkiverse/langchain4j/bam/deployment/CacheConfigTest.java b/model-providers/bam/deployment/src/test/java/io/quarkiverse/langchain4j/bam/deployment/CacheConfigTest.java new file mode 100644 index 000000000..4d039c82b --- /dev/null +++ b/model-providers/bam/deployment/src/test/java/io/quarkiverse/langchain4j/bam/deployment/CacheConfigTest.java @@ -0,0 +1,147 @@ +package io.quarkiverse.langchain4j.bam.deployment; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import java.util.List; +import java.util.stream.Collectors; +import org.jboss.shrinkwrap.api.ShrinkWrap; +import org.jboss.shrinkwrap.api.spec.JavaArchive; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.chat.ChatLanguageModel; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.output.Response; +import dev.langchain4j.service.UserMessage; +import io.quarkiverse.langchain4j.CacheResult; +import io.quarkiverse.langchain4j.RegisterAiService; +import io.quarkiverse.langchain4j.runtime.cache.AiCacheStore; +import io.quarkus.test.QuarkusUnitTest; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; + +public class CacheConfigTest { + + @RegisterExtension + static QuarkusUnitTest unitTest = new QuarkusUnitTest() + .overrideRuntimeConfigKey("quarkus.langchain4j.bam.base-url", WireMockUtil.URL) + .overrideRuntimeConfigKey("quarkus.langchain4j.bam.api-key", WireMockUtil.API_KEY) + .overrideRuntimeConfigKey("quarkus.langchain4j.cache.ttl", "2s") + .overrideRuntimeConfigKey("quarkus.langchain4j.cache.max-size", "3") + .setArchiveProducer( + () -> ShrinkWrap.create(JavaArchive.class).addClass(WireMockUtil.class)); + + @RegisterAiService(chatMemoryProviderSupplier = RegisterAiService.NoChatMemoryProviderSupplier.class) + @Singleton + interface LLMService { + + @UserMessage("{text}") + @CacheResult + String chat(String text); + + @Singleton + public class CustomChatModel implements ChatLanguageModel { + @Override + public Response generate(List messages) { + String m = messages.stream().map(ChatMessage::text).collect(Collectors.joining("")); + return Response.from(AiMessage.from("cache: " + m)); + } + } + + @Singleton + public class CustomEmbedding implements EmbeddingModel { + + @Override + public Response> embedAll(List textSegments) { + if (textSegments.get(0).text().equals("FIRST")) + return Response.from(List.of(Embedding.from(first))); + else if (textSegments.get(0).text().equals("SECOND")) + return Response.from(List.of(Embedding.from(second))); + else if (textSegments.get(0).text().equals("THIRD")) + return Response.from(List.of(Embedding.from(third))); + else if (textSegments.get(0).text().equals("FOURTH")) + return Response.from(List.of(Embedding.from(fourth))); + return null; + } + } + } + + @Inject + LLMService service; + + @Inject + AiCacheStore aiCacheStore; + + @Test + @Order(1) + void cache_ttl_test() throws InterruptedException { + + String cacheId = "default"; + aiCacheStore.deleteCache(cacheId); + + service.chat("FIRST"); + service.chat("SECOND"); + assertEquals(2, aiCacheStore.getAll(cacheId).size()); + assertEquals("cache: FIRST", aiCacheStore.getAll(cacheId).get(0).response().text()); + assertEquals(first, aiCacheStore.getAll(cacheId).get(0).embedded().vector()); + assertEquals("cache: SECOND", aiCacheStore.getAll(cacheId).get(1).response().text()); + assertEquals(second, aiCacheStore.getAll(cacheId).get(1).embedded().vector()); + + Thread.sleep(3000); + service.chat("THIRD"); + assertEquals(1, aiCacheStore.getAll(cacheId).size()); + assertEquals("cache: THIRD", aiCacheStore.getAll(cacheId).get(0).response().text()); + assertEquals(third, aiCacheStore.getAll(cacheId).get(0).embedded().vector()); + } + + @Test + @Order(2) + void cache_max_size_test() { + + String cacheId = "default"; + aiCacheStore.deleteCache(cacheId); + + service.chat("FIRST"); + assertEquals(1, aiCacheStore.getAll(cacheId).size()); + assertEquals("cache: FIRST", aiCacheStore.getAll(cacheId).get(0).response().text()); + assertEquals(first, aiCacheStore.getAll(cacheId).get(0).embedded().vector()); + + service.chat("SECOND"); + service.chat("THIRD"); + service.chat("FOURTH"); + assertEquals(3, aiCacheStore.getAll(cacheId).size()); + assertEquals("cache: SECOND", aiCacheStore.getAll(cacheId).get(0).response().text()); + assertEquals(second, aiCacheStore.getAll(cacheId).get(0).embedded().vector()); + assertEquals("cache: THIRD", aiCacheStore.getAll(cacheId).get(1).response().text()); + assertEquals(third, aiCacheStore.getAll(cacheId).get(1).embedded().vector()); + assertEquals("cache: FOURTH", aiCacheStore.getAll(cacheId).get(2).response().text()); + assertEquals(fourth, aiCacheStore.getAll(cacheId).get(2).embedded().vector()); + } + + static float[] first = { + 0.039016734808683395f, + 0.010098248720169067f, + -0.02687959559261799f, + }; + + static float[] second = { + 0.139016734108685515f, + 0.211198249720169167f, + 0.62687959559261799f, + }; + + static float[] third = { + -0.229016734199685515f, + -0.211198249721169127f, + -0.62999959559261719f, + }; + + static float[] fourth = { + -1.229016734199685515f, + 0.211198249721169127f, + 3.62999959559261719f, + }; +} diff --git a/model-providers/bam/deployment/src/test/java/io/quarkiverse/langchain4j/bam/deployment/CacheMethodTest.java b/model-providers/bam/deployment/src/test/java/io/quarkiverse/langchain4j/bam/deployment/CacheMethodTest.java new file mode 100644 index 000000000..2081b2ede --- /dev/null +++ b/model-providers/bam/deployment/src/test/java/io/quarkiverse/langchain4j/bam/deployment/CacheMethodTest.java @@ -0,0 +1,1139 @@ +package io.quarkiverse.langchain4j.bam.deployment; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.List; + +import jakarta.enterprise.context.control.ActivateRequestContext; +import jakarta.inject.Inject; +import jakarta.inject.Singleton; + +import org.jboss.shrinkwrap.api.ShrinkWrap; +import org.jboss.shrinkwrap.api.spec.JavaArchive; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.message.AiMessage; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.segment.TextSegment; +import dev.langchain4j.model.chat.ChatLanguageModel; +import dev.langchain4j.model.embedding.EmbeddingModel; +import dev.langchain4j.model.output.Response; +import dev.langchain4j.service.SystemMessage; +import dev.langchain4j.service.UserMessage; +import io.quarkiverse.langchain4j.CacheResult; +import io.quarkiverse.langchain4j.RegisterAiService; +import io.quarkiverse.langchain4j.runtime.cache.AiCacheStore; +import io.quarkus.arc.Arc; +import io.quarkus.arc.ArcContainer; +import io.quarkus.arc.ManagedContext; +import io.quarkus.test.QuarkusUnitTest; + +public class CacheMethodTest { + + @RegisterExtension + static QuarkusUnitTest unitTest = new QuarkusUnitTest() + .overrideRuntimeConfigKey("quarkus.langchain4j.bam.base-url", WireMockUtil.URL) + .overrideRuntimeConfigKey("quarkus.langchain4j.bam.api-key", WireMockUtil.API_KEY) + .setArchiveProducer( + () -> ShrinkWrap.create(JavaArchive.class).addClass(WireMockUtil.class)); + + @RegisterAiService + @Singleton + interface LLMService { + + @SystemMessage("This is a systemMessage") + @UserMessage("This is a userMessage {text}") + @CacheResult + String chat(String text); + + @SystemMessage("This is a systemMessage") + @UserMessage("This is a userMessage {text}") + String chatNoCache(String text); + + @Singleton + public class CustomChatModel implements ChatLanguageModel { + @Override + public Response generate(List messages) { + return Response.from(AiMessage.from("result")); + } + } + + @Singleton + public class CustomEmbedding implements EmbeddingModel { + + @Override + public Response> embedAll(List textSegments) { + return Response.from(List.of(Embedding.from(es))); + } + } + } + + @Inject + LLMService service; + + @Inject + AiCacheStore aiCacheStore; + + @Test + void cache_test() { + + String cacheId = "default"; + + assertEquals(0, aiCacheStore.getAll(cacheId).size()); + service.chatNoCache("noCache"); + assertEquals(0, aiCacheStore.getAll(cacheId).size()); + + service.chat("cache"); + assertEquals(1, aiCacheStore.getAll(cacheId).size()); + assertEquals("result", aiCacheStore.getAll(cacheId).get(0).response().text()); + assertEquals(es, aiCacheStore.getAll(cacheId).get(0).embedded().vector()); + } + + @Test + @ActivateRequestContext + void cache_test_with_request_context() { + + ArcContainer container = Arc.container(); + ManagedContext requestContext = container.requestContext(); + String chatNoCacheId = requestContext.getState() + "#" + LLMService.class.getName() + ".chatNoCache"; + String chatCacheId = requestContext.getState() + "#" + LLMService.class.getName() + ".chat"; + + assertEquals(0, aiCacheStore.getAll(chatNoCacheId).size()); + service.chatNoCache("noCache"); + assertEquals(0, aiCacheStore.getAll(chatNoCacheId).size()); + + service.chat("cache"); + assertEquals(1, aiCacheStore.getAll(chatCacheId).size()); + assertEquals("result", aiCacheStore.getAll(chatCacheId).get(0).response().text()); + assertEquals(es, aiCacheStore.getAll(chatCacheId).get(0).embedded().vector()); + } + + static float[] es = { + 0.039016734808683395f, + 0.010098248720169067f, + -0.02687959559261799f, + -0.0656861960887909f, + 0.026491783559322357f, + -0.034191329032182693f, + -0.01605447754263878f, + 0.05859170854091644f, + 0.05299557000398636f, + -0.037048257887363434f, + 0.033948030322790146f, + 0.021606875583529472f, + -0.02515285648405552f, + -0.026005540043115616f, + -0.03924171254038811f, + -0.018618367612361908f, + 0.019430793821811676f, + 0.018351968377828598f, + 0.023682260885834694f, + 0.009436847642064095f, + 0.02903357706964016f, + -0.0030121582094579935f, + -0.014792232774198055f, + -0.03732290863990784f, + -0.015421096235513687f, + -0.012878560461103916f, + -0.03516802936792374f, + -0.028987467288970947f, + 0.0025460650213062763f, + -0.023227907717227936f, + 0.016212934628129005f, + 0.04003201052546501f, + -0.0716291293501854f, + -0.024192918092012405f, + -0.022297894582152367f, + 0.04087153077125549f, + 0.018859276548027992f, + -0.0024456740356981754f, + -0.02141951397061348f, + 0.013890198431909084f, + -0.024088209494948387f, + 0.03852872923016548f, + -0.01587112806737423f, + -0.0343363992869854f, + -0.004860903602093458f, + 0.011112665757536888f, + 0.0342666395008564f, + 0.003387244651094079f, + -0.011313678696751595f, + 0.009971032850444317f, + 0.038495250046253204f, + 0.025818120688199997f, + 0.0005494438228197396f, + -0.018540427088737488f, + -0.04263493791222572f, + 0.04554981738328934f, + -0.004352741874754429f, + -0.008359842002391815f, + -0.06736010313034058f, + 0.02220807410776615f, + -0.039974700659513474f, + 0.013279488310217857f, + 0.03903115540742874f, + -0.008271003141999245f, + -0.0400996208190918f, + 0.0388149656355381f, + 0.01530807837843895f, + 0.000992060056887567f, + -0.05355864390730858f, + 0.03955431655049324f, + -0.012402032501995564f, + 0.025374174118041992f, + -0.04465978592634201f, + -0.0018460184801369905f, + -0.025601740926504135f, + -0.010232556611299515f, + 0.04121308773756027f, + -0.014194999821484089f, + 0.0003599336196202785f, + -0.031029604375362396f, + 0.07545553147792816f, + -0.023090437054634094f, + 0.011858539655804634f, + -0.0059988973662257195f, + 0.040219854563474655f, + 0.008092803880572319f, + 0.013441123999655247f, + 0.07714559137821198f, + 0.027036814019083977f, + 0.010920023545622826f, + -0.014534782618284225f, + 0.04086093232035637f, + 0.023914894089102745f, + 0.012763736769557f, + -0.04254480451345444f, + 0.011305714957416058f, + 0.014120567589998245f, + 0.0196896493434906f, + -0.017100676894187927f, + -0.0013453649589791894f, + -0.007299546152353287f, + -0.0016225805738940835f, + 0.02249031327664852f, + -0.035537295043468475f, + -0.03191295638680458f, + 0.04402138665318489f, + 0.045873962342739105f, + 0.033952441066503525f, + 0.004062877967953682f, + -0.016052523627877235f, + 0.033559925854206085f, + 0.040684811770915985f, + 0.0027443845756351948f, + -0.036945641040802f, + -0.03932597115635872f, + 0.014583739452064037f, + 0.009190680459141731f, + -0.01264051254838705f, + -0.005108111072331667f, + 0.04355351999402046f, + 0.03141055628657341f, + 0.04495703801512718f, + -0.009294744580984116f, + -0.035847608000040054f, + 0.019878702238202095f, + -0.02355729602277279f, + -0.014673866331577301f, + 0.021822089329361916f, + -0.0028103822842240334f, + 0.028539221733808517f, + 0.0329912044107914f, + 0.04047539830207825f, + -0.014253382571041584f, + -0.00660439720377326f, + -0.0639570951461792f, + -0.03520612418651581f, + 0.011432294733822346f, + 0.018043745309114456f, + -0.014185379259288311f, + 0.006499403156340122f, + 0.012511038221418858f, + 0.03146800398826599f, + 0.02470427192747593f, + -0.027075596153736115f, + -0.023012971505522728f, + -0.03645343333482742f, + -0.03147031366825104f, + 0.03326469659805298f, + -0.02293357253074646f, + 0.00007493131852243096f, + 0.027926260605454445f, + 0.0010919274063780904f, + -0.03810613974928856f, + -0.0009479214786551893f, + 0.01032050047069788f, + -0.02284139208495617f, + 0.022822747007012367f, + 0.013322929851710796f, + 0.01755167357623577f, + 0.006281423382461071f, + -0.05161968991160393f, + -0.03454432263970375f, + -0.015171839855611324f, + -0.013086988590657711f, + 0.009599068202078342f, + -0.023829564452171326f, + 0.03170377388596535f, + 0.025455616414546967f, + 0.006359482184052467f, + -0.006833407562226057f, + -0.02959762141108513f, + -0.04312501475214958f, + 0.020601991564035416f, + -0.04397542029619217f, + -0.011911196634173393f, + 0.030813507735729218f, + 0.04656203091144562f, + 0.043676819652318954f, + 0.06412668526172638f, + -0.02376391552388668f, + 0.020222248509526253f, + 0.048504240810871124f, + 0.04187004268169403f, + -0.03268779441714287f, + -0.013356110081076622f, + 0.003990362398326397f, + 0.03669435530900955f, + 0.03567170351743698f, + 0.02201189659535885f, + 0.007053814362734556f, + 0.02911461517214775f, + -0.016084877774119377f, + -0.013361542485654354f, + -0.0025507789105176926f, + -0.017344053834676743f, + 0.03305065259337425f, + 0.03494741767644882f, + 0.02428996004164219f, + -0.024909833446145058f, + -0.03706767037510872f, + -0.014623625203967094f, + 0.025253746658563614f, + -0.04022902622818947f, + 0.013688928447663784f, + 0.020935285836458206f, + -0.03215555474162102f, + -0.01734493114054203f, + -0.016057977452874184f, + 0.0332186259329319f, + -0.062192827463150024f, + -0.06338762491941452f, + -0.013384060002863407f, + 0.0031143389642238617f, + 0.007829300127923489f, + -0.05590514838695526f, + -0.028918664902448654f, + -0.018310556188225746f, + -0.02395123988389969f, + 0.0227365605533123f, + -0.037892408668994904f, + 0.01014055497944355f, + 0.056368228048086166f, + -0.000996291171759367f, + 0.05579328536987305f, + 0.018668418750166893f, + 0.027242204174399376f, + 0.05181897431612015f, + 0.02047695778310299f, + 0.037248481065034866f, + 0.048321932554244995f, + 0.0017545935697853565f, + 0.010634876787662506f, + -0.030261456966400146f, + -0.026554463431239128f, + 0.0051453569903969765f, + -0.013943018391728401f, + -0.04623245820403099f, + 0.0003986647061537951f, + 0.03525836020708084f, + 0.049685895442962646f, + -0.03695899248123169f, + 0.02713131159543991f, + 0.010237785056233406f, + -0.03279371187090874f, + -0.02466234378516674f, + 0.009988408535718918f, + 0.018033919855952263f, + -0.0013647808227688074f, + 0.03687773644924164f, + -0.0029583261348307133f, + 0.0067709567956626415f, + 0.03685348480939865f, + -0.031049998477101326f, + -0.001257769064977765f, + 0.04537429288029671f, + 0.010919678024947643f, + 0.04005173593759537f, + -0.013691303320229053f, + -0.002520656678825617f, + -0.01983926258981228f, + 0.02462667040526867f, + -0.01412965264171362f, + -0.01593015529215336f, + 0.056297656148672104f, + 0.03142832964658737f, + 0.009161954745650291f, + -0.02579585276544094f, + -0.031644586473703384f, + -0.01380220428109169f, + -0.04127073660492897f, + -0.04070030152797699f, + -0.048543404787778854f, + -0.05622323229908943f, + -0.0021230385173112154f, + -0.03724157437682152f, + -0.017728988081216812f, + -0.049515388906002045f, + -0.013393918052315712f, + 0.00653441995382309f, + -0.008645026944577694f, + -0.03468145802617073f, + -0.008628074079751968f, + 0.035358525812625885f, + -0.030219580978155136f, + 0.017552250996232033f, + 0.017619483172893524f, + 0.00875946693122387f, + -0.019581124186515808f, + 0.003319726325571537f, + -0.0051649208180606365f, + 0.03903667628765106f, + -0.0009742019465193152f, + 0.05749356374144554f, + 0.011132277548313141f, + -0.016711536794900894f, + 0.04648770019412041f, + -0.021648593246936798f, + -0.10148133337497711f, + -0.03178001940250397f, + 0.01529928483068943f, + -0.032578423619270325f, + -0.05097171664237976f, + 0.01338517852127552f, + 0.012569229118525982f, + -0.006618269253522158f, + -0.03592976555228233f, + 0.02533770725131035f, + 0.03580918163061142f, + -0.06759073585271835f, + 0.027629327028989792f, + 0.04527544975280762f, + 0.029171645641326904f, + 0.02645142935216427f, + -0.01734251342713833f, + -0.06954346597194672f, + 0.013422731310129166f, + -0.03871454671025276f, + 0.018256206065416336f, + -0.024370983242988586f, + 0.05545681715011597f, + -0.039528828114271164f, + -0.013660108670592308f, + -0.059706974774599075f, + 0.016853436827659607f, + 0.031912483274936676f, + 0.015487623400986195f, + 0.011767021380364895f, + -0.05857180431485176f, + -0.005341519135981798f, + -0.04033682122826576f, + 0.025871943682432175f, + -0.02402888983488083f, + 0.02561469003558159f, + 0.030010027810931206f, + -0.03678053244948387f, + 0.03396481275558472f, + -0.07291311025619507f, + 0.02396109327673912f, + 0.025626540184020996f, + 0.00955990795046091f, + 0.013724202290177345f, + -0.014750544913113117f, + -0.032967016100883484f, + 0.02058098278939724f, + 0.0033076023682951927f, + -0.029527032747864723f, + -0.0009443678427487612f, + -0.02781200222671032f, + 0.02402203157544136f, + 0.014563879929482937f, + -0.011585783213376999f, + -0.026427732780575752f, + 0.019133128225803375f, + 0.021863577887415886f, + 0.006209978833794594f, + 0.07440700381994247f, + 0.008599634282290936f, + -0.04177406057715416f, + 0.02020418830215931f, + -0.004323258996009827f, + 0.005790286231786013f, + -0.025323275476694107f, + 0.058458585292100906f, + -0.03278389200568199f, + -0.011032545939087868f, + -0.0036148857325315475f, + -0.0004926437395624816f, + -0.022332727909088135f, + -0.04997304454445839f, + -0.0007595937349833548f, + 0.03481508418917656f, + 0.020037902519106865f, + -0.021608928218483925f, + 0.01581759564578533f, + -0.011867176741361618f, + -0.00947039108723402f, + -0.07732773572206497f, + -0.011982443742454052f, + 0.023730168119072914f, + -0.04481136426329613f, + -0.011057991534471512f, + -0.04485539346933365f, + 0.01307082362473011f, + -0.03840833529829979f, + -0.005540383048355579f, + -0.03217718377709389f, + 0.19306251406669617f, + 0.01557007897645235f, + 0.03433694690465927f, + -0.015490150079131126f, + 0.014035001397132874f, + 0.008068409748375416f, + 0.029123784974217415f, + 0.010411231778562069f, + 0.03180669620633125f, + 0.016272393986582756f, + -0.03405269607901573f, + 0.008909656666219234f, + 0.026794541627168655f, + -0.007454108912497759f, + 0.016099030151963234f, + 0.05538764223456383f, + 0.008271883241832256f, + 0.023237742483615875f, + 0.034973062574863434f, + -0.0081742312759161f, + 0.04468708857893944f, + -0.03461684659123421f, + 0.03345729038119316f, + 0.0005239890888333321f, + -0.056889843195676804f, + -0.042187951505184174f, + -0.007005332037806511f, + 0.031517598778009415f, + -0.057308562099933624f, + 0.03141198307275772f, + 0.0007111141458153725f, + 0.028522223234176636f, + -0.04995636269450188f, + -0.022562386468052864f, + 0.039314135909080505f, + 0.015615178272128105f, + 0.02232501097023487f, + -0.03433450683951378f, + 0.012382658198475838f, + -0.02625468373298645f, + -0.0083118611946702f, + 0.017250170931220055f, + -0.02019653283059597f, + 0.03392040356993675f, + 0.0013320763828232884f, + 0.04296816885471344f, + 0.05378912016749382f, + -0.018141886219382286f, + -0.02347896434366703f, + 0.03130969777703285f, + 0.021074768155813217f, + 0.011518022045493126f, + -0.004174298606812954f, + -0.03582172095775604f, + -0.015318387188017368f, + -0.007444401737302542f, + -0.03830752149224281f, + -0.06777879595756531f, + -0.006020883098244667f, + 0.008447406813502312f, + 0.062179744243621826f, + 0.01918203756213188f, + -0.016178149729967117f, + -0.023090755566954613f, + 0.028789259493350983f, + 0.040439531207084656f, + -0.02672472968697548f, + -0.04095323011279106f, + -0.05732221528887749f, + -0.0020660210866481066f, + -0.055687226355075836f, + -0.008511627092957497f, + -0.039763759821653366f, + 0.03596954047679901f, + 0.033216822892427444f, + -0.005118542350828648f, + -0.03271672502160072f, + 0.018839454278349876f, + 0.016132114455103874f, + 0.028364259749650955f, + 0.006845584139227867f, + -0.0012978515587747097f, + -0.018664628267288208f, + 0.03485922887921333f, + 0.04039550945162773f, + -0.031199388206005096f, + 0.01957610994577408f, + 0.03947604075074196f, + -0.04612560197710991f, + 0.022226179018616676f, + 0.03061433508992195f, + 0.007180668413639069f, + -0.033031608909368515f, + 0.05610867962241173f, + 0.0284646637737751f, + -0.05204550921916962f, + 0.017941653728485107f, + 0.022062553092837334f, + -0.04683050885796547f, + -0.023560887202620506f, + -0.0276334211230278f, + 0.0041246069595217705f, + -0.032277919352054596f, + -0.01940121315419674f, + 0.0013466235250234604f, + -0.024347569793462753f, + 0.01447154488414526f, + 0.03692981228232384f, + 0.02556624449789524f, + -0.013659082353115082f, + 0.030253291130065918f, + 0.004965346772223711f, + 0.023664172738790512f, + -0.01723453216254711f, + 0.05455326661467552f, + 0.02362307533621788f, + 0.022859739139676094f, + -0.054154444485902786f, + 0.008361054584383965f, + 0.009449029341340065f, + -0.014727693982422352f, + 0.010806793347001076f, + -0.04166347533464432f, + 0.07135552912950516f, + -0.007770725060254335f, + 0.030580557882785797f, + -0.04844602942466736f, + 0.0008229002705775201f, + 0.026535481214523315f, + 0.021868199110031128f, + 0.004678168799728155f, + -0.03931296244263649f, + -0.04464258253574371f, + 0.03712749108672142f, + 0.014044600538909435f, + -0.0023832779843360186f, + 0.08839696645736694f, + -0.011742178350687027f, + -0.02589561603963375f, + 0.04511994495987892f, + -0.011089873500168324f, + 0.04424068331718445f, + -0.031209222972393036f, + 0.018096735700964928f, + 0.0276931282132864f, + 0.032378826290369034f, + -0.031537868082523346f, + -0.027904240414500237f, + -0.015190032310783863f, + -0.004481968469917774f, + 0.04267747700214386f, + -0.028845272958278656f, + 0.019086001440882683f, + 0.009203453548252583f, + -0.039411865174770355f, + -0.06638588011264801f, + 0.019709186628460884f, + 0.03591395914554596f, + 0.05008409172296524f, + -0.011427966877818108f, + -0.04971498250961304f, + -0.004772470332682133f, + 0.01432930026203394f, + 0.1095755398273468f, + 0.02603503316640854f, + 0.019698046147823334f, + -0.032637741416692734f, + 0.033184606581926346f, + 0.02692507766187191f, + 0.010238883085548878f, + -0.010183892212808132f, + -0.029479920864105225f, + -0.019146805629134178f, + 0.02746221050620079f, + 0.021641163155436516f, + -0.024658724665641785f, + -0.01630176603794098f, + 0.02958461083471775f, + -0.04108947888016701f, + 0.032371439039707184f, + -0.027287479490041733f, + 0.020171169191598892f, + 0.03383186459541321f, + -0.015322552062571049f, + -0.010307767428457737f, + 0.013115078210830688f, + 0.005613786168396473f, + -0.02910318784415722f, + -0.021252978593111038f, + 0.005560233723372221f, + -0.04231780767440796f, + 0.08192117512226105f, + -0.029788488522171974f, + -0.0016719227423891425f, + -0.03858398273587227f, + -0.027913816273212433f, + 0.012290316633880138f, + 0.015825187787413597f, + -0.026750769466161728f, + -0.039809394627809525f, + 0.010853637009859085f, + -0.04720485955476761f, + -0.010609184391796589f, + -0.01278828363865614f, + -0.03623141720890999f, + -0.020634643733501434f, + 0.030923841521143913f, + 0.03285284712910652f, + 0.01452653668820858f, + 0.03416169807314873f, + 0.010913162492215633f, + -0.024978244677186012f, + 0.03296514227986336f, + 0.02161172777414322f, + 0.030312156304717064f, + 0.03071526065468788f, + 0.04639757424592972f, + 0.031048797070980072f, + 0.03407664597034454f, + -0.041457146406173706f, + 0.011761465109884739f, + 0.012112592346966267f, + 0.019076142460107803f, + -0.059248920530080795f, + -0.04760892689228058f, + -0.0314653106033802f, + 0.016123836860060692f, + 0.006126987282186747f, + -0.031283795833587646f, + -0.0035350737161934376f, + 0.012261965312063694f, + -0.024649467319250107f, + -0.009447114542126656f, + -0.08359000086784363f, + 0.005851270165294409f, + -0.030149782076478004f, + 0.027616258710622787f, + 0.03182379528880119f, + 0.010887391865253448f, + -0.027072472497820854f, + 0.020552828907966614f, + 0.017932042479515076f, + 0.04406345263123512f, + -0.028712112456560135f, + -0.008050181902945042f, + 0.04549504071474075f, + -0.06913027167320251f, + -0.004572774283587933f, + 0.00011219691077712923f, + -0.006029843352735043f, + -0.012317972257733345f, + -0.04016095772385597f, + 0.030960263684391975f, + 0.039273906499147415f, + -0.015845293179154396f, + -0.011702924966812134f, + 0.011656280606985092f, + -0.03036012314260006f, + -0.08103150129318237f, + -0.023546282202005386f, + 0.016684504225850105f, + 0.0006793781649321318f, + -0.030003944411873817f, + 0.017032481729984283f, + -0.023390373215079308f, + 0.018340930342674255f, + 0.024219363927841187f, + -0.03645186498761177f, + -0.018286602571606636f, + -0.048507433384656906f, + 0.046701669692993164f, + -0.020781058818101883f, + -0.01605192758142948f, + -0.0903584435582161f, + 0.06904299557209015f, + 0.015601576305925846f, + -0.018518496304750443f, + -0.038458120077848434f, + -0.019991688430309296f, + -0.024300318211317062f, + -0.012263032607734203f, + 0.024427302181720734f, + -0.024809060618281364f, + 0.019496100023388863f, + 0.01758641004562378f, + 0.006886046379804611f, + -0.03860282525420189f, + 0.009409677237272263f, + -0.013964204117655754f, + -0.04017601162195206f, + -0.016977611929178238f, + 0.02575852908194065f, + -0.013815070502460003f, + 0.006740113254636526f, + -0.04932718351483345f, + -0.05205217003822327f, + -0.04415235295891762f, + -0.01197398267686367f, + 0.011467654258012772f, + -0.03154896944761276f, + 0.0028590233996510506f, + -0.03314562886953354f, + -0.038118354976177216f, + -0.03914812579751015f, + -0.0005633080145344138f, + 0.005159502383321524f, + -0.009560227394104004f, + -0.025696488097310066f, + 0.03951350972056389f, + 0.04184769093990326f, + -0.010419424623250961f, + -0.03487123176455498f, + 0.001081227557733655f, + 0.031209642067551613f, + 0.01224304549396038f, + -0.012603535316884518f, + 0.029348352923989296f, + -0.026243818923830986f, + -0.023612577468156815f, + 0.04481798782944679f, + -0.04039709270000458f, + -0.01403393130749464f, + -0.022715067490935326f, + 0.008543731644749641f, + 0.04796065390110016f, + 0.00940101407468319f, + 0.03649118170142174f, + 0.016881216317415237f, + 0.013232183642685413f, + 0.014445126056671143f, + 0.010655111633241177f, + -0.015311007387936115f, + -0.03107623942196369f, + -0.012181959114968777f, + -0.030173713341355324f, + -0.03136519715189934f, + -0.00968134868890047f, + -0.004136156756430864f, + 0.0005978004774078727f, + -0.05459468439221382f, + -0.015934104099869728f, + 0.053266167640686035f, + 0.03784262016415596f, + -0.010738855227828026f, + 0.03190033510327339f, + -0.001496695913374424f, + -0.024366231635212898f, + 0.03414757177233696f, + -0.0077161891385912895f, + -0.044079799205064774f, + -0.015896553173661232f, + 0.03637973219156265f, + -0.02161337621510029f, + 0.006903046742081642f, + -0.006962891202419996f, + 0.05574609339237213f, + -0.029158806428313255f, + 0.03076884150505066f, + -0.009158081375062466f, + -0.04707910493016243f, + -0.024784710258245468f, + 0.015435277484357357f, + 0.007692893035709858f, + 0.0006725761923007667f, + -0.038610704243183136f, + 0.009134447202086449f, + 0.009849862195551395f, + 0.024519752711057663f, + -0.03863726183772087f, + 0.007874547503888607f, + -0.020069731399416924f, + 0.007399733178317547f, + -0.05068531632423401f, + -0.009004154242575169f, + -0.04068838059902191f, + 0.040781937539577484f, + -0.02937493473291397f, + 0.016979916021227837f, + 0.037655118852853775f, + -0.01305563747882843f, + 0.013344315811991692f, + -0.05640937015414238f, + -0.020350581035017967f, + -0.02501261793076992f, + 0.038834795355796814f, + -0.04604274779558182f, + 0.0004385723441373557f, + 0.023745035752654076f, + 0.003656078828498721f, + 0.039705485105514526f, + -0.011241079308092594f, + 0.044178713113069534f, + -0.011368539184331894f, + 0.012771987356245518f, + -0.08492542803287506f, + -0.009417480789124966f, + -0.010677497833967209f, + -0.03395388647913933f, + -0.011490026488900185f, + -0.006386809982359409f, + -0.014871163293719292f, + 0.004204850643873215f, + 0.029643211513757706f, + -0.04543495550751686f, + 0.02437104657292366f, + 0.015322508290410042f, + 0.004863101989030838f, + -0.002539763692766428f, + 0.015903716906905174f, + 0.016624106094241142f, + 0.023341473191976547f, + -0.030201323330402374f, + 0.007918000221252441f, + 0.0009107357473112643f, + 0.03916248679161072f, + -0.010109283030033112f, + 0.018451586365699768f, + 0.0062872618436813354f, + 0.023154068738222122f, + 0.028111852705478668f, + -0.014428956434130669f, + 0.036793746054172516f, + -0.03575288504362106f, + -0.057246819138526917f, + -0.021768808364868164f, + -0.023564910516142845f, + 0.02397366799414158f, + 0.020718568935990334f, + -0.015275984071195126f, + 0.03703267127275467f, + -0.010308920405805111f, + -0.04211723431944847f, + -0.01663627289235592f, + 0.02286018803715706f, + -0.03881242126226425f, + 0.03525247052311897f, + -0.042307011783123016f, + -0.003567407839000225f, + 0.001457449048757553f, + 0.05936582013964653f, + 0.002895828103646636f, + -0.025598157197237015f, + 0.01283274032175541f, + -0.05007452890276909f, + -0.03235140070319176f, + -0.011465230956673622f, + 0.026079369708895683f, + 0.058149781078100204f, + 0.024806367233395576f, + 0.01770002953708172f, + 0.0005065941368229687f, + -0.026064587756991386f, + 0.015047438442707062f, + 0.03801387920975685f, + 0.01752178929746151f, + -0.02259741723537445f, + -0.03879963234066963f, + -0.02445252798497677f, + -0.029194211587309837f, + -0.04118459299206734f, + -0.07230067998170853f, + -0.009569251909852028f, + -0.01829957403242588f, + 0.03917815536260605f, + -0.019530808553099632f, + -0.03650574013590813f, + 0.028397979214787483f, + -0.026918869465589523f, + 0.023724351078271866f, + -0.011466676369309425f, + -0.040093325078487396f, + 0.010017112828791142f, + 0.012075957842171192f, + 0.010774342343211174f, + -0.016421642154455185f, + -0.04688034579157829f, + 0.02431156113743782f, + -0.003046917961910367f, + -0.0010734435636550188f, + 0.022572355344891548f, + 0.011197290383279324f, + -0.025137169286608696f, + -0.049277711659669876f, + 0.04823236167430878f, + -0.04765353724360466f, + -0.018735116347670555f, + 0.0003122256603091955f, + -0.008810674771666527f, + -0.017892984673380852f, + 0.00021120262681506574f, + -0.031895555555820465f, + -0.023480713367462158f, + 0.010128460824489594f, + 0.016516825184226036f, + -0.019813979044556618f, + -0.0044202678836882114f, + -0.00702977180480957f, + -0.002187698846682906f, + 0.013112161308526993f, + 0.008308488875627518f, + 0.03817952051758766f, + -0.020533859729766846f, + 0.028264036402106285f, + -0.04136377573013306f, + -0.05692916363477707f, + -0.021915340796113014f, + -0.019034316763281822f, + -0.015118561685085297f, + -0.0319315642118454f, + 0.03268538787961006f, + -0.0010479907505214214f, + 0.00024182928609661758f, + -0.0012983399210497737f, + -0.0035084697883576155f, + -0.01018751785159111f, + -0.06536149233579636f, + 0.03195962309837341f, + -0.004426909144967794f, + 0.0029187293257564306f, + -0.0256631039083004f, + 0.036579765379428864f, + -0.06405307352542877f, + 0.013227395713329315f, + -0.03624671325087547f, + 0.037127405405044556f, + -0.06633934378623962f, + 0.023713771253824234f, + 0.001657120301388204f, + -0.030938217416405678f, + -0.005628079175949097f, + -0.053462110459804535f, + 0.07506952434778214f, + -0.007784669753164053f, + 0.028080616146326065f, + -0.01574862003326416f, + -0.04546362906694412f, + -0.05016990005970001f, + -0.05019865557551384f, + -0.039244186133146286f, + 0.030151229351758957f, + -0.05171814560890198f, + -0.03881967440247536f, + -0.009908814914524555f, + -0.016938813030719757f, + 0.011704900301992893f, + -0.014907843433320522f, + -0.03325396403670311f, + 0.047350019216537476f, + 0.0045817457139492035f, + 0.028241490945219994f, + -0.003765946952626109f, + 0.01571817509829998f, + 0.03741828724741936f, + 0.0156949982047081f, + -0.01970336213707924f, + 0.009402992203831673f, + -0.03331432864069939f, + 0.025959284976124763f, + -0.0041138906963169575f, + -0.040316756814718246f, + 0.0012001245049759746f, + 0.009792031720280647f, + -0.047768693417310715f, + -0.02163655124604702f, + 0.018602421507239342f, + 0.000602972402703017f, + -0.03311364725232124f, + 0.026526229456067085f, + 0.04202568158507347f, + 0.008449001237750053f, + 0.06475643813610077f, + 0.0035848424304276705f, + 0.0488557443022728f, + 0.020431524142622948f, + 0.010165947489440441f, + 0.00012580514885485172f, + 0.011490631848573685f, + 0.049949027597904205f, + 0.03915407881140709f, + 0.059800226241350174f, + 0.024663494899868965f, + 0.02410218119621277f, + -0.014562365598976612f, + 0.02789120003581047f, + -0.03704062104225159f, + 0.02282542549073696f, + -0.047931917011737823f, + 0.05969615653157234f, + 0.038760412484407425f, + -0.007845507934689522f, + -0.0011439616791903973f, + 0.014877899549901485f, + -0.033575110137462616f, + -0.021374600008130074f, + 0.014881900511682034f, + 0.01631966605782509f, + 0.04771406203508377f, + -0.023484190925955772f, + -0.03400464728474617f, + -0.018462782725691795f, + -0.056461531668901443f, + 0.02125520072877407f, + -0.04083051532506943f, + 0.04490841552615166f, + 0.05638275668025017f, + -0.03922900930047035f, + 0.015876512974500656f, + 0.02240809239447117f, + -0.00000876183094078442f, + 0.041717130690813065f, + 0.041573211550712585f, + -0.014516608789563179f, + -0.020156629383563995f, + 0.03390810638666153f, + 0.04974587634205818f, + 0.011070906184613705f, + -0.0019445940852165222f, + 0.014449193142354488f, + -0.02405550517141819f, + 0.021492162719368935f, + -0.05518610030412674f, + 0.01799919828772545f, + -0.039734020829200745f, + -0.0037850206717848778f, + 0.007841100916266441f, + 0.014386530965566635f, + 0.02055184915661812f, + -0.0069200219586491585f, + -0.036611758172512054f, + -0.044122129678726196f, + 0.045821838080883026f, + 0.012124829925596714f, + -0.00812555756419897f, + 0.02017771080136299f, + -0.0007992296013981104f, + 0.003344304393976927f, + -0.007832828909158707f, + 0.0202498622238636f + }; +}