Merge branch 'main' into dependabot/pip/code/streamlit-1.31.0

Azure-Samples · Feb 16, 2024 · f11e08f · f11e08f
2 parents b40e733 + f7dd41e
commit f11e08f
Show file tree

Hide file tree

Showing 21 changed files with 363 additions and 146 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -23,6 +23,7 @@
 				"ms-azuretools.vscode-bicep",
 				"ms-azuretools.vscode-docker",
 				"ms-python.python",
+				"ms-python.black-formatter",
 				"ms-python.vscode-pylance",
 				"ms-vscode.vscode-node-azure-pack",
 				"TeamsDevApp.ms-teams-vscode-extension"

diff --git a/.github/workflows/AdminWebApp.yml b/.github/workflows/AdminWebApp.yml
@@ -13,7 +13,7 @@ jobs:
     steps:
 
     - name: Docker Login
-      uses: docker/login-action@v3.0.0
+      uses: docker/login-action@v3
       with:
         registry: fruoccopublic.azurecr.io
         username: fruoccopublic

diff --git a/.github/workflows/Backend.yml b/.github/workflows/Backend.yml
@@ -13,7 +13,7 @@ jobs:
     steps:
 
     - name: Docker Login
-      uses: docker/login-action@v3.0.0
+      uses: docker/login-action@v3
       with:
         registry: fruoccopublic.azurecr.io
         username: fruoccopublic

diff --git a/.github/workflows/CwyodBackend.yml b/.github/workflows/CwyodBackend.yml
@@ -13,7 +13,7 @@ jobs:
     steps:
 
     - name: Docker Login
-      uses: docker/login-action@v3.0.0
+      uses: docker/login-action@v3
       with:
         registry: fruoccopublic.azurecr.io
         username: fruoccopublic

diff --git a/.github/workflows/WebApp.yml b/.github/workflows/WebApp.yml
@@ -13,7 +13,7 @@ jobs:
     steps:
 
     - name: Docker Login
-      uses: docker/login-action@v3.0.0
+      uses: docker/login-action@v3
       with:
         registry: fruoccopublic.azurecr.io
         username: fruoccopublic

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -16,10 +16,10 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2 
+        uses: docker/login-action@v3 
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}

diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml
@@ -20,6 +20,6 @@ jobs:
             architecture: x64
         - name: Install dependencies
           run: |
-            pip install -r code/requirements.txt -r code/dev-requirements.txt
+            pip install -r code/requirements.txt -r code/dev-requirements.txt -r code/app/requirements.txt
         - name: Run Python tests
           run: python -m pytest --rootdir=code -m "not azure"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -4,9 +4,22 @@ repos:
     hooks:
       - id: black
         language_version: python3
-  
+
   - repo: https://github.com/pycqa/flake8
     rev: 7.0.0
     hooks:
-    -   id: flake8
-        args: [--extend-ignore=E501]
+      - id: flake8
+        args: [--extend-ignore=E501]
+
+  - repo: local
+    hooks:
+      - id: bicep
+        name: bicep
+        description: Lint and build Bicep files
+        entry: ./scripts/generate_arm_templates.sh
+        language: script
+        files: \.bicep$
+        require_serial: true
+        args: # Bicep files that we want to generate ARM templates from
+          - -f=./infra/deployment.bicep
+          - -f=./extensions/infrastructure/main.bicep
diff --git a/code/app/app.py b/code/app/app.py
@@ -313,11 +313,21 @@ def conversation_azure_byod():
         )
 
 
-@app.route("/api/conversation/custom", methods=["GET", "POST"])
-def conversation_custom():
+def get_message_orchestrator():
     from utilities.helpers.OrchestratorHelper import Orchestrator
 
-    message_orchestrator = Orchestrator()
+    return Orchestrator()
+
+
+def get_orchestrator_config():
+    from utilities.helpers.ConfigHelper import ConfigHelper
+
+    return ConfigHelper.get_active_config_or_default().orchestrator
+
+
+@app.route("/api/conversation/custom", methods=["GET", "POST"])
+def conversation_custom():
+    message_orchestrator = get_message_orchestrator()
 
     try:
         user_message = request.json["messages"][-1]["content"]
@@ -328,22 +338,12 @@ def conversation_custom():
                 request.json["messages"][0:-1],
             )
         )
-        chat_history = []
-        for i, k in enumerate(user_assistant_messages):
-            if i % 2 == 0:
-                chat_history.append(
-                    (
-                        user_assistant_messages[i]["content"],
-                        user_assistant_messages[i + 1]["content"],
-                    )
-                )
-        from utilities.helpers.ConfigHelper import ConfigHelper
 
         messages = message_orchestrator.handle_message(
             user_message=user_message,
-            chat_history=chat_history,
+            chat_history=user_assistant_messages,
             conversation_id=conversation_id,
-            orchestrator=ConfigHelper.get_active_config_or_default().orchestrator,
+            orchestrator=get_orchestrator_config(),
         )
 
         response_obj = {

diff --git a/code/app/requirements.txt b/code/app/requirements.txt
@@ -1,5 +1,5 @@
 azure-identity==1.15.0
-Flask==3.0.0
+Flask==3.0.2
 openai==0.27.8
 azure-storage-blob==12.19.0
 python-dotenv==1.0.1

diff --git a/code/dev-requirements.txt b/code/dev-requirements.txt
@@ -1,5 +1,5 @@
 pytest==8.0.0
 pytest-cov==4.1.0
 flake8==7.0.0
-pre-commit==3.5.0
-black==23.12.1
+pre-commit==3.6.0
+black==24.1.1
diff --git a/code/utilities/helpers/ConfigHelper.py b/code/utilities/helpers/ConfigHelper.py
@@ -184,6 +184,24 @@ def get_default_config():
                     },
                     "loading": {"strategy": LoadingStrategy.DOCX},
                 },
+                {
+                    "document_type": "jpg",
+                    "chunking": {
+                        "strategy": ChunkingStrategy.LAYOUT,
+                        "size": 500,
+                        "overlap": 100,
+                    },
+                    "loading": {"strategy": LoadingStrategy.LAYOUT},
+                },
+                {
+                    "document_type": "png",
+                    "chunking": {
+                        "strategy": ChunkingStrategy.LAYOUT,
+                        "size": 500,
+                        "overlap": 100,
+                    },
+                    "loading": {"strategy": LoadingStrategy.LAYOUT},
+                },
             ],
             "logging": {"log_user_interactions": True, "log_tokens": True},
             "orchestrator": {"strategy": "openai_function"},

diff --git a/code/utilities/orchestrator/LangChainAgent.py b/code/utilities/orchestrator/LangChainAgent.py
@@ -89,8 +89,10 @@ def orchestrate(
             memory_key="chat_history", return_messages=True
         )
         for message in chat_history:
-            memory.chat_memory.add_user_message(message[0])
-            memory.chat_memory.add_ai_message(message[1])
+            if message["role"] == "user":
+                memory.chat_memory.add_user_message(message["content"])
+            elif message["role"] == "assistant":
+                memory.chat_memory.add_ai_message(message["content"])
         # Define Agent and Agent Chain
         llm_chain = LLMChain(llm=llm_helper.get_llm(), prompt=prompt)
         agent = ZeroShotAgent(llm_chain=llm_chain, tools=self.tools, verbose=True)

diff --git a/code/utilities/orchestrator/OpenAIFunctions.py b/code/utilities/orchestrator/OpenAIFunctions.py
@@ -81,8 +81,7 @@ def orchestrate(
         # Create conversation history
         messages = [{"role": "system", "content": system_message}]
         for message in chat_history:
-            messages.append({"role": "user", "content": message[0]})
-            messages.append({"role": "assistant", "content": message[1]})
+            messages.append({"role": message["role"], "content": message["content"]})
         messages.append({"role": "user", "content": user_message})
 
         result = llm_helper.get_chat_completion_with_functions(

diff --git a/extensions/infrastructure/main.bicep b/extensions/infrastructure/main.bicep
@@ -119,7 +119,7 @@ param AzureFormRecognizerEndpoint string
 @secure()
 param AzureFormRecognizerKey string
 
-@description('Storage Account Name - Created during the "Chat with your data" Solution Accelerator') 
+@description('Storage Account Name - Created during the "Chat with your data" Solution Accelerator')
 param AzureBlobAccountName string
 
 @description('Storage Account Key - Created during the "Chat with your data" Solution Accelerator')
@@ -151,39 +151,39 @@ resource Function 'Microsoft.Web/sites@2018-11-01' = {
   properties: {
     siteConfig: {
       appSettings: [
-        { name: 'FUNCTIONS_EXTENSION_VERSION', value: '~4'}
-        { name: 'WEBSITES_ENABLE_APP_SERVICE_STORAGE', value: 'false'}
-        { name: 'APPINSIGHTS_CONNECTION_STRING', value: AppInsightsConnectionString}
-        { name: 'AZURE_SEARCH_SERVICE', value: 'https://${AzureSearchName}.search.windows.net'}
-        { name: 'AZURE_SEARCH_INDEX', value: AzureSearchIndex}
-        { name: 'AZURE_SEARCH_CONVERSATIONS_LOG_INDEX', value: AzureSearchConversationLogIndex}
-        { name: 'AZURE_SEARCH_KEY', value: AzureSearchKey}
-        { name: 'AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG', value: AzureSearchSemanticSearchConfig}
-        { name: 'AZURE_SEARCH_INDEX_IS_PRECHUNKED', value: AzureSearchIndexIsPrechunked}
-        { name: 'AZURE_SEARCH_TOP_K', value: AzureSearchTopK}
-        { name: 'AZURE_SEARCH_ENABLE_IN_DOMAIN', value: AzureSearchEnableInDomain}
-        { name: 'AZURE_SEARCH_CONTENT_COLUMNS', value: AzureSearchContentColumns}
-        { name: 'AZURE_SEARCH_FILENAME_COLUMN', value: AzureSearchFilenameColumn}
-        { name: 'AZURE_SEARCH_TITLE_COLUMN', value: AzureSearchTitleColumn}
-        { name: 'AZURE_SEARCH_URL_COLUMN', value: AzureSearchUrlColumn}
-        { name: 'AZURE_OPENAI_RESOURCE', value: AzureOpenAIResource}
-        { name: 'AZURE_OPENAI_KEY', value: AzureOpenAIKey}
-        { name: 'AZURE_OPENAI_MODEL', value: AzureOpenAIModel}
-        { name: 'AZURE_OPENAI_MODEL_NAME', value: AzureOpenAIModelName}
-        { name: 'AZURE_OPENAI_TEMPERATURE', value: AzureOpenAITemperature}
-        { name: 'AZURE_OPENAI_TOP_P', value: AzureOpenAITopP}
-        { name: 'AZURE_OPENAI_MAX_TOKENS', value: AzureOpenAIMaxTokens}
-        { name: 'AZURE_OPENAI_STOP_SEQUENCE', value: AzureOpenAIStopSequence}
-        { name: 'AZURE_OPENAI_SYSTEM_MESSAGE', value: AzureOpenAISystemMessage}
-        { name: 'AZURE_OPENAI_API_VERSION', value: AzureOpenAIApiVersion}
-        { name: 'AZURE_OPENAI_STREAM', value: AzureOpenAIStream}
-        { name: 'AZURE_OPENAI_EMBEDDING_MODEL', value: AzureOpenAIEmbeddingModel}
-        { name: 'AZURE_FORM_RECOGNIZER_ENDPOINT', value: AzureFormRecognizerEndpoint}
-        { name: 'AZURE_FORM_RECOGNIZER_KEY', value: AzureFormRecognizerKey}
-        { name: 'AZURE_BLOB_ACCOUNT_NAME', value: AzureBlobAccountName}
-        { name: 'AZURE_BLOB_ACCOUNT_KEY', value: AzureBlobAccountKey}
-        { name: 'AZURE_BLOB_CONTAINER_NAME', value: AzureBlobContainerName}
-        { name: 'ORCHESTRATION_STRATEGY', value: OrchestrationStrategy}
+        { name: 'FUNCTIONS_EXTENSION_VERSION', value: '~4' }
+        { name: 'WEBSITES_ENABLE_APP_SERVICE_STORAGE', value: 'false' }
+        { name: 'APPINSIGHTS_CONNECTION_STRING', value: AppInsightsConnectionString }
+        { name: 'AZURE_SEARCH_SERVICE', value: 'https://${AzureSearchName}.search.windows.net' }
+        { name: 'AZURE_SEARCH_INDEX', value: AzureSearchIndex }
+        { name: 'AZURE_SEARCH_CONVERSATIONS_LOG_INDEX', value: AzureSearchConversationLogIndex }
+        { name: 'AZURE_SEARCH_KEY', value: AzureSearchKey }
+        { name: 'AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG', value: AzureSearchSemanticSearchConfig }
+        { name: 'AZURE_SEARCH_INDEX_IS_PRECHUNKED', value: AzureSearchIndexIsPrechunked }
+        { name: 'AZURE_SEARCH_TOP_K', value: AzureSearchTopK }
+        { name: 'AZURE_SEARCH_ENABLE_IN_DOMAIN', value: AzureSearchEnableInDomain }
+        { name: 'AZURE_SEARCH_CONTENT_COLUMNS', value: AzureSearchContentColumns }
+        { name: 'AZURE_SEARCH_FILENAME_COLUMN', value: AzureSearchFilenameColumn }
+        { name: 'AZURE_SEARCH_TITLE_COLUMN', value: AzureSearchTitleColumn }
+        { name: 'AZURE_SEARCH_URL_COLUMN', value: AzureSearchUrlColumn }
+        { name: 'AZURE_OPENAI_RESOURCE', value: AzureOpenAIResource }
+        { name: 'AZURE_OPENAI_KEY', value: AzureOpenAIKey }
+        { name: 'AZURE_OPENAI_MODEL', value: AzureOpenAIModel }
+        { name: 'AZURE_OPENAI_MODEL_NAME', value: AzureOpenAIModelName }
+        { name: 'AZURE_OPENAI_TEMPERATURE', value: AzureOpenAITemperature }
+        { name: 'AZURE_OPENAI_TOP_P', value: AzureOpenAITopP }
+        { name: 'AZURE_OPENAI_MAX_TOKENS', value: AzureOpenAIMaxTokens }
+        { name: 'AZURE_OPENAI_STOP_SEQUENCE', value: AzureOpenAIStopSequence }
+        { name: 'AZURE_OPENAI_SYSTEM_MESSAGE', value: AzureOpenAISystemMessage }
+        { name: 'AZURE_OPENAI_API_VERSION', value: AzureOpenAIApiVersion }
+        { name: 'AZURE_OPENAI_STREAM', value: AzureOpenAIStream }
+        { name: 'AZURE_OPENAI_EMBEDDING_MODEL', value: AzureOpenAIEmbeddingModel }
+        { name: 'AZURE_FORM_RECOGNIZER_ENDPOINT', value: AzureFormRecognizerEndpoint }
+        { name: 'AZURE_FORM_RECOGNIZER_KEY', value: AzureFormRecognizerKey }
+        { name: 'AZURE_BLOB_ACCOUNT_NAME', value: AzureBlobAccountName }
+        { name: 'AZURE_BLOB_ACCOUNT_KEY', value: AzureBlobAccountKey }
+        { name: 'AZURE_BLOB_CONTAINER_NAME', value: AzureBlobContainerName }
+        { name: 'ORCHESTRATION_STRATEGY', value: OrchestrationStrategy }
       ]
       cors: {
         allowedOrigins: [
@@ -214,4 +214,4 @@ resource WaitFunctionDeploymentSection 'Microsoft.Resources/deploymentScripts@20
   dependsOn: [
     Function
   ]
-}
+}
diff --git a/extensions/teams/infra/azure.bicep b/extensions/teams/infra/azure.bicep
@@ -79,4 +79,4 @@ module azureBotRegistration './botRegistration/azurebot.bicep' = {
 
 // The output will be persisted in .env.{envName}. Visit https://aka.ms/teamsfx-actions/arm-deploy for more details.
 output BOT_AZURE_APP_SERVICE_RESOURCE_ID string = webApp.id
-output BOT_DOMAIN string = webApp.properties.defaultHostName
+output BOT_DOMAIN string = webApp.properties.defaultHostName
diff --git a/extensions/teams/infra/botRegistration/azurebot.bicep b/extensions/teams/infra/botRegistration/azurebot.bicep
@@ -34,4 +34,4 @@ resource botServiceMsTeamsChannel 'Microsoft.BotService/botServices/channels@202
   properties: {
     channelName: 'MsTeamsChannel'
   }
-}
+}
-Original file line number
+Diff line change
@@ Expand Up @@
       properties: {
         channelName: 'MsTeamsChannel'
       }
-    }
+    }