Update to latest version of most dependencies (#255)

* Bump to latest versions

* Downgrade azure-search-documents latest version is not compatible with
langchaig
- Discussion: langchain-ai/langchain#13245
- PR to fix: langchain-ai/langchain#15659

* Migrate to new content safety api

- Old: https://github.com/Azure-Samples/AzureAIContentSafety/blob/main/python/1.0.0b1/sample_analyze_text.py#L37
- New: https://github.com/Azure-Samples/AzureAIContentSafety/blob/main/python/1.0.0/sample_analyze_text.py#L39

* Downgrade openai

The changes are too vast to be included in this PR

* Bump dependencies

* Fix test and bump langchain to latest version

code/app/requirements.txt

@@ -1,18 +1,18 @@
 azure-identity==1.15.0
-Flask==2.3.2
+Flask==3.0.0
 openai==0.27.8
 azure-storage-blob==12.19.0
-python-dotenv==1.0.0
-langchain==0.0.329
+python-dotenv==1.0.1
+langchain==0.1.4
 azure-ai-formrecognizer==3.3.2
 requests==2.31.0
 tiktoken==0.5.2
 azure-storage-queue==12.9.0
-beautifulsoup4==4.12.2
+beautifulsoup4==4.12.3
 fake-useragent==1.4.0
 chardet==5.2.0
 --extra-index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/
 azure-search-documents==11.4.0b8
 opencensus-ext-azure==1.1.13
-azure-ai-contentsafety==1.0.0b1
+azure-ai-contentsafety==1.0.0
 python-docx==1.1.0

code/requirements.txt

@@ -3,25 +3,25 @@ streamlit==1.30.0
 openai==0.27.8
 matplotlib==3.8.2
 plotly==5.18.0
-scipy==1.11.4
-scikit-learn==1.3.2
-transformers==4.36.2
-python-dotenv==1.0.0
+scipy==1.12.0
+scikit-learn==1.4.0
+transformers==4.37.1
+python-dotenv==1.0.1
 azure-ai-formrecognizer==3.3.2
 azure-storage-blob==12.19.0
 azure-identity==1.15.0
-azure-ai-contentsafety==1.0.0b1
+azure-ai-contentsafety==1.0.0
 requests==2.31.0
 tiktoken==0.5.2
 azure-storage-queue==12.9.0
-langchain==0.0.329
-beautifulsoup4==4.12.2
+langchain==0.1.4
+beautifulsoup4==4.12.3
 fake-useragent==1.4.0
 chardet==5.2.0
 --extra-index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/
 azure-search-documents==11.4.0b8
 opencensus-ext-azure==1.1.13
-pandas==1.5.1
+pandas==2.2.0
 python-docx==1.1.0
 
 # Add dev dependencies here - this will be refactored out by Poetry

code/utilities/tools/ContentSafetyChecker.py

@@ -1,7 +1,7 @@
 from azure.ai.contentsafety import ContentSafetyClient
 from azure.core.credentials import AzureKeyCredential
 from azure.core.exceptions import HttpResponseError
-from azure.ai.contentsafety.models import AnalyzeTextOptions
+from azure.ai.contentsafety.models import AnalyzeTextOptions, TextCategory
 from ..helpers.EnvHelper import EnvHelper
 from .AnswerProcessingBase import AnswerProcessingBase
 from ..common.Answer import Answer
@@ -48,12 +48,17 @@ def _filter_text_and_replace(self, text, response_template):
             raise
 
         filtered_text = text
-
+
+        hate_result = next(item for item in response.categories_analysis if item.category == TextCategory.HATE)
+        self_harm_result = next(item for item in response.categories_analysis if item.category == TextCategory.SELF_HARM)
+        sexual_result = next(item for item in response.categories_analysis if item.category == TextCategory.SEXUAL)
+        violence_result = next(item for item in response.categories_analysis if item.category == TextCategory.VIOLENCE)
+
         if (
-            response.hate_result.severity > 0
-            or response.self_harm_result.severity > 0
-            or response.sexual_result.severity > 0
-            or response.violence_result.severity > 0
+            hate_result.severity > 0
+            or self_harm_result.severity > 0
+            or sexual_result.severity > 0
+            or violence_result.severity > 0
         ):
             filtered_text = response_template
 

extensions/backend/requirements.txt

@@ -7,23 +7,23 @@ streamlit==1.30.0
 openai==0.27.8
 matplotlib==3.8.2
 plotly==5.18.0
-scipy==1.11.4
-scikit-learn==1.3.2
-transformers==4.36.2
-python-dotenv==1.0.0
+scipy==1.12.0
+scikit-learn==1.4.0
+transformers==4.37.1
+python-dotenv==1.0.1
 azure-ai-formrecognizer==3.3.2
 azure-storage-blob==12.19.0
 azure-identity==1.15.0
-azure-ai-contentsafety==1.0.0b1
+azure-ai-contentsafety==1.0.0
 requests==2.31.0
 tiktoken==0.5.2
 azure-storage-queue==12.9.0
-langchain==0.0.329
-beautifulsoup4==4.12.2
+langchain==0.1.4
+beautifulsoup4==4.12.3
 fake-useragent==1.4.0
 chardet==5.2.0
 --extra-index-url https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-python/pypi/simple/
 azure-search-documents==11.4.0b8
 opencensus-ext-azure==1.1.13
-pandas==1.5.1
+pandas==2.2.0
 python-docx==1.1.0

tests/test_DocumentChunking.py

@@ -91,7 +91,7 @@ def test_document_chunking_fixed_size_overlap():
     )
     document_chunking = DocumentChunking()
     chunked_documents = document_chunking.chunk(documents, chunking)
-    assert len(chunked_documents) == 8
+    assert len(chunked_documents) == 7
     assert chunked_documents[0].content == "PAGE 1: This short sentence with 20 tokens"
     assert (
         chunked_documents[1].content
@@ -111,4 +111,3 @@ def test_document_chunking_fixed_size_overlap():
         chunked_documents[6].content
         == " shows how the different chunking strategies work now!"
     )
-    assert chunked_documents[7].content == "ing strategies work now!"