Skip to content

Commit

Permalink
Merge branch 'PSL-8488-UT-ANSWERS' of https://github.com/Azure-Sample…
Browse files Browse the repository at this point in the history 
…s/chat-with-your-data-solution-accelerator into PSL-8488-UT-ANSWERS
  • Loading branch information
@Kiran-Siluveru-Microsoft
Kiran-Siluveru-Microsoft committed Oct 17, 2024
2 parents 9606928 + 597fddc commit 4299876
Show file tree
Hide file tree
Showing 9 changed files with 175 additions and 169 deletions.
16 changes: 16 additions & 0 deletions azure.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,24 @@ services:
project: ./code/backend
language: py
host: appservice
hooks:
prepackage:
windows:
shell: pwsh
run: poetry install; poetry export -o requirements.txt
posix:
shell: sh
run: poetry install; poetry export -o requirements.txt

function:
project: ./code/backend/batch
language: py
host: function
hooks:
prepackage:
windows:
shell: pwsh
run: poetry export -o requirements.txt; pip install -r requirements.txt
posix:
shell: sh
run: poetry export -o requirements.txt; pip install -r requirements.txt
8 changes: 7 additions & 1 deletion code/backend/pages/01_Ingest_Data.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from os import path
import re
import streamlit as st
import traceback
import requests
Expand Down Expand Up @@ -56,6 +57,11 @@ def add_urls():
add_url_embeddings(urls)


def sanitize_metadata_value(value):
# Remove invalid characters
return re.sub(r"[^a-zA-Z0-9-_ .]", "?", value)


def add_url_embeddings(urls: list[str]):
params = {}
if env_helper.FUNCTION_KEY is not None:
Expand Down Expand Up @@ -89,7 +95,7 @@ def add_url_embeddings(urls: list[str]):
for up in uploaded_files:
# To read file as bytes:
bytes_data = up.getvalue()
title = up.name.encode("latin-1", "replace").decode("latin-1")
title = sanitize_metadata_value(up.name)
if st.session_state.get("filename", "") != up.name:
# Upload a new file
st.session_state["filename"] = up.name
Expand Down
158 changes: 72 additions & 86 deletions code/backend/pages/04_Configuration.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,96 +391,82 @@ def validate_documents():
st.checkbox("Log tokens", key="log_tokens")

if st.form_submit_button("Save configuration"):
valid = all(
row["document_type"]
and row["chunking_strategy"]
and row["loading_strategy"]
for row in edited_document_processors
)

if not valid:
st.error(
"Please ensure all fields are selected and not left blank in Document processing configuration."
document_processors = []
if env_helper.AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION is False:
valid = all(
row["document_type"]
and row["chunking_strategy"]
and row["loading_strategy"]
for row in edited_document_processors
)
else:
document_processors = (
list(
map(
lambda x: {
"document_type": x["document_type"],
"chunking": {
"strategy": x["chunking_strategy"],
"size": x["chunking_size"],
"overlap": x["chunking_overlap"],
},
"loading": {
"strategy": x["loading_strategy"],
},
"use_advanced_image_processing": x[
"use_advanced_image_processing"
],
if not valid:
st.error(
"Please ensure all fields are selected and not left blank in Document processing configuration."
)
document_processors = list(
map(
lambda x: {
"document_type": x["document_type"],
"chunking": {
"strategy": x["chunking_strategy"],
"size": x["chunking_size"],
"overlap": x["chunking_overlap"],
},
edited_document_processors,
)
"loading": {
"strategy": x["loading_strategy"],
},
"use_advanced_image_processing": x[
"use_advanced_image_processing"
],
},
edited_document_processors,
)
if env_helper.AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION is False
else []
)
current_config = {
"prompts": {
"condense_question_prompt": "", # st.session_state['condense_question_prompt'],
"answering_system_prompt": st.session_state[
"answering_system_prompt"
],
"answering_user_prompt": st.session_state[
"answering_user_prompt"
],
"use_on_your_data_format": st.session_state[
"use_on_your_data_format"
],
"post_answering_prompt": st.session_state[
"post_answering_prompt"
],
"enable_post_answering_prompt": st.session_state[
"enable_post_answering_prompt"
],
"enable_content_safety": st.session_state[
"enable_content_safety"
],
"ai_assistant_type": st.session_state["ai_assistant_type"],
"conversational_flow": st.session_state["conversational_flow"],
},
"messages": {
"post_answering_filter": st.session_state[
"post_answering_filter_message"
]
},
"example": {
"documents": st.session_state["example_documents"],
"user_question": st.session_state["example_user_question"],
"answer": st.session_state["example_answer"],
},
"document_processors": document_processors,
"logging": {
"log_user_interactions": st.session_state[
"log_user_interactions"
],
"log_tokens": st.session_state["log_tokens"],
},
"orchestrator": {
"strategy": st.session_state["orchestrator_strategy"]
},
"integrated_vectorization_config": (
integrated_vectorization_config
if env_helper.AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION
else None
),
"enable_chat_history": st.session_state["enable_chat_history"],
}
ConfigHelper.save_config_as_active(current_config)
st.success(
"Configuration saved successfully! Please restart the chat service for these changes to take effect."
)
current_config = {
"prompts": {
"condense_question_prompt": "", # st.session_state['condense_question_prompt'],
"answering_system_prompt": st.session_state[
"answering_system_prompt"
],
"answering_user_prompt": st.session_state["answering_user_prompt"],
"use_on_your_data_format": st.session_state[
"use_on_your_data_format"
],
"post_answering_prompt": st.session_state["post_answering_prompt"],
"enable_post_answering_prompt": st.session_state[
"enable_post_answering_prompt"
],
"enable_content_safety": st.session_state["enable_content_safety"],
"ai_assistant_type": st.session_state["ai_assistant_type"],
"conversational_flow": st.session_state["conversational_flow"],
},
"messages": {
"post_answering_filter": st.session_state[
"post_answering_filter_message"
]
},
"example": {
"documents": st.session_state["example_documents"],
"user_question": st.session_state["example_user_question"],
"answer": st.session_state["example_answer"],
},
"document_processors": document_processors,
"logging": {
"log_user_interactions": st.session_state["log_user_interactions"],
"log_tokens": st.session_state["log_tokens"],
},
"orchestrator": {"strategy": st.session_state["orchestrator_strategy"]},
"integrated_vectorization_config": (
integrated_vectorization_config
if env_helper.AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION
else None
),
"enable_chat_history": st.session_state["enable_chat_history"],
}
ConfigHelper.save_config_as_active(current_config)
st.success(
"Configuration saved successfully! Please restart the chat service for these changes to take effect."
)

with st.popover(":red[Reset configuration to defaults]"):

Expand Down
Loading

0 comments on commit 4299876

Please sign in to comment.