Skip to content

Commit

Permalink
Merge branch 'main' into komal/iv-backendfunctional
Browse files Browse the repository at this point in the history
  • Loading branch information
@komalg1
komalg1 committed May 17, 2024
2 parents d985433 + 21064e7 commit e3dcd3a
Show file tree
Hide file tree
Showing 7 changed files with 534 additions and 110 deletions.
12 changes: 12 additions & 0 deletions code/backend/batch/utilities/common/answer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,18 @@ def __init__(
self.prompt_tokens = prompt_tokens
self.completion_tokens = completion_tokens

def __eq__(self, value: object) -> bool:
if not isinstance(value, Answer):
return False

return (
self.question == value.question
and self.answer == value.answer
and self.source_documents == value.source_documents
and self.prompt_tokens == value.prompt_tokens
and self.completion_tokens == value.completion_tokens
)

def to_json(self):
return json.dumps(self, cls=AnswerEncoder)

Expand Down
48 changes: 19 additions & 29 deletions code/backend/batch/utilities/tools/post_prompt_tool.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,3 @@
from langchain.chains.llm import LLMChain
from langchain.prompts import PromptTemplate
from langchain_community.callbacks import get_openai_callback
from ..common.answer import Answer
from ..helpers.llm_helper import LLMHelper
from ..helpers.config.config_helper import ConfigHelper
Expand All @@ -14,53 +11,46 @@ def validate_answer(self, answer: Answer) -> Answer:
config = ConfigHelper.get_active_config_or_default()
llm_helper = LLMHelper()

was_message_filtered = False
post_answering_prompt = PromptTemplate(
template=config.prompts.post_answering_prompt,
input_variables=["question", "answer", "sources"],
)
post_answering_chain = LLMChain(
llm=llm_helper.get_llm(),
prompt=post_answering_prompt,
output_key="correct",
verbose=True,
)

sources = "\n".join(
[
f"[doc{i+1}]: {source.content}"
for i, source in enumerate(answer.source_documents)
]
)

with get_openai_callback() as cb:
post_result = post_answering_chain(
message = config.prompts.post_answering_prompt.format(
question=answer.question,
answer=answer.answer,
sources=sources,
)

response = llm_helper.get_chat_completion(
[
{
"question": answer.question,
"answer": answer.answer,
"sources": sources,
"role": "user",
"content": message,
}
)

was_message_filtered = not (
post_result["correct"].lower() == "true"
or post_result["correct"].lower() == "yes"
]
)

result = response.choices[0].message.content

was_message_filtered = result.lower() not in ["true", "yes"]

# Return filtered answer or just the original one
if was_message_filtered:
return Answer(
question=answer.question,
answer=config.messages.post_answering_filter,
source_documents=[],
prompt_tokens=cb.prompt_tokens,
completion_tokens=cb.completion_tokens,
prompt_tokens=response.usage.prompt_tokens,
completion_tokens=response.usage.completion_tokens,
)
else:
return Answer(
question=answer.question,
answer=answer.answer,
source_documents=answer.source_documents,
prompt_tokens=cb.prompt_tokens,
completion_tokens=cb.completion_tokens,
prompt_tokens=response.usage.prompt_tokens,
completion_tokens=response.usage.completion_tokens,
)
164 changes: 84 additions & 80 deletions code/tests/functional/conftest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,86 +16,6 @@ def setup_default_mocking(httpserver: HTTPServer, app_config: AppConfig):
method="HEAD",
).respond_with_data()

httpserver.expect_request(
f"/{AZURE_STORAGE_CONFIG_CONTAINER_NAME}/{AZURE_STORAGE_CONFIG_FILE_NAME}",
method="GET",
).respond_with_json(
{
"prompts": {
"condense_question_prompt": "",
"answering_system_prompt": "system prompt",
"answering_user_prompt": "## Retrieved Documents\n{sources}\n\n## User Question\n{question}",
"use_on_your_data_format": True,
"post_answering_prompt": "post answering prompt",
"enable_post_answering_prompt": False,
"enable_content_safety": True,
},
"messages": {"post_answering_filter": "post answering filer"},
"example": {
"documents": '{"retrieved_documents":[{"[doc1]":{"content":"content"}}]}',
"user_question": "user question",
"answer": "answer",
},
"document_processors": [
{
"document_type": "pdf",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "layout"},
"use_advanced_image_processing": False,
},
{
"document_type": "txt",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "web"},
"use_advanced_image_processing": False,
},
{
"document_type": "url",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "web"},
"use_advanced_image_processing": False,
},
{
"document_type": "md",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "web"},
"use_advanced_image_processing": False,
},
{
"document_type": "html",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "web"},
"use_advanced_image_processing": False,
},
{
"document_type": "docx",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "docx"},
"use_advanced_image_processing": False,
},
{
"document_type": "jpg",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "layout"},
"use_advanced_image_processing": True,
},
{
"document_type": "png",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "layout"},
"use_advanced_image_processing": False,
},
],
"logging": {"log_user_interactions": True, "log_tokens": True},
"orchestrator": {"strategy": "openai_function"},
"integrated_vectorization_config": None,
},
headers={
"Content-Type": "application/json",
"Content-Range": "bytes 0-12882/12883",
},
)

httpserver.expect_request(
f"/openai/deployments/{app_config.get('AZURE_OPENAI_EMBEDDING_MODEL')}/embeddings",
method="POST",
Expand Down Expand Up @@ -315,3 +235,87 @@ def prime_search_to_trigger_creation_of_index(
]
}
)


# This fixture can be overriden
@pytest.fixture(autouse=True)
def setup_config_mocking(httpserver: HTTPServer):
httpserver.expect_request(
f"/{AZURE_STORAGE_CONFIG_CONTAINER_NAME}/{AZURE_STORAGE_CONFIG_FILE_NAME}",
method="GET",
).respond_with_json(
{
"prompts": {
"condense_question_prompt": "",
"answering_system_prompt": "system prompt",
"answering_user_prompt": "## Retrieved Documents\n{sources}\n\n## User Question\n{question}",
"use_on_your_data_format": True,
"post_answering_prompt": "post answering prompt\n{question}\n{answer}\n{sources}",
"enable_post_answering_prompt": False,
"enable_content_safety": True,
},
"messages": {"post_answering_filter": "post answering filter"},
"example": {
"documents": '{"retrieved_documents":[{"[doc1]":{"content":"content"}}]}',
"user_question": "user question",
"answer": "answer",
},
"document_processors": [
{
"document_type": "pdf",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "layout"},
"use_advanced_image_processing": False,
},
{
"document_type": "txt",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "web"},
"use_advanced_image_processing": False,
},
{
"document_type": "url",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "web"},
"use_advanced_image_processing": False,
},
{
"document_type": "md",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "web"},
"use_advanced_image_processing": False,
},
{
"document_type": "html",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "web"},
"use_advanced_image_processing": False,
},
{
"document_type": "docx",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "docx"},
"use_advanced_image_processing": False,
},
{
"document_type": "jpg",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "layout"},
"use_advanced_image_processing": True,
},
{
"document_type": "png",
"chunking": {"strategy": "layout", "size": 500, "overlap": 100},
"loading": {"strategy": "layout"},
"use_advanced_image_processing": False,
},
],
"logging": {"log_user_interactions": True, "log_tokens": True},
"orchestrator": {"strategy": "openai_function"},
"integrated_vectorization_config": None,
},
headers={
"Content-Type": "application/json",
"Content-Range": "bytes 0-12882/12883",
},
)
Loading

0 comments on commit e3dcd3a

Please sign in to comment.