diff --git a/code/backend/batch/utilities/helpers/azure_computer_vision_client.py b/code/backend/batch/utilities/helpers/azure_computer_vision_client.py index c20b339eb..6ab0733f3 100644 --- a/code/backend/batch/utilities/helpers/azure_computer_vision_client.py +++ b/code/backend/batch/utilities/helpers/azure_computer_vision_client.py @@ -1,5 +1,4 @@ import logging -from typing import List from urllib.parse import urljoin from azure.identity import DefaultAzureCredential, get_bearer_token_provider @@ -15,6 +14,7 @@ class AzureComputerVisionClient: __TOKEN_SCOPE = "https://cognitiveservices.azure.com/.default" __VECTORIZE_IMAGE_PATH = "computervision/retrieval:vectorizeImage" + __VECTORIZE_TEXT_PATH = "computervision/retrieval:vectorizeText" __RESPONSE_VECTOR_KEY = "vector" def __init__(self, env_helper: EnvHelper) -> None: @@ -27,15 +27,29 @@ def __init__(self, env_helper: EnvHelper) -> None: env_helper.AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION ) - def vectorize_image(self, image_url: str) -> List[float]: + def vectorize_image(self, image_url: str) -> list[float]: logger.info(f"Making call to computer vision to vectorize image: {image_url}") - response = self.__make_request(image_url) + response = self.__make_request( + self.__VECTORIZE_IMAGE_PATH, + body={"url": image_url}, + ) + self.__validate_response(response) + + response_json = self.__get_json_body(response) + return self.__get_vectors(response_json) + + def vectorize_text(self, text: str) -> list[float]: + logger.debug(f"Making call to computer vision to vectorize text: {text}") + response = self.__make_request( + self.__VECTORIZE_TEXT_PATH, + body={"text": text}, + ) self.__validate_response(response) response_json = self.__get_json_body(response) return self.__get_vectors(response_json) - def __make_request(self, image_url: str) -> Response: + def __make_request(self, path: str, body) -> Response: try: headers = {} if self.use_keys: @@ -47,22 +61,22 @@ def __make_request(self, image_url: str) -> Response: headers["Authorization"] = "Bearer " + token_provider() return requests.post( - url=urljoin(self.host, self.__VECTORIZE_IMAGE_PATH), + url=urljoin(self.host, path), params={ "api-version": self.api_version, "model-version": self.model_version, }, - json={"url": image_url}, + json=body, headers=headers, timeout=self.timeout, ) except Exception as e: - raise Exception(f"Call to vectorize image failed: {image_url}") from e + raise Exception("Call to Azure Computer Vision failed") from e def __validate_response(self, response: Response): if response.status_code != 200: raise Exception( - f"Call to vectorize image failed with status: {response.status_code} body: {response.text}" + f"Call to Azure Computer Vision failed with status: {response.status_code}, body: {response.text}" ) def __get_json_body(self, response: Response) -> dict: @@ -70,13 +84,13 @@ def __get_json_body(self, response: Response) -> dict: return response.json() except Exception as e: raise Exception( - f"Call to vectorize image returned malformed response body: {response.text}", + f"Call to Azure Computer Vision returned malformed response body: {response.text}", ) from e - def __get_vectors(self, response_json: dict) -> List[float]: + def __get_vectors(self, response_json: dict) -> list[float]: if self.__RESPONSE_VECTOR_KEY in response_json: return response_json[self.__RESPONSE_VECTOR_KEY] else: raise Exception( - f"Call to vectorize image returned no vector: {response_json}" + f"Call to Azure Computer Vision returned no vector: {response_json}" ) diff --git a/code/backend/batch/utilities/helpers/azure_search_helper.py b/code/backend/batch/utilities/helpers/azure_search_helper.py index c48360e71..3ae804609 100644 --- a/code/backend/batch/utilities/helpers/azure_search_helper.py +++ b/code/backend/batch/utilities/helpers/azure_search_helper.py @@ -24,6 +24,8 @@ VectorSearchAlgorithmMetric, VectorSearchProfile, ) + +from ..helpers.azure_computer_vision_client import AzureComputerVisionClient from .llm_helper import LLMHelper from .env_helper import EnvHelper @@ -32,6 +34,7 @@ class AzureSearchHelper: _search_dimension: int | None = None + _image_search_dimension: int | None = None def __init__(self): self.llm_helper = LLMHelper() @@ -40,6 +43,7 @@ def __init__(self): search_credential = self._search_credential() self.search_client = self._create_search_client(search_credential) self.search_index_client = self._create_search_index_client(search_credential) + self.azure_computer_vision_client = AzureComputerVisionClient(self.env_helper) def _search_credential(self): if self.env_helper.is_auth_type_keys(): @@ -75,6 +79,14 @@ def search_dimensions(self) -> int: ) return AzureSearchHelper._search_dimension + @property + def image_search_dimensions(self) -> int: + if AzureSearchHelper._image_search_dimension is None: + AzureSearchHelper._image_search_dimension = len( + self.azure_computer_vision_client.vectorize_text("Text") + ) + return AzureSearchHelper._image_search_dimension + def create_index(self): fields = [ SimpleField( @@ -128,7 +140,7 @@ def create_index(self): name="image_vector", type=SearchFieldDataType.Collection(SearchFieldDataType.Single), searchable=True, - vector_search_dimensions=1024, + vector_search_dimensions=self.image_search_dimensions, vector_search_profile_name="myHnswProfile", ), ) diff --git a/code/backend/batch/utilities/helpers/embedders/push_embedder.py b/code/backend/batch/utilities/helpers/embedders/push_embedder.py index 58ba2f682..ab2fdc1f4 100644 --- a/code/backend/batch/utilities/helpers/embedders/push_embedder.py +++ b/code/backend/batch/utilities/helpers/embedders/push_embedder.py @@ -59,8 +59,6 @@ def __embed( and file_extension in self.config.get_advanced_image_processing_image_types() ): - logger.warning("Advanced image processing is not supported yet") - caption = self.__generate_image_caption(source_url) caption_vector = self.llm_helper.generate_embeddings(caption) diff --git a/code/backend/batch/utilities/search/azure_search_handler.py b/code/backend/batch/utilities/search/azure_search_handler.py index 815e6a89f..4b384c126 100644 --- a/code/backend/batch/utilities/search/azure_search_handler.py +++ b/code/backend/batch/utilities/search/azure_search_handler.py @@ -1,6 +1,7 @@ from typing import List from .search_handler_base import SearchHandlerBase from ..helpers.llm_helper import LLMHelper +from ..helpers.azure_computer_vision_client import AzureComputerVisionClient from ..helpers.azure_search_helper import AzureSearchHelper from ..common.source_document import SourceDocument import json @@ -9,13 +10,12 @@ class AzureSearchHandler(SearchHandlerBase): - _ENCODER_NAME = "cl100k_base" - _VECTOR_FIELD = "content_vector" def __init__(self, env_helper): super().__init__(env_helper) self.llm_helper = LLMHelper() + self.azure_computer_vision_client = AzureComputerVisionClient(env_helper) def create_search_client(self): return AzureSearchHelper().get_search_client() @@ -66,14 +66,31 @@ def delete_files(self, files): def query_search(self, question) -> List[SourceDocument]: encoding = tiktoken.get_encoding(self._ENCODER_NAME) tokenised_question = encoding.encode(question) + + if self.env_helper.USE_ADVANCED_IMAGE_PROCESSING: + vectorized_question = self.azure_computer_vision_client.vectorize_text( + question + ) + else: + vectorized_question = None + if self.env_helper.AZURE_SEARCH_USE_SEMANTIC_SEARCH: - results = self._semantic_search(question, tokenised_question) + results = self._semantic_search( + question, tokenised_question, vectorized_question + ) else: - results = self._hybrid_search(question, tokenised_question) + results = self._hybrid_search( + question, tokenised_question, vectorized_question + ) return self._convert_to_source_documents(results) - def _semantic_search(self, question: str, tokenised_question: list[int]): + def _semantic_search( + self, + question: str, + tokenised_question: list[int], + vectorized_question: list[float] | None, + ): return self.search_client.search( search_text=question, vector_queries=[ @@ -81,7 +98,18 @@ def _semantic_search(self, question: str, tokenised_question: list[int]): vector=self.llm_helper.generate_embeddings(tokenised_question), k_nearest_neighbors=self.env_helper.AZURE_SEARCH_TOP_K, fields=self._VECTOR_FIELD, - ) + ), + *( + [ + VectorizedQuery( + vector=vectorized_question, + k_nearest_neighbors=self.env_helper.AZURE_SEARCH_TOP_K, + fields=self._IMAGE_VECTOR_FIELD, + ) + ] + if vectorized_question is not None + else [] + ), ], filter=self.env_helper.AZURE_SEARCH_FILTER, query_type="semantic", @@ -91,7 +119,12 @@ def _semantic_search(self, question: str, tokenised_question: list[int]): top=self.env_helper.AZURE_SEARCH_TOP_K, ) - def _hybrid_search(self, question: str, tokenised_question: list[int]): + def _hybrid_search( + self, + question: str, + tokenised_question: list[int], + vectorized_question: list[float] | None, + ): return self.search_client.search( search_text=question, vector_queries=[ @@ -100,7 +133,18 @@ def _hybrid_search(self, question: str, tokenised_question: list[int]): k_nearest_neighbors=self.env_helper.AZURE_SEARCH_TOP_K, filter=self.env_helper.AZURE_SEARCH_FILTER, fields=self._VECTOR_FIELD, - ) + ), + *( + [ + VectorizedQuery( + vector=vectorized_question, + k_nearest_neighbors=self.env_helper.AZURE_SEARCH_TOP_K, + fields=self._IMAGE_VECTOR_FIELD, + ) + ] + if vectorized_question is not None + else [] + ), ], query_type="simple", # this is the default value filter=self.env_helper.AZURE_SEARCH_FILTER, diff --git a/code/backend/batch/utilities/search/integrated_vectorization_search_handler.py b/code/backend/batch/utilities/search/integrated_vectorization_search_handler.py index bba5c4106..1e9a891ee 100644 --- a/code/backend/batch/utilities/search/integrated_vectorization_search_handler.py +++ b/code/backend/batch/utilities/search/integrated_vectorization_search_handler.py @@ -81,7 +81,7 @@ def _hybrid_search(self, question: str): vector_query = VectorizableTextQuery( text=question, k_nearest_neighbors=self.env_helper.AZURE_SEARCH_TOP_K, - fields="content_vector", + fields=self._VECTOR_FIELD, exhaustive=True, ) return self.search_client.search( @@ -94,7 +94,7 @@ def _semantic_search(self, question: str): vector_query = VectorizableTextQuery( text=question, k_nearest_neighbors=self.env_helper.AZURE_SEARCH_TOP_K, - fields="content_vector", + fields=self._VECTOR_FIELD, exhaustive=True, ) return self.search_client.search( diff --git a/code/backend/batch/utilities/search/search.py b/code/backend/batch/utilities/search/search.py index 1c0a37789..6a5eed95e 100644 --- a/code/backend/batch/utilities/search/search.py +++ b/code/backend/batch/utilities/search/search.py @@ -2,18 +2,21 @@ from ..search.integrated_vectorization_search_handler import ( IntegratedVectorizationSearchHandler, ) +from ..search.search_handler_base import SearchHandlerBase from ..common.source_document import SourceDocument from ..helpers.env_helper import EnvHelper class Search: @staticmethod - def get_search_handler(env_helper: EnvHelper): + def get_search_handler(env_helper: EnvHelper) -> SearchHandlerBase: if env_helper.AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION: return IntegratedVectorizationSearchHandler(env_helper) else: return AzureSearchHandler(env_helper) @staticmethod - def get_source_documents(search_handler, question) -> list[SourceDocument]: + def get_source_documents( + search_handler: SearchHandlerBase, question: str + ) -> list[SourceDocument]: return search_handler.query_search(question) diff --git a/code/backend/batch/utilities/search/search_handler_base.py b/code/backend/batch/utilities/search/search_handler_base.py index 5e3443e5c..f497410c7 100644 --- a/code/backend/batch/utilities/search/search_handler_base.py +++ b/code/backend/batch/utilities/search/search_handler_base.py @@ -1,10 +1,13 @@ from abc import ABC, abstractmethod from ..helpers.env_helper import EnvHelper - from ..common.source_document import SourceDocument +from azure.search.documents import SearchClient class SearchHandlerBase(ABC): + _VECTOR_FIELD = "content_vector" + _IMAGE_VECTOR_FIELD = "image_vector" + def __init__(self, env_helper: EnvHelper): self.env_helper = env_helper self.search_client = self.create_search_client() @@ -20,7 +23,7 @@ def get_unique_files(self, results, facet_key: str): return [] @abstractmethod - def create_search_client(self): + def create_search_client(self) -> SearchClient: pass @abstractmethod diff --git a/code/tests/constants.py b/code/tests/constants.py index d29977139..0961aa086 100644 --- a/code/tests/constants.py +++ b/code/tests/constants.py @@ -3,3 +3,5 @@ COMPUTER_VISION_VECTORIZE_IMAGE_PATH = "/computervision/retrieval:vectorizeImage" COMPUTER_VISION_VECTORIZE_IMAGE_REQUEST_METHOD = "POST" +COMPUTER_VISION_VECTORIZE_TEXT_PATH = "/computervision/retrieval:vectorizeText" +COMPUTER_VISION_VECTORIZE_TEXT_REQUEST_METHOD = "POST" diff --git a/code/tests/functional/conftest.py b/code/tests/functional/conftest.py index 18a35d7f2..b4a3cb18f 100644 --- a/code/tests/functional/conftest.py +++ b/code/tests/functional/conftest.py @@ -6,6 +6,8 @@ AZURE_STORAGE_CONFIG_FILE_NAME, COMPUTER_VISION_VECTORIZE_IMAGE_PATH, COMPUTER_VISION_VECTORIZE_IMAGE_REQUEST_METHOD, + COMPUTER_VISION_VECTORIZE_TEXT_PATH, + COMPUTER_VISION_VECTORIZE_TEXT_REQUEST_METHOD, ) @@ -128,6 +130,11 @@ def setup_default_mocking(httpserver: HTTPServer, app_config: AppConfig): COMPUTER_VISION_VECTORIZE_IMAGE_REQUEST_METHOD, ).respond_with_json({"modelVersion": "2022-04-11", "vector": [1.0, 2.0, 3.0]}) + httpserver.expect_request( + COMPUTER_VISION_VECTORIZE_TEXT_PATH, + COMPUTER_VISION_VECTORIZE_TEXT_REQUEST_METHOD, + ).respond_with_json({"modelVersion": "2022-04-11", "vector": [1.0, 2.0, 3.0]}) + httpserver.expect_request( f"/indexes('{app_config.get('AZURE_SEARCH_INDEX')}')/docs/search.index", method="POST", diff --git a/code/tests/functional/tests/backend_api/default/conftest.py b/code/tests/functional/tests/backend_api/default/conftest.py index 4c8aac922..8a5b286c6 100644 --- a/code/tests/functional/tests/backend_api/default/conftest.py +++ b/code/tests/functional/tests/backend_api/default/conftest.py @@ -30,6 +30,8 @@ def app_config(make_httpserver, ca): "AZURE_CONTENT_SAFETY_ENDPOINT": f"https://localhost:{make_httpserver.port}/", "AZURE_SPEECH_REGION_ENDPOINT": f"https://localhost:{make_httpserver.port}/", "AZURE_STORAGE_ACCOUNT_ENDPOINT": f"https://localhost:{make_httpserver.port}/", + "AZURE_COMPUTER_VISION_ENDPOINT": f"https://localhost:{make_httpserver.port}/", + "USE_ADVANCED_IMAGE_PROCESSING": "True", "SSL_CERT_FILE": ca_temp_path, "CURL_CA_BUNDLE": ca_temp_path, } diff --git a/code/tests/functional/tests/backend_api/default/test_advanced_image_processing.py b/code/tests/functional/tests/backend_api/default/test_advanced_image_processing.py new file mode 100644 index 000000000..64e359a9a --- /dev/null +++ b/code/tests/functional/tests/backend_api/default/test_advanced_image_processing.py @@ -0,0 +1,171 @@ +import pytest +import requests +from pytest_httpserver import HTTPServer +from tests.constants import ( + COMPUTER_VISION_VECTORIZE_TEXT_PATH, + COMPUTER_VISION_VECTORIZE_TEXT_REQUEST_METHOD, +) +from tests.functional.app_config import AppConfig +from tests.request_matching import RequestMatcher, verify_request_made + +pytestmark = pytest.mark.functional + +path = "/api/conversation/custom" +body = { + "conversation_id": "123", + "messages": [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi, how can I help?"}, + {"role": "user", "content": "What is the meaning of life?"}, + ], +} + + +@pytest.fixture(autouse=True) +def completions_mocking(httpserver: HTTPServer, app_config: AppConfig): + httpserver.expect_oneshot_request( + f"/openai/deployments/{app_config.get('AZURE_OPENAI_MODEL')}/chat/completions", + method="POST", + ).respond_with_json( + { + "choices": [ + { + "content_filter_results": {}, + "finish_reason": "function_call", + "index": 0, + "message": { + "content": None, + "role": "assistant", + "function_call": { + "arguments": '{"question":"What is the meaning of life?"}', + "name": "search_documents", + }, + }, + } + ], + "created": 1714576877, + "id": "chatcmpl-9K63hMvVH1DyQJqqM7rFE4oRPFCeR", + "model": app_config.get("AZURE_OPENAI_MODEL"), + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "system_fingerprint": "fp_2f57f81c11", + "usage": { + "completion_tokens": 21, + "prompt_tokens": 256, + "total_tokens": 277, + }, + } + ) + + httpserver.expect_oneshot_request( + f"/openai/deployments/{app_config.get('AZURE_OPENAI_MODEL')}/chat/completions", + method="POST", + ).respond_with_json( + { + "choices": [ + { + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + "finish_reason": "stop", + "index": 0, + "message": { + "content": "42 is the meaning of life[doc1].", + "role": "assistant", + }, + } + ], + "created": 1714576891, + "id": "chatcmpl-9K63vDGs3slJFynnpi2K6RcVPwgrT", + "model": app_config.get("AZURE_OPENAI_MODEL"), + "object": "chat.completion", + "prompt_filter_results": [ + { + "prompt_index": 0, + "content_filter_results": { + "hate": {"filtered": False, "severity": "safe"}, + "self_harm": {"filtered": False, "severity": "safe"}, + "sexual": {"filtered": False, "severity": "safe"}, + "violence": {"filtered": False, "severity": "safe"}, + }, + } + ], + "system_fingerprint": "fp_2f57f81c11", + "usage": { + "completion_tokens": 101, + "prompt_tokens": 4288, + "total_tokens": 4389, + }, + } + ) + + +def test_post_responds_successfully(app_url: str, app_config: AppConfig): + # when + response = requests.post(f"{app_url}{path}", json=body) + + # then + assert response.status_code == 200 + assert response.json() == { + "choices": [ + { + "messages": [ + { + "content": r'{"citations": [{"content": "[/documents/doc.pdf](https://source)\n\n\ncontent", "id": "doc_1", "chunk_id": 95, "title": "/documents/doc.pdf", "filepath": "source", "url": "[/documents/doc.pdf](https://source)", "metadata": {"offset": 202738, "source": "https://source", "markdown_url": "[/documents/doc.pdf](https://source)", "title": "/documents/doc.pdf", "original_url": "https://source", "chunk": 95, "key": "doc_1", "filename": "source"}}], "intent": "What is the meaning of life?"}', + "end_turn": False, + "role": "tool", + }, + { + "content": "42 is the meaning of life[doc1].", + "end_turn": True, + "role": "assistant", + }, + ] + } + ], + "created": "response.created", + "id": "response.id", + "model": app_config.get("AZURE_OPENAI_MODEL"), + "object": "response.object", + } + assert response.headers["Content-Type"] == "application/json" + + +def test_text_passed_to_computer_vision_to_generate_text_embeddings( + app_url: str, httpserver: HTTPServer, app_config: AppConfig +): + # when + requests.post(f"{app_url}{path}", json=body) + + # then + verify_request_made( + httpserver, + RequestMatcher( + path=COMPUTER_VISION_VECTORIZE_TEXT_PATH, + method=COMPUTER_VISION_VECTORIZE_TEXT_REQUEST_METHOD, + json={ + "text": "What is the meaning of life?", + }, + query_string="api-version=2024-02-01&model-version=2023-04-15", + headers={ + "Content-Type": "application/json", + "Ocp-Apim-Subscription-Key": app_config.get( + "AZURE_COMPUTER_VISION_KEY" + ), + }, + times=1, + ), + ) diff --git a/code/tests/functional/tests/backend_api/default/test_conversation_custom.py b/code/tests/functional/tests/backend_api/default/test_conversation_custom.py index 1341d6878..6f94a9a0a 100644 --- a/code/tests/functional/tests/backend_api/default/test_conversation_custom.py +++ b/code/tests/functional/tests/backend_api/default/test_conversation_custom.py @@ -441,6 +441,13 @@ def test_post_makes_correct_call_to_create_documents_search_index( "sortable": False, "facetable": False, }, + { + "name": "image_vector", + "type": "Collection(Edm.Single)", + "searchable": True, + "dimensions": 3, + "vectorSearchProfile": "myHnswProfile", + }, ], "semantic": { "configurations": [ @@ -509,7 +516,13 @@ def test_post_makes_correct_call_to_search_documents_search_index( "k": int(app_config.get("AZURE_SEARCH_TOP_K")), "fields": "content_vector", "vector": [0.018990106880664825, -0.0073809814639389515], - } + }, + { + "kind": "vector", + "k": int(app_config.get("AZURE_SEARCH_TOP_K")), + "fields": "image_vector", + "vector": [1.0, 2.0, 3.0], + }, ], }, headers={ diff --git a/code/tests/functional/tests/functions/advanced_image_processing/test_advanced_image_processing.py b/code/tests/functional/tests/functions/advanced_image_processing/test_advanced_image_processing.py index ac7748d42..f18708f08 100644 --- a/code/tests/functional/tests/functions/advanced_image_processing/test_advanced_image_processing.py +++ b/code/tests/functional/tests/functions/advanced_image_processing/test_advanced_image_processing.py @@ -344,7 +344,7 @@ def test_makes_correct_call_to_create_documents_search_index( "name": "image_vector", "type": "Collection(Edm.Single)", "searchable": True, - "dimensions": 1024, + "dimensions": 3, "vectorSearchProfile": "myHnswProfile", }, ], diff --git a/code/tests/search_utilities/test_azure_search_handler.py b/code/tests/search_utilities/test_azure_search_handler.py index fa23780ae..8ba75edaf 100644 --- a/code/tests/search_utilities/test_azure_search_handler.py +++ b/code/tests/search_utilities/test_azure_search_handler.py @@ -7,16 +7,17 @@ from backend.batch.utilities.common.source_document import SourceDocument -@pytest.fixture +@pytest.fixture(autouse=True) def env_helper_mock(): mock = Mock() mock.AZURE_SEARCH_USE_SEMANTIC_SEARCH = False + mock.USE_ADVANCED_IMAGE_PROCESSING = False mock.AZURE_SEARCH_TOP_K = 3 mock.AZURE_SEARCH_FILTER = "some-search-filter" return mock -@pytest.fixture +@pytest.fixture(autouse=True) def mock_search_client(): with patch( "backend.batch.utilities.search.azure_search_handler.AzureSearchHelper" @@ -25,13 +26,24 @@ def mock_search_client(): yield search_client -@pytest.fixture +@pytest.fixture(autouse=True) def mock_llm_helper(): with patch("backend.batch.utilities.search.azure_search_handler.LLMHelper") as mock: mock_llm_helper = mock.return_value yield mock_llm_helper +@pytest.fixture(autouse=True) +def mock_azure_computer_vision_client(): + with patch( + "backend.batch.utilities.search.azure_search_handler.AzureComputerVisionClient" + ) as mock: + azure_computer_vision_client = mock.return_value + azure_computer_vision_client.vectorize_text.return_value = [3, 2, 1] + + yield azure_computer_vision_client + + @pytest.fixture def handler(env_helper_mock, mock_search_client, mock_llm_helper): with patch( @@ -192,7 +204,7 @@ def test_query_search_performs_semantic_search( mock_llm_helper.generate_embeddings.return_value = [1, 2, 3] env_helper_mock.AZURE_SEARCH_USE_SEMANTIC_SEARCH = True - env_helper_mock.AZURE_SEARCH_SEMANTIC_CONFIG_NAME = "some-semantic-config" + env_helper_mock.AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG = "some-semantic-config" # when handler.query_search(question) @@ -269,3 +281,85 @@ def test_query_search_converts_results_to_source_documents( # then assert actual_results == expected_results + + +def test_hybrid_search_with_advanced_image_processing( + handler: AzureSearchHandler, + mock_llm_helper: MagicMock, + mock_azure_computer_vision_client: MagicMock, + env_helper_mock: MagicMock, +): + # given + env_helper_mock.USE_ADVANCED_IMAGE_PROCESSING = True + mock_llm_helper.generate_embeddings.return_value = [1, 2, 3] + + question = "What is the answer?" + + # when + handler.query_search(question) + + # then + mock_azure_computer_vision_client.vectorize_text.assert_called_once_with(question) + + handler.search_client.search.assert_called_once_with( + search_text=question, + vector_queries=[ + VectorizedQuery( + vector=[1, 2, 3], + k_nearest_neighbors=handler.env_helper.AZURE_SEARCH_TOP_K, + filter=handler.env_helper.AZURE_SEARCH_FILTER, + fields="content_vector", + ), + VectorizedQuery( + vector=[3, 2, 1], + k_nearest_neighbors=handler.env_helper.AZURE_SEARCH_TOP_K, + fields="image_vector", + ), + ], + query_type="simple", + filter=handler.env_helper.AZURE_SEARCH_FILTER, + top=handler.env_helper.AZURE_SEARCH_TOP_K, + ) + + +def test_semantic_search_with_advanced_image_processing( + handler: AzureSearchHandler, + mock_llm_helper: MagicMock, + mock_azure_computer_vision_client: MagicMock, + env_helper_mock: MagicMock, +): + # given + env_helper_mock.USE_ADVANCED_IMAGE_PROCESSING = True + env_helper_mock.AZURE_SEARCH_USE_SEMANTIC_SEARCH = True + env_helper_mock.AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG = "some-semantic-config" + mock_llm_helper.generate_embeddings.return_value = [1, 2, 3] + + question = "What is the answer?" + + # when + handler.query_search(question) + + # then + mock_azure_computer_vision_client.vectorize_text.assert_called_once_with(question) + + handler.search_client.search.assert_called_once_with( + search_text=question, + vector_queries=[ + VectorizedQuery( + vector=[1, 2, 3], + k_nearest_neighbors=handler.env_helper.AZURE_SEARCH_TOP_K, + fields="content_vector", + ), + VectorizedQuery( + vector=[3, 2, 1], + k_nearest_neighbors=handler.env_helper.AZURE_SEARCH_TOP_K, + fields="image_vector", + ), + ], + filter=handler.env_helper.AZURE_SEARCH_FILTER, + query_type="semantic", + semantic_configuration_name=handler.env_helper.AZURE_SEARCH_SEMANTIC_SEARCH_CONFIG, + query_caption="extractive", + query_answer="extractive", + top=handler.env_helper.AZURE_SEARCH_TOP_K, + ) diff --git a/code/tests/utilities/helpers/test_azure_computer_vision_client.py b/code/tests/utilities/helpers/test_azure_computer_vision_client.py index 40f9d530d..97e291fb4 100644 --- a/code/tests/utilities/helpers/test_azure_computer_vision_client.py +++ b/code/tests/utilities/helpers/test_azure_computer_vision_client.py @@ -16,6 +16,8 @@ from tests.constants import ( COMPUTER_VISION_VECTORIZE_IMAGE_PATH, COMPUTER_VISION_VECTORIZE_IMAGE_REQUEST_METHOD, + COMPUTER_VISION_VECTORIZE_TEXT_PATH, + COMPUTER_VISION_VECTORIZE_TEXT_REQUEST_METHOD, ) @@ -30,6 +32,7 @@ # and we switch to it, we should consider switching back to convential test mocking. IMAGE_URL = "some-image-url.jpg" +TEXT = "some text" AZURE_COMPUTER_VISION_KEY = "some-api-key" @@ -155,6 +158,24 @@ def test_returns_image_vectors( assert actual_vectors == expected_vectors +def test_returns_text_vectors( + httpserver: HTTPServer, azure_computer_vision_client: AzureComputerVisionClient +): + # given + expected_vectors = [3.0, 2.0, 1.0] + + httpserver.expect_request( + COMPUTER_VISION_VECTORIZE_TEXT_PATH, + COMPUTER_VISION_VECTORIZE_TEXT_REQUEST_METHOD, + ).respond_with_json({"modelVersion": "2022-04-11", "vector": expected_vectors}) + + # when + actual_vectors = azure_computer_vision_client.vectorize_text(TEXT) + + # then + assert actual_vectors == expected_vectors + + def test_vectorize_image_calls_computer_vision_timeout( httpserver: HTTPServer, azure_computer_vision_client: AzureComputerVisionClient ): @@ -175,7 +196,7 @@ def handler(_) -> werkzeug.Response: with pytest.raises(Exception) as exec_info: azure_computer_vision_client.vectorize_image(IMAGE_URL) - assert exec_info.value.args[0] == "Call to vectorize image failed: " + IMAGE_URL + assert exec_info.value.args[0] == "Call to Azure Computer Vision failed" assert isinstance(exec_info.value.__cause__, ReadTimeout) @@ -197,7 +218,7 @@ def test_raises_exception_if_bad_response_code( # then assert ( exec_info.value.args[0] - == f"Call to vectorize image failed with status: {response_status} body: {json.dumps(response_body, indent=4)}" + == f"Call to Azure Computer Vision failed with status: {response_status}, body: {json.dumps(response_body, indent=4)}" ) @@ -218,7 +239,7 @@ def test_raises_exception_if_non_json_response( # then assert ( exec_info.value.args[0] - == f"Call to vectorize image returned malformed response body: {response_body}" + == f"Call to Azure Computer Vision returned malformed response body: {response_body}" ) assert isinstance(exec_info.value.__cause__, JSONDecodeError) @@ -240,5 +261,5 @@ def test_raises_exception_if_vector_not_in_response( # then assert ( exec_info.value.args[0] - == f"Call to vectorize image returned no vector: {response_body}" + == f"Call to Azure Computer Vision returned no vector: {response_body}" ) diff --git a/code/tests/utilities/helpers/test_azure_search_helper.py b/code/tests/utilities/helpers/test_azure_search_helper.py index 11e808a19..f16f07792 100644 --- a/code/tests/utilities/helpers/test_azure_search_helper.py +++ b/code/tests/utilities/helpers/test_azure_search_helper.py @@ -30,6 +30,9 @@ AZURE_SEARCH_CONVERSATIONS_LOG_INDEX = "mock-log-index" USE_ADVANCED_IMAGE_PROCESSING = False +SEARCH_EMBEDDINGS = [0, 0, 0, 0] +IMAGE_SEARCH_EMBEDDINGS = [0, 0, 0] + @pytest.fixture(autouse=True) def azure_search_mock(): @@ -43,9 +46,9 @@ def azure_search_mock(): def llm_helper_mock(): with patch("backend.batch.utilities.helpers.azure_search_helper.LLMHelper") as mock: llm_helper = mock.return_value - llm_helper.get_embedding_model.return_value.embed_query.return_value = [ - 0 - ] * 1536 + llm_helper.get_embedding_model.return_value.embed_query.return_value = ( + SEARCH_EMBEDDINGS + ) yield llm_helper @@ -75,8 +78,21 @@ def env_helper_mock(): @pytest.fixture(autouse=True) def reset_search_dimensions(): AzureSearchHelper._search_dimension = None + AzureSearchHelper._image_search_dimension = None yield AzureSearchHelper._search_dimension = None + AzureSearchHelper._image_search_dimension = None + + +@pytest.fixture(autouse=True) +def azure_computer_vision_client_mock(): + with patch( + "backend.batch.utilities.helpers.azure_search_helper.AzureComputerVisionClient" + ) as mock: + client = mock.return_value + client.vectorize_text.return_value = IMAGE_SEARCH_EMBEDDINGS + + yield client @patch("backend.batch.utilities.helpers.azure_search_helper.SearchClient") @@ -170,7 +186,7 @@ def test_creates_search_index_if_not_exists( name="content_vector", type=SearchFieldDataType.Collection(SearchFieldDataType.Single), searchable=True, - vector_search_dimensions=1536, + vector_search_dimensions=len(SEARCH_EMBEDDINGS), vector_search_profile_name="myHnswProfile", ), SearchableField( @@ -272,7 +288,7 @@ def test_creates_search_index_with_image_embeddings_when_advanced_image_processi name="image_vector", type=SearchFieldDataType.Collection(SearchFieldDataType.Single), searchable=True, - vector_search_dimensions=1024, + vector_search_dimensions=len(IMAGE_SEARCH_EMBEDDINGS), vector_search_profile_name="myHnswProfile", ) diff --git a/infra/app/adminweb.bicep b/infra/app/adminweb.bicep index 8303892cc..d2e993282 100644 --- a/infra/app/adminweb.bicep +++ b/infra/app/adminweb.bicep @@ -14,6 +14,7 @@ param keyVaultName string = '' param azureOpenAIName string = '' param azureAISearchName string = '' param speechServiceName string = '' +param computerVisionName string = '' @secure() param appSettings object = {} param useKeyVault bool @@ -21,6 +22,7 @@ param openAIKeyName string = '' param storageAccountKeyName string = '' param formRecognizerKeyName string = '' param searchKeyName string = '' +param computerVisionKeyName string = '' param contentSafetyKeyName string = '' param speechKeyName string = '' param authType string @@ -111,6 +113,17 @@ module adminweb '../core/host/appservice.bicep' = { ), '2023-05-01' ).key1 + AZURE_COMPUTER_VISION_KEY: (useKeyVault || computerVisionName == '') + ? computerVisionKeyName + : listKeys( + resourceId( + subscription().subscriptionId, + resourceGroup().name, + 'Microsoft.CognitiveServices/accounts', + computerVisionName + ), + '2023-05-01' + ).key1 }) } } diff --git a/infra/app/web.bicep b/infra/app/web.bicep index 9bba279e2..142062a46 100644 --- a/infra/app/web.bicep +++ b/infra/app/web.bicep @@ -14,6 +14,7 @@ param storageAccountName string = '' param formRecognizerName string = '' param contentSafetyName string = '' param speechServiceName string = '' +param computerVisionName string = '' @secure() param appSettings object = {} param useKeyVault bool @@ -21,6 +22,7 @@ param openAIKeyName string = '' param storageAccountKeyName string = '' param formRecognizerKeyName string = '' param searchKeyName string = '' +param computerVisionKeyName string = '' param contentSafetyKeyName string = '' param speechKeyName string = '' param authType string @@ -107,6 +109,17 @@ module web '../core/host/appservice.bicep' = { ), '2023-05-01' ).key1 + AZURE_COMPUTER_VISION_KEY: (useKeyVault || computerVisionName == '') + ? computerVisionKeyName + : listKeys( + resourceId( + subscription().subscriptionId, + resourceGroup().name, + 'Microsoft.CognitiveServices/accounts', + computerVisionName + ), + '2023-05-01' + ).key1 }) keyVaultName: keyVaultName runtimeName: runtimeName diff --git a/infra/main.bicep b/infra/main.bicep index 81a59711b..f57d93075 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -495,18 +495,23 @@ module web './app/web.bicep' = if (hostingModel == 'code') { formRecognizerName: formrecognizer.outputs.name contentSafetyName: contentsafety.outputs.name speechServiceName: speechService.outputs.name + computerVisionName: useAdvancedImageProcessing ? computerVision.outputs.name : '' openAIKeyName: useKeyVault ? storekeys.outputs.OPENAI_KEY_NAME : '' storageAccountKeyName: useKeyVault ? storekeys.outputs.STORAGE_ACCOUNT_KEY_NAME : '' formRecognizerKeyName: useKeyVault ? storekeys.outputs.FORM_RECOGNIZER_KEY_NAME : '' searchKeyName: useKeyVault ? storekeys.outputs.SEARCH_KEY_NAME : '' contentSafetyKeyName: useKeyVault ? storekeys.outputs.CONTENT_SAFETY_KEY_NAME : '' speechKeyName: useKeyVault ? storekeys.outputs.SPEECH_KEY_NAME : '' + computerVisionKeyName: useKeyVault ? storekeys.outputs.COMPUTER_VISION_KEY_NAME : '' useKeyVault: useKeyVault keyVaultName: useKeyVault || authType == 'rbac' ? keyvault.outputs.name : '' authType: authType appSettings: { AZURE_BLOB_ACCOUNT_NAME: storageAccountName AZURE_BLOB_CONTAINER_NAME: blobContainerName + AZURE_COMPUTER_VISION_ENDPOINT: useAdvancedImageProcessing ? computerVision.outputs.endpoint : '' + AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_API_VERSION: computerVisionVectorizeImageApiVersion + AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION: computerVisionVectorizeImageModelVersion AZURE_CONTENT_SAFETY_ENDPOINT: contentsafety.outputs.endpoint AZURE_FORM_RECOGNIZER_ENDPOINT: formrecognizer.outputs.endpoint AZURE_OPENAI_RESOURCE: azureOpenAIResourceName @@ -537,6 +542,7 @@ module web './app/web.bicep' = if (hostingModel == 'code') { AZURE_SPEECH_SERVICE_NAME: speechServiceName AZURE_SPEECH_SERVICE_REGION: location AZURE_SPEECH_RECOGNIZER_LANGUAGES: recognizedLanguages + USE_ADVANCED_IMAGE_PROCESSING: useAdvancedImageProcessing ORCHESTRATION_STRATEGY: orchestrationStrategy LOGLEVEL: logLevel } @@ -560,10 +566,12 @@ module web_docker './app/web.bicep' = if (hostingModel == 'container') { formRecognizerName: formrecognizer.outputs.name contentSafetyName: contentsafety.outputs.name speechServiceName: speechService.outputs.name + computerVisionName: useAdvancedImageProcessing ? computerVision.outputs.name : '' openAIKeyName: useKeyVault ? storekeys.outputs.OPENAI_KEY_NAME : '' storageAccountKeyName: useKeyVault ? storekeys.outputs.STORAGE_ACCOUNT_KEY_NAME : '' formRecognizerKeyName: useKeyVault ? storekeys.outputs.FORM_RECOGNIZER_KEY_NAME : '' searchKeyName: useKeyVault ? storekeys.outputs.SEARCH_KEY_NAME : '' + computerVisionKeyName: useKeyVault ? storekeys.outputs.COMPUTER_VISION_KEY_NAME : '' contentSafetyKeyName: useKeyVault ? storekeys.outputs.CONTENT_SAFETY_KEY_NAME : '' speechKeyName: useKeyVault ? storekeys.outputs.SPEECH_KEY_NAME : '' useKeyVault: useKeyVault @@ -572,6 +580,9 @@ module web_docker './app/web.bicep' = if (hostingModel == 'container') { appSettings: { AZURE_BLOB_ACCOUNT_NAME: storageAccountName AZURE_BLOB_CONTAINER_NAME: blobContainerName + AZURE_COMPUTER_VISION_ENDPOINT: useAdvancedImageProcessing ? computerVision.outputs.endpoint : '' + AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_API_VERSION: computerVisionVectorizeImageApiVersion + AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION: computerVisionVectorizeImageModelVersion AZURE_CONTENT_SAFETY_ENDPOINT: contentsafety.outputs.endpoint AZURE_FORM_RECOGNIZER_ENDPOINT: formrecognizer.outputs.endpoint AZURE_OPENAI_RESOURCE: azureOpenAIResourceName @@ -602,6 +613,7 @@ module web_docker './app/web.bicep' = if (hostingModel == 'container') { AZURE_SPEECH_SERVICE_NAME: speechServiceName AZURE_SPEECH_SERVICE_REGION: location AZURE_SPEECH_RECOGNIZER_LANGUAGES: recognizedLanguages + USE_ADVANCED_IMAGE_PROCESSING: useAdvancedImageProcessing ORCHESTRATION_STRATEGY: orchestrationStrategy LOGLEVEL: logLevel } @@ -625,10 +637,12 @@ module adminweb './app/adminweb.bicep' = if (hostingModel == 'code') { formRecognizerName: formrecognizer.outputs.name contentSafetyName: contentsafety.outputs.name speechServiceName: speechService.outputs.name + computerVisionName: useAdvancedImageProcessing ? computerVision.outputs.name : '' openAIKeyName: useKeyVault ? storekeys.outputs.OPENAI_KEY_NAME : '' storageAccountKeyName: useKeyVault ? storekeys.outputs.STORAGE_ACCOUNT_KEY_NAME : '' formRecognizerKeyName: useKeyVault ? storekeys.outputs.FORM_RECOGNIZER_KEY_NAME : '' searchKeyName: useKeyVault ? storekeys.outputs.SEARCH_KEY_NAME : '' + computerVisionKeyName: useKeyVault ? storekeys.outputs.COMPUTER_VISION_KEY_NAME : '' contentSafetyKeyName: useKeyVault ? storekeys.outputs.CONTENT_SAFETY_KEY_NAME : '' speechKeyName: useKeyVault ? storekeys.outputs.SPEECH_KEY_NAME : '' useKeyVault: useKeyVault @@ -637,6 +651,9 @@ module adminweb './app/adminweb.bicep' = if (hostingModel == 'code') { appSettings: { AZURE_BLOB_ACCOUNT_NAME: storageAccountName AZURE_BLOB_CONTAINER_NAME: blobContainerName + AZURE_COMPUTER_VISION_ENDPOINT: useAdvancedImageProcessing ? computerVision.outputs.endpoint : '' + AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_API_VERSION: computerVisionVectorizeImageApiVersion + AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION: computerVisionVectorizeImageModelVersion AZURE_CONTENT_SAFETY_ENDPOINT: contentsafety.outputs.endpoint AZURE_FORM_RECOGNIZER_ENDPOINT: formrecognizer.outputs.endpoint AZURE_OPENAI_RESOURCE: azureOpenAIResourceName @@ -691,18 +708,23 @@ module adminweb_docker './app/adminweb.bicep' = if (hostingModel == 'container') formRecognizerName: formrecognizer.outputs.name contentSafetyName: contentsafety.outputs.name speechServiceName: speechService.outputs.name + computerVisionName: useAdvancedImageProcessing ? computerVision.outputs.name : '' openAIKeyName: useKeyVault ? storekeys.outputs.OPENAI_KEY_NAME : '' storageAccountKeyName: useKeyVault ? storekeys.outputs.STORAGE_ACCOUNT_KEY_NAME : '' formRecognizerKeyName: useKeyVault ? storekeys.outputs.FORM_RECOGNIZER_KEY_NAME : '' searchKeyName: useKeyVault ? storekeys.outputs.SEARCH_KEY_NAME : '' contentSafetyKeyName: useKeyVault ? storekeys.outputs.CONTENT_SAFETY_KEY_NAME : '' speechKeyName: useKeyVault ? storekeys.outputs.SPEECH_KEY_NAME : '' + computerVisionKeyName: useKeyVault ? storekeys.outputs.COMPUTER_VISION_KEY_NAME : '' useKeyVault: useKeyVault keyVaultName: useKeyVault || authType == 'rbac' ? keyvault.outputs.name : '' authType: authType appSettings: { AZURE_BLOB_ACCOUNT_NAME: storageAccountName AZURE_BLOB_CONTAINER_NAME: blobContainerName + AZURE_COMPUTER_VISION_ENDPOINT: useAdvancedImageProcessing ? computerVision.outputs.endpoint : '' + AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_API_VERSION: computerVisionVectorizeImageApiVersion + AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION: computerVisionVectorizeImageModelVersion AZURE_CONTENT_SAFETY_ENDPOINT: contentsafety.outputs.endpoint AZURE_FORM_RECOGNIZER_ENDPOINT: formrecognizer.outputs.endpoint AZURE_OPENAI_RESOURCE: azureOpenAIResourceName @@ -807,6 +829,7 @@ module function './app/function.bicep' = if (hostingModel == 'code') { appSettings: { AZURE_BLOB_ACCOUNT_NAME: storageAccountName AZURE_BLOB_CONTAINER_NAME: blobContainerName + AZURE_COMPUTER_VISION_ENDPOINT: useAdvancedImageProcessing ? computerVision.outputs.endpoint : '' AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_API_VERSION: computerVisionVectorizeImageApiVersion AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION: computerVisionVectorizeImageModelVersion AZURE_CONTENT_SAFETY_ENDPOINT: contentsafety.outputs.endpoint @@ -821,7 +844,6 @@ module function './app/function.bicep' = if (hostingModel == 'code') { AZURE_SEARCH_INDEXER_NAME: azureSearchIndexer AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION: azureSearchUseIntegratedVectorization USE_ADVANCED_IMAGE_PROCESSING: useAdvancedImageProcessing - AZURE_COMPUTER_VISION_ENDPOINT: useAdvancedImageProcessing ? computerVision.outputs.endpoint : '' DOCUMENT_PROCESSING_QUEUE_NAME: queueName ORCHESTRATION_STRATEGY: orchestrationStrategy LOGLEVEL: logLevel @@ -860,6 +882,7 @@ module function_docker './app/function.bicep' = if (hostingModel == 'container') appSettings: { AZURE_BLOB_ACCOUNT_NAME: storageAccountName AZURE_BLOB_CONTAINER_NAME: blobContainerName + AZURE_COMPUTER_VISION_ENDPOINT: useAdvancedImageProcessing ? computerVision.outputs.endpoint : '' AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_API_VERSION: computerVisionVectorizeImageApiVersion AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION: computerVisionVectorizeImageModelVersion AZURE_CONTENT_SAFETY_ENDPOINT: contentsafety.outputs.endpoint @@ -874,7 +897,6 @@ module function_docker './app/function.bicep' = if (hostingModel == 'container') AZURE_SEARCH_INDEXER_NAME: azureSearchIndexer AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION: azureSearchUseIntegratedVectorization USE_ADVANCED_IMAGE_PROCESSING: useAdvancedImageProcessing - AZURE_COMPUTER_VISION_ENDPOINT: useAdvancedImageProcessing ? computerVision.outputs.endpoint : '' DOCUMENT_PROCESSING_QUEUE_NAME: queueName ORCHESTRATION_STRATEGY: orchestrationStrategy LOGLEVEL: logLevel diff --git a/infra/main.json b/infra/main.json index 4939310fd..3a9584621 100644 --- a/infra/main.json +++ b/infra/main.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.27.1.19265", - "templateHash": "5735866947841378196" + "templateHash": "11889969251281398200" } }, "parameters": { @@ -1959,12 +1959,14 @@ "speechServiceName": { "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('speechServiceName')), '2022-09-01').outputs.name.value]" }, + "computerVisionName": "[if(parameters('useAdvancedImageProcessing'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision'), '2022-09-01').outputs.name.value), createObject('value', ''))]", "openAIKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.OPENAI_KEY_NAME.value), createObject('value', ''))]", "storageAccountKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.STORAGE_ACCOUNT_KEY_NAME.value), createObject('value', ''))]", "formRecognizerKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.FORM_RECOGNIZER_KEY_NAME.value), createObject('value', ''))]", "searchKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.SEARCH_KEY_NAME.value), createObject('value', ''))]", "contentSafetyKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.CONTENT_SAFETY_KEY_NAME.value), createObject('value', ''))]", "speechKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.SPEECH_KEY_NAME.value), createObject('value', ''))]", + "computerVisionKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.COMPUTER_VISION_KEY_NAME.value), createObject('value', ''))]", "useKeyVault": { "value": "[parameters('useKeyVault')]" }, @@ -1976,6 +1978,9 @@ "value": { "AZURE_BLOB_ACCOUNT_NAME": "[parameters('storageAccountName')]", "AZURE_BLOB_CONTAINER_NAME": "[variables('blobContainerName')]", + "AZURE_COMPUTER_VISION_ENDPOINT": "[if(parameters('useAdvancedImageProcessing'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision'), '2022-09-01').outputs.endpoint.value, '')]", + "AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_API_VERSION": "[parameters('computerVisionVectorizeImageApiVersion')]", + "AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION": "[parameters('computerVisionVectorizeImageModelVersion')]", "AZURE_CONTENT_SAFETY_ENDPOINT": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('contentSafetyName')), '2022-09-01').outputs.endpoint.value]", "AZURE_FORM_RECOGNIZER_ENDPOINT": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('formRecognizerName')), '2022-09-01').outputs.endpoint.value]", "AZURE_OPENAI_RESOURCE": "[parameters('azureOpenAIResourceName')]", @@ -2006,6 +2011,7 @@ "AZURE_SPEECH_SERVICE_NAME": "[parameters('speechServiceName')]", "AZURE_SPEECH_SERVICE_REGION": "[parameters('location')]", "AZURE_SPEECH_RECOGNIZER_LANGUAGES": "[parameters('recognizedLanguages')]", + "USE_ADVANCED_IMAGE_PROCESSING": "[parameters('useAdvancedImageProcessing')]", "ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]", "LOGLEVEL": "[parameters('logLevel')]" } @@ -2018,7 +2024,7 @@ "_generator": { "name": "bicep", "version": "0.27.1.19265", - "templateHash": "1163226522178490219" + "templateHash": "3523301306282440498" } }, "parameters": { @@ -2084,6 +2090,10 @@ "type": "string", "defaultValue": "" }, + "computerVisionName": { + "type": "string", + "defaultValue": "" + }, "appSettings": { "type": "secureObject", "defaultValue": {} @@ -2107,6 +2117,10 @@ "type": "string", "defaultValue": "" }, + "computerVisionKeyName": { + "type": "string", + "defaultValue": "" + }, "contentSafetyKeyName": { "type": "string", "defaultValue": "" @@ -2162,7 +2176,7 @@ "value": "[parameters('appServicePlanId')]" }, "appSettings": { - "value": "[union(parameters('appSettings'), createObject('AZURE_AUTH_TYPE', parameters('authType'), 'USE_KEY_VAULT', if(parameters('useKeyVault'), parameters('useKeyVault'), ''), 'AZURE_OPENAI_API_KEY', if(parameters('useKeyVault'), parameters('openAIKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('azureOpenAIName')), '2023-05-01').key1), 'AZURE_SEARCH_KEY', if(parameters('useKeyVault'), parameters('searchKeyName'), listAdminKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Search/searchServices', parameters('azureAISearchName')), '2021-04-01-preview').primaryKey), 'AZURE_BLOB_ACCOUNT_KEY', if(parameters('useKeyVault'), parameters('storageAccountKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2021-09-01').keys[0].value), 'AZURE_FORM_RECOGNIZER_KEY', if(parameters('useKeyVault'), parameters('formRecognizerKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('formRecognizerName')), '2023-05-01').key1), 'AZURE_CONTENT_SAFETY_KEY', if(parameters('useKeyVault'), parameters('contentSafetyKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('contentSafetyName')), '2023-05-01').key1), 'AZURE_SPEECH_SERVICE_KEY', if(parameters('useKeyVault'), parameters('speechKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('speechServiceName')), '2023-05-01').key1)))]" + "value": "[union(parameters('appSettings'), createObject('AZURE_AUTH_TYPE', parameters('authType'), 'USE_KEY_VAULT', if(parameters('useKeyVault'), parameters('useKeyVault'), ''), 'AZURE_OPENAI_API_KEY', if(parameters('useKeyVault'), parameters('openAIKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('azureOpenAIName')), '2023-05-01').key1), 'AZURE_SEARCH_KEY', if(parameters('useKeyVault'), parameters('searchKeyName'), listAdminKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Search/searchServices', parameters('azureAISearchName')), '2021-04-01-preview').primaryKey), 'AZURE_BLOB_ACCOUNT_KEY', if(parameters('useKeyVault'), parameters('storageAccountKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2021-09-01').keys[0].value), 'AZURE_FORM_RECOGNIZER_KEY', if(parameters('useKeyVault'), parameters('formRecognizerKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('formRecognizerName')), '2023-05-01').key1), 'AZURE_CONTENT_SAFETY_KEY', if(parameters('useKeyVault'), parameters('contentSafetyKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('contentSafetyName')), '2023-05-01').key1), 'AZURE_SPEECH_SERVICE_KEY', if(parameters('useKeyVault'), parameters('speechKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('speechServiceName')), '2023-05-01').key1), 'AZURE_COMPUTER_VISION_KEY', if(or(parameters('useKeyVault'), equals(parameters('computerVisionName'), '')), parameters('computerVisionKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('computerVisionName')), '2023-05-01').key1)))]" }, "keyVaultName": { "value": "[parameters('keyVaultName')]" @@ -2831,6 +2845,7 @@ } }, "dependsOn": [ + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision')]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('contentSafetyName'))]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('formRecognizerName'))]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('hostingPlanName'))]", @@ -2895,10 +2910,12 @@ "speechServiceName": { "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('speechServiceName')), '2022-09-01').outputs.name.value]" }, + "computerVisionName": "[if(parameters('useAdvancedImageProcessing'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision'), '2022-09-01').outputs.name.value), createObject('value', ''))]", "openAIKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.OPENAI_KEY_NAME.value), createObject('value', ''))]", "storageAccountKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.STORAGE_ACCOUNT_KEY_NAME.value), createObject('value', ''))]", "formRecognizerKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.FORM_RECOGNIZER_KEY_NAME.value), createObject('value', ''))]", "searchKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.SEARCH_KEY_NAME.value), createObject('value', ''))]", + "computerVisionKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.COMPUTER_VISION_KEY_NAME.value), createObject('value', ''))]", "contentSafetyKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.CONTENT_SAFETY_KEY_NAME.value), createObject('value', ''))]", "speechKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.SPEECH_KEY_NAME.value), createObject('value', ''))]", "useKeyVault": { @@ -2912,6 +2929,9 @@ "value": { "AZURE_BLOB_ACCOUNT_NAME": "[parameters('storageAccountName')]", "AZURE_BLOB_CONTAINER_NAME": "[variables('blobContainerName')]", + "AZURE_COMPUTER_VISION_ENDPOINT": "[if(parameters('useAdvancedImageProcessing'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision'), '2022-09-01').outputs.endpoint.value, '')]", + "AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_API_VERSION": "[parameters('computerVisionVectorizeImageApiVersion')]", + "AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION": "[parameters('computerVisionVectorizeImageModelVersion')]", "AZURE_CONTENT_SAFETY_ENDPOINT": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('contentSafetyName')), '2022-09-01').outputs.endpoint.value]", "AZURE_FORM_RECOGNIZER_ENDPOINT": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('formRecognizerName')), '2022-09-01').outputs.endpoint.value]", "AZURE_OPENAI_RESOURCE": "[parameters('azureOpenAIResourceName')]", @@ -2942,6 +2962,7 @@ "AZURE_SPEECH_SERVICE_NAME": "[parameters('speechServiceName')]", "AZURE_SPEECH_SERVICE_REGION": "[parameters('location')]", "AZURE_SPEECH_RECOGNIZER_LANGUAGES": "[parameters('recognizedLanguages')]", + "USE_ADVANCED_IMAGE_PROCESSING": "[parameters('useAdvancedImageProcessing')]", "ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]", "LOGLEVEL": "[parameters('logLevel')]" } @@ -2954,7 +2975,7 @@ "_generator": { "name": "bicep", "version": "0.27.1.19265", - "templateHash": "1163226522178490219" + "templateHash": "3523301306282440498" } }, "parameters": { @@ -3020,6 +3041,10 @@ "type": "string", "defaultValue": "" }, + "computerVisionName": { + "type": "string", + "defaultValue": "" + }, "appSettings": { "type": "secureObject", "defaultValue": {} @@ -3043,6 +3068,10 @@ "type": "string", "defaultValue": "" }, + "computerVisionKeyName": { + "type": "string", + "defaultValue": "" + }, "contentSafetyKeyName": { "type": "string", "defaultValue": "" @@ -3098,7 +3127,7 @@ "value": "[parameters('appServicePlanId')]" }, "appSettings": { - "value": "[union(parameters('appSettings'), createObject('AZURE_AUTH_TYPE', parameters('authType'), 'USE_KEY_VAULT', if(parameters('useKeyVault'), parameters('useKeyVault'), ''), 'AZURE_OPENAI_API_KEY', if(parameters('useKeyVault'), parameters('openAIKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('azureOpenAIName')), '2023-05-01').key1), 'AZURE_SEARCH_KEY', if(parameters('useKeyVault'), parameters('searchKeyName'), listAdminKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Search/searchServices', parameters('azureAISearchName')), '2021-04-01-preview').primaryKey), 'AZURE_BLOB_ACCOUNT_KEY', if(parameters('useKeyVault'), parameters('storageAccountKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2021-09-01').keys[0].value), 'AZURE_FORM_RECOGNIZER_KEY', if(parameters('useKeyVault'), parameters('formRecognizerKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('formRecognizerName')), '2023-05-01').key1), 'AZURE_CONTENT_SAFETY_KEY', if(parameters('useKeyVault'), parameters('contentSafetyKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('contentSafetyName')), '2023-05-01').key1), 'AZURE_SPEECH_SERVICE_KEY', if(parameters('useKeyVault'), parameters('speechKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('speechServiceName')), '2023-05-01').key1)))]" + "value": "[union(parameters('appSettings'), createObject('AZURE_AUTH_TYPE', parameters('authType'), 'USE_KEY_VAULT', if(parameters('useKeyVault'), parameters('useKeyVault'), ''), 'AZURE_OPENAI_API_KEY', if(parameters('useKeyVault'), parameters('openAIKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('azureOpenAIName')), '2023-05-01').key1), 'AZURE_SEARCH_KEY', if(parameters('useKeyVault'), parameters('searchKeyName'), listAdminKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Search/searchServices', parameters('azureAISearchName')), '2021-04-01-preview').primaryKey), 'AZURE_BLOB_ACCOUNT_KEY', if(parameters('useKeyVault'), parameters('storageAccountKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2021-09-01').keys[0].value), 'AZURE_FORM_RECOGNIZER_KEY', if(parameters('useKeyVault'), parameters('formRecognizerKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('formRecognizerName')), '2023-05-01').key1), 'AZURE_CONTENT_SAFETY_KEY', if(parameters('useKeyVault'), parameters('contentSafetyKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('contentSafetyName')), '2023-05-01').key1), 'AZURE_SPEECH_SERVICE_KEY', if(parameters('useKeyVault'), parameters('speechKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('speechServiceName')), '2023-05-01').key1), 'AZURE_COMPUTER_VISION_KEY', if(or(parameters('useKeyVault'), equals(parameters('computerVisionName'), '')), parameters('computerVisionKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('computerVisionName')), '2023-05-01').key1)))]" }, "keyVaultName": { "value": "[parameters('keyVaultName')]" @@ -3767,6 +3796,7 @@ } }, "dependsOn": [ + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision')]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('contentSafetyName'))]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('formRecognizerName'))]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('hostingPlanName'))]", @@ -3831,10 +3861,12 @@ "speechServiceName": { "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('speechServiceName')), '2022-09-01').outputs.name.value]" }, + "computerVisionName": "[if(parameters('useAdvancedImageProcessing'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision'), '2022-09-01').outputs.name.value), createObject('value', ''))]", "openAIKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.OPENAI_KEY_NAME.value), createObject('value', ''))]", "storageAccountKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.STORAGE_ACCOUNT_KEY_NAME.value), createObject('value', ''))]", "formRecognizerKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.FORM_RECOGNIZER_KEY_NAME.value), createObject('value', ''))]", "searchKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.SEARCH_KEY_NAME.value), createObject('value', ''))]", + "computerVisionKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.COMPUTER_VISION_KEY_NAME.value), createObject('value', ''))]", "contentSafetyKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.CONTENT_SAFETY_KEY_NAME.value), createObject('value', ''))]", "speechKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.SPEECH_KEY_NAME.value), createObject('value', ''))]", "useKeyVault": { @@ -3848,6 +3880,9 @@ "value": { "AZURE_BLOB_ACCOUNT_NAME": "[parameters('storageAccountName')]", "AZURE_BLOB_CONTAINER_NAME": "[variables('blobContainerName')]", + "AZURE_COMPUTER_VISION_ENDPOINT": "[if(parameters('useAdvancedImageProcessing'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision'), '2022-09-01').outputs.endpoint.value, '')]", + "AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_API_VERSION": "[parameters('computerVisionVectorizeImageApiVersion')]", + "AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION": "[parameters('computerVisionVectorizeImageModelVersion')]", "AZURE_CONTENT_SAFETY_ENDPOINT": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('contentSafetyName')), '2022-09-01').outputs.endpoint.value]", "AZURE_FORM_RECOGNIZER_ENDPOINT": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('formRecognizerName')), '2022-09-01').outputs.endpoint.value]", "AZURE_OPENAI_RESOURCE": "[parameters('azureOpenAIResourceName')]", @@ -3892,7 +3927,7 @@ "_generator": { "name": "bicep", "version": "0.27.1.19265", - "templateHash": "6071122860249879835" + "templateHash": "9122829587959554559" } }, "parameters": { @@ -3958,6 +3993,10 @@ "type": "string", "defaultValue": "" }, + "computerVisionName": { + "type": "string", + "defaultValue": "" + }, "appSettings": { "type": "secureObject", "defaultValue": {} @@ -3981,6 +4020,10 @@ "type": "string", "defaultValue": "" }, + "computerVisionKeyName": { + "type": "string", + "defaultValue": "" + }, "contentSafetyKeyName": { "type": "string", "defaultValue": "" @@ -4045,7 +4088,7 @@ "value": "[parameters('appServicePlanId')]" }, "appSettings": { - "value": "[union(parameters('appSettings'), createObject('AZURE_AUTH_TYPE', parameters('authType'), 'USE_KEY_VAULT', if(parameters('useKeyVault'), parameters('useKeyVault'), ''), 'AZURE_OPENAI_API_KEY', if(parameters('useKeyVault'), parameters('openAIKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('azureOpenAIName')), '2023-05-01').key1), 'AZURE_SEARCH_KEY', if(parameters('useKeyVault'), parameters('searchKeyName'), listAdminKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Search/searchServices', parameters('azureAISearchName')), '2021-04-01-preview').primaryKey), 'AZURE_BLOB_ACCOUNT_KEY', if(parameters('useKeyVault'), parameters('storageAccountKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2021-09-01').keys[0].value), 'AZURE_FORM_RECOGNIZER_KEY', if(parameters('useKeyVault'), parameters('formRecognizerKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('formRecognizerName')), '2023-05-01').key1), 'AZURE_CONTENT_SAFETY_KEY', if(parameters('useKeyVault'), parameters('contentSafetyKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('contentSafetyName')), '2023-05-01').key1), 'AZURE_SPEECH_SERVICE_KEY', if(parameters('useKeyVault'), parameters('speechKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('speechServiceName')), '2023-05-01').key1)))]" + "value": "[union(parameters('appSettings'), createObject('AZURE_AUTH_TYPE', parameters('authType'), 'USE_KEY_VAULT', if(parameters('useKeyVault'), parameters('useKeyVault'), ''), 'AZURE_OPENAI_API_KEY', if(parameters('useKeyVault'), parameters('openAIKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('azureOpenAIName')), '2023-05-01').key1), 'AZURE_SEARCH_KEY', if(parameters('useKeyVault'), parameters('searchKeyName'), listAdminKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Search/searchServices', parameters('azureAISearchName')), '2021-04-01-preview').primaryKey), 'AZURE_BLOB_ACCOUNT_KEY', if(parameters('useKeyVault'), parameters('storageAccountKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2021-09-01').keys[0].value), 'AZURE_FORM_RECOGNIZER_KEY', if(parameters('useKeyVault'), parameters('formRecognizerKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('formRecognizerName')), '2023-05-01').key1), 'AZURE_CONTENT_SAFETY_KEY', if(parameters('useKeyVault'), parameters('contentSafetyKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('contentSafetyName')), '2023-05-01').key1), 'AZURE_SPEECH_SERVICE_KEY', if(parameters('useKeyVault'), parameters('speechKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('speechServiceName')), '2023-05-01').key1), 'AZURE_COMPUTER_VISION_KEY', if(or(parameters('useKeyVault'), equals(parameters('computerVisionName'), '')), parameters('computerVisionKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('computerVisionName')), '2023-05-01').key1)))]" } }, "template": { @@ -4698,6 +4741,7 @@ } }, "dependsOn": [ + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision')]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('contentSafetyName'))]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('formRecognizerName'))]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('hostingPlanName'))]", @@ -4759,12 +4803,14 @@ "speechServiceName": { "value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('speechServiceName')), '2022-09-01').outputs.name.value]" }, + "computerVisionName": "[if(parameters('useAdvancedImageProcessing'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision'), '2022-09-01').outputs.name.value), createObject('value', ''))]", "openAIKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.OPENAI_KEY_NAME.value), createObject('value', ''))]", "storageAccountKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.STORAGE_ACCOUNT_KEY_NAME.value), createObject('value', ''))]", "formRecognizerKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.FORM_RECOGNIZER_KEY_NAME.value), createObject('value', ''))]", "searchKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.SEARCH_KEY_NAME.value), createObject('value', ''))]", "contentSafetyKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.CONTENT_SAFETY_KEY_NAME.value), createObject('value', ''))]", "speechKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.SPEECH_KEY_NAME.value), createObject('value', ''))]", + "computerVisionKeyName": "[if(parameters('useKeyVault'), createObject('value', reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'storekeys'), '2022-09-01').outputs.COMPUTER_VISION_KEY_NAME.value), createObject('value', ''))]", "useKeyVault": { "value": "[parameters('useKeyVault')]" }, @@ -4776,6 +4822,9 @@ "value": { "AZURE_BLOB_ACCOUNT_NAME": "[parameters('storageAccountName')]", "AZURE_BLOB_CONTAINER_NAME": "[variables('blobContainerName')]", + "AZURE_COMPUTER_VISION_ENDPOINT": "[if(parameters('useAdvancedImageProcessing'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision'), '2022-09-01').outputs.endpoint.value, '')]", + "AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_API_VERSION": "[parameters('computerVisionVectorizeImageApiVersion')]", + "AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION": "[parameters('computerVisionVectorizeImageModelVersion')]", "AZURE_CONTENT_SAFETY_ENDPOINT": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('contentSafetyName')), '2022-09-01').outputs.endpoint.value]", "AZURE_FORM_RECOGNIZER_ENDPOINT": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('formRecognizerName')), '2022-09-01').outputs.endpoint.value]", "AZURE_OPENAI_RESOURCE": "[parameters('azureOpenAIResourceName')]", @@ -4820,7 +4869,7 @@ "_generator": { "name": "bicep", "version": "0.27.1.19265", - "templateHash": "6071122860249879835" + "templateHash": "9122829587959554559" } }, "parameters": { @@ -4886,6 +4935,10 @@ "type": "string", "defaultValue": "" }, + "computerVisionName": { + "type": "string", + "defaultValue": "" + }, "appSettings": { "type": "secureObject", "defaultValue": {} @@ -4909,6 +4962,10 @@ "type": "string", "defaultValue": "" }, + "computerVisionKeyName": { + "type": "string", + "defaultValue": "" + }, "contentSafetyKeyName": { "type": "string", "defaultValue": "" @@ -4973,7 +5030,7 @@ "value": "[parameters('appServicePlanId')]" }, "appSettings": { - "value": "[union(parameters('appSettings'), createObject('AZURE_AUTH_TYPE', parameters('authType'), 'USE_KEY_VAULT', if(parameters('useKeyVault'), parameters('useKeyVault'), ''), 'AZURE_OPENAI_API_KEY', if(parameters('useKeyVault'), parameters('openAIKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('azureOpenAIName')), '2023-05-01').key1), 'AZURE_SEARCH_KEY', if(parameters('useKeyVault'), parameters('searchKeyName'), listAdminKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Search/searchServices', parameters('azureAISearchName')), '2021-04-01-preview').primaryKey), 'AZURE_BLOB_ACCOUNT_KEY', if(parameters('useKeyVault'), parameters('storageAccountKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2021-09-01').keys[0].value), 'AZURE_FORM_RECOGNIZER_KEY', if(parameters('useKeyVault'), parameters('formRecognizerKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('formRecognizerName')), '2023-05-01').key1), 'AZURE_CONTENT_SAFETY_KEY', if(parameters('useKeyVault'), parameters('contentSafetyKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('contentSafetyName')), '2023-05-01').key1), 'AZURE_SPEECH_SERVICE_KEY', if(parameters('useKeyVault'), parameters('speechKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('speechServiceName')), '2023-05-01').key1)))]" + "value": "[union(parameters('appSettings'), createObject('AZURE_AUTH_TYPE', parameters('authType'), 'USE_KEY_VAULT', if(parameters('useKeyVault'), parameters('useKeyVault'), ''), 'AZURE_OPENAI_API_KEY', if(parameters('useKeyVault'), parameters('openAIKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('azureOpenAIName')), '2023-05-01').key1), 'AZURE_SEARCH_KEY', if(parameters('useKeyVault'), parameters('searchKeyName'), listAdminKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Search/searchServices', parameters('azureAISearchName')), '2021-04-01-preview').primaryKey), 'AZURE_BLOB_ACCOUNT_KEY', if(parameters('useKeyVault'), parameters('storageAccountKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.Storage/storageAccounts', parameters('storageAccountName')), '2021-09-01').keys[0].value), 'AZURE_FORM_RECOGNIZER_KEY', if(parameters('useKeyVault'), parameters('formRecognizerKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('formRecognizerName')), '2023-05-01').key1), 'AZURE_CONTENT_SAFETY_KEY', if(parameters('useKeyVault'), parameters('contentSafetyKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('contentSafetyName')), '2023-05-01').key1), 'AZURE_SPEECH_SERVICE_KEY', if(parameters('useKeyVault'), parameters('speechKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('speechServiceName')), '2023-05-01').key1), 'AZURE_COMPUTER_VISION_KEY', if(or(parameters('useKeyVault'), equals(parameters('computerVisionName'), '')), parameters('computerVisionKeyName'), listKeys(resourceId(subscription().subscriptionId, resourceGroup().name, 'Microsoft.CognitiveServices/accounts', parameters('computerVisionName')), '2023-05-01').key1)))]" } }, "template": { @@ -5626,6 +5683,7 @@ } }, "dependsOn": [ + "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision')]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('contentSafetyName'))]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('formRecognizerName'))]", "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('hostingPlanName'))]", @@ -7445,6 +7503,7 @@ "value": { "AZURE_BLOB_ACCOUNT_NAME": "[parameters('storageAccountName')]", "AZURE_BLOB_CONTAINER_NAME": "[variables('blobContainerName')]", + "AZURE_COMPUTER_VISION_ENDPOINT": "[if(parameters('useAdvancedImageProcessing'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision'), '2022-09-01').outputs.endpoint.value, '')]", "AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_API_VERSION": "[parameters('computerVisionVectorizeImageApiVersion')]", "AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION": "[parameters('computerVisionVectorizeImageModelVersion')]", "AZURE_CONTENT_SAFETY_ENDPOINT": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('contentSafetyName')), '2022-09-01').outputs.endpoint.value]", @@ -7459,7 +7518,6 @@ "AZURE_SEARCH_INDEXER_NAME": "[parameters('azureSearchIndexer')]", "AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION": "[parameters('azureSearchUseIntegratedVectorization')]", "USE_ADVANCED_IMAGE_PROCESSING": "[parameters('useAdvancedImageProcessing')]", - "AZURE_COMPUTER_VISION_ENDPOINT": "[if(parameters('useAdvancedImageProcessing'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision'), '2022-09-01').outputs.endpoint.value, '')]", "DOCUMENT_PROCESSING_QUEUE_NAME": "[variables('queueName')]", "ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]", "LOGLEVEL": "[parameters('logLevel')]" @@ -8680,6 +8738,7 @@ "value": { "AZURE_BLOB_ACCOUNT_NAME": "[parameters('storageAccountName')]", "AZURE_BLOB_CONTAINER_NAME": "[variables('blobContainerName')]", + "AZURE_COMPUTER_VISION_ENDPOINT": "[if(parameters('useAdvancedImageProcessing'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision'), '2022-09-01').outputs.endpoint.value, '')]", "AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_API_VERSION": "[parameters('computerVisionVectorizeImageApiVersion')]", "AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION": "[parameters('computerVisionVectorizeImageModelVersion')]", "AZURE_CONTENT_SAFETY_ENDPOINT": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', parameters('contentSafetyName')), '2022-09-01').outputs.endpoint.value]", @@ -8694,7 +8753,6 @@ "AZURE_SEARCH_INDEXER_NAME": "[parameters('azureSearchIndexer')]", "AZURE_SEARCH_USE_INTEGRATED_VECTORIZATION": "[parameters('azureSearchUseIntegratedVectorization')]", "USE_ADVANCED_IMAGE_PROCESSING": "[parameters('useAdvancedImageProcessing')]", - "AZURE_COMPUTER_VISION_ENDPOINT": "[if(parameters('useAdvancedImageProcessing'), reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, variables('rgName')), 'Microsoft.Resources/deployments', 'computerVision'), '2022-09-01').outputs.endpoint.value, '')]", "DOCUMENT_PROCESSING_QUEUE_NAME": "[variables('queueName')]", "ORCHESTRATION_STRATEGY": "[parameters('orchestrationStrategy')]", "LOGLEVEL": "[parameters('logLevel')]"