From fe405227bf70b4081308dc2e924a67a40995e241 Mon Sep 17 00:00:00 2001 From: XiangRao <131976564+ricardrao@users.noreply.github.com> Date: Thu, 7 Mar 2024 11:02:46 +0800 Subject: [PATCH] Fix DBCopilot environment vulnerabilities (#2448) * Fix DBCopilot environment vulnerabilities * fix * fix * fix * fix * fix * fix * fix * fix * fix * fix --- .../data_ingestion_db_to_acs/spec.yaml | 6 +++--- .../data_ingestion_db_to_faiss/spec.yaml | 6 +++--- .../spec.yaml | 6 +++--- .../spec.yaml | 6 +++--- .../llm_dbcopilot_create_promptflow/spec.yaml | 2 +- .../llm_dbcopilot_deploy_endpoint/spec.yaml | 2 +- .../llm_dbcopilot_grounding/spec.yaml | 2 +- .../spec.yaml | 2 +- .../dbcopilot_embeddings/context/Dockerfile | 19 +++++++++++++------ .../context/requirements.txt | 4 ++-- .../context/extra_requirements.txt | 4 ++-- 11 files changed, 33 insertions(+), 26 deletions(-) diff --git a/assets/large_language_models/components_pipelines/data_ingestion_db_to_acs/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_db_to_acs/spec.yaml index 41118e481f..2df4bfba22 100644 --- a/assets/large_language_models/components_pipelines/data_ingestion_db_to_acs/spec.yaml +++ b/assets/large_language_models/components_pipelines/data_ingestion_db_to_acs/spec.yaml @@ -128,7 +128,7 @@ jobs: output_chunk_file: type: uri_folder output_grounding_context_file: ${{parent.outputs.db_context}} - component: "azureml:llm_dbcopilot_grounding:0.0.47" + component: "azureml:llm_dbcopilot_grounding:0.0.48" type: command generate_meta_embeddings: type: command @@ -195,7 +195,7 @@ jobs: ######################################### db_sample_loading_generator: type: command - component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.23" + component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.24" resources: instance_count: ${{parent.inputs.serverless_instance_count}} instance_type: ${{parent.inputs.serverless_instance_type}} @@ -303,5 +303,5 @@ jobs: path: ${{parent.inputs.llm_config}} runtime: path: ${{parent.inputs.runtime}} - component: "azureml:llm_dbcopilot_create_promptflow:0.0.47" + component: "azureml:llm_dbcopilot_create_promptflow:0.0.48" type: command diff --git a/assets/large_language_models/components_pipelines/data_ingestion_db_to_faiss/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_db_to_faiss/spec.yaml index 31fbcb583d..42226adf72 100644 --- a/assets/large_language_models/components_pipelines/data_ingestion_db_to_faiss/spec.yaml +++ b/assets/large_language_models/components_pipelines/data_ingestion_db_to_faiss/spec.yaml @@ -118,7 +118,7 @@ jobs: output_chunk_file: type: uri_folder output_grounding_context_file: ${{parent.outputs.db_context}} - component: "azureml:llm_dbcopilot_grounding:0.0.47" + component: "azureml:llm_dbcopilot_grounding:0.0.48" type: command generate_meta_embeddings: type: command @@ -183,7 +183,7 @@ jobs: ######################################### db_sample_loading_generator: type: command - component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.23" + component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.24" resources: instance_count: ${{parent.inputs.serverless_instance_count}} instance_type: ${{parent.inputs.serverless_instance_type}} @@ -287,5 +287,5 @@ jobs: path: ${{parent.inputs.llm_config}} runtime: path: ${{parent.inputs.runtime}} - component: "azureml:llm_dbcopilot_create_promptflow:0.0.47" + component: "azureml:llm_dbcopilot_create_promptflow:0.0.48" type: command diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dbcopilot_acs_e2e/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dbcopilot_acs_e2e/spec.yaml index 1dbc95a8fc..0f9631ba30 100644 --- a/assets/large_language_models/components_pipelines/data_ingestion_dbcopilot_acs_e2e/spec.yaml +++ b/assets/large_language_models/components_pipelines/data_ingestion_dbcopilot_acs_e2e/spec.yaml @@ -120,7 +120,7 @@ jobs: ######################################### db_meta_loading_generator: type: command - component: "azureml:llm_dbcopilot_grounding:0.0.47" + component: "azureml:llm_dbcopilot_grounding:0.0.48" resources: instance_count: ${{parent.inputs.serverless_instance_count}} instance_type: ${{parent.inputs.serverless_instance_type}} @@ -188,7 +188,7 @@ jobs: ######################################### db_sample_loading_generator: type: command - component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.23" + component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.24" resources: instance_count: ${{parent.inputs.serverless_instance_count}} instance_type: ${{parent.inputs.serverless_instance_type}} @@ -250,7 +250,7 @@ jobs: ######################################### endpoint_deployment_job: type: command - component: "azureml:llm_dbcopilot_deploy_endpoint:0.0.23" + component: "azureml:llm_dbcopilot_deploy_endpoint:0.0.24" resources: instance_count: ${{parent.inputs.serverless_instance_count}} instance_type: ${{parent.inputs.serverless_instance_type}} diff --git a/assets/large_language_models/components_pipelines/data_ingestion_dbcopilot_faiss_e2e/spec.yaml b/assets/large_language_models/components_pipelines/data_ingestion_dbcopilot_faiss_e2e/spec.yaml index f39bc1aca1..4022f8e6b4 100644 --- a/assets/large_language_models/components_pipelines/data_ingestion_dbcopilot_faiss_e2e/spec.yaml +++ b/assets/large_language_models/components_pipelines/data_ingestion_dbcopilot_faiss_e2e/spec.yaml @@ -110,7 +110,7 @@ jobs: ######################################### db_meta_loading_generator: type: command - component: "azureml:llm_dbcopilot_grounding:0.0.47" + component: "azureml:llm_dbcopilot_grounding:0.0.48" resources: instance_count: ${{parent.inputs.serverless_instance_count}} instance_type: ${{parent.inputs.serverless_instance_type}} @@ -176,7 +176,7 @@ jobs: ######################################### db_sample_loading_generator: type: command - component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.23" + component: "azureml:llm_dbcopilot_grounding_ground_samples:0.0.24" resources: instance_count: ${{parent.inputs.serverless_instance_count}} instance_type: ${{parent.inputs.serverless_instance_type}} @@ -234,7 +234,7 @@ jobs: ######################################### endpoint_deployment_job: type: command - component: "azureml:llm_dbcopilot_deploy_endpoint:0.0.23" + component: "azureml:llm_dbcopilot_deploy_endpoint:0.0.24" resources: instance_count: ${{parent.inputs.serverless_instance_count}} instance_type: ${{parent.inputs.serverless_instance_type}} diff --git a/assets/large_language_models/dbcopilot/components/llm_dbcopilot_create_promptflow/spec.yaml b/assets/large_language_models/dbcopilot/components/llm_dbcopilot_create_promptflow/spec.yaml index 837d7bfc3a..d6ee3da95a 100644 --- a/assets/large_language_models/dbcopilot/components/llm_dbcopilot_create_promptflow/spec.yaml +++ b/assets/large_language_models/dbcopilot/components/llm_dbcopilot_create_promptflow/spec.yaml @@ -4,7 +4,7 @@ tags: Preview: "" name: llm_dbcopilot_create_promptflow display_name: LLM - Create DBCopilot Prompt Flow -version: 0.0.47 +version: 0.0.48 inputs: index_name: type: string diff --git a/assets/large_language_models/dbcopilot/components/llm_dbcopilot_deploy_endpoint/spec.yaml b/assets/large_language_models/dbcopilot/components/llm_dbcopilot_deploy_endpoint/spec.yaml index 5a35f4c43a..14f026bfce 100644 --- a/assets/large_language_models/dbcopilot/components/llm_dbcopilot_deploy_endpoint/spec.yaml +++ b/assets/large_language_models/dbcopilot/components/llm_dbcopilot_deploy_endpoint/spec.yaml @@ -3,7 +3,7 @@ type: command tags: {} name: llm_dbcopilot_deploy_endpoint display_name: LLM - DBCopilot Deploy Endpoint Component -version: 0.0.23 +version: 0.0.24 inputs: deployment_name: type: string diff --git a/assets/large_language_models/dbcopilot/components/llm_dbcopilot_grounding/spec.yaml b/assets/large_language_models/dbcopilot/components/llm_dbcopilot_grounding/spec.yaml index b514d94f16..0e4856b0b5 100644 --- a/assets/large_language_models/dbcopilot/components/llm_dbcopilot_grounding/spec.yaml +++ b/assets/large_language_models/dbcopilot/components/llm_dbcopilot_grounding/spec.yaml @@ -3,7 +3,7 @@ type: command tags: Preview: "" name: llm_dbcopilot_grounding -version: 0.0.47 +version: 0.0.48 inputs: asset_uri: type: string diff --git a/assets/large_language_models/dbcopilot/components/llm_dbcopilot_grounding_ground_samples/spec.yaml b/assets/large_language_models/dbcopilot/components/llm_dbcopilot_grounding_ground_samples/spec.yaml index 2a5e212203..3b25790837 100644 --- a/assets/large_language_models/dbcopilot/components/llm_dbcopilot_grounding_ground_samples/spec.yaml +++ b/assets/large_language_models/dbcopilot/components/llm_dbcopilot_grounding_ground_samples/spec.yaml @@ -3,7 +3,7 @@ type: command tags: {} name: llm_dbcopilot_grounding_ground_samples display_name: LLM - DBCopilot Grounding Ground Samples Component -version: 0.0.23 +version: 0.0.24 inputs: grounding_context: type: uri_folder diff --git a/assets/large_language_models/dbcopilot/environments/dbcopilot_embeddings/context/Dockerfile b/assets/large_language_models/dbcopilot/environments/dbcopilot_embeddings/context/Dockerfile index 5044a3bddd..cd85a05c3b 100644 --- a/assets/large_language_models/dbcopilot/environments/dbcopilot_embeddings/context/Dockerfile +++ b/assets/large_language_models/dbcopilot/environments/dbcopilot_embeddings/context/Dockerfile @@ -1,10 +1,10 @@ FROM mcr.microsoft.com/azureml/promptflow/promptflow-runtime-stable:{{latest-image-tag}} USER root + # Set the shared environment path ENV AZUREML_CONDA_ENVIRONMENT_PATH /azureml-envs/dbcopilot ENV PATH $AZUREML_CONDA_ENVIRONMENT_PATH/bin:$PATH - # Create conda environment COPY conda_dependencies.yaml . RUN conda env create -p $AZUREML_CONDA_ENVIRONMENT_PATH -f conda_dependencies.yaml -q && \ @@ -16,10 +16,13 @@ RUN apt-get update && apt-get install -y \ curl \ gnupg \ unixodbc-dev \ - # Upgrade libldap to the required version + openssl=1.1.1f-1ubuntu2.22 \ + libssl1.1=1.1.1f-1ubuntu2.22 \ libldap-2.4-2=2.4.49+dfsg-2ubuntu1.10 \ - libldap-common=2.4.49+dfsg-2ubuntu1.10 && \ - apt-get install --only-upgrade openssl=1.1.1f-1ubuntu2.21 libssl1.1=1.1.1f-1ubuntu2.21 && \ + libldap-common=2.4.49+dfsg-2ubuntu1.10 \ + login=1:4.8.1-1ubuntu5.20.04.5 \ + passwd=1:4.8.1-1ubuntu5.20.04.5 \ + uidmap=1:4.8.1-1ubuntu5.20.04.5 && \ rm -rf /var/lib/apt/lists/* # Import the public repository GPG keys @@ -46,7 +49,11 @@ RUN pip install opentelemetry-instrumentation-django==0.42b0 promptflow-image-bu ## Install dbcopilot RUN pip install https://ragsample.blob.core.windows.net/ragdata/wheels/dbcopilot/db_copilot_tool-0.1.16-py3-none-any.whl -RUN pip install cryptography==42.0.0 langchain==0.0.348 pyarrow==14.0.1 pymongo==4.6.1 +RUN /bin/bash -c "source /opt/miniconda/etc/profile.d/conda.sh && \ + conda activate azureml-envs/prompt-flow/runtime && \ + pip install cryptography==42.0.0 && \ + conda deactivate" && \ + pip install cryptography==42.0.0 langchain==0.0.348 pyarrow==14.0.1 pymongo==4.6.1 # Install certificates and other utilities RUN set -eux; \ @@ -70,4 +77,4 @@ ENV LANG C.UTF-8 ENV JAVA_VERSION 21-ea+22 COPY setup.sh . -RUN bash setup.sh \ No newline at end of file +RUN bash setup.sh diff --git a/assets/large_language_models/dbcopilot/environments/dbcopilot_embeddings/context/requirements.txt b/assets/large_language_models/dbcopilot/environments/dbcopilot_embeddings/context/requirements.txt index 0766aa4537..25abecbd99 100644 --- a/assets/large_language_models/dbcopilot/environments/dbcopilot_embeddings/context/requirements.txt +++ b/assets/large_language_models/dbcopilot/environments/dbcopilot_embeddings/context/requirements.txt @@ -1,3 +1,3 @@ -promptflow[azure]==1.5.0 -promptflow-tools==1.2.0 +promptflow[azure]==1.6.0 +promptflow-tools==1.3.0 promptflow-vectordb==0.2.4 \ No newline at end of file diff --git a/assets/large_language_models/dbcopilot/environments/dbcopilot_mir/context/extra_requirements.txt b/assets/large_language_models/dbcopilot/environments/dbcopilot_mir/context/extra_requirements.txt index 0766aa4537..25abecbd99 100644 --- a/assets/large_language_models/dbcopilot/environments/dbcopilot_mir/context/extra_requirements.txt +++ b/assets/large_language_models/dbcopilot/environments/dbcopilot_mir/context/extra_requirements.txt @@ -1,3 +1,3 @@ -promptflow[azure]==1.5.0 -promptflow-tools==1.2.0 +promptflow[azure]==1.6.0 +promptflow-tools==1.3.0 promptflow-vectordb==0.2.4 \ No newline at end of file