From 9d815d572d9b8aa5ce0cccc4181fcad517c0dd41 Mon Sep 17 00:00:00 2001 From: Milind Waykole Date: Fri, 19 Apr 2024 13:58:38 +0530 Subject: [PATCH 1/5] Add Test and Kw for authorino Signed-off-by: Milind Waykole --- .../Resources/CLI/ModelServing/llm.resource | 24 ++++++++++ .../Resources/Files/llm/rolebinding_view.yaml | 12 +++++ .../overlay/authorino/kustomization.yaml | 10 ++++ .../LLMs/422__model_serving_llm.robot | 29 ++++++++++++ .../LLMs/422__model_serving_llm_tgis.robot | 47 +++++++++++++++++++ 5 files changed, 122 insertions(+) create mode 100644 ods_ci/tests/Resources/Files/llm/rolebinding_view.yaml create mode 100644 ods_ci/tests/Resources/Files/llm/serving_runtimes/overlay/authorino/kustomization.yaml diff --git a/ods_ci/tests/Resources/CLI/ModelServing/llm.resource b/ods_ci/tests/Resources/CLI/ModelServing/llm.resource index 15034615f..7c4dc3bda 100644 --- a/ods_ci/tests/Resources/CLI/ModelServing/llm.resource +++ b/ods_ci/tests/Resources/CLI/ModelServing/llm.resource @@ -13,8 +13,10 @@ ${INFERENCESERVICE_FILEPATH}= ${LLM_RESOURCES_DIRPATH}/serving_runtimes/base/ ${INFERENCESERVICE_FILLED_FILEPATH}= ${LLM_RESOURCES_DIRPATH}/serving_runtimes/isvc_filled.yaml ${DEFAULT_BUCKET_SECRET_NAME}= models-bucket-secret ${DEFAULT_BUCKET_SA_NAME}= models-bucket-sa +${DEFAULT_BUCKET_PREFIX}= models-bucket ${BUCKET_SECRET_FILEPATH}= ${LLM_RESOURCES_DIRPATH}/bucket_secret.yaml ${BUCKET_SA_FILEPATH}= ${LLM_RESOURCES_DIRPATH}/bucket_sa.yaml +${ROLE_BINDING_FILEPATH}= ${LLM_RESOURCES_DIRPATH}/rolebinding_view.yaml ${USE_BUCKET_HTTPS}= "1" ${MODELS_BUCKET}= ${S3.BUCKET_3} ${SERVICEMESH_CR_NS}= istio-system @@ -315,6 +317,7 @@ Query Model Multiple Times ... ${port}=443 ... ${body_params}=&{EMPTY} ... ${cert}=${False} + ... ${token}=${None} ... &{args} IF "${inference_type}" == "streaming" ${streamed_response}= Set Variable ${TRUE} @@ -339,6 +342,10 @@ Query Model Multiple Times ${body} ${header} ${extra_args}= llm.Prepare Payload runtime=${runtime} protocol=${protocol} ... inference_type=${inference_type} model_name=${model_name} body_params=${body_params} ... query_text=${EXP_RESPONSES}[queries][${query_idx}][query_text] + IF "${token}" != "${None}" + ${header}= Set Variable "Authorization: Bearer ${token}" -H ${header} + Log To Console ${header} + END ${runtime_details}= Set Variable ${RUNTIME_FORMATS}[${runtime}][${inference_type}][${protocol}] ${endpoint}= Set Variable ${runtime_details}[endpoint] Set To Dictionary ${args} &{extra_args} @@ -774,3 +781,20 @@ Start Port-forwarding ... alias=${process_alias} stderr=STDOUT shell=yes Process Should Be Running ${process} sleep 7s + +Create Inference Access Token + [Arguments] ${test_namespace} ${DEFAULT_BUCKET_SA_NAME} + ${rc} ${out}= Run And Return Rc And Output oc create token -n ${test_namespace} ${DEFAULT_BUCKET_SA_NAME} + Should Be Equal As Strings ${rc} 0 + [Return] ${out} + +Create Role Binding For Authorino + [Arguments] ${name} ${namespace} + Set Test Variable ${name} + Set Test Variable ${namespace} + Set Test Variable ${nameview} ${name}-view + Set Test Variable ${namesa} ${name}-sa + Create File From Template ${ROLE_BINDING_FILEPATH} ${LLM_RESOURCES_DIRPATH}/rb.filled.yaml + ${rc} ${out}= Run And Return Rc And Output + ... oc apply -f ${LLM_RESOURCES_DIRPATH}/rb.filled.yaml + Should Be Equal As Integers ${rc} ${0} diff --git a/ods_ci/tests/Resources/Files/llm/rolebinding_view.yaml b/ods_ci/tests/Resources/Files/llm/rolebinding_view.yaml new file mode 100644 index 000000000..9e8b2ee45 --- /dev/null +++ b/ods_ci/tests/Resources/Files/llm/rolebinding_view.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ${NAMEVIEW} + namespace: ${NAMESPACE} +subjects: + - kind: ServiceAccount + name: ${NAMESA} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: view diff --git a/ods_ci/tests/Resources/Files/llm/serving_runtimes/overlay/authorino/kustomization.yaml b/ods_ci/tests/Resources/Files/llm/serving_runtimes/overlay/authorino/kustomization.yaml new file mode 100644 index 000000000..33dd636cc --- /dev/null +++ b/ods_ci/tests/Resources/Files/llm/serving_runtimes/overlay/authorino/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +metadata: + name: authorino +resources: + - ../../base/ + +commonAnnotations: + security.opendatahub.io/enable-auth: "true" + diff --git a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm.robot b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm.robot index 9ac65dacc..88d117d35 100644 --- a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm.robot +++ b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm.robot @@ -550,6 +550,35 @@ Verify User Can Query A Model Using HTTP Calls [Teardown] Clean Up Test Project test_ns=${test_namespace} ... isvc_names=${models_names} wait_prj_deletion=${FALSE} +Verify User Can Serve And Query A Model With Token + [Documentation] Basic tests for preparing, deploying and querying a LLM model + ... using Kserve and Caikit+TGIS runtime + [Tags] authz-2 + [Setup] Set Project And Runtime namespace=${TEST_NS}-cli + ${test_namespace}= Set Variable ${TEST_NS}-cli + ${flan_model_name}= Set Variable flan-t5-small-caikit + ${models_names}= Create List ${flan_model_name} + ${overlays}= Create List authz + Compile Inference Service YAML isvc_name=${flan_model_name} + ... sa_name=${DEFAULT_BUCKET_SA_NAME} + ... model_storage_uri=${FLAN_STORAGE_URI} + ... overlays=${overlays} + + Deploy Model Via CLI isvc_filepath=${INFERENCESERVICE_FILLED_FILEPATH} + ... namespace=${test_namespace} + Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${flan_model_name} + ... namespace=${test_namespace} + Create Role Binding For Authorino name=${DEFAULT_BUCKET_PREFIX} namespace=${test_namespace} + ${inf_token} Create Inference Access Token ${test_namespace} ${DEFAULT_BUCKET_SA_NAME} + Query Model Multiple Times model_name=${flan_model_name} + ... inference_type=all-tokens n_times=1 + ... namespace=${test_namespace} token=${inf_token} + Query Model Multiple Times model_name=${flan_model_name} + ... inference_type=streaming n_times=1 + ... namespace=${test_namespace} token=${inf_token} + + [Teardown] Clean Up Test Project test_ns=${test_namespace} + ... isvc_names=${models_names} wait_prj_deletion=${FALSE} *** Keywords *** Install Model Serving Stack Dependencies diff --git a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot index b25750455..7b97a687b 100644 --- a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot +++ b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot @@ -625,6 +625,53 @@ Verify User Can Query A Model Using HTTP Calls [Teardown] Clean Up Test Project test_ns=${test_namespace} ... isvc_names=${models_names} wait_prj_deletion=${FALSE} +Verify User Can Serve And Query A Model With Token + [Documentation] Basic tests for preparing, deploying and querying a LLM model + ... with token using Kserve and TGIS runtime + [Tags] Tier1 ODS-authz + + [Setup] Set Project And Runtime runtime=${TGIS_RUNTIME_NAME} namespace=${TEST_NS}-cli + ${test_namespace}= Set Variable ${TEST_NS}-cli + ${flan_model_name}= Set Variable flan-t5-small-caikit + ${models_names}= Create List ${flan_model_name} + ${overlays}= Create List authz + + Compile Inference Service YAML isvc_name=${flan_model_name} + ... sa_name=${DEFAULT_BUCKET_SA_NAME} + ... model_storage_uri=${FLAN_STORAGE_URI} + ... model_format=pytorch serving_runtime=${TGIS_RUNTIME_NAME} + ... limits_dict=${GPU_LIMITS} + ... overlays=${overlays} + Deploy Model Via CLI isvc_filepath=${INFERENCESERVICE_FILLED_FILEPATH} + ... namespace=${test_namespace} + Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${flan_model_name} + ... namespace=${test_namespace} + Create Role Binding For Authorino name=${DEFAULT_BUCKET_PREFIX} namespace=tgis-standalone-cli + ${inf_token} Create Inference Access Token ${test_namespace} ${DEFAULT_BUCKET_SA_NAME} + ${pod_name}= Get Pod Name namespace=${test_namespace} label_selector=serving.kserve.io/inferenceservice=${flan_model_name} + IF ${IS_KSERVE_RAW} Start Port-forwarding namespace=${test_namespace} pod_name=${pod_name} + Query Model Multiple Times model_name=${flan_model_name} runtime=${TGIS_RUNTIME_NAME} + ... inference_type=all-tokens n_times=1 + ... namespace=${test_namespace} port_forwarding=${IS_KSERVE_RAW} token=${inf_token} + Query Model Multiple Times model_name=${flan_model_name} runtime=${TGIS_RUNTIME_NAME} + ... inference_type=tokenize n_times=1 port_forwarding=${IS_KSERVE_RAW} + ... namespace=${test_namespace} validate_response=${TRUE} string_check_only=${TRUE} + ... token=${inf_token} + Query Model Multiple Times model_name=${flan_model_name} runtime=${TGIS_RUNTIME_NAME} + ... inference_type=model-info n_times=1 port_forwarding=${IS_KSERVE_RAW} + ... namespace=${test_namespace} validate_response=${TRUE} string_check_only=${TRUE} + ... token=${inf_token} + Query Model Multiple Times model_name=${flan_model_name} runtime=${TGIS_RUNTIME_NAME} + ... inference_type=streaming n_times=1 port_forwarding=${IS_KSERVE_RAW} + ... namespace=${test_namespace} validate_response=${FALSE} + ... token=${inf_token} + + [Teardown] Run Keywords + ... Clean Up Test Project test_ns=${test_namespace} + ... isvc_names=${models_names} wait_prj_deletion=${FALSE} + ... AND + ... Run Keyword If ${IS_KSERVE_RAW} Terminate Process llm-query-process kill=true + *** Keywords *** Suite Setup From 381658cc395b1529cd486c63e7e09bedb8745c85 Mon Sep 17 00:00:00 2001 From: Milind Waykole Date: Sat, 20 Apr 2024 23:15:34 +0530 Subject: [PATCH 2/5] Add Test and Kw for authorino Signed-off-by: Milind Waykole --- ods_ci/tests/Resources/CLI/ModelServing/llm.resource | 7 ------- ods_ci/tests/Resources/RHOSi.resource | 6 ++++++ .../420__model_serving/LLMs/422__model_serving_llm.robot | 5 +++-- .../LLMs/422__model_serving_llm_tgis.robot | 5 +++-- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/ods_ci/tests/Resources/CLI/ModelServing/llm.resource b/ods_ci/tests/Resources/CLI/ModelServing/llm.resource index 7c4dc3bda..d3669fdb7 100644 --- a/ods_ci/tests/Resources/CLI/ModelServing/llm.resource +++ b/ods_ci/tests/Resources/CLI/ModelServing/llm.resource @@ -344,7 +344,6 @@ Query Model Multiple Times ... query_text=${EXP_RESPONSES}[queries][${query_idx}][query_text] IF "${token}" != "${None}" ${header}= Set Variable "Authorization: Bearer ${token}" -H ${header} - Log To Console ${header} END ${runtime_details}= Set Variable ${RUNTIME_FORMATS}[${runtime}][${inference_type}][${protocol}] ${endpoint}= Set Variable ${runtime_details}[endpoint] @@ -782,12 +781,6 @@ Start Port-forwarding Process Should Be Running ${process} sleep 7s -Create Inference Access Token - [Arguments] ${test_namespace} ${DEFAULT_BUCKET_SA_NAME} - ${rc} ${out}= Run And Return Rc And Output oc create token -n ${test_namespace} ${DEFAULT_BUCKET_SA_NAME} - Should Be Equal As Strings ${rc} 0 - [Return] ${out} - Create Role Binding For Authorino [Arguments] ${name} ${namespace} Set Test Variable ${name} diff --git a/ods_ci/tests/Resources/RHOSi.resource b/ods_ci/tests/Resources/RHOSi.resource index 957ce6857..a57584c5c 100644 --- a/ods_ci/tests/Resources/RHOSi.resource +++ b/ods_ci/tests/Resources/RHOSi.resource @@ -97,3 +97,9 @@ Assign Vars According To Product Set Suite Variable ${OPERATOR_DEPLOYMENT_NAME} opendatahub-operator-controller-manager Set Suite Variable ${OPERATOR_LABEL_SELECTOR} control-plane=controller-manager END + +Create Inference Access Token + [Arguments] ${test_namespace} ${bucket_sa_name} + ${rc} ${out}= Run And Return Rc And Output oc create token -n ${test_namespace} ${bucket_sa_name} + Should Be Equal As Strings ${rc} 0 + [Return] ${out} \ No newline at end of file diff --git a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm.robot b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm.robot index 88d117d35..ade576cde 100644 --- a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm.robot +++ b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm.robot @@ -552,8 +552,9 @@ Verify User Can Query A Model Using HTTP Calls Verify User Can Serve And Query A Model With Token [Documentation] Basic tests for preparing, deploying and querying a LLM model - ... using Kserve and Caikit+TGIS runtime - [Tags] authz-2 + ... With Token using Kserve and Caikit+TGIS runtime + [Tags] RHOAIENG-6333 + ... Tier1 [Setup] Set Project And Runtime namespace=${TEST_NS}-cli ${test_namespace}= Set Variable ${TEST_NS}-cli ${flan_model_name}= Set Variable flan-t5-small-caikit diff --git a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot index 7b97a687b..7b7861cdd 100644 --- a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot +++ b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot @@ -628,8 +628,9 @@ Verify User Can Query A Model Using HTTP Calls Verify User Can Serve And Query A Model With Token [Documentation] Basic tests for preparing, deploying and querying a LLM model ... with token using Kserve and TGIS runtime - [Tags] Tier1 ODS-authz - + ... With Token using Kserve and Caikit+TGIS runtime + [Tags] RHOAIENG-6306 + ... Tier1 [Setup] Set Project And Runtime runtime=${TGIS_RUNTIME_NAME} namespace=${TEST_NS}-cli ${test_namespace}= Set Variable ${TEST_NS}-cli ${flan_model_name}= Set Variable flan-t5-small-caikit From be2ddc9bcdc2007f82f5ce655668b2c52b2e0aac Mon Sep 17 00:00:00 2001 From: Milind Waykole Date: Mon, 6 May 2024 11:46:40 +0530 Subject: [PATCH 3/5] Add Test and Kw for authorino Signed-off-by: Milind Waykole --- ods_ci/tests/Resources/CLI/ModelServing/llm.resource | 6 ++++++ ods_ci/tests/Resources/RHOSi.resource | 7 +------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ods_ci/tests/Resources/CLI/ModelServing/llm.resource b/ods_ci/tests/Resources/CLI/ModelServing/llm.resource index d3669fdb7..b3c7eb5f3 100644 --- a/ods_ci/tests/Resources/CLI/ModelServing/llm.resource +++ b/ods_ci/tests/Resources/CLI/ModelServing/llm.resource @@ -791,3 +791,9 @@ Create Role Binding For Authorino ${rc} ${out}= Run And Return Rc And Output ... oc apply -f ${LLM_RESOURCES_DIRPATH}/rb.filled.yaml Should Be Equal As Integers ${rc} ${0} + +Create Inference Access Token + [Arguments] ${test_namespace} ${bucket_sa_name} + ${rc} ${out}= Run And Return Rc And Output oc create token -n ${test_namespace} ${bucket_sa_name} + Should Be Equal As Strings ${rc} 0 + [Return] ${out} \ No newline at end of file diff --git a/ods_ci/tests/Resources/RHOSi.resource b/ods_ci/tests/Resources/RHOSi.resource index a57584c5c..56b61ae9a 100644 --- a/ods_ci/tests/Resources/RHOSi.resource +++ b/ods_ci/tests/Resources/RHOSi.resource @@ -40,6 +40,7 @@ Resource Common.robot ... Set Thanos Credentials Variables ... Generate Minio Random Credentials ... Get Minio Credentials +... Create Inference Access Token *** Keywords *** @@ -97,9 +98,3 @@ Assign Vars According To Product Set Suite Variable ${OPERATOR_DEPLOYMENT_NAME} opendatahub-operator-controller-manager Set Suite Variable ${OPERATOR_LABEL_SELECTOR} control-plane=controller-manager END - -Create Inference Access Token - [Arguments] ${test_namespace} ${bucket_sa_name} - ${rc} ${out}= Run And Return Rc And Output oc create token -n ${test_namespace} ${bucket_sa_name} - Should Be Equal As Strings ${rc} 0 - [Return] ${out} \ No newline at end of file From 2006dc98cbe2c1d6b9593c12c6c40ca50e8d50bd Mon Sep 17 00:00:00 2001 From: Milind Waykole Date: Mon, 6 May 2024 14:47:27 +0530 Subject: [PATCH 4/5] Add Test and Kw for authorino Signed-off-by: Milind Waykole --- ods_ci/tests/Resources/CLI/ModelServing/llm.resource | 2 +- .../420__model_serving/LLMs/422__model_serving_llm.robot | 2 +- .../420__model_serving/LLMs/422__model_serving_llm_tgis.robot | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ods_ci/tests/Resources/CLI/ModelServing/llm.resource b/ods_ci/tests/Resources/CLI/ModelServing/llm.resource index 82a263617..2b502357d 100644 --- a/ods_ci/tests/Resources/CLI/ModelServing/llm.resource +++ b/ods_ci/tests/Resources/CLI/ModelServing/llm.resource @@ -797,7 +797,7 @@ Create Inference Access Token ${rc} ${out}= Run And Return Rc And Output oc create token -n ${test_namespace} ${bucket_sa_name} Should Be Equal As Strings ${rc} 0 [Return] ${out} -======= + Wait For Model KServe Deployment To Be Ready [Documentation] Waits for the Pod to be Ready (i.e., Running status) and checks that ... the deployment has the expected pods and containers diff --git a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm.robot b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm.robot index f879a6593..0afce48e1 100644 --- a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm.robot +++ b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm.robot @@ -561,7 +561,7 @@ Verify User Can Serve And Query A Model With Token ${test_namespace}= Set Variable ${TEST_NS}-cli ${flan_model_name}= Set Variable flan-t5-small-caikit ${models_names}= Create List ${flan_model_name} - ${overlays}= Create List authz + ${overlays}= Create List authorino Compile Inference Service YAML isvc_name=${flan_model_name} ... sa_name=${DEFAULT_BUCKET_SA_NAME} ... model_storage_uri=${FLAN_STORAGE_URI} diff --git a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot index aeda70bab..ef190a0e7 100644 --- a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot +++ b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot @@ -636,7 +636,7 @@ Verify User Can Serve And Query A Model With Token ${test_namespace}= Set Variable ${TEST_NS}-cli ${flan_model_name}= Set Variable flan-t5-small-caikit ${models_names}= Create List ${flan_model_name} - ${overlays}= Create List authz + ${overlays}= Create List authorino Compile Inference Service YAML isvc_name=${flan_model_name} ... sa_name=${DEFAULT_BUCKET_SA_NAME} From 3990774776da562b288105903e3ad2a5e916ae40 Mon Sep 17 00:00:00 2001 From: Milind Waykole Date: Mon, 6 May 2024 15:41:52 +0530 Subject: [PATCH 5/5] Add Test and Kw for authorino Signed-off-by: Milind Waykole --- .../420__model_serving/LLMs/422__model_serving_llm_tgis.robot | 1 - 1 file changed, 1 deletion(-) diff --git a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot index ef190a0e7..03e9e45b3 100644 --- a/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot +++ b/ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis.robot @@ -628,7 +628,6 @@ Verify User Can Query A Model Using HTTP Calls Verify User Can Serve And Query A Model With Token [Documentation] Basic tests for preparing, deploying and querying a LLM model - ... with token using Kserve and TGIS runtime ... With Token using Kserve and Caikit+TGIS runtime [Tags] RHOAIENG-6306 ... Tier1