From 3f86a41d84207b9332c300adb902606557a3e41a Mon Sep 17 00:00:00 2001 From: Kobi Hakimi Date: Tue, 10 Sep 2024 22:51:31 +0300 Subject: [PATCH 1/2] add the option to install previous rhoai version on psi operatorhub then to upgrade Signed-off-by: Kobi Hakimi --- .../RHODS_OLM/install/cs_template.yaml | 3 +- .../Resources/RHODS_OLM/install/install.robot | 6 +- .../RHODS_OLM/install/oc_install.robot | 65 ++++++++++++------- .../pre-tasks/oc_is_operator_installed.robot | 2 +- ods_ci/tasks/Tasks/rhods_olm.robot | 17 ++--- ods_ci/tests/Resources/Common.robot | 22 ++++++- ods_ci/tests/Resources/OCP.resource | 27 +++++++- .../Resources/Page/Operators/ISVs.resource | 4 +- ods_ci/tests/Resources/RHOSi.resource | 2 + .../120__upgrades/120__pre_upgrade.robot | 7 +- .../120__upgrades/121__during_upgrade.robot | 19 ++++-- .../120__upgrades/122__post_ugrade.robot | 7 +- .../133__kservewaw_rhoai_installation.robot | 2 + .../200__metrics/200__metrics.robot | 5 ++ .../testconfig/generateTestConfigFile.py | 11 ++++ 15 files changed, 150 insertions(+), 49 deletions(-) diff --git a/ods_ci/tasks/Resources/RHODS_OLM/install/cs_template.yaml b/ods_ci/tasks/Resources/RHODS_OLM/install/cs_template.yaml index 6b14b84f5..920d97d89 100644 --- a/ods_ci/tasks/Resources/RHODS_OLM/install/cs_template.yaml +++ b/ods_ci/tasks/Resources/RHODS_OLM/install/cs_template.yaml @@ -13,10 +13,11 @@ items: namespace: spec: channel: - installPlanApproval: Automatic + installPlanApproval: name: source: sourceNamespace: openshift-marketplace + startingCSV: - apiVersion: operators.coreos.com/v1 kind: OperatorGroup metadata: diff --git a/ods_ci/tasks/Resources/RHODS_OLM/install/install.robot b/ods_ci/tasks/Resources/RHODS_OLM/install/install.robot index 565f46f27..0851cf83d 100644 --- a/ods_ci/tasks/Resources/RHODS_OLM/install/install.robot +++ b/ods_ci/tasks/Resources/RHODS_OLM/install/install.robot @@ -5,13 +5,15 @@ Resource oc_install.robot *** Keywords *** -Installing RHODS Operator ${image_url} +Installing RHODS Operator + [Documentation] Installing the RHOAI operator + [Arguments] ${image_url} ${install_plan_approval} ${rhoai_version}=${EMPTY} ${is_operator_installed} = Is RHODS Installed IF not ${is_operator_installed} Run Keywords ... Log Installing RHODS operator in ${cluster_type} console=yes AND ... Set Suite Variable ${image_url} AND ... Set Test Variable ${RHODS_OSD_INSTALL_REPO} AND - ... Install RHODS ${cluster_type} ${image_url} + ... Install RHODS ${cluster_type} ${image_url} ${install_plan_approval} ${rhoai_version} RHODS Operator Should Be installed Verify RHODS Installation diff --git a/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot b/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot index d2b9e3e13..3bb8b6f60 100644 --- a/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot +++ b/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot @@ -27,36 +27,49 @@ ${SERVICEMESH_SUB_NAME}= servicemeshoperator ${AUTHORINO_OP_NAME}= authorino-operator ${AUTHORINO_SUB_NAME}= authorino-operator ${AUTHORINO_CHANNEL_NAME}= tech-preview-v1 -${RHODS_CSV_DISPLAY}= Red Hat OpenShift AI -${ODH_CSV_DISPLAY}= Open Data Hub Operator ${CUSTOM_MANIFESTS}= ${EMPTY} +${RHODS_OSD_INSTALL_REPO} ${EMPTY} +${OLM_DIR} rhodsolm +@{SUPPORTED_TEST_ENV} AWS AWS_DIS GCP PSI PSI_DIS ROSA IBM_CLOUD CRC AZURE ROSA_HCP +${install_plan_approval} Manual +${rhoai_version} ${EMPTY} *** Keywords *** Install RHODS - [Arguments] ${cluster_type} ${image_url} + [Arguments] ${cluster_type} ${image_url} ${install_plan_approval} ${rhoai_version}=${EMPTY} ${is_upgrade}=False + Log To Console Start installing RHOAI with:\n\- cluster type: ${cluster_type}\n\- image_url: ${image_url}\n\- update_channel: ${UPDATE_CHANNEL} + Log To Console \- rhoai_version: ${rhoai_version}\n\- is_upgrade: ${is_upgrade}\n\- install_plan_approval: ${install_plan_approval} + Assign Vars According To Product Install Kserve Dependencies Clone OLM Install Repo - IF "${PRODUCT}" == "ODH" - ${csv_display_name} = Set Variable ${ODH_CSV_DISPLAY} - ELSE - ${csv_display_name} = Set Variable ${RHODS_CSV_DISPLAY} - END IF "${cluster_type}" == "selfmanaged" IF "${TEST_ENV}" in "${SUPPORTED_TEST_ENV}" and "${INSTALL_TYPE}" == "CLi" Install RHODS In Self Managed Cluster Using CLI ${cluster_type} ${image_url} ELSE IF "${TEST_ENV}" in "${SUPPORTED_TEST_ENV}" and "${INSTALL_TYPE}" == "OperatorHub" - ${file_path} = Set Variable tasks/Resources/RHODS_OLM/install/ - Copy File source=${file_path}cs_template.yaml destination=${file_path}cs_apply.yaml - IF "${PRODUCT}" == "ODH" - Run sed -i'' -e 's//community-operators/' ${file_path}cs_apply.yaml + IF "${is_upgrade}" == "False" + ${file_path} = Set Variable tasks/Resources/RHODS_OLM/install/ + ${starting_csv} = Set Variable "" + IF "${rhoai_version}" != "${EMPTY}" + Log To Console Start installing "${OPERATOR_NAME}" with version: ${rhoai_version} + ${starting_csv} = Set Variable ${OPERATOR_DEPLOYMENT_NAME}.${rhoai_version} + END + Copy File source=${file_path}cs_template.yaml destination=${file_path}cs_apply.yaml + Run sed -i'' -e 's//${CATALOG_SOURCE}/' ${file_path}cs_apply.yaml + Run sed -i'' -e 's//${OPERATOR_DEPLOYMENT_NAME}/' ${file_path}cs_apply.yaml + Run sed -i'' -e 's//${OPERATOR_NAMESPACE}/' ${file_path}cs_apply.yaml + Run sed -i'' -e 's//${UPDATE_CHANNEL}/' ${file_path}cs_apply.yaml + Run sed -i'' -e 's//${starting_csv}/' ${file_path}cs_apply.yaml + Run sed -i'' -e 's//${install_plan_approval}/' ${file_path}cs_apply.yaml + Oc Apply kind=List src=${file_path}cs_apply.yaml + Remove File ${file_path}cs_apply.yml ELSE - Run sed -i'' -e 's//redhat-operators/' ${file_path}cs_apply.yaml + ${patch_update_channel_status} = Run And Return Rc oc patch subscription ${OPERATOR_DEPLOYMENT_NAME} -n ${OPERATOR_NAMESPACE} --type='json' -p='[{"op": "replace", "path": "/spec/channel", "value": ${UPDATE_CHANNEL}}]' #robocop:disable + Should Be Equal As Integers ${patch_update_channel_status} 0 msg=Error while changing the UPDATE_CHANNEL + Sleep 30s reason=wait for thirty seconds until old CSV is removed and new one is ready + END + IF "${rhoai_version}" != "${EMPTY}" + Wait For Installplan And Approve It ${OPERATOR_NAMESPACE} ${OPERATOR_DEPLOYMENT_NAME} ${rhoai_version} END - Run sed -i'' -e 's//${OPERATOR_NAME}/' ${file_path}cs_apply.yaml - Run sed -i'' -e 's//${OPERATOR_NAMESPACE}/' ${file_path}cs_apply.yaml - Run sed -i'' -e 's//${UPDATE_CHANNEL}/' ${file_path}cs_apply.yaml - Oc Apply kind=List src=${file_path}cs_apply.yaml - Remove File ${file_path}cs_apply.yml ELSE FAIL Provided test environment and install type is not supported END @@ -71,7 +84,7 @@ Install RHODS FAIL Provided test environment is not supported END END - Wait Until Csv Is Ready display_name=${csv_display_name} operators_namespace=${OPERATOR_NAMESPACE} + Wait Until Csv Is Ready display_name=${OPERATOR_NAME} operators_namespace=${OPERATOR_NAMESPACE} Verify RHODS Installation Set Global Variable ${DASHBOARD_APP_NAME} ${PRODUCT.lower()}-dashboard @@ -79,7 +92,7 @@ Verify RHODS Installation Log To Console Waiting for all RHODS resources to be up and running Wait For Pods Numbers 1 ... namespace=${OPERATOR_NAMESPACE} - ... label_selector=name=${OPERATOR_NAME} + ... label_selector=name=${OPERATOR_DEPLOYMENT_NAME} ... timeout=2000 Wait For Pods Status namespace=${OPERATOR_NAMESPACE} timeout=1200 Log Verified ${OPERATOR_NAMESPACE} console=yes @@ -190,9 +203,14 @@ Verify Builds In redhat-ods-applications Clone OLM Install Repo [Documentation] Clone OLM git repo - ${return_code} ${output} Run And Return Rc And Output git clone ${RHODS_OSD_INSTALL_REPO} ${EXECDIR}/${OLM_DIR} - Log To Console ${output} - Should Be Equal As Integers ${return_code} 0 + ${status} = Run Keyword And Return Status Directory Should Exist ${EXECDIR}/${OLM_DIR} + IF ${status} + Log To Console "The directory ${EXECDIR}/${OLM_DIR} already exist, skipping clone of the repo." + ELSE + ${return_code} ${output} Run And Return Rc And Output git clone ${RHODS_OSD_INSTALL_REPO} ${EXECDIR}/${OLM_DIR} + Log To Console ${output} + Should Be Equal As Integers ${return_code} 0 + END Install RHODS In Self Managed Cluster Using CLI [Documentation] Install rhods on self managed cluster using cli @@ -213,6 +231,7 @@ Wait For Pods Numbers ${status} Set Variable False FOR ${counter} IN RANGE ${timeout} ${return_code} ${output} Run And Return Rc And Output oc get pod -n ${namespace} -l ${label_selector} | tail -n +2 | wc -l + Log To Console Output: ${output} RC: ${return_code} counter: ${counter} timeout: ${timeout} NS: ${namespace} label: ${label_selector} IF ${output} == ${count} ${status} Set Variable True Log To Console pods ${label_selector} created diff --git a/ods_ci/tasks/Resources/RHODS_OLM/pre-tasks/oc_is_operator_installed.robot b/ods_ci/tasks/Resources/RHODS_OLM/pre-tasks/oc_is_operator_installed.robot index 989e4c6d5..96b432fb9 100644 --- a/ods_ci/tasks/Resources/RHODS_OLM/pre-tasks/oc_is_operator_installed.robot +++ b/ods_ci/tasks/Resources/RHODS_OLM/pre-tasks/oc_is_operator_installed.robot @@ -53,7 +53,7 @@ Check A RHODS Family Operator Is Installed [Arguments] ${namespace} ${subscription} Log Getting CSV from subscription ${subscription} namespace ${namespace} console=yes ${rc} ${current_csv_name} = Run And Return Rc And Output - ... oc get subscription ${subscription} -n ${namespace} -ojson | jq '.status.currentCSV' | tr -d '"' + ... oc get subscription ${subscription} -n ${namespace} -ojson | jq '.status.installedCSV' | tr -d '"' Log Got CSV ${current_csv_name} from subscription ${subscription}, result: ${rc} console=yes IF "${rc}" == "0" and "${current_csv_name}" != "${EMPTY}" ${result} = Run Keyword And Return Status diff --git a/ods_ci/tasks/Tasks/rhods_olm.robot b/ods_ci/tasks/Tasks/rhods_olm.robot index babf2080d..43abeeadc 100644 --- a/ods_ci/tasks/Tasks/rhods_olm.robot +++ b/ods_ci/tasks/Tasks/rhods_olm.robot @@ -10,12 +10,9 @@ Library String ***Variables*** ${cluster_type} selfmanaged ${image_url} ${EMPTY} -${RHODS_OSD_INSTALL_REPO} None -@{SUPPORTED_TEST_ENV} AWS AWS_DIS GCP PSI PSI_DIS ROSA IBM_CLOUD CRC AZURE ROSA_HCP ${TEST_ENV} AWS ${INSTALL_TYPE} OperatorHub ${UPDATE_CHANNEL} odh-nightlies -${OLM_DIR} rhodsolm ${RHODS_VERSION} None *** Tasks *** @@ -24,15 +21,15 @@ Can Install RHODS Operator IF "${PRODUCT}" == "ODH" Set Global Variable ${OPERATOR_NAMESPACE} opendatahub-operators IF "${UPDATE_CHANNEL}" == "odh-nightlies" - Set Global Variable ${OPERATOR_NAME} rhods-operator + Set Global Variable ${OPERATOR_DEPLOYMENT_NAME} rhods-operator ELSE - Set Global Variable ${OPERATOR_NAME} opendatahub-operator + Set Global Variable ${OPERATOR_DEPLOYMENT_NAME} opendatahub-operator END ELSE - Set Global Variable ${OPERATOR_NAME} rhods-operator + Set Global Variable ${OPERATOR_DEPLOYMENT_NAME} rhods-operator END Given Selected Cluster Type ${cluster_type} - When Installing RHODS Operator ${image_url} + When Installing RHODS Operator ${image_url} ${install_plan_approval} ${RHOAI_VERSION} Then RHODS Operator Should Be Installed [Teardown] Install Teardown @@ -41,12 +38,12 @@ Can Uninstall RHODS Operator IF "${PRODUCT}" == "ODH" Set Global Variable ${OPERATOR_NAMESPACE} opendatahub-operators IF "${UPDATE_CHANNEL}" == "odh-nightlies" - Set Global Variable ${OPERATOR_NAME} rhods-operator + Set Global Variable ${OPERATOR_DEPLOYMENT_NAME} rhods-operator ELSE - Set Global Variable ${OPERATOR_NAME} opendatahub-operator + Set Global Variable ${OPERATOR_DEPLOYMENT_NAME} opendatahub-operator END ELSE - Set Global Variable ${OPERATOR_NAME} rhods-operator + Set Global Variable ${OPERATOR_DEPLOYMENT_NAME} rhods-operator END Given Selected Cluster Type ${cluster_type} When Uninstalling RHODS Operator diff --git a/ods_ci/tests/Resources/Common.robot b/ods_ci/tests/Resources/Common.robot index 998fba49e..84e36ada5 100644 --- a/ods_ci/tests/Resources/Common.robot +++ b/ods_ci/tests/Resources/Common.robot @@ -155,7 +155,7 @@ Get RHODS Version ${RHODS_VERSION}= Run oc get csv -n ${OPERATOR_NAMESPACE} | grep "opendatahub" | awk -F ' {2,}' '{print $3}' END END - Log Product:${PRODUCT} Version:${RHODS_VERSION} + Log To Console Product:${PRODUCT} Version:${RHODS_VERSION} RETURN ${RHODS_VERSION} #robocop: disable: line-too-long @@ -176,17 +176,33 @@ Get CodeFlare Version #robocop: disable: line-too-long Wait Until Csv Is Ready [Documentation] Waits ${timeout} for Operators CSV '${display_name}' to have status phase 'Succeeded' - [Arguments] ${display_name} ${timeout}=10m ${operators_namespace}=openshift-operators + [Arguments] ${display_name} ${timeout}=10m ${operators_namespace}=openshift-operators ${check_interval}=5s Log Waiting ${timeout} for Operator CSV '${display_name}' in ${operators_namespace} to have status phase 'Succeeded' console=yes WHILE True limit=${timeout} ... on_limit_message=${timeout} Timeout exceeded waiting for CSV '${display_name}' to be created + Sleep ${check_interval} ${csv_created}= Run Process oc get csv --no-headers -n ${operators_namespace} | awk '/${display_name}/ {print \$1}' shell=yes - IF "${csv_created.stdout}" == "${EMPTY}" CONTINUE + Log To Console The Result of csv_created Output is: ${csv_created.stdout} RC: ${csv_created.rc} + IF '${csv_created.stdout}' == '${EMPTY}' CONTINUE + # In case of upgrade there are 2 operators, we need to wait until only one will be available + ${lines}= Split String ${csv_created.stdout} \n + ${line_count}= Get Length ${lines} + IF ${line_count} > 1 CONTINUE ${csv_ready}= Run Process ... oc wait --timeout\=${timeout} --for jsonpath\='{.status.phase}'\=Succeeded csv -n ${operators_namespace} ${csv_created.stdout} shell=yes + Log To Console The Result of csv_ready Output is: ${csv_ready.stdout} RC: ${csv_ready.rc} IF ${csv_ready.rc} == ${0} BREAK END +Check If Resource Exists By Command Output + [Documentation] Check if resource exists with specific command, + ... Fail if the output is empty or contains the string 'No resources found' + [Arguments] ${command} + ${result}= Run Process ${command} shell=True stdout=PIPE + Log To Console The Result of Check If Resource Exists By Command Output is: ${result.stdout} RC: ${result.rc} + Should Not Be Empty ${result.stdout} + Should Not Contain ${result.stdout} 'No resources found' + Get Cluster ID [Documentation] Retrieves the ID of the currently connected cluster ${cluster_id}= Run oc get clusterversion -o json | jq .items[].spec.clusterID diff --git a/ods_ci/tests/Resources/OCP.resource b/ods_ci/tests/Resources/OCP.resource index f04899f91..9c6fbc511 100644 --- a/ods_ci/tests/Resources/OCP.resource +++ b/ods_ci/tests/Resources/OCP.resource @@ -284,7 +284,32 @@ Get Resource Attribute Should Be Equal "${rc}" "0" msg=${value} IF "${value}" != "${EMPTY}" Log To Console ${resource_type} ${resource_name} in the namespace ${namespace} attribute ${attribute_path} value is ${value} - ELSE + ELSE Log To Console ${resource_type} ${resource_name} in the namespace ${namespace} attribute ${attribute_path} value is EMPTY END RETURN ${value} + +Wait For Installplan And Approve It + [Documentation] Wait for the operator's installplan to appear then approve it in case of Manual installplan approval + [Arguments] ${namespace} ${operator_name} ${operator_version}=${EMPTY} + Wait Until Keyword Succeeds 10m 5s Check If Resource Exists By Command Output oc get installplans -n ${namespace} -o yaml | grep "${operator_name}" + ${installplan_name}= Run oc get subscription ${operator_name} -n ${namespace} -o json | jq '.status.installplan.name' + IF "${operator_version}" != "${EMPTY}" + ${installplan_version_exist}= Run And Return Rc oc get installplan -n ${namespace} | grep "${operator_version}" + IF ${installplan_version_exist} != 0 + ${installplans}= Run oc get installplan -n ${namespace} + FAIL Failed to find installplan for operator: ${operator_name} version: ${operator_version}\nExisting installplans:\n${installplans} + END + END + ${installplan_approval}= Get Resource Attribute ${namespace} + ... InstallPlan ${installplan_name} .spec.approval + Log To Console The installplan approval for: ${installplan_name} is ${installplan_approval} + IF "${installplan_approval}" == "Manual" + Log To Console Start approving the installplan: ${installplan_name}... + ${patch_status} = Run And Return Rc oc patch installplan ${installplan_name} -n ${namespace} -p '{"spec":{"approved": true}}' --type=merge + IF ${patch_status} == 0 + Log To Console Approving the installplan ${installplan_name} successfully!! + ELSE + Log To Console Failed to approving the installplan ${installplan_name} + END + END diff --git a/ods_ci/tests/Resources/Page/Operators/ISVs.resource b/ods_ci/tests/Resources/Page/Operators/ISVs.resource index a65036f13..de35980f2 100644 --- a/ods_ci/tests/Resources/Page/Operators/ISVs.resource +++ b/ods_ci/tests/Resources/Page/Operators/ISVs.resource @@ -38,6 +38,7 @@ Install ISV Operator From OperatorHub Via CLI # robocop: disable Oc Apply kind=Subscription src=${operator_sub_filepath} Wait Until Keyword Succeeds 1 min 0 sec ... Is Resource Present Subscription ${subscription_name} ${namespace} ${IS_PRESENT} +# Wait For Installplan And Approve It ${namespace} ${operator_name} WHILE "${installplan_name}" == "${EMPTY}" limit=5m ${installplan_name}= Get Resource Attribute ${namespace} ... Subscription ${subscription_name} .status.installPlanRef.name @@ -49,6 +50,7 @@ Install ISV Operator From OperatorHub Via CLI # robocop: disable ${return_code} ${out}= Run And Return Rc And Output oc patch installplan ${installplan_name} -n ${namespace} --type='json' -p '[{"op": "replace", "path": "/spec/approved", "value": true}]' #robocop:disable Should Be Equal As Integers ${return_code} 0 msg=Error while approving installplan END +# until here Create Operator Group [Documentation] Creates the Operator Group object which might be needed by an operator. @@ -83,7 +85,7 @@ Wait Until Operator Subscription Last Condition Is # robocop: disable [Arguments] ${type} ${status} ${reason} ... ${subcription_name} ${namespace}=openshift-operators ... ${retry}=60 - ... ${retry_interval}=3s + ... ${retry_interval}=5s Wait Until Keyword Succeeds ${retry} times ${retry_interval} Operator Subscription Last Condition Should Be # robocop: disable ... type=${type} status=${status} ... reason=${reason} subcription_name=${subcription_name} diff --git a/ods_ci/tests/Resources/RHOSi.resource b/ods_ci/tests/Resources/RHOSi.resource index a017d0195..b9bc54866 100644 --- a/ods_ci/tests/Resources/RHOSi.resource +++ b/ods_ci/tests/Resources/RHOSi.resource @@ -90,6 +90,7 @@ Assign Vars According To Product IF "${PRODUCT}" == "RHODS" Set Suite Variable ${OPERATOR_APPNAME} Red Hat OpenShift AI Set Suite Variable ${OPERATOR_NAME} Red Hat OpenShift AI + Set Suite Variable ${CATALOG_SOURCE} redhat-operators Set Suite Variable ${OPERATOR_DEPLOYMENT_NAME} rhods-operator Set Suite Variable ${OPERATOR_LABEL_SELECTOR} name=rhods-operator Set Suite Variable ${AUTHORINO_CR_NS} redhat-ods-applications-auth-provider @@ -99,6 +100,7 @@ Assign Vars According To Product ELSE IF "${PRODUCT}" == "ODH" Set Suite Variable ${OPERATOR_APPNAME} Open Data Hub Operator Set Suite Variable ${OPERATOR_NAME} Open Data Hub Operator + Set Suite Variable ${CATALOG_SOURCE} community-operators Set Suite Variable ${OPERATOR_DEPLOYMENT_NAME} opendatahub-operator-controller-manager Set Suite Variable ${OPERATOR_LABEL_SELECTOR} control-plane=controller-manager Set Suite Variable ${AUTHORINO_CR_NS} opendatahub-auth-provider diff --git a/ods_ci/tests/Tests/100__deploy/120__upgrades/120__pre_upgrade.robot b/ods_ci/tests/Tests/100__deploy/120__upgrades/120__pre_upgrade.robot index 0f9445be7..ec7b72920 100644 --- a/ods_ci/tests/Tests/100__deploy/120__upgrades/120__pre_upgrade.robot +++ b/ods_ci/tests/Tests/100__deploy/120__upgrades/120__pre_upgrade.robot @@ -22,7 +22,7 @@ Resource ../../../Resources/Page/DistributedWorkloads/DistributedWorkl Resource ../../../Resources/Page/DistributedWorkloads/WorkloadMetricsUI.resource Suite Setup Dashboard Suite Setup Suite Teardown RHOSi Teardown -Test Tags PreUpgrade +Test Tags PreUpgradeKobi *** Variables *** @@ -39,6 +39,11 @@ ${MODEL_CREATED}= ${FALSE} ${RUNTIME_NAME}= Model Serving Test *** Test Cases *** +Test PreUpgrade + [Documentation] Test for pre upgrade + [Tags] PreUpgrade + Log To Console This is test of PreUpgrade + Set PVC Size Via UI [Documentation] Sets a Pod toleration via the admin UI [Tags] Upgrade diff --git a/ods_ci/tests/Tests/100__deploy/120__upgrades/121__during_upgrade.robot b/ods_ci/tests/Tests/100__deploy/120__upgrades/121__during_upgrade.robot index 008317447..522851c48 100644 --- a/ods_ci/tests/Tests/100__deploy/120__upgrades/121__during_upgrade.robot +++ b/ods_ci/tests/Tests/100__deploy/120__upgrades/121__during_upgrade.robot @@ -9,14 +9,20 @@ Resource ../../../Resources/Page/ODH/ODHDashboard/ODHDashboardSetting Resource ../../../Resources/Page/ODH/JupyterHub/ODHJupyterhub.resource Library DebugLibrary Library JupyterLibrary -Test Tags DuringUpgrade +Test Tags DuringUpgradeKobi *** Variables *** ${CODE} while True: import time ; time.sleep(10); print ("Hello") +${UPGRADE_TO_IIB}= ${EMPTY} *** Test Cases *** +Test DuringUpgrade + [Documentation] Test for during upgrade + [Tags] DuringUpgrade + Log To Console This is test of DuringUpgrade + Long Running Jupyter Notebook [Documentation] Launch a long running notebook before the upgrade [Tags] Upgrade @@ -31,11 +37,11 @@ Upgrade RHODS [Documentation] Approve the install plan for the upgrade and make sure that upgrade has completed [Tags] ODS-1766 ... Upgrade + ... DuringUpgrade ${initial_version} = Get RHODS Version ${initial_creation_date} = Get Operator Pod Creation Date - ${return_code} ${output} Run And Return Rc And Output oc patch installplan $(oc get installplans -n ${OPERATOR_NAMESPACE} | grep -v NAME | awk '{print $1}') -n ${OPERATOR_NAMESPACE} --type='json' -p '[{"op": "replace", "path": "/spec/approved", "value": true}]' #robocop:disable - Should Be Equal As Integers ${return_code} 0 msg=Error while upgrading RHODS - Sleep 30s reason=wait for thirty seconds until old CSV is removed and new one is ready + Set Suite Variable ${UPDATE_CHANNEL} ${UPGRADE_TO_UPDATE_CHANNEL} + Install RHODS ${CLUSTER_TYPE} ${UPGRADE_TO_IIB} Manual ${UPGRADE_TO_VERSION} True RHODS Version Should Be Greater Than ${initial_version} Operator Pod Creation Date Should Be Updated ${initial_creation_date} OpenShiftLibrary.Wait For Pods Status namespace=${OPERATOR_NAMESPACE} timeout=300 @@ -93,8 +99,9 @@ RHODS Version Should Be Greater Than [Documentation] Checks if the RHODS version is greater than the given initial version. ... Fails if the version is not greater. [Arguments] ${initial_version} - ${ver} = Get RHODS Version + ${ver} = Get RHODS Version True ${ver} = Fetch From Left ${ver} - + Log To Console The initial RHODS operator version is: ${initial_version} the current version is: ${ver} Should Be True '${ver}' > '${initial_version}' msg=Version wasn't greater than initial one ${initial_version} Get Operator Pod Creation Date @@ -104,6 +111,7 @@ Get Operator Pod Creation Date ${return_code} ${creation_date} = Run And Return Rc And Output ... oc get pod -n ${OPERATOR_NAMESPACE} -l name=rhods-operator --no-headers -o jsonpath='{.items[0].metadata.creationTimestamp}' Should Be Equal As Integers ${return_code} 0 msg=Error while getting creation date of the operator pod + Log To Console Creation date of the RHODS operator pod is: ${creation_date} RETURN ${creation_date} Operator Pod Creation Date Should Be Updated @@ -111,5 +119,6 @@ Operator Pod Creation Date Should Be Updated ... Fails if the updated creation date is not more recent than the initial creation date. [Arguments] ${initial_creation_date} ${updated_creation_date} = Get Operator Pod Creation Date + Log To Console The initial Creation date of the RHODS operator pod is: ${initial_creation_date} the updated date is: ${updated_creation_date} Should Be True '${updated_creation_date}' > '${initial_creation_date}' ... msg=Operator pod creation date was not updated after upgrade diff --git a/ods_ci/tests/Tests/100__deploy/120__upgrades/122__post_ugrade.robot b/ods_ci/tests/Tests/100__deploy/120__upgrades/122__post_ugrade.robot index 0d5dc3bb3..7e8c29e02 100644 --- a/ods_ci/tests/Tests/100__deploy/120__upgrades/122__post_ugrade.robot +++ b/ods_ci/tests/Tests/100__deploy/120__upgrades/122__post_ugrade.robot @@ -18,7 +18,7 @@ Resource ../../../Resources/Page/OCPDashboard/Pods/Pods.robot Resource ../../../Resources/Page/OCPDashboard/Builds/Builds.robot Resource ../../../Resources/Page/HybridCloudConsole/OCM.robot Resource ../../../Resources/Page/DistributedWorkloads/DistributedWorkloads.resource -Test Tags PostUpgrade +Test Tags PostUpgradeKobi *** Variables *** @@ -35,6 +35,11 @@ ${RUNTIME_NAME}= Model Serving Test *** Test Cases *** +Test PostUpgrade + [Documentation] Test for post upgrade + [Tags] PostUpgrade + Log To Console This is test of PostUpgrade + Verify PVC Size [Documentation] Verify PVC Size after the upgrade [Tags] Upgrade diff --git a/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/133__kservewaw_rhoai_installation.robot b/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/133__kservewaw_rhoai_installation.robot index cb3486cb5..121797f81 100644 --- a/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/133__kservewaw_rhoai_installation.robot +++ b/ods_ci/tests/Tests/100__deploy/130__operators/130__rhods_operator/133__kservewaw_rhoai_installation.robot @@ -39,6 +39,8 @@ Install RHOAI For KserveRaw Copy File source=${file_path}cs_template.yaml destination=${file_path}cs_apply.yaml Run sed -i'' -e 's//openshift-marketplace/' ${file_path}cs_apply.yaml Run sed -i'' -e 's//${image_escaped}/' ${file_path}cs_apply.yaml + Run sed -i'' -e 's///' ${file_path}cs_apply.yaml + Run sed -i'' -e 's//Automatic/' ${file_path}cs_apply.yaml ${rc} ${out} = Run And Return Rc And Output oc apply -f ${file_path}cs_apply.yaml IF ${rc}!=0 Fail Log ${out} console=yes diff --git a/ods_ci/tests/Tests/200__monitor_and_manage/200__metrics/200__metrics.robot b/ods_ci/tests/Tests/200__monitor_and_manage/200__metrics/200__metrics.robot index f46a50921..ab73fd3c3 100644 --- a/ods_ci/tests/Tests/200__monitor_and_manage/200__metrics/200__metrics.robot +++ b/ods_ci/tests/Tests/200__monitor_and_manage/200__metrics/200__metrics.robot @@ -23,6 +23,11 @@ Test Tags ExcludeOnODH *** Test Cases *** +Test Smoke + [Documentation] Test for smoke + [Tags] SmokeKobi + Log To Console This is test of SmokeKobi + Test Existence of Prometheus Alerting Rules [Documentation] Verifies the prometheus alerting rules [Tags] Smoke diff --git a/ods_ci/utils/scripts/testconfig/generateTestConfigFile.py b/ods_ci/utils/scripts/testconfig/generateTestConfigFile.py index d60e4b70f..67392cb9b 100755 --- a/ods_ci/utils/scripts/testconfig/generateTestConfigFile.py +++ b/ods_ci/utils/scripts/testconfig/generateTestConfigFile.py @@ -261,6 +261,17 @@ def generate_test_config_file( data["OPERATOR_NAMESPACE"] = config_data["OPERATOR_NAMESPACE"] data["NOTEBOOKS_NAMESPACE"] = config_data["NOTEBOOKS_NAMESPACE"] data["OPENSHIFT_PIPELINES_CHANNEL"] = config_data["OPENSHIFT_PIPELINES_CHANNEL"] + + data["CLUSTER_TYPE"] = config_data["CLUSTER_TYPE"] + data["TEST_ENV"] = config_data["TEST_ENV"] + data["INSTALL_TYPE"] = config_data["INSTALL_TYPE"] if config_data.get("INSTALL_TYPE") else "OperatorHub" + data["UPDATE_CHANNEL"] = config_data["UPDATE_CHANNEL"] + if config_data.get("RHOAI_VERSION"): + data["RHOAI_VERSION"] = config_data["RHOAI_VERSION"] + if config_data.get("UPGRADE_TO_UPDATE_CHANNEL"): + data["UPGRADE_TO_UPDATE_CHANNEL"] = config_data["UPGRADE_TO_UPDATE_CHANNEL"] + if config_data.get("UPGRADE_TO_VERSION"): + data["UPGRADE_TO_VERSION"] = config_data["UPGRADE_TO_VERSION"] if config_data.get("PIP_INDEX_URL"): data["PIP_INDEX_URL"] = config_data["PIP_INDEX_URL"] if config_data.get("PIP_TRUSTED_HOST"): From 3547f0430a2c819c7ad6f7360283b38e84a63be0 Mon Sep 17 00:00:00 2001 From: Kobi Hakimi Date: Tue, 26 Nov 2024 01:39:31 +0200 Subject: [PATCH 2/2] remove test tags --- .../Resources/RHODS_OLM/install/oc_install.robot | 1 - ods_ci/tests/Resources/Common.robot | 2 +- ods_ci/tests/Resources/OCP.resource | 4 +++- .../tests/Resources/Page/Operators/ISVs.resource | 14 +------------- .../120__upgrades/120__pre_upgrade.robot | 7 +------ .../120__upgrades/121__during_upgrade.robot | 8 +------- .../120__upgrades/122__post_ugrade.robot | 7 +------ .../200__metrics/200__metrics.robot | 5 ----- 8 files changed, 8 insertions(+), 40 deletions(-) diff --git a/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot b/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot index 3bb8b6f60..6849555c8 100644 --- a/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot +++ b/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot @@ -231,7 +231,6 @@ Wait For Pods Numbers ${status} Set Variable False FOR ${counter} IN RANGE ${timeout} ${return_code} ${output} Run And Return Rc And Output oc get pod -n ${namespace} -l ${label_selector} | tail -n +2 | wc -l - Log To Console Output: ${output} RC: ${return_code} counter: ${counter} timeout: ${timeout} NS: ${namespace} label: ${label_selector} IF ${output} == ${count} ${status} Set Variable True Log To Console pods ${label_selector} created diff --git a/ods_ci/tests/Resources/Common.robot b/ods_ci/tests/Resources/Common.robot index 84e36ada5..e63c5e847 100644 --- a/ods_ci/tests/Resources/Common.robot +++ b/ods_ci/tests/Resources/Common.robot @@ -183,7 +183,7 @@ Wait Until Csv Is Ready Sleep ${check_interval} ${csv_created}= Run Process oc get csv --no-headers -n ${operators_namespace} | awk '/${display_name}/ {print \$1}' shell=yes Log To Console The Result of csv_created Output is: ${csv_created.stdout} RC: ${csv_created.rc} - IF '${csv_created.stdout}' == '${EMPTY}' CONTINUE + IF '$csv_created.stdout' == '${EMPTY}' CONTINUE # In case of upgrade there are 2 operators, we need to wait until only one will be available ${lines}= Split String ${csv_created.stdout} \n ${line_count}= Get Length ${lines} diff --git a/ods_ci/tests/Resources/OCP.resource b/ods_ci/tests/Resources/OCP.resource index 9c6fbc511..dbee6497e 100644 --- a/ods_ci/tests/Resources/OCP.resource +++ b/ods_ci/tests/Resources/OCP.resource @@ -292,7 +292,9 @@ Get Resource Attribute Wait For Installplan And Approve It [Documentation] Wait for the operator's installplan to appear then approve it in case of Manual installplan approval [Arguments] ${namespace} ${operator_name} ${operator_version}=${EMPTY} - Wait Until Keyword Succeeds 10m 5s Check If Resource Exists By Command Output oc get installplans -n ${namespace} -o yaml | grep "${operator_name}" + ${csv} = Evaluate '${operator_name}.${operator_version}' if '${operator_version}' else '${operator_name}' + Log To Console The csv is: ${csv} + Wait Until Keyword Succeeds 10m 5s Check If Resource Exists By Command Output oc get installplans -n ${namespace} -o yaml | grep "${csv}" ${installplan_name}= Run oc get subscription ${operator_name} -n ${namespace} -o json | jq '.status.installplan.name' IF "${operator_version}" != "${EMPTY}" ${installplan_version_exist}= Run And Return Rc oc get installplan -n ${namespace} | grep "${operator_version}" diff --git a/ods_ci/tests/Resources/Page/Operators/ISVs.resource b/ods_ci/tests/Resources/Page/Operators/ISVs.resource index de35980f2..a8a5836e1 100644 --- a/ods_ci/tests/Resources/Page/Operators/ISVs.resource +++ b/ods_ci/tests/Resources/Page/Operators/ISVs.resource @@ -38,19 +38,7 @@ Install ISV Operator From OperatorHub Via CLI # robocop: disable Oc Apply kind=Subscription src=${operator_sub_filepath} Wait Until Keyword Succeeds 1 min 0 sec ... Is Resource Present Subscription ${subscription_name} ${namespace} ${IS_PRESENT} -# Wait For Installplan And Approve It ${namespace} ${operator_name} - WHILE "${installplan_name}" == "${EMPTY}" limit=5m - ${installplan_name}= Get Resource Attribute ${namespace} - ... Subscription ${subscription_name} .status.installPlanRef.name - Sleep 20s - END - ${installplan_approval}= Get Resource Attribute ${namespace} - ... InstallPlan ${installplan_name} .spec.approval - IF "${installplan_approval}" == "Manual" - ${return_code} ${out}= Run And Return Rc And Output oc patch installplan ${installplan_name} -n ${namespace} --type='json' -p '[{"op": "replace", "path": "/spec/approved", "value": true}]' #robocop:disable - Should Be Equal As Integers ${return_code} 0 msg=Error while approving installplan - END -# until here + Wait For Installplan And Approve It ${namespace} ${operator_name} Create Operator Group [Documentation] Creates the Operator Group object which might be needed by an operator. diff --git a/ods_ci/tests/Tests/100__deploy/120__upgrades/120__pre_upgrade.robot b/ods_ci/tests/Tests/100__deploy/120__upgrades/120__pre_upgrade.robot index ec7b72920..0f9445be7 100644 --- a/ods_ci/tests/Tests/100__deploy/120__upgrades/120__pre_upgrade.robot +++ b/ods_ci/tests/Tests/100__deploy/120__upgrades/120__pre_upgrade.robot @@ -22,7 +22,7 @@ Resource ../../../Resources/Page/DistributedWorkloads/DistributedWorkl Resource ../../../Resources/Page/DistributedWorkloads/WorkloadMetricsUI.resource Suite Setup Dashboard Suite Setup Suite Teardown RHOSi Teardown -Test Tags PreUpgradeKobi +Test Tags PreUpgrade *** Variables *** @@ -39,11 +39,6 @@ ${MODEL_CREATED}= ${FALSE} ${RUNTIME_NAME}= Model Serving Test *** Test Cases *** -Test PreUpgrade - [Documentation] Test for pre upgrade - [Tags] PreUpgrade - Log To Console This is test of PreUpgrade - Set PVC Size Via UI [Documentation] Sets a Pod toleration via the admin UI [Tags] Upgrade diff --git a/ods_ci/tests/Tests/100__deploy/120__upgrades/121__during_upgrade.robot b/ods_ci/tests/Tests/100__deploy/120__upgrades/121__during_upgrade.robot index 522851c48..b69fc5ff3 100644 --- a/ods_ci/tests/Tests/100__deploy/120__upgrades/121__during_upgrade.robot +++ b/ods_ci/tests/Tests/100__deploy/120__upgrades/121__during_upgrade.robot @@ -9,7 +9,7 @@ Resource ../../../Resources/Page/ODH/ODHDashboard/ODHDashboardSetting Resource ../../../Resources/Page/ODH/JupyterHub/ODHJupyterhub.resource Library DebugLibrary Library JupyterLibrary -Test Tags DuringUpgradeKobi +Test Tags DuringUpgrade *** Variables *** @@ -18,11 +18,6 @@ ${UPGRADE_TO_IIB}= ${EMPTY} *** Test Cases *** -Test DuringUpgrade - [Documentation] Test for during upgrade - [Tags] DuringUpgrade - Log To Console This is test of DuringUpgrade - Long Running Jupyter Notebook [Documentation] Launch a long running notebook before the upgrade [Tags] Upgrade @@ -37,7 +32,6 @@ Upgrade RHODS [Documentation] Approve the install plan for the upgrade and make sure that upgrade has completed [Tags] ODS-1766 ... Upgrade - ... DuringUpgrade ${initial_version} = Get RHODS Version ${initial_creation_date} = Get Operator Pod Creation Date Set Suite Variable ${UPDATE_CHANNEL} ${UPGRADE_TO_UPDATE_CHANNEL} diff --git a/ods_ci/tests/Tests/100__deploy/120__upgrades/122__post_ugrade.robot b/ods_ci/tests/Tests/100__deploy/120__upgrades/122__post_ugrade.robot index 7e8c29e02..0d5dc3bb3 100644 --- a/ods_ci/tests/Tests/100__deploy/120__upgrades/122__post_ugrade.robot +++ b/ods_ci/tests/Tests/100__deploy/120__upgrades/122__post_ugrade.robot @@ -18,7 +18,7 @@ Resource ../../../Resources/Page/OCPDashboard/Pods/Pods.robot Resource ../../../Resources/Page/OCPDashboard/Builds/Builds.robot Resource ../../../Resources/Page/HybridCloudConsole/OCM.robot Resource ../../../Resources/Page/DistributedWorkloads/DistributedWorkloads.resource -Test Tags PostUpgradeKobi +Test Tags PostUpgrade *** Variables *** @@ -35,11 +35,6 @@ ${RUNTIME_NAME}= Model Serving Test *** Test Cases *** -Test PostUpgrade - [Documentation] Test for post upgrade - [Tags] PostUpgrade - Log To Console This is test of PostUpgrade - Verify PVC Size [Documentation] Verify PVC Size after the upgrade [Tags] Upgrade diff --git a/ods_ci/tests/Tests/200__monitor_and_manage/200__metrics/200__metrics.robot b/ods_ci/tests/Tests/200__monitor_and_manage/200__metrics/200__metrics.robot index ab73fd3c3..f46a50921 100644 --- a/ods_ci/tests/Tests/200__monitor_and_manage/200__metrics/200__metrics.robot +++ b/ods_ci/tests/Tests/200__monitor_and_manage/200__metrics/200__metrics.robot @@ -23,11 +23,6 @@ Test Tags ExcludeOnODH *** Test Cases *** -Test Smoke - [Documentation] Test for smoke - [Tags] SmokeKobi - Log To Console This is test of SmokeKobi - Test Existence of Prometheus Alerting Rules [Documentation] Verifies the prometheus alerting rules [Tags] Smoke