From 930e0e93f6ecd7e91155ee0badb1927ba82dc12d Mon Sep 17 00:00:00 2001 From: Jan Stourac Date: Wed, 11 Dec 2024 19:07:13 +0100 Subject: [PATCH] [fix] the Verify Notebook Has Not Restarted upgrade test This refactores this test - the first test part is moved to "pre_upgrade" phase, because this phase is run on the pre-upgrade version with the pre-upgrade code-base (branch). This is crucial since the further phases will be run with the code-base of the upgraded product and we may hit some issues in the differences in the product otherwise. There are done other changes and fixes to the test to make it more robust including the notebook deletion in the post upgrade phase. Apart from that, to propagate the timestamp value between the phases, this introduces a configmap in the "upgrade" namespace, where we can put some context that should be passed to the followup phase. --- .../ODH/JupyterHub/JupyterLabLauncher.robot | 3 +- .../0201__pre_upgrade.robot | 49 ++++++++++++++++++- .../0202__during_upgrade.robot | 28 +---------- .../0203__post_upgrade.robot | 34 ++++++++++--- 4 files changed, 78 insertions(+), 36 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot index d6509e2e4..a50071984 100644 --- a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot +++ b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot @@ -178,10 +178,9 @@ Clean Up Server Should Match "${ls_server}" "${EMPTY}" Get User Notebook Pod Name - [Documentation] Returns notebook pod name for given username (e.g. for user ldap-admin1 it will be jupyterhub-nb-ldap-2dadmin1) + [Documentation] Returns notebook pod name for given username (e.g. for user ldap-admin1 it will be jupyter-nb-ldap-2dadmin1-0) [Arguments] ${username} ${safe_username}= Get Safe Username ${username} - #${notebook_pod_name}= Set Variable jupyterhub-nb-${safe_username} ${notebook_pod_name}= Set Variable jupyter-nb-${safe_username}-0 RETURN ${notebook_pod_name} diff --git a/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot b/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot index 9b482db80..2f7e8a369 100644 --- a/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot +++ b/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot @@ -24,7 +24,7 @@ Resource ../../Resources/Page/DistributedWorkloads/DistributedWorkloa Resource ../../Resources/Page/DistributedWorkloads/WorkloadMetricsUI.resource Resource ../../Resources/Page/ModelRegistry/ModelRegistry.resource -Suite Setup Dashboard Suite Setup +Suite Setup Upgrade Suite Setup Suite Teardown RHOSi Teardown Test Tags PreUpgrade @@ -34,6 +34,9 @@ Test Tags PreUpgrade ${CUSTOM_CULLER_TIMEOUT} 60000 ${S_SIZE} 25 ${DW_PROJECT_CREATED} False +${CODE} while True: import time ; time.sleep(10); print ("Hello") +${UPGRADE_NS} upgrade +${UPGRADE_CONFIG_MAP} upgrade-config-map *** Test Cases *** @@ -246,12 +249,54 @@ Model Registry Pre Upgrade Set Up [Tags] Upgrade ModelRegistryUpgrade Model Registry Pre Upgrade Scenario +Long Running Jupyter Notebook + [Documentation] Launch a long running notebook before the upgrade + [Tags] Upgrade + Launch Notebook + Add And Run JupyterLab Code Cell In Active Notebook ${CODE} + + # Get the notebook pod creation timestamp + ${notebook_pod_name}= Get User Notebook Pod Name ${TEST_USER2.USERNAME} + ${return_code} ${ntb_creation_timestamp} = Run And Return Rc And Output + ... oc get pod -n ${NOTEBOOKS_NAMESPACE} ${notebook_pod_name} --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' # robocop: disable: line-too-long + Should Be Equal As Integers ${return_code} 0 msg=${ntb_creation_timestamp} + + # Save the timestamp to the OpenShift ConfigMap so it can be used in test in the next phase + ${return_code} ${cmd_output} = Run And Return Rc And Output + ... oc create configmap ${UPGRADE_CONFIG_MAP} -n ${UPGRADE_NS} --from-literal=ntb_creation_timestamp=${ntb_creation_timestamp} # robocop: disable: line-too-long + Should Be Equal As Integers ${return_code} 0 msg=${cmd_output} + + Close Browser + *** Keywords *** -Dashboard Suite Setup +Launch Notebook + [Documentation] Launch notebook for the suite + [Arguments] ${notebook_image}=minimal-notebook + ... ${username}=${TEST_USER2.USERNAME} + ... ${password}=${TEST_USER2.PASSWORD} + ... ${auth_type}=${TEST_USER2.AUTH_TYPE} + Begin Web Test username=${username} password=${password} auth_type=${auth_type} + Launch Jupyter From RHODS Dashboard Link + Spawn Notebook With Arguments + ... image=${notebook_image} + ... username=${username} + ... password=${password} + ... auth_type=${auth_type} + +Upgrade Suite Setup [Documentation] Basic suite setup Set Library Search Order SeleniumLibrary RHOSi Setup + # Prepare a namespace for storing values that should be shared between different upgrade test phases + # 1. if the namespace exists already, let's remove it + ${return_code} ${cmd_output} = Run And Return Rc And Output + ... oc delete namespace --wait --ignore-not-found ${UPGRADE_NS} + Should Be Equal As Integers ${return_code} 0 msg=${cmd_output} + # 2. create the namespace now + ${return_code} ${cmd_output} = Run And Return Rc And Output + ... oc create namespace ${UPGRADE_NS} + Should Be Equal As Integers ${return_code} 0 msg=${cmd_output} Dashboard Test Teardown [Documentation] Basic suite teardown diff --git a/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot b/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot index b0c467722..0ebce6811 100644 --- a/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot +++ b/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot @@ -14,23 +14,7 @@ Library JupyterLibrary Test Tags DuringUpgrade -*** Variables *** -${CODE} while True: import time ; time.sleep(10); print ("Hello") - - *** Test Cases *** -Long Running Jupyter Notebook - [Documentation] Launch a long running notebook before the upgrade - [Tags] Upgrade - Launch Notebook - Add And Run JupyterLab Code Cell In Active Notebook ${CODE} - # robocop:disable - ${return_code} ${timestamp} = Run And Return Rc And Output - ... oc get pod -n ${NOTEBOOKS_NAMESPACE} jupyter-nb-ldap-2dadmin2-0 --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' - Should Be Equal As Integers ${return_code} 0 - Set Global Variable ${timestamp} # robocop: disable - Close Browser - Upgrade RHODS [Documentation] Approve the install plan for the upgrade and make sure that upgrade has completed [Tags] ODS-1766 Upgrade @@ -79,22 +63,14 @@ PyTorch Image Workload Test *** Keywords *** Launch Notebook [Documentation] Launch notebook for the suite - [Arguments] ${notbook_image}=minimal-notebook + [Arguments] ${notebook_image}=minimal-notebook ... ${username}=${TEST_USER2.USERNAME} ... ${password}=${TEST_USER2.PASSWORD} ... ${auth_type}=${TEST_USER2.AUTH_TYPE} - # robocop: disable Begin Web Test username=${username} password=${password} auth_type=${auth_type} - Login To RHODS Dashboard ${username} ${password} ${auth_type} - Wait For RHODS Dashboard To Load Launch Jupyter From RHODS Dashboard Link - Login To Jupyterhub ${username} ${password} ${auth_type} - ${authorization_required} = Is Service Account Authorization Required - IF ${authorization_required} Authorize JupyterLab Service Account - Fix Spawner Status - # robocop: disable Spawn Notebook With Arguments - ... image=${notbook_image} + ... image=${notebook_image} ... username=${username} ... password=${password} ... auth_type=${auth_type} diff --git a/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot b/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot index d912e756a..27f62cd62 100644 --- a/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot +++ b/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot @@ -34,6 +34,8 @@ Test Tags PostUpgrade *** Variables *** ${S_SIZE} 25 ${DW_PROJECT_CREATED} False +${UPGRADE_NS} upgrade +${UPGRADE_CONFIG_MAP} upgrade-config-map *** Test Cases *** @@ -71,13 +73,26 @@ Verify Culler is Enabled END Verify Notebook Has Not Restarted - [Documentation] Verify Notbook pod has not restarted after the upgrade - [Tags] Upgrade AutomationBug RHOAIENG-14840 - # robocop:disable + [Documentation] Verify Notebook pod has not restarted after the upgrade + [Tags] Upgrade + ${notebook_name}= Get User CR Notebook Name ${TEST_USER2.USERNAME} + ${notebook_pod_name}= Get User Notebook Pod Name ${TEST_USER2.USERNAME} + + # Get the running notebook creation timestamp ${return_code} ${new_timestamp} Run And Return Rc And Output - ... oc get pod -n ${NOTEBOOKS_NAMESPACE} jupyter-nb-ldap-2dadmin2-0 --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' - Should Be Equal As Integers ${return_code} 0 - Should Be Equal ${timestamp} ${new_timestamp} msg=Running notebook pod has restarted + ... oc get pod -n ${NOTEBOOKS_NAMESPACE} ${notebook_pod_name} --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' # robocop: disable: line-too-long + Should Be Equal As Integers ${return_code} 0 msg=${new_timestamp} + + # Get the running notebook creation timestamp from the upgrade ConfigMap safed in the previous + # phase (before the actual RHOAI upgrade) + ${return_code} ${ntb_creation_timestamp} Run And Return Rc And Output + ... oc get configmap ${UPGRADE_CONFIG_MAP} -n ${UPGRADE_NS} -o jsonpath='{.data.ntb_creation_timestamp}' + Should Be Equal As Integers ${return_code} 0 msg=${ntb_creation_timestamp} + + # The timestamps should be equal + Should Be Equal ${ntb_creation_timestamp} ${new_timestamp} msg=Running notebook pod has restarted + + [Teardown] Terminate Running Notebook ${notebook_name} Verify Custom Image Is Present [Documentation] Verify Custom Noteboook is not deleted after the upgrade @@ -325,6 +340,13 @@ Delete OOTB Image IF not ${status} Fail Notebook image is deleted after the upgrade IF not ${IS_SELF_MANAGED} Managed RHOAI Upgrade Test Teardown +Terminate Running Notebook + [Documentation] Terminates the running notebook instance + [Arguments] ${notebook_name} + ${return_code} ${cmd_output} Run And Return Rc And Output + ... oc delete Notebook.kubeflow.org -n ${NOTEBOOKS_NAMESPACE} ${notebook_name} + Should Be Equal As Integers ${return_code} 0 msg=${cmd_output} + Managed RHOAI Upgrade Test Teardown # robocop: off=too-many-calls-in-keyword [Documentation] Check rhods_aggregate_availability metric when RHOAI is installed as managed