diff --git a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot index d6509e2e4..a50071984 100644 --- a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot +++ b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot @@ -178,10 +178,9 @@ Clean Up Server Should Match "${ls_server}" "${EMPTY}" Get User Notebook Pod Name - [Documentation] Returns notebook pod name for given username (e.g. for user ldap-admin1 it will be jupyterhub-nb-ldap-2dadmin1) + [Documentation] Returns notebook pod name for given username (e.g. for user ldap-admin1 it will be jupyter-nb-ldap-2dadmin1-0) [Arguments] ${username} ${safe_username}= Get Safe Username ${username} - #${notebook_pod_name}= Set Variable jupyterhub-nb-${safe_username} ${notebook_pod_name}= Set Variable jupyter-nb-${safe_username}-0 RETURN ${notebook_pod_name} diff --git a/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot b/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot index 9b482db80..2f7e8a369 100644 --- a/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot +++ b/ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot @@ -24,7 +24,7 @@ Resource ../../Resources/Page/DistributedWorkloads/DistributedWorkloa Resource ../../Resources/Page/DistributedWorkloads/WorkloadMetricsUI.resource Resource ../../Resources/Page/ModelRegistry/ModelRegistry.resource -Suite Setup Dashboard Suite Setup +Suite Setup Upgrade Suite Setup Suite Teardown RHOSi Teardown Test Tags PreUpgrade @@ -34,6 +34,9 @@ Test Tags PreUpgrade ${CUSTOM_CULLER_TIMEOUT} 60000 ${S_SIZE} 25 ${DW_PROJECT_CREATED} False +${CODE} while True: import time ; time.sleep(10); print ("Hello") +${UPGRADE_NS} upgrade +${UPGRADE_CONFIG_MAP} upgrade-config-map *** Test Cases *** @@ -246,12 +249,54 @@ Model Registry Pre Upgrade Set Up [Tags] Upgrade ModelRegistryUpgrade Model Registry Pre Upgrade Scenario +Long Running Jupyter Notebook + [Documentation] Launch a long running notebook before the upgrade + [Tags] Upgrade + Launch Notebook + Add And Run JupyterLab Code Cell In Active Notebook ${CODE} + + # Get the notebook pod creation timestamp + ${notebook_pod_name}= Get User Notebook Pod Name ${TEST_USER2.USERNAME} + ${return_code} ${ntb_creation_timestamp} = Run And Return Rc And Output + ... oc get pod -n ${NOTEBOOKS_NAMESPACE} ${notebook_pod_name} --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' # robocop: disable: line-too-long + Should Be Equal As Integers ${return_code} 0 msg=${ntb_creation_timestamp} + + # Save the timestamp to the OpenShift ConfigMap so it can be used in test in the next phase + ${return_code} ${cmd_output} = Run And Return Rc And Output + ... oc create configmap ${UPGRADE_CONFIG_MAP} -n ${UPGRADE_NS} --from-literal=ntb_creation_timestamp=${ntb_creation_timestamp} # robocop: disable: line-too-long + Should Be Equal As Integers ${return_code} 0 msg=${cmd_output} + + Close Browser + *** Keywords *** -Dashboard Suite Setup +Launch Notebook + [Documentation] Launch notebook for the suite + [Arguments] ${notebook_image}=minimal-notebook + ... ${username}=${TEST_USER2.USERNAME} + ... ${password}=${TEST_USER2.PASSWORD} + ... ${auth_type}=${TEST_USER2.AUTH_TYPE} + Begin Web Test username=${username} password=${password} auth_type=${auth_type} + Launch Jupyter From RHODS Dashboard Link + Spawn Notebook With Arguments + ... image=${notebook_image} + ... username=${username} + ... password=${password} + ... auth_type=${auth_type} + +Upgrade Suite Setup [Documentation] Basic suite setup Set Library Search Order SeleniumLibrary RHOSi Setup + # Prepare a namespace for storing values that should be shared between different upgrade test phases + # 1. if the namespace exists already, let's remove it + ${return_code} ${cmd_output} = Run And Return Rc And Output + ... oc delete namespace --wait --ignore-not-found ${UPGRADE_NS} + Should Be Equal As Integers ${return_code} 0 msg=${cmd_output} + # 2. create the namespace now + ${return_code} ${cmd_output} = Run And Return Rc And Output + ... oc create namespace ${UPGRADE_NS} + Should Be Equal As Integers ${return_code} 0 msg=${cmd_output} Dashboard Test Teardown [Documentation] Basic suite teardown diff --git a/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot b/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot index b0c467722..0ebce6811 100644 --- a/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot +++ b/ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot @@ -14,23 +14,7 @@ Library JupyterLibrary Test Tags DuringUpgrade -*** Variables *** -${CODE} while True: import time ; time.sleep(10); print ("Hello") - - *** Test Cases *** -Long Running Jupyter Notebook - [Documentation] Launch a long running notebook before the upgrade - [Tags] Upgrade - Launch Notebook - Add And Run JupyterLab Code Cell In Active Notebook ${CODE} - # robocop:disable - ${return_code} ${timestamp} = Run And Return Rc And Output - ... oc get pod -n ${NOTEBOOKS_NAMESPACE} jupyter-nb-ldap-2dadmin2-0 --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' - Should Be Equal As Integers ${return_code} 0 - Set Global Variable ${timestamp} # robocop: disable - Close Browser - Upgrade RHODS [Documentation] Approve the install plan for the upgrade and make sure that upgrade has completed [Tags] ODS-1766 Upgrade @@ -79,22 +63,14 @@ PyTorch Image Workload Test *** Keywords *** Launch Notebook [Documentation] Launch notebook for the suite - [Arguments] ${notbook_image}=minimal-notebook + [Arguments] ${notebook_image}=minimal-notebook ... ${username}=${TEST_USER2.USERNAME} ... ${password}=${TEST_USER2.PASSWORD} ... ${auth_type}=${TEST_USER2.AUTH_TYPE} - # robocop: disable Begin Web Test username=${username} password=${password} auth_type=${auth_type} - Login To RHODS Dashboard ${username} ${password} ${auth_type} - Wait For RHODS Dashboard To Load Launch Jupyter From RHODS Dashboard Link - Login To Jupyterhub ${username} ${password} ${auth_type} - ${authorization_required} = Is Service Account Authorization Required - IF ${authorization_required} Authorize JupyterLab Service Account - Fix Spawner Status - # robocop: disable Spawn Notebook With Arguments - ... image=${notbook_image} + ... image=${notebook_image} ... username=${username} ... password=${password} ... auth_type=${auth_type} diff --git a/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot b/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot index d912e756a..27f62cd62 100644 --- a/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot +++ b/ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot @@ -34,6 +34,8 @@ Test Tags PostUpgrade *** Variables *** ${S_SIZE} 25 ${DW_PROJECT_CREATED} False +${UPGRADE_NS} upgrade +${UPGRADE_CONFIG_MAP} upgrade-config-map *** Test Cases *** @@ -71,13 +73,26 @@ Verify Culler is Enabled END Verify Notebook Has Not Restarted - [Documentation] Verify Notbook pod has not restarted after the upgrade - [Tags] Upgrade AutomationBug RHOAIENG-14840 - # robocop:disable + [Documentation] Verify Notebook pod has not restarted after the upgrade + [Tags] Upgrade + ${notebook_name}= Get User CR Notebook Name ${TEST_USER2.USERNAME} + ${notebook_pod_name}= Get User Notebook Pod Name ${TEST_USER2.USERNAME} + + # Get the running notebook creation timestamp ${return_code} ${new_timestamp} Run And Return Rc And Output - ... oc get pod -n ${NOTEBOOKS_NAMESPACE} jupyter-nb-ldap-2dadmin2-0 --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' - Should Be Equal As Integers ${return_code} 0 - Should Be Equal ${timestamp} ${new_timestamp} msg=Running notebook pod has restarted + ... oc get pod -n ${NOTEBOOKS_NAMESPACE} ${notebook_pod_name} --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp' # robocop: disable: line-too-long + Should Be Equal As Integers ${return_code} 0 msg=${new_timestamp} + + # Get the running notebook creation timestamp from the upgrade ConfigMap safed in the previous + # phase (before the actual RHOAI upgrade) + ${return_code} ${ntb_creation_timestamp} Run And Return Rc And Output + ... oc get configmap ${UPGRADE_CONFIG_MAP} -n ${UPGRADE_NS} -o jsonpath='{.data.ntb_creation_timestamp}' + Should Be Equal As Integers ${return_code} 0 msg=${ntb_creation_timestamp} + + # The timestamps should be equal + Should Be Equal ${ntb_creation_timestamp} ${new_timestamp} msg=Running notebook pod has restarted + + [Teardown] Terminate Running Notebook ${notebook_name} Verify Custom Image Is Present [Documentation] Verify Custom Noteboook is not deleted after the upgrade @@ -325,6 +340,13 @@ Delete OOTB Image IF not ${status} Fail Notebook image is deleted after the upgrade IF not ${IS_SELF_MANAGED} Managed RHOAI Upgrade Test Teardown +Terminate Running Notebook + [Documentation] Terminates the running notebook instance + [Arguments] ${notebook_name} + ${return_code} ${cmd_output} Run And Return Rc And Output + ... oc delete Notebook.kubeflow.org -n ${NOTEBOOKS_NAMESPACE} ${notebook_name} + Should Be Equal As Integers ${return_code} 0 msg=${cmd_output} + Managed RHOAI Upgrade Test Teardown # robocop: off=too-many-calls-in-keyword [Documentation] Check rhods_aggregate_availability metric when RHOAI is installed as managed