Merge branch 'master' into ods-195-dashboard-metrics-code

ods_ci/tests/Resources/Files/triton/kserve-triton-onnx-grpc-output.json

@@ -0,0 +1 @@
+{"modelName":"densenetonnx","modelVersion":"1","id":"test1","outputs":[{"name":"fc6_1","datatype":"FP32","shape":["1000"]}],"rawOutputContents":["crbswY2eFMHy2ljBr3gZwk41nsGFaUfAwRMFwmtDScGn5h7BrHeHwdxaF8GCdt7A4DwkwezErcF98U/BQW4Iwbxw88BHxz7BmN4fwVPN7sGNiG7B5blvwUd0P8F3MqTB4RxUwbKIucHzFATCn6r7wTvnpcH5wczBxbHcwZmxvsE2KabByCZewaYnrkAMlIzBQuTfv1Mozz84gQ9B+IBkwanLXMGIUhXBIDmSwG6UfcHxIofBeiZKwTf1KEDOJ0PBAunpwdgtwMF3dCLB7/CNQDjkUEEnZu5AYr9PQCD7DUATXPlByjflwEOTasGPwhs95/zYQfriPcEGbXTBrvKpQHfmI8F/0ofBT3UfQZVHAEBHosw+kavuwUTvScGkB8a/m3UEwWWaT0B+3q3ApvsUQfTd6b/5zw5A6lbTwN7mC0Eq3wvCr6kJwovtc8DlbVjBtE8RwjkXA8JVDobBiAFfwUy7h8HLr9vBOZ5/waKBC8KYJojAJNBDwSJe6D/kgLrBVIeNwDm/z8BMv2HByQWnwFvF3sArnAnCQ0/QwSzPAcKd45PBIta0wYjICsLMab/BMNVWwb8KC8AVgiTCbRhSwSSN60BG/ePAse6rwXwZvMGfAgbCeYuTvvCQLcDvJonAP6qNwUsRFj9HFY1ARftxwaXOAsE46/rB8EmFwaBmz8FmkaPB/XuXwd+XTcEIDQbCXAW8wdX6ZcDhwQTCy3VCwWxDQcFKUszA6aYCwrWFpUDvVpjB6ondP0pUQcE9BWvAG6TYwcASsMEJa5nBsIC6waho8cGww+PB6ta6wbxBxjyvvOlAnwBXQeRlTcDGOCvA5cOpwJj/KkLaobG+18rdwDbLMkC0W4jALF6uwP5p2cBdSqbBh512wAbP1sGLRZrB2UxhwcfuGcFKOMnBPlh+wRB3p8Gl1jbBEQ+/v4NMJkGoerG+ApdbwcTZDcExhpPB7NQLwpOJ777S7IHBOLQNQUUNo79UXRrAc8wNwKkJCkG5X0BAFNUSwchLTsDHdTlAqJxSwHkrlkDtUI+8MjQ1wX4DVD5nY/xAmfPhQHht8T+0+4hBlbOEwCi+V0Gy8au9WvGtQAHNO8FG+6TBt100wV4Dk8GOEp3BxxekwT/52b92r8rB3CUmwby4LcFJdnHB0cwJQbTjv8CeYh3B82xmQAfyS0HNhSlBWaenwOTCW0AoQB3Bcd7xwIXQrUHo3ie/LGRiQXphF0EytZNBkQ8DQTASp8BfMo9AiR8iwaMbmEAkx9nAyzfyP1xFTsENAGNBzM+Tv37pjb5UCM7APYwVwW0UukCEoc5ACX+BwVWM/cBEcZxA56qHP2UiNUCSYzrBfIGswEJn1kHwkpnA501/weD5m0Bt8CzBz5r8QNNSzUB+x/s/9ymbP48J5b6ae4ZBzri0QcTDp0FCtGZBrvSlQaGtrcFRjJ7BmdHawf0zYMHKkjnB+ECGwQxM78DugSbBTOnJwZPTPsDMZMDB1QwzwbaenMFAwo9BXFwVQRrQmz/0OL/BCHB/P84i5sFUmP7AEAsswWX0hcHqmJrBfS+TwXUyicEepmVA1vS2wQOJ88GHK7/BZs/BwclFm8GRoNLBn+QDwXdfqsB+nmFAFqxNQRmHkMFU5qLBJCnOwOB7q0DQS/bAzTeDwdm0uEGFOEVBJLugQVeFtsCqObTA9YWtwKTqEMLAyprBPml2wdO0WUC/t+rA5mMbQXFtkMFeyZRBwfWywSstE8FdGsC/1+eKQRPUq7+c/ObBCZBBwNaKsMDpu9hADkbHv29S1MF/iTrBqs4TwvL5BsJd3pTBmITIwSpjZ8Flo9/Bbm20wXY3HMI29RPCpuICwup/LMI3wPXBZ9DOwUjeEMLMa/7Bz7gFwp9wfMFHxjPBIB20wTVmoMHR7KvBeEiWwQhSU8HJXwTBRmEOwvt/n8He5AvCEmkzwRFYLMFP2jjBB2jNwUhwCcHCUx/BkZ1BwS/r+8HIX+zBp2bgwBtyGMFz3uzBfjsdwrhsBsL9H4XBFm/gwCr328F+uDXBG+PVwfXgMsBxZoXB3uK7wUlJy8G5tZfBrr9WwdpAB8Kf25jBFjRTwZYhFMLJqETBSQqKwVzvgMG0h8LBQviZwaHCvMF6YfFBG99XwYO7rED1LIpAw8RDQfYE2MES+hxB+zNGQZVjV8GiYZlBWhUewdLeg0L2raXBFWJuQYfGiEFtzPA+5Vv9QLZDQEBAPNvAqOAdQhkNt0GtM2Y/0DAKwcJfiDz6aK9Bq5PjQZgApL+vrarBAM9XQWDnfkGLJypAEarxQGiMM8Fl3AVCa1cFQRak7sG81P/BAda1vyO3rUG3kZ9AtBOFQaFkj0BBB3FBcM2xQcErfUFgZpnA8hztQGiOUz+2zH9Btr0OQU0dIcF5D8nBPl0NwFnHEMHbNTXBctmqQOcoVkEZTpe/rm6oQf3lMkLBYKzB4n1PQWOmTsGL/9pAMKOjQNblqEHonttBRH0TwTtsyr9PEVjBamqFQTvAjEE58LlBySEPwc+MMsE5Jp1AtB+EwfsaQkEyvwDBhYkvwSrkG0CDGjhCel49vrQNw0DUXDRBUSHpwHhJh0An1ctBvQYLQUOvykANcZtB1qA7QVZ5xkDnAWHB69YawQXDB8HiuYJBA0SPQN78tMHHIA3BGP23QTiktUDfiIjB0iqFwReXqr8LJwBCaoNxQmRIVkLdtxxC9Cb0QJj8VUHmOUJAigJvwayBaMDxXHNBeQHaQOArPcH5hae/qLkJQnlKnsDest1BgmlkwZx2X0Gi8dVBQ+vLQTCFAUHa30C/bEbDwCJ/BEJbCRlCA9HXP57QnMHOtmVCJoeXQT9qHkJgGhs/lNXDQXrE6UEC7ZjBAoGRwZgLw0H6aXS+BQ3YwaUI+kGe48hBScYqQjw9wkDdMvJBYfOBQUmNq8FQlCtBaX2YQbwUTUJt4sxBDwSAQcIHl0Fz9TDBNTFeQbCzGkHQqZ1A6h3wvvt5L0KPAbRBA4QrwGhX1cADW9NAosItwaelbkFq6NVADy28QUEfkMBafZDBOolTQYt7K0HGhfJBFUwXQDKI7kEfBBrAlcpMwW4+vkFuS2FB8HBJQFVhQMGlU6ZB9gqAQeOtUMBztutBXfxUQVQRAUG4jG9ARCeSQVaX1b4hUZZAWkwNQSdF7UBYOb7ALOyPP8YQScGvYbdACM+bwYfXwUEPd8lBLp/RQdnsWEHwDcNAOCTiwewGx0G2VhtCSYSLvzaHIr/ybLZATErGQc9jE0GQ3cs/EH7WQGaGEkEkcra/R9J4Qau7AkIgKx9B3VH6QFOPcEKp3nVBWTxMwaXdl0C7BM3AjxejQdpShcE73zBBThCcQdPZ28FKZDZBVJMBwfualEG7WEBCCLNWQaDgEsKZ9b9BqbYeQT7PGcC1a+hBGEX2wH7NpMEAnhhCq9yxQQfWAkHXJ9NBoAQvvgcs1UFtvsZBR8bswOr5C8F8mwFCNsbHQaS2xsAWWzrBakuBQYEmIkF1uyBA1kIZQcIQpUHjLqxB11u2wdDaGEGki2rBqQvowFKE/UEF3uTA/QlOQbfkREHF1qnAcfJSwfn9JEHNnAW/em6hQLmFvEHho/hAhGgGwrxrskGMrYdAq+dJPqW0ikEI1KhBNEIGQnMRU8FS+YxBPWttQFaPhEFDiB0/KnFAwed2ocDA6pLA2/j6wVup3762GYBBzCsYP2mouL/+DYM+ifhJwExf3cApzf/Ad6imQHPbaMDP3qlBWtGwwPB2RLxF8MQ/z1edPwelI0FAOb7AooLSQCTktcAEG+tBE6KHQe6s6sAForFB2UikQPlGAsIRfDrBgw2/QVpveUG6WAQ/uC4kQWlphEG8ZlNCVlA3QoLjZMECaQRCqrgewRwT20H8iBzBZCHcQYkcvsDs+BVBKRO5Pnq13EHsvGNB1JHXwMqe8UHmsEBB+kcgwK+nukC0giPBgSumwd7f2EEZSVfBf5IwQfsLmkFQs5ZB24+6QaRrtcHkLLM/L4jVPpzdc0EnNt9A05X6wJyRT0IZe+S/++BfwRv3JsAaKppBA+MoQTSR08EF1NtBJn8dQRmRAMLHw8vAwRgIQkc6A0L2PgvAtHKnQV0/m0HgNDLBpomjvxt1eT/eovVB6YYPQdx4XsGZrR5Ah7vBwL5Zt0GfdYNADTknwWzvBkJHyhNCrmB9QYycWsGf94NBKJ8GQfZSuUHyNjHALCVfQS9jyEBaKmBB9PeuQfytnMHmA2vAqiOPQdwq1MB6DCfApxEDwVuxP0FgqKRByOkUQDtUPsF3HUPASOiHQRsLdEEu8JG/fk+eQcgZG78hyFxBnvP+P9u3hkE2CxLBh6moQW7lJcFvYgRBvgyiwMaYlUAblTtCcEYEQfXyXsHC6oFA7QrSQfCLokEdI19B4F/ywMjGyUG1RPhB6NDIQYXc+MBE7idBzFMtQcZtRMHG7JbBEEp9wUOI28GxW/1BZ/10QVdqfkEbsaBA8H28QYEzg0BsYi9BlmH0QArmQ0F99D5Bcu84Qq6mvMHDENBAaqJqQtEfOEF6EhlC60T8QSHrQkAcDtJA0yKrQCTNAcFA2FPAVTKkwbfrK0Lqs+BAogAev5E7DkFiBInARb0bQaMw0D/FrhfBn6dnwSlKOULEP5TAnYCkQXvhFcGkDrVBdn0fQFGpqsGYLDBBp6VxwRKY0EFBFo/BZ6WlQYSpgsAaAKPAi164QXWXlUFE76g/k1uhwUpLW0HuHXTB+IstQfpDzUC9XW/ACTA9QclM8kGxhwzBDwQTQRLxk0ELp4BBqc3LQQPHFEKkxiZCKy7RQPK6AcF0HCNCwHm7QfOCQ0E1ylFBBTgiwV7njEGt8rVB9abWQV4XzT/6Fu5AXX0kwedACML+Ev3BUKUGQC7KQULFOA9BMpncQb/n40FiF9pBwQclwRE6UkH+mjRBWJalQMcx/EArevXAODoLwXKIVUFfzYBAO1nWQG0+kEEjWqhBtsuvwD+qCEFhaRZA7nn6voqpwEHRZBZBt89Dv1JEOMFfYavB1u4tQO5TlD8xF1vA7k2ZvpIQG8FB629B6KemQVoelUFl67VBfZqKQU6UV0EAhBHBNCwEQin06UBfib1AsfsGQbsUb8BKyAlBSaN0QW63zcDjTAhBvaAGwZD+RkAOGxvBKlIIQo1aLEJMy39CqhEAQoOoxsCtt+NADdXiPymzDMGoeoDBnjwJwYaWQcF4GShB+8sdvzs1JcGnHjHByDiAwBwcl0EWbVvAegd1wO9soEG9BwrCBmWiQd2Bib+8jYjBw2Jhv5Adu8G4BIHBv0z5wWZ3BcKElV/BMe6wQAPnvcGmp5JAqHhTwQ=="]}

ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot

@@ -178,10 +178,9 @@ Clean Up Server
     Should Match    "${ls_server}"    "${EMPTY}"
 
 Get User Notebook Pod Name
-  [Documentation]   Returns notebook pod name for given username  (e.g. for user ldap-admin1 it will be jupyterhub-nb-ldap-2dadmin1)
+  [Documentation]   Returns notebook pod name for given username  (e.g. for user ldap-admin1 it will be jupyter-nb-ldap-2dadmin1-0)
   [Arguments]  ${username}
   ${safe_username}=  Get Safe Username    ${username}
-  #${notebook_pod_name}=   Set Variable  jupyterhub-nb-${safe_username}
   ${notebook_pod_name}=   Set Variable  jupyter-nb-${safe_username}-0
   RETURN  ${notebook_pod_name}
 

ods_ci/tests/Tests/0200__rhoai_upgrade/0201__pre_upgrade.robot

@@ -24,7 +24,7 @@ Resource            ../../Resources/Page/DistributedWorkloads/DistributedWorkloa
 Resource            ../../Resources/Page/DistributedWorkloads/WorkloadMetricsUI.resource
 Resource            ../../Resources/Page/ModelRegistry/ModelRegistry.resource
 
-Suite Setup         Dashboard Suite Setup
+Suite Setup         Upgrade Suite Setup
 Suite Teardown      RHOSi Teardown
 
 Test Tags           PreUpgrade
@@ -34,6 +34,9 @@ Test Tags           PreUpgrade
 ${CUSTOM_CULLER_TIMEOUT}    60000
 ${S_SIZE}                   25
 ${DW_PROJECT_CREATED}       False
+${CODE}     while True: import time ; time.sleep(10); print ("Hello")
+${UPGRADE_NS}    upgrade
+${UPGRADE_CONFIG_MAP}    upgrade-config-map
 
 
 *** Test Cases ***
@@ -246,12 +249,54 @@ Model Registry Pre Upgrade Set Up
     [Tags]      Upgrade     ModelRegistryUpgrade
     Model Registry Pre Upgrade Scenario
 
+Long Running Jupyter Notebook
+    [Documentation]    Launch a long running notebook before the upgrade
+    [Tags]      Upgrade
+    Launch Notebook
+    Add And Run JupyterLab Code Cell In Active Notebook     ${CODE}
+
+    # Get the notebook pod creation timestamp
+    ${notebook_pod_name}=    Get User Notebook Pod Name    ${TEST_USER2.USERNAME}
+    ${return_code}    ${ntb_creation_timestamp} =    Run And Return Rc And Output
+    ...    oc get pod -n ${NOTEBOOKS_NAMESPACE} ${notebook_pod_name} --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp'    # robocop: disable: line-too-long
+    Should Be Equal As Integers     ${return_code}    0    msg=${ntb_creation_timestamp}
+
+    # Save the timestamp to the OpenShift ConfigMap so it can be used in test in the next phase
+    ${return_code}    ${cmd_output} =    Run And Return Rc And Output
+    ...    oc create configmap ${UPGRADE_CONFIG_MAP} -n ${UPGRADE_NS} --from-literal=ntb_creation_timestamp=${ntb_creation_timestamp}    # robocop: disable: line-too-long
+    Should Be Equal As Integers     ${return_code}    0    msg=${cmd_output}
+
+    Close Browser
+
 
 *** Keywords ***
-Dashboard Suite Setup
+Launch Notebook
+    [Documentation]    Launch notebook for the suite
+    [Arguments]     ${notebook_image}=minimal-notebook
+    ...    ${username}=${TEST_USER2.USERNAME}
+    ...    ${password}=${TEST_USER2.PASSWORD}
+    ...    ${auth_type}=${TEST_USER2.AUTH_TYPE}
+    Begin Web Test    username=${username}    password=${password}    auth_type=${auth_type}
+    Launch Jupyter From RHODS Dashboard Link
+    Spawn Notebook With Arguments
+    ...    image=${notebook_image}
+    ...    username=${username}
+    ...    password=${password}
+    ...    auth_type=${auth_type}
+
+Upgrade Suite Setup
     [Documentation]    Basic suite setup
     Set Library Search Order    SeleniumLibrary
     RHOSi Setup
+    # Prepare a namespace for storing values that should be shared between different upgrade test phases
+    # 1. if the namespace exists already, let's remove it
+    ${return_code}    ${cmd_output} =    Run And Return Rc And Output
+    ...    oc delete namespace --wait --ignore-not-found ${UPGRADE_NS}
+    Should Be Equal As Integers     ${return_code}    0    msg=${cmd_output}
+    # 2. create the namespace now
+    ${return_code}    ${cmd_output} =    Run And Return Rc And Output
+    ...    oc create namespace ${UPGRADE_NS}
+    Should Be Equal As Integers     ${return_code}    0    msg=${cmd_output}
 
 Dashboard Test Teardown
     [Documentation]    Basic suite teardown

ods_ci/tests/Tests/0200__rhoai_upgrade/0202__during_upgrade.robot

@@ -14,23 +14,7 @@ Library             JupyterLibrary
 Test Tags           DuringUpgrade
 
 
-*** Variables ***
-${CODE}     while True: import time ; time.sleep(10); print ("Hello")
-
-
 *** Test Cases ***
-Long Running Jupyter Notebook
-    [Documentation]    Launch a long running notebook before the upgrade
-    [Tags]      Upgrade
-    Launch Notebook
-    Add And Run JupyterLab Code Cell In Active Notebook     ${CODE}
-    # robocop:disable
-    ${return_code}    ${timestamp} =    Run And Return Rc And Output
-    ...    oc get pod -n ${NOTEBOOKS_NAMESPACE} jupyter-nb-ldap-2dadmin2-0 --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp'
-    Should Be Equal As Integers     ${return_code}      0
-    Set Global Variable     ${timestamp}  # robocop: disable
-    Close Browser
-
 Upgrade RHODS
     [Documentation]    Approve the install plan for the upgrade and make sure that upgrade has completed
     [Tags]      ODS-1766        Upgrade
@@ -79,22 +63,14 @@ PyTorch Image Workload Test
 *** Keywords ***
 Launch Notebook
     [Documentation]    Launch notebook for the suite
-    [Arguments]     ${notbook_image}=minimal-notebook
+    [Arguments]     ${notebook_image}=minimal-notebook
     ...    ${username}=${TEST_USER2.USERNAME}
     ...    ${password}=${TEST_USER2.PASSWORD}
     ...    ${auth_type}=${TEST_USER2.AUTH_TYPE}
-    # robocop: disable
     Begin Web Test    username=${username}    password=${password}    auth_type=${auth_type}
-    Login To RHODS Dashboard    ${username}    ${password}    ${auth_type}
-    Wait For RHODS Dashboard To Load
     Launch Jupyter From RHODS Dashboard Link
-    Login To Jupyterhub    ${username}    ${password}    ${auth_type}
-    ${authorization_required} =    Is Service Account Authorization Required
-    IF    ${authorization_required}    Authorize JupyterLab Service Account
-    Fix Spawner Status
-    # robocop: disable
     Spawn Notebook With Arguments
-    ...    image=${notbook_image}
+    ...    image=${notebook_image}
     ...    username=${username}
     ...    password=${password}
     ...    auth_type=${auth_type}

ods_ci/tests/Tests/0200__rhoai_upgrade/0203__post_upgrade.robot

@@ -34,6 +34,8 @@ Test Tags           PostUpgrade
 *** Variables ***
 ${S_SIZE}                   25
 ${DW_PROJECT_CREATED}       False
+${UPGRADE_NS}    upgrade
+${UPGRADE_CONFIG_MAP}    upgrade-config-map
 
 
 *** Test Cases ***
@@ -71,13 +73,26 @@ Verify Culler is Enabled
     END
 
 Verify Notebook Has Not Restarted
-    [Documentation]    Verify Notbook pod has not restarted after the upgrade
-    [Tags]      Upgrade     AutomationBug       RHOAIENG-14840
-    # robocop:disable
+    [Documentation]    Verify Notebook pod has not restarted after the upgrade
+    [Tags]      Upgrade
+    ${notebook_name}=    Get User CR Notebook Name    ${TEST_USER2.USERNAME}
+    ${notebook_pod_name}=    Get User Notebook Pod Name    ${TEST_USER2.USERNAME}
+
+    # Get the running notebook creation timestamp
     ${return_code}    ${new_timestamp}    Run And Return Rc And Output
-    ...    oc get pod -n ${NOTEBOOKS_NAMESPACE} jupyter-nb-ldap-2dadmin2-0 --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp'
-    Should Be Equal As Integers     ${return_code}      0
-    Should Be Equal     ${timestamp}        ${new_timestamp}        msg=Running notebook pod has restarted
+    ...    oc get pod -n ${NOTEBOOKS_NAMESPACE} ${notebook_pod_name} --no-headers --output='custom-columns=TIMESTAMP:.metadata.creationTimestamp'    # robocop: disable: line-too-long
+    Should Be Equal As Integers    ${return_code}    0    msg=${new_timestamp}
+
+    # Get the running notebook creation timestamp from the upgrade ConfigMap safed in the previous
+    # phase (before the actual RHOAI upgrade)
+    ${return_code}    ${ntb_creation_timestamp}    Run And Return Rc And Output
+    ...    oc get configmap ${UPGRADE_CONFIG_MAP} -n ${UPGRADE_NS} -o jsonpath='{.data.ntb_creation_timestamp}'
+    Should Be Equal As Integers    ${return_code}    0    msg=${ntb_creation_timestamp}
+
+    # The timestamps should be equal
+    Should Be Equal    ${ntb_creation_timestamp}    ${new_timestamp}    msg=Running notebook pod has restarted
+
+    [Teardown]    Terminate Running Notebook    ${notebook_name}
 
 Verify Custom Image Is Present
     [Documentation]    Verify Custom Noteboook is not deleted after the upgrade
@@ -325,6 +340,13 @@ Delete OOTB Image
     IF    not ${status}    Fail    Notebook image is deleted after the upgrade
     IF    not ${IS_SELF_MANAGED}    Managed RHOAI Upgrade Test Teardown
 
+Terminate Running Notebook
+    [Documentation]    Terminates the running notebook instance
+    [Arguments]     ${notebook_name}
+    ${return_code}    ${cmd_output}    Run And Return Rc And Output
+    ...    oc delete Notebook.kubeflow.org -n ${NOTEBOOKS_NAMESPACE} ${notebook_name}
+    Should Be Equal As Integers    ${return_code}    0    msg=${cmd_output}
+
 Managed RHOAI Upgrade Test Teardown
     # robocop: off=too-many-calls-in-keyword
     [Documentation]    Check rhods_aggregate_availability metric when RHOAI is installed as managed