RHOAIENG-5435 - Distributed Workloads Components (#1392)

Included are test cases for DSC Distributed Workloads Components Ray, Codeflare and Kueue, tests Manage/Remove management states. These components are restored to original management state.
red-hat-data-services · May 8, 2024 · 34ff049 · 34ff049
2 parents 569367b + 288f7b1
commit 34ff049
Show file tree

Hide file tree

Showing 4 changed files with 166 additions and 1 deletion.
diff --git a/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot b/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot
@@ -420,6 +420,16 @@ Set Component State
     END
     Log To Console    Component ${component} state was set to ${state}
 
+Get DSC Component State
+    [Documentation]    Get component management state
+    [Arguments]    ${dsc}    ${component}    ${namespace}
+
+    ${rc}   ${state}=    Run And Return Rc And Output
+    ...    oc get DataScienceCluster/${dsc} -n ${namespace} -o 'jsonpath={.spec.components.${component}.managementState}'
+    Should Be Equal    "${rc}"    "0"    msg=${state}
+    Log To Console    Component ${component} state ${state}
+
+    RETURN    ${state}
 Enable Component
     [Documentation]    Enables a component in Data Science Cluster
     [Arguments]    ${component}

diff --git a/ods_ci/tests/Resources/OCP.resource b/ods_ci/tests/Resources/OCP.resource
@@ -220,7 +220,7 @@ Verify Pod Logs Do Not Contain
     [Documentation]    Checks there are no errors on Pod Logs
     [Arguments]    ${pod_name}    ${pod_namespace}    ${regex_pattern}    ${container}
     ${pod_logs}=    Oc Get Pod Logs  name=${pod_name}  namespace=${pod_namespace}  container=${container}
-    ${match_list} 	 Get Regexp Matches   ${pod_logs}     ${regex_pattern}
+    ${match_list}    Get Regexp Matches   ${pod_logs}     ${regex_pattern}
     ${entry_msg}      Remove Duplicates      ${match_list}
     ${length}         Get Length   ${entry_msg}
     IF   ${length} != ${0}    FAIL    Pod ${pod_name} logs should not contain regexp ${regex_pattern}. Matching log entry: ${entry_msg}
@@ -250,3 +250,11 @@ Delete Namespace From Openshift
         ${rc}   ${output}=    Run And Return Rc And Output    oc delete project ${namespace}
         Should Be Equal    "${rc}"    "0"   msg=${output}
     END
+
+Check If Pod Does Not Exist
+    [Documentation]    Check if pod does not exist using pod label-selector
+    [Arguments]        ${label_selector}    ${namespace}
+
+    ${rc}   ${output}=    Run And Return Rc And Output
+    ...    oc get pod -l {label_selector} -n ${namespace}
+    Should Be Equal    "${rc}"    "1"    msg=${output}
diff --git a/ods_ci/tests/Resources/ODS.robot b/ods_ci/tests/Resources/ODS.robot
@@ -403,3 +403,10 @@ Wait For DSCI Ready State
     ${rc}   ${output}=    Run And Return Rc And Output
     ...    oc wait --timeout=${wait_time} --for jsonpath='{.status.conditions[].reason}'=ReconcileCompleted -n ${namespace} dsci ${dsci}
     Should Be Equal    "${rc}"    "0"     msg=${output}
+Wait For DSC Conditions Reconciled
+    [Documentation]    Checks all DSC conditions to be successfully reconciled
+    [Arguments]    ${namespace}    ${dsc_name}    ${wait_time}=3m
+    ${rc}    ${out}=    Run And Return Rc And Output
+    ...    oc wait --timeout=${wait_time} --for jsonpath='{.status.conditions[].reason}'=ReconcileCompleted -n ${namespace} dsc ${dsc_name}    # robocop: disable
+    Should Be Equal As Integers    ${rc}     ${0}
+    Log    ${out}    console=${out}
diff --git a/...100__deploy/130__operators/130__rhods_operator/142__distributed_workload_components.robot b/...100__deploy/130__operators/130__rhods_operator/142__distributed_workload_components.robot
@@ -0,0 +1,140 @@
+*** Settings ***
+Documentation       Test Cases to verify DSC Distributed Workloads Components
+
+Library             Collections
+Resource            ../../../../Resources/OCP.resource
+Resource            ../../../../Resources/ODS.robot
+Resource            ../../../../../tasks/Resources/RHODS_OLM/install/oc_install.robot
+Suite Setup         Suite Setup
+Suite Teardown      Suite Teardown
+
+
+*** Variables ***
+${OPERATOR_NS}                  ${OPERATOR_NAMESPACE}
+${APPLICATIONS_NS}              ${APPLICATIONS_NAMESPACE}
+${DSC_NAME}                     default-dsc
+${KUEUE_LABEL_SELECTOR}         app.kubernetes.io/name=kueue
+${KUEUE_DEPLOYMENT_NAME}        kueue-controller-manager
+${CODEFLARE_LABEL_SELECTOR}     app.kubernetes.io/name=codeflare-operator
+${CODEFLARE_DEPLOYMENT_NAME}    codeflare-operator-manager
+${RAY_LABEL_SELECTOR}           app.kubernetes.io/name=kuberay
+${RAY_DEPLOYMENT_NAME}          kuberay-operator
+${IS_PRESENT}        0
+${IS_NOT_PRESENT}    1
+&{SAVED_MANAGEMENT_STATES}
+...  RAY=${EMPTY}
+...  KUEUE=${EMPTY}
+...  CODEFLARE=${EMPTY}
+
+
+*** Test Cases ***
+Validate Kueue Managed State
+    [Documentation]    Validate that the DSC by default sets component 'kueue' to state Managed,
+    ...    check that kueue deployment and pod are created
+    [Tags]    Operator    Tier1    RHOAIENG-5435    kueue-managed
+
+    Set DSC Component Managed State And Wait For Completion   kueue    ${KUEUE_DEPLOYMENT_NAME}    ${KUEUE_LABEL_SELECTOR}
+
+    [Teardown]     Restore DSC Component State    kueue    ${KUEUE_DEPLOYMENT_NAME}    ${KUEUE_LABEL_SELECTOR}    ${SAVED_MANAGEMENT_STATES.KUEUE}
+
+Validate Kueue Removed State
+    [Documentation]    Validate that Kueue management state Removed does remove relevant resources.
+    [Tags]    Operator    Tier1    RHOAIENG-5435    kueue-removed
+
+    Set DSC Component Removed State And Wait For Completion   kueue    ${KUEUE_DEPLOYMENT_NAME}    ${KUEUE_LABEL_SELECTOR}
+
+    [Teardown]     Restore DSC Component State    kueue    ${KUEUE_DEPLOYMENT_NAME}    ${KUEUE_LABEL_SELECTOR}    ${SAVED_MANAGEMENT_STATES.KUEUE}
+
+ Validate Codeflare Managed State
+    [Documentation]    Validate that the DSC by default sets component 'Codeflare' to state Managed,
+    ...    check that Codeflare deployment and pod are created
+    [Tags]    Operator    Tier1    RHOAIENG-5435    codeflare-managed
+
+    Set DSC Component Managed State And Wait For Completion   codeflare    ${CODEFLARE_DEPLOYMENT_NAME}    ${CODEFLARE_LABEL_SELECTOR}
+
+    [Teardown]     Restore DSC Component State    codeflare    ${CODEFLARE_DEPLOYMENT_NAME}    ${CODEFLARE_LABEL_SELECTOR}    ${SAVED_MANAGEMENT_STATES.CODEFLARE}
+
+Validate Codeflare Removed State
+    [Documentation]    Validate that Codeflare management state Removed does remove relevant resources.
+    [Tags]    Operator    Tier1    RHOAIENG-5435    codeflare-removed
+
+    Set DSC Component Removed State And Wait For Completion   codeflare    ${CODEFLARE_DEPLOYMENT_NAME}    ${CODEFLARE_LABEL_SELECTOR}
+
+    [Teardown]     Restore DSC Component State    codeflare    ${CODEFLARE_DEPLOYMENT_NAME}    ${CODEFLARE_LABEL_SELECTOR}    ${SAVED_MANAGEMENT_STATES.CODEFLARE}
+
+Validate Ray Managed State
+    [Documentation]    Validate that the DSC by default sets component 'Ray' to state Managed,
+    ...    check that Ray deployment and pod are created
+    [Tags]    Operator    Tier1    RHOAIENG-5435    ray-managed
+
+    Set DSC Component Managed State And Wait For Completion   ray    ${RAY_DEPLOYMENT_NAME}    ${RAY_LABEL_SELECTOR}
+
+    [Teardown]     Restore DSC Component State    ray    ${RAY_DEPLOYMENT_NAME}    ${RAY_LABEL_SELECTOR}    ${SAVED_MANAGEMENT_STATES.RAY}
+
+Validate Ray Removed State
+    [Documentation]    Validate that Ray management state Removed does remove relevant resources.
+    [Tags]    Operator    Tier1    RHOAIENG-5435    ray-removed
+
+    Set DSC Component Removed State And Wait For Completion   ray    ${RAY_DEPLOYMENT_NAME}    ${RAY_LABEL_SELECTOR}
+
+    [Teardown]     Restore DSC Component State    ray    ${RAY_DEPLOYMENT_NAME}    ${RAY_LABEL_SELECTOR}    ${SAVED_MANAGEMENT_STATES.RAY}
+
+
+*** Keywords ***
+Suite Setup
+    [Documentation]    Suite Setup
+    RHOSi Setup
+    Wait For DSC Conditions Reconciled    ${OPERATOR_NS}     ${DSC_NAME}
+    ${SAVED_MANAGEMENT_STATES.RAY}=     Get DSC Component State    ${DSC_NAME}    ray    ${OPERATOR_NS}
+    ${SAVED_MANAGEMENT_STATES.KUEUE}=     Get DSC Component State    ${DSC_NAME}    kueue    ${OPERATOR_NS}
+    ${SAVED_MANAGEMENT_STATES.CODEFLARE}=     Get DSC Component State    ${DSC_NAME}    codeflare    ${OPERATOR_NS}
+    Set Suite Variable    ${SAVED_MANAGEMENT_STATES}
+
+Suite Teardown
+    [Documentation]    Suite Teardown
+    RHOSi Teardown
+
+Set DSC Component Removed State And Wait For Completion
+    [Documentation]    Set component management state to 'Removed', and wait for deployment and pod to be removed.
+    [Arguments]    ${component}    ${deployment_name}    ${label_selector}
+
+    ${management_state}=    Get DSC Component State    ${DSC_NAME}    ${component}    ${OPERATOR_NS}
+    IF    "${management_state}" != "Removed"
+            Set Component State    ${component}    Removed
+    END
+
+    Wait Until Keyword Succeeds    5 min    0 sec
+    ...    Is Resource Present     Deployment    ${deployment_name}    ${APPLICATIONS_NS}    ${IS_NOT_PRESENT}
+
+    Wait Until Keyword Succeeds    3 min    0 sec
+    ...    Check If Pod Does Not Exist    ${label_selector}    ${APPLICATIONS_NS}
+
+Set DSC Component Managed State And Wait For Completion
+    [Documentation]    Set component management state to 'Managed', and wait for deployment and pod to be available.
+    [Arguments]    ${component}    ${deployment_name}    ${label_selector}
+
+    ${management_state}=    Get DSC Component State    ${DSC_NAME}    ${component}    ${OPERATOR_NS}
+    IF    "${management_state}" != "Managed"
+            Set Component State    ${component}    Managed
+    END
+
+    Wait Until Keyword Succeeds    5 min    0 sec
+    ...    Is Resource Present     Deployment    ${deployment_name}    ${APPLICATIONS_NS}    ${IS_PRESENT}
+
+    Wait Until Keyword Succeeds    3 min    0 sec
+    ...    Check If Pod Exists    ${APPLICATIONS_NS}    ${label_selector}    ${FALSE}
+
+Restore DSC Component State
+    [Documentation]    Set component management state to original state, wait for component resources to be available.
+    [Arguments]    ${component}    ${deployment_name}    ${LABEL_SELECTOR}    ${saved_state}
+
+    ${current_state}=    Get DSC Component State    ${DSC_NAME}    ${component}    ${OPERATOR_NS}
+    IF    "${current_state}" != "${saved_state}"
+        IF    "${saved_state}" == "Managed"
+            Set DSC Component Managed State And Wait For Completion    ${component}    ${deployment_name}    ${LABEL_SELECTOR}
+        ELSE IF    "${saved_state}" == "Removed"
+            Set DSC Component Removed State And Wait For Completion    ${component}    ${deployment_name}    ${LABEL_SELECTOR}
+        ELSE
+            FAIL    Component ${component} state "${saved_state}" not supported at this time
+        END
+    END