Add Custom Serving Runtime UI test suite (red-hat-data-services#964)

* make open settings kw generic pt1 Signed-off-by: bdattoma <[email protected]> * add custom serving runtime suite Signed-off-by: bdattoma <[email protected]> * apply changes to Open dashboard settings kw calls Signed-off-by: bdattoma <[email protected]> * add basic keyword to upload a runtime template Signed-off-by: bdattoma <[email protected]> * add ods-2276 Signed-off-by: bdattoma <[email protected]> * add ovms-custom template Signed-off-by: bdattoma <[email protected]> * add click action common kw Signed-off-by: bdattoma <[email protected]> * add serving runtime deletion kw Signed-off-by: bdattoma <[email protected]> * add ods-2279 Signed-off-by: bdattoma <[email protected]> * check if details page is already open Signed-off-by: bdattoma <[email protected]> * make model server kw working on DSProject page Signed-off-by: bdattoma <[email protected]> * add ODS-2281 and final suite setup Signed-off-by: bdattoma <[email protected]> * add kw to delete single dsproject from cli Signed-off-by: bdattoma <[email protected]> * final suite setup/teardown Signed-off-by: bdattoma <[email protected]> * move runtime deletion to suite teardown Signed-off-by: bdattoma <[email protected]> * remove continue keyword Signed-off-by: bdattoma <[email protected]> * remove wrong arg from call Signed-off-by: bdattoma <[email protected]> * fix setup and teardown Signed-off-by: bdattoma <[email protected]> * fix eventual runtime creation Signed-off-by: bdattoma <[email protected]> * fix open section options menu Signed-off-by: bdattoma <[email protected]> * fix runtime deletion from cli Signed-off-by: bdattoma <[email protected]> * fix dryrun Signed-off-by: bdattoma <[email protected]> * fix dryrun (2) Signed-off-by: bdattoma <[email protected]> * fix dryrun (3) Signed-off-by: bdattoma <[email protected]> * increase pod timeout for runtime Signed-off-by: bdattoma <[email protected]> --------- Signed-off-by: bdattoma <[email protected]>
ChughShilpa · Jan 2, 2024 · 62220aa · 62220aa
1 parent 8a5862e
commit 62220aa
Show file tree

Hide file tree

Showing 12 changed files with 297 additions and 41 deletions.
diff --git a/ods_ci/tests/Resources/Files/ovms_servingruntime.yaml b/ods_ci/tests/Resources/Files/ovms_servingruntime.yaml
@@ -0,0 +1,41 @@
+apiVersion: serving.kserve.io/v1alpha1
+kind: ServingRuntime
+metadata:
+  annotations:
+    opendatahub.io/disable-gpu: "true"
+    openshift.io/display-name: ODS-CI Custom OpenVINO Model Server
+  labels:
+    opendatahub.io/dashboard: "true"
+  name: ovms-ods-ci
+spec:
+  builtInAdapter:
+    memBufferBytes: 134217728
+    modelLoadingTimeoutMillis: 90000
+    runtimeManagementPort: 8888
+    serverType: ovms
+  containers:
+    - args:
+        - --port=8001
+        - --rest_port=8888
+        - --config_path=/models/model_config_list.json
+        - --file_system_poll_wait_seconds=0
+        - --grpc_bind_address=127.0.0.1
+        - --rest_bind_address=127.0.0.1
+      image: quay.io/opendatahub/openvino_model_server:2022.3-release
+      name: ovms
+  grpcDataEndpoint: port:8001
+  grpcEndpoint: port:8085
+  multiModel: true
+  protocolVersions:
+    - grpc-v1
+  replicas: 1
+  supportedModelFormats:
+    - autoSelect: true
+      name: openvino_ir
+      version: opset1
+    - autoSelect: true
+      name: onnx
+      version: "1"
+    - autoSelect: true
+      name: tensorflow
+      version: "2"
diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDashboard.robot b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDashboard.robot
@@ -825,3 +825,13 @@ Handle Deletion Confirmation Modal
         Click Button    ${delete_btn_xp}
     END
     Wait Until Generic Modal Disappears
+
+Click Action From Actions Menu
+    [Documentation]    Clicks an action from Actions menu (3-dots menu on the right)
+    [Arguments]    ${item_title}    ${action}    ${item_type}=${NONE}
+    Click Element       xpath=//tr[td[@data-label="Name"]//*[text()="${item_title}"]]/td[@class="pf-c-table__action"]/div/button[@aria-label="Actions"]
+    IF    "${item_type}" != "${NONE}"
+        ${action}=    Catenate    ${action}    ${item_type}
+    END
+    Wait Until Page Contains Element       xpath=//tr[td[@data-label="Name"]//*[text()="${item_title}"]]/td[@class="pf-c-table__action"]/div/ul/li/button[text()="${action}"]
+    Click Element       xpath=//tr[td[@data-label="Name"]//*[text()="${item_title}"]]/td[@class="pf-c-table__action"]/div/ul/li/button[text()="${action}"]
diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDashboardSettings.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDashboardSettings.resource
@@ -189,7 +189,7 @@ Set Notebook Culler Timeout
 
 Disable Notebook Culler
     [Documentation]    Disables the culler (i.e. sets the default timeout of 1 year)
-    Open Dashboard Cluster Settings
+    Open Dashboard Settings    settings_page=Cluster settings
     Sleep  5
     ${disabled_field} =  Run Keyword And Return Status  Page Should Contain Element
     ...    xpath://input[@id="hour-input"][@disabled=""]
@@ -201,12 +201,13 @@ Disable Notebook Culler
 Modify Notebook Culler Timeout
     [Documentation]    Modifies the culler timeout via UI
     [Arguments]    ${new_timeout}
-    Open Dashboard Cluster Settings
+    Open Dashboard Settings    settings_page=Cluster settings
     Set Notebook Culler Timeout  ${new_timeout}
     Sleep  10s  msg=Give time for rollout
 
-Open Dashboard Cluster Settings
+Open Dashboard Settings
     [Documentation]    Opens the RHODS dashboard and navigates to the Cluster settings page
+    [Arguments]    ${settings_page}
     Launch Dashboard    ${TEST_USER.USERNAME}    ${TEST_USER.PASSWORD}    ${TEST_USER.AUTH_TYPE}
     ...    ${ODH_DASHBOARD_URL}    ${BROWSER.NAME}    ${BROWSER.OPTIONS}
     Sleep  1s
@@ -215,6 +216,15 @@ Open Dashboard Cluster Settings
     IF  ${settings_hidden}==True
         Click Element  xpath://button[@id="settings"]
     END
-    Click Element  xpath://a[.="Cluster settings"]
-    Wait For RHODS Dashboard To Load    expected_page=Cluster Settings
+    Click Element  xpath://a[.="${settings_page}"]
+    IF    "${settings_page}" == "Notebook images"
+        ${exp_page_title}=    Set Variable    Notebook images settings
+    ELSE IF     "${settings_page}" == "Cluster settings"
+        ${exp_page_title}=    Set Variable    Cluster Settings
+    ELSE IF     "${settings_page}" == "Serving runtimes"
+        ${exp_page_title}=    Set Variable    Serving runtimes
+    ELSE IF     "${settings_page}" == "User management"
+        ${exp_page_title}=    Set Variable    User and group settings
+    END
+    Wait For RHODS Dashboard To Load    expected_page=${exp_page_title}
     ...    wait_for_cards=${FALSE}
diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDashboardSettingsRuntimes.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDashboardSettingsRuntimes.resource
@@ -0,0 +1,70 @@
+*** Settings ***
+Documentation    Collection of keywords to interact with Serving runtimes page
+Resource       ../../../Common.robot
+
+
+*** Variables ***
+${ADD_RUNTIME_BTN_XP}=    //button[text()="Add serving runtime"]
+${SUBMIT_RUNTIME_BTN_XP}=    //button[text()="Add"]
+${UPLOAD_RUNTIME_BTN_XP}=    //button[text()="Upload files"]
+${SCRATCH_RUNTIME_BTN_XP}=    //button[text()="Start from scratch"]
+${EDITOR_RUNTIME_BTN_XP}=    //div[contains(@class, "odh-dashboard__code-editor")]
+
+
+*** Keywords ***
+Click Add Serving Runtime Template Button
+    SeleniumLibrary.Wait Until Page Contains Element    ${ADD_RUNTIME_BTN_XP}
+    SeleniumLibrary.Click Element    ${ADD_RUNTIME_BTN_XP}
+    Run Keyword And Continue On Failure    SeleniumLibrary.Page Should Contain    Add a serving runtime
+    Run Keyword And Continue On Failure    SeleniumLibrary.Page Should Contain    Drag a file here, upload files, or start from scratch.
+    Run Keyword And Continue On Failure    SeleniumLibrary.Page Should Contain Element    ${UPLOAD_RUNTIME_BTN_XP}
+    Run Keyword And Continue On Failure    SeleniumLibrary.Page Should Contain Element    ${SCRATCH_RUNTIME_BTN_XP}
+    Run Keyword And Continue On Failure    SeleniumLibrary.Page Should Contain Element    ${EDITOR_RUNTIME_BTN_XP}
+
+Upload Serving Runtime Template
+    [Documentation]    Uploads via UI a YAML file containing a custom Serving Runtime definition
+    [Arguments]    ${runtime_filepath}
+    Click Add Serving Runtime Template Button
+    SeleniumLibrary.Click Element   ${UPLOAD_RUNTIME_BTN_XP}
+    ${rc}    ${pwd}=    Run And Return Rc And Output    echo $PWD
+    Should Be Equal As Integers    ${rc}    ${0}
+    SeleniumLibrary.Choose File    ${EDITOR_RUNTIME_BTN_XP}//input[@type="file"]    ${pwd}/${runtime_filepath}
+    Run Keyword And Continue On Failure    SeleniumLibrary.Wait Until Page Contains Element    //span[text()="kind"]
+    Run Keyword And Continue On Failure    SeleniumLibrary.Wait Until Page Contains Element    //span[text()="ServingRuntime"]
+    Submit Serving Runtime Template
+
+Submit Serving Runtime Template
+    SeleniumLibrary.Click Element    ${SUBMIT_RUNTIME_BTN_XP}
+    Wait For RHODS Dashboard To Load    expected_page=Serving runtimes
+    ...    wait_for_cards=${FALSE}
+
+Serving Runtime Template Should Be Listed
+    [Arguments]    ${displayed_name}
+    Run Keyword And Continue On Failure
+    ...    SeleniumLibrary.Wait Until Page Contains Element
+    ...    //table//tr/td[@data-label="Name"]//div[text()="${displayed_name}"]
+    ...    timeout=10s
+
+Delete Serving Runtime Template
+    [Arguments]    ${displayed_name}    ${press_cancel}=${FALSE}
+    ODHDashboard.Click Action From Actions Menu    item_title=${displayed_name}
+    ...    action=Delete
+    Handle Deletion Confirmation Modal    item_title=${displayed_name}    item_type=serving runtime    press_cancel=${press_cancel}
+
+Delete Serving Runtime Template From CLI
+    [Arguments]    ${displayed_name}
+    ${resource_name}=    Get OpenShift Template Resource Name By Displayed Name
+    ...    displayed_name=${displayed_name}
+    IF    "${resource_name}" == "${EMPTY}"
+        Log   No Templates corresponding to ${displayed_name} Runtime.
+    ELSE
+        Oc Delete    kind=Template    name=${resource_name}    namespace=${APPLICATIONS_NAMESPACE}
+    END
+
+Get OpenShift Template Resource Name By Displayed Name
+    [Arguments]    ${displayed_name}
+    ${rc}    ${resource_name}=    Run And Return Rc And Output
+    ...    oc get templates -ojson -n redhat-ods-applications | jq '.items[] | select(.objects[].metadata.annotations."openshift.io/display-name"=="${displayed_name}") | .metadata.name' | tr -d '"'
+    Should Be Equal As Integers    ${rc}    ${0}
+    RETURN    ${resource_name}
+
diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/DataConnections.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/DataConnections.resource
@@ -38,7 +38,7 @@ Edit S3 Data Connection
     [Arguments]    ${project_title}    ${dc_name}    ${aws_access_key}    ${aws_secret_access}
     ...            ${aws_s3_endpoint}=${S3.AWS_DEFAULT_ENDPOINT}    ${aws_region}=${S3.AWS_DEFAULT_REGION}
     ...            ${connected_workbench}=${NONE}    ${press_cancel}=${FALSE}    ${aws_bucket_name}=${NONE}
-    Click Action From Actions Menu    item_title=${dc_name}    item_type=data connection    action=Edit
+    Workbenches.Click Action From Actions Menu    item_title=${dc_name}    item_type=data connection    action=Edit
     Fill Data Connection Form    ${project_title}    ${dc_name}    ${aws_access_key}    ${aws_secret_access}
     ...            ${S3_DC_EDIT_BTN}    aws_s3_endpoint=${aws_s3_endpoint}    aws_region=${aws_region}
     ...            connected_workbench=${connected_workbench}    press_cancel=${press_cancel}
@@ -86,7 +86,7 @@ Get Openshift Secret From Data Connection
 Delete Data Connection
     [Documentation]    Deletes a Data Connection from DS Project details page
     [Arguments]     ${name}    ${press_cancel}=${FALSE}
-    Click Action From Actions Menu    item_title=${name}    item_type=data connection    action=Delete
+    Workbenches.Click Action From Actions Menu    item_title=${name}    item_type=data connection    action=Delete
     Handle Deletion Confirmation Modal    item_title=${name}    item_type=data connection    press_cancel=${press_cancel}
 
 Fill Data Connection Form
@@ -126,7 +126,7 @@ Fill Data Connection Form
 
 Get Data Connection Form Values
     [Arguments]    ${dc_name}
-    Click Action From Actions Menu    item_title=${dc_name}    item_type=data connection    action=Edit
+    Workbenches.Click Action From Actions Menu    item_title=${dc_name}    item_type=data connection    action=Edit
     Wait Until Page Contains Element    ${S3_NAME_DC_INPUT_XP}
     ${s3_name}   get element attribute    ${S3_NAME_DC_INPUT_XP}      value
     ${s3_key}   get element attribute    ${S3_KEY_DC_INPUT_XP}      value

diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource
@@ -35,12 +35,15 @@ Is Data Science Projects Page Open
 Open Data Science Project Details Page
     [Documentation]    Verifies submenu Settings > "Data Science Projects" is visible
     [Arguments]     ${project_title}
-    ${is_open}=    Is Data Science Projects Page Open
-    IF    ${is_open} == ${FALSE}
-        Open Data Science Projects Home Page
+    ${is_project_open}=    Is Data Science Project Details Page Open    project_title=${project_title}
+    IF    ${is_project_open} == ${FALSE}
+        ${is_open}=    Is Data Science Projects Page Open
+        IF    ${is_open} == ${FALSE}
+            Open Data Science Projects Home Page
+        END
+        Click Link    ${project_title}
+        Wait Until Project Is Open    project_title=${project_title}
     END
-    Click Link    ${project_title}
-    Wait Until Project Is Open    project_title=${project_title}
 
 Delete Data Science Project
     [Documentation]    Deletes a given DS project. It allows cancelling the operation
@@ -71,6 +74,12 @@ Project Should Not Exist In Openshift
         Fail   msg=The project ${project_title} exists!
     END
 
+Is Data Science Project Details Page Open
+    [Arguments]     ${project_title}
+    ${page_open}=   Run Keyword And Return Status
+    ...    SeleniumLibrary.Page Should Contain Element     xpath=//h1[contains(text(),"${project_title}")]
+    [Return]    ${page_open}   
+
 Wait Until Project Is Open
     [Documentation]    Waits until a DS Project Details page is laoded
     [Arguments]     ${project_title}    ${timeout-pre-spinner}=3s    ${timeout-spinner}=5s
@@ -149,18 +158,23 @@ Delete Data Science Projects From CLI
     ...                 It expects to receive a list of project (i.e,${ocp_projects}) as argument
     [Arguments]     ${ocp_projects}
     FOR   ${displayed_name}    IN  @{ocp_projects}
-        ${project_k8s_name}=     Run     oc get projects -o json | jq '.items[] | select((.metadata.annotations."openshift.io/display-name" != null) and (.metadata.labels."opendatahub.io/dashboard"=="true") and (.metadata.annotations."openshift.io/display-name"=="${displayed_name}")) | .metadata.name'    # robocop: disable
-        ${project_k8s_name}=    Replace String    ${project_k8s_name}    "    ${EMPTY}
-        IF    "${project_k8s_name}" == "${EMPTY}"
-            Log    msg=There is probably no DS Projects with Diplay Name equal to ${displayed_name}
-            Continue For Loop
-        ELSE
-            Oc Delete    kind=Project   name=${project_k8s_name}
-            WHILE    ${TRUE}
-                ${exists} =    Run And Return Rc    oc get project ${project_k8s_name}
-                IF    "${exists}" == "1"    BREAK
-                Sleep    5s    reason=let's not overload the API
-            END
+        Delete Data Science Project From CLI    displayed_name=${displayed_name}
+    END
+
+Delete Data Science Project From CLI
+    [Documentation]    Deletes a single DS project usig CLI command
+    [Arguments]     ${displayed_name}
+    ${project_k8s_name}=     Run     oc get projects -o json | jq '.items[] | select((.metadata.annotations."openshift.io/display-name" != null) and (.metadata.labels."opendatahub.io/dashboard"=="true") and (.metadata.annotations."openshift.io/display-name"=="${displayed_name}")) | .metadata.name'    # robocop: disable
+    ${project_k8s_name}=    Replace String    ${project_k8s_name}    "    ${EMPTY}
+    IF    "${project_k8s_name}" == "${EMPTY}"
+        Log    msg=There is probably no DS Projects with Diplay Name equal to ${displayed_name}
+        ...    level=WARN
+    ELSE
+        Oc Delete    kind=Project   name=${project_k8s_name}
+        WHILE    ${TRUE}
+            ${exists} =    Run And Return Rc    oc get project ${project_k8s_name}
+            IF    "${exists}" == "1"    BREAK
+            Sleep    5s    reason=let's not overload the API
         END
     END
 
@@ -371,3 +385,14 @@ Move To Tab
     [Documentation]    Open one of the tabs in a DS Project details page (e.g., Permissions)
     [Arguments]    ${tab_title}
     Click Element    xpath://span[text()="${tab_title}"]
+
+Create Data Science Project If Not Exists
+    [Documentation]    If the given ${project_title} DS Project does not exist, it creates one.
+    ...                Useful as test setup.
+    [Arguments]    ${project_title}    ${username}    ${description}=${EMPTY}
+    ${rc}    ${resource_name}=    Run And Return Rc And Output     oc get projects -o json | jq '.items[] | select((.metadata.annotations."openshift.io/display-name" != null) and (.metadata.labels."opendatahub.io/dashboard"=="true") and (.metadata.annotations."openshift.io/display-name"=="${project_title}")) | .metadata.name' | tr -d '"'   # robocop: disable
+    IF    "${resource_name}" == "${EMPTY}"
+        Log    msg=There is no DS Projects with Diplay Name equal to ${project_title}. Creating it now.
+        Launch Data Science Project Main Page    username=${username}
+        Create Data Science Project    title=${PRJ_TITLE}    description=${description}
+    END
diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Storages.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Storages.resource
@@ -111,7 +111,7 @@ Set Connection Between PV And Workbench
 Delete Storage
     [Documentation]    Deletes a cluster storage from DS Project details page
     [Arguments]     ${name}    ${press_cancel}=${FALSE}
-    Click Action From Actions Menu    item_title=${name}    item_type=storage    action=Delete
+    Workbenches.Click Action From Actions Menu    item_title=${name}    item_type=storage    action=Delete
     Handle Deletion Confirmation Modal    item_title=${name}    item_type=storage    press_cancel=${press_cancel}
 
 Get Openshift PVC From Storage

diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Workbenches.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Workbenches.resource
@@ -357,7 +357,7 @@ Click Action From Actions Menu
 Delete Workbench
     [Documentation]    Deletes a workbench
     [Arguments]     ${workbench_title}    ${press_cancel}=${FALSE}
-    Click Action From Actions Menu    item_title=${workbench_title}    item_type=workbench    action=Delete
+    Workbenches.Click Action From Actions Menu    item_title=${workbench_title}    item_type=workbench    action=Delete
     Handle Deletion Confirmation Modal    item_title=${workbench_title}    item_type=workbench    press_cancel=${press_cancel}
 
 Edit Workbench
@@ -463,7 +463,7 @@ Select Workbench Number Of GPUs
 Edit GPU Number
     [Documentation]    Edit a workbench
     [Arguments]     ${workbench_title}    ${gpus}=1
-    Click Action From Actions Menu    item_title=${workbench_title}    item_type=workbench    action=Edit
+    Workbenches.Click Action From Actions Menu    item_title=${workbench_title}    item_type=workbench    action=Edit
     ${status}=    Run Keyword And Return Status    Select Workbench Number Of GPUs    gpus=${gpus}
     IF    ${status} == ${FALSE}
         Click Button    ${GENERIC_CANCEL_BTN_XP}
@@ -484,7 +484,7 @@ Delete Workbench From CLI
 GPU Dropdown Should Be Disabled
     [Documentation]    Checks if the GPU dropdown is not able editable
     [Arguments]     ${workbench_title}
-    Click Action From Actions Menu    item_title=${workbench_title}    item_type=workbench    action=Edit
+    Workbenches.Click Action From Actions Menu    item_title=${workbench_title}    item_type=workbench    action=Edit
     Wait Until Page Contains Element    ${WORKBENCH_GPU_MENU_BTN_XP}
     Element Should Be Disabled    ${WORKBENCH_GPU_MENU_BTN_XP}