From d4c2069a93a1f10bda7cb1b2ccdd5d9147a3fc9a Mon Sep 17 00:00:00 2001 From: lugi0 Date: Tue, 1 Oct 2024 18:23:08 +0200 Subject: [PATCH 01/10] e2e UI test + breaking ui changes Signed-off-by: lugi0 --- .../DataConnections.resource | 24 +++++----- .../ODHDataScienceProject/Projects.resource | 12 ++--- .../Workbenches.resource | 37 ++++++++++----- .../1302_model_registry_model_serving.robot | 45 ++++++++++++++++--- 4 files changed, 83 insertions(+), 35 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/DataConnections.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/DataConnections.resource index 2f8af04c1..d8c4c309c 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/DataConnections.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/DataConnections.resource @@ -16,7 +16,7 @@ ${DC_ADD_BTN_1_XP}= xpath=//*[@data-testid="add-data-connection-butt ${S3_BUCKET_DC_INPUT_XP}= xpath=//*[@data-testid="field AWS_S3_BUCKET"] ${S3_DC_ADD_BTN}= xpath=//*[@data-testid="data-connection-submit-button"] ${S3_DC_EDIT_BTN}= xpath=//footer/button[.="Update data connection"] -${DC_WORKBENCH_SELECTOR_XP}= xpath=//ul[@aria-label="Notebook select"]/li +${DC_WORKBENCH_SELECTOR_XP}= xpath=//div[@data-testid="connect-existing-workbench-group"]//ul/li *** Keywords *** @@ -47,25 +47,25 @@ Edit S3 Data Connection Set Connection Between Data Connection And Workbench [Documentation] Connects a DataConnection to an existent workbench [Arguments] ${workbench_title} - Wait Until Element Is Enabled xpath=//button[@aria-label="Options menu"] - Click Element xpath=//button[@aria-label="Options menu"] - Wait Until Page Contains Element ${DC_WORKBENCH_SELECTOR_XP}/button[text()="${workbench_title}"] - Click Element ${DC_WORKBENCH_SELECTOR_XP}/button[text()="${workbench_title}"] + Wait Until Element Is Enabled xpath=//button[@aria-label="Notebook select"] + Click Element xpath=//button[@aria-label="Notebook select"] + Wait Until Page Contains Element ${DC_WORKBENCH_SELECTOR_XP}/button[.="${workbench_title}"] + Click Element ${DC_WORKBENCH_SELECTOR_XP}/button[.="${workbench_title}"] Data Connection Should Be Listed [Documentation] Checks a Data Connection is listed in DS Project details page [Arguments] ${name} ${type} ${connected_workbench} - ${connection_xp}= Set Variable ${DC_SECTION_XP}//tr[td[@data-label="Name"]//div[text()="${name}"]] + ${connection_xp}= Set Variable ${DC_SECTION_XP}//tr[td[@data-label="Name"]//div[.="${name}"]] Run Keyword And Continue On Failure Page Should Contain Element ${connection_xp} - Run Keyword And Continue On Failure Page Should Contain Element ${connection_xp}/td[text()=" ${type}"] + Run Keyword And Continue On Failure Page Should Contain Element ${connection_xp}/td[.=" ${type}"] IF "${connected_workbench}" == "${NONE}" Run Keyword And Continue On Failure Page Should Contain Element - ... ${connection_xp}/td[text()="No connections"] + ... ${connection_xp}/td[.="No connections"] ELSE FOR ${index} ${workbench_title} IN ENUMERATE @{connected_workbench} Log ${index}: ${workbench_title} Run Keyword And Continue On Failure Page Should Contain Element - ... ${connection_xp}/td[@data-label="Connected workbenches"]//*[text()="${workbench_title}"] + ... ${connection_xp}/td[@data-label="Connected workbenches"]//*[.="${workbench_title}"] END END @@ -73,7 +73,7 @@ Data Connection Should Not Be Listed [Documentation] Checks a Data Connection is not listed in DS Project details page [Arguments] ${name} Run Keyword And Continue On Failure Wait Until Page Does Not Contain Element - ... ${DC_SECTION_XP}//tr[td[@data-label="Name"]/*/div[text()="${name}"]] + ... ${DC_SECTION_XP}//tr[td[@data-label="Name"]/*/div[.="${name}"]] Get Openshift Secret From Data Connection [Documentation] Retrieves name of Openshift secret corresponding to a given S3 Data Connection based on Dispayed name in DS Project details page @@ -103,7 +103,7 @@ Fill Data Connection Form Log msg=you are not connecting any workbenchs to ${dc_name} DataConnection ELSE Run Keyword And Continue On Failure - ... Element Should Be Enabled xpath=//div[contains(@class,"modal")]//button[@aria-label="Options menu"] + ... Element Should Be Enabled xpath=//div[contains(@class,"modal")]//button[@aria-label="Notebook select"] FOR ${workbench_title} IN @{connected_workbench} Set Connection Between Data Connection And Workbench ${workbench_title} Run Keyword And Continue On Failure Element Should Be Enabled ${S3_DC_ADD_BTN} @@ -137,7 +137,7 @@ Recreate S3 Data Connection ... ${connected_workbench}=${NONE} ${press_cancel}=${FALSE} ${aws_bucket_name}=${NONE} Open Data Science Project Details Page project_title=${project_title} tab_id=data-connections ${is_exist}= Run Keyword And Return Status - ... Wait Until Page Contains Element ${DC_SECTION_XP}//tr[td[@data-label="Name"]/*/div[text()="${dc_name}"]] + ... Wait Until Page Contains Element ${DC_SECTION_XP}//tr[td[@data-label="Name"]/*/div[.="${dc_name}"]] IF ${is_exist} Log Data Connection '${dc_name}' already exists, updating connection console=True Edit S3 Data Connection project_title=${project_title} dc_name=${dc_name} new_dc_name=${dc_name}${SPACE} diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource index d8085440c..2dabfdbd3 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource @@ -143,12 +143,12 @@ Create Data Science Project Run Keyword And Warn On Failure Element Should Be Disabled ${GENERIC_CREATE_BTN_XP} Input Text ${TITLE_INPUT_XP} ${title} Input Text ${DESCR_INPUT_XP} ${description} - Run Keyword And Continue On Failure Validate Generated Resource Name project_title=${title} - IF "${resource_name}" == "${NONE}" - ${resource_name}= Get Value ${RESOURCE_INPUT_XP} - ${resource_name}= Generate Unique Namespace ${resource_name} - END - Input Text ${RESOURCE_INPUT_XP} ${resource_name} + #Run Keyword And Continue On Failure Validate Generated Resource Name project_title=${title} + #IF "${resource_name}" == "${NONE}" + # ${resource_name}= Get Value ${RESOURCE_INPUT_XP} + # ${resource_name}= Generate Unique Namespace ${resource_name} + #END + #Input Text ${RESOURCE_INPUT_XP} ${resource_name} Wait Until Element Is Enabled ${GENERIC_CREATE_BTN_XP} Click Button ${GENERIC_CREATE_BTN_XP} Wait Until Generic Modal Disappears diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Workbenches.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Workbenches.resource index 4e70d06f6..c74625de8 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Workbenches.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Workbenches.resource @@ -264,14 +264,14 @@ Workbench Should Be Listed [Arguments] ${workbench_title} ${timeout}=10s Run keyword And Continue On Failure ... Wait Until Page Contains Element - ... ${WORKBENCH_SECTION_XP}//td[@data-label="Name"]/*[div[text()="${workbench_title}"]] timeout=${timeout} + ... ${WORKBENCH_SECTION_XP}//td[@data-label="Name"]//*[.="${workbench_title}"] timeout=${timeout} Workbench With Description Should Be Listed [Documentation] Checks a workbench with particular name and description is listed in the DS Project details page [Arguments] ${workbench_title} ${workbench_description} ${timeout}=10s Run keyword And Continue On Failure ... Wait Until Page Contains Element - ... ${WORKBENCH_SECTION_XP}//td[@data-label="Name"][//div[text()="${workbench_title}"] and //p[text()="${workbench_description}"]] + ... ${WORKBENCH_SECTION_XP}//td[@data-label="Name"][//*[text()="${workbench_title}"] and //p[text()="${workbench_description}"]] ... timeout=${timeout} Workbench Should Not Be Listed @@ -286,7 +286,7 @@ Workbench Status Should Be [Arguments] ${workbench_title} ${status} ${timeout}=10 Run keyword And Continue On Failure ... Wait Until Page Contains Element - ... ${WORKBENCH_SECTION_XP}//tr[td[@data-label="Name"]/*[div[text()="${workbench_title}"]]]/td[@data-label="Status"]//p[text()="${status}"] + ... ${WORKBENCH_SECTION_XP}//td[@data-label="Name"]//*[.="${workbench_title}"]/../../../td[@data-label="Status"]//span[text()="${status}"] ... timeout=${timeout} Status Error Icon Should Appear @@ -301,7 +301,7 @@ Wait Until Workbench Is Started [Documentation] Waits until workbench status is "RUNNING" in the DS Project details page [Arguments] ${workbench_title} ${timeout}=60s ${status}=${WORKBENCH_STATUS_RUNNING} Wait Until Page Contains Element - ... ${WORKBENCH_SECTION_XP}//tr[td[@data-label="Name"]/*[div[text()="${workbench_title}"]]]/td[@data-label="Status"]//p[text()="${status}"] + ... ${WORKBENCH_SECTION_XP}//td[@data-label="Name"]//*[.="${workbench_title}"]/../../../td[@data-label="Status"]//span[text()="${status}"] ... timeout=${timeout} Wait Until Workbench Is Stopped @@ -350,27 +350,42 @@ Access To Workbench END Launch And Access Workbench - [Documentation] Launches a workbench from DS Project details page + [Documentation] Launches a workbench from DS Project details page and returns the handle of the current tab [Arguments] ${workbench_title} ${username}=${TEST_USER_3.USERNAME} ... ${password}=${TEST_USER_3.PASSWORD} ${auth_type}=${TEST_USER_3.AUTH_TYPE} ${is_started}= Run Keyword And Return Status Workbench Status Should Be ... workbench_title=${workbench_title} status=${WORKBENCH_STATUS_RUNNING} IF ${is_started} == ${TRUE} - Open Workbench workbench_title=${workbench_title} + ${previous_handle}= Open Workbench workbench_title=${workbench_title} Access To Workbench username=${username} password=${password} ... auth_type=${auth_type} ELSE Fail msg=Cannot Launch And Access Workbench ${workbench_title} because it is not running... END + RETURN ${previous_handle} Open Workbench - [Documentation] Clicks on "open" link for a given workbench + [Documentation] Clicks on "open" link for a given workbench and returns the handle of the current tab [Arguments] ${workbench_title} + ${desc}= Does Workbench Have A Description ${workbench_title} ${open_workbench_link}= Set Variable - ... ${WORKBENCH_SECTION_XP}//tr[td[@data-label="Name"]/*[div[text()="${workbench_title}"]]]/td//a[text()="Open"] - Wait Until Page Contains Element ${open_workbench_link} timeout=30s - Click Link ${open_workbench_link} - Switch Window NEW + ... ${WORKBENCH_SECTION_XP}//td[@data-label="Name"]//*[.="${workbench_title}"] + IF ${desc} + ${open_workbench_link}= Catenate ${open_workbench_link} /../../../td//a[.="Open"] + ELSE + ${open_workbench_link}= Catenate ${open_workbench_link} /../../td//a[.="Open"] + END + SeleniumLibrary.Wait Until Page Contains Element ${open_workbench_link} timeout=30s + SeleniumLibrary.Click Link ${open_workbench_link} + ${previous_handle}= SeleniumLibrary.Switch Window NEW + RETURN ${previous_handle} + +Does Workbench Have A Description + [Documentation] Checks if a Workbench has a description in the list view of the project + [Arguments] ${workbench_title} + ${has_description}= Run Keyword And Return Status SeleniumLibrary.Page Should Contain Element + ... ${WORKBENCH_SECTION_XP}//td[@data-label="Name"]//*[.="${workbench_title}"]/../span[@data-testid="table-row-title-description"] # robocop: disable + RETURN ${has_description} Stop Workbench [Documentation] Stops a workbench from DS Project details page diff --git a/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot b/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot index b61b9256a..099e2d365 100644 --- a/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot +++ b/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot @@ -38,6 +38,11 @@ ${SECRET_PART_NAME_1}= modelregistry-sample-rest ${SECRET_PART_NAME_2}= modelregistry-sample-grpc ${SECRET_PART_NAME_3}= model-registry-db ${BROWSER.NAME}= chrome +${MR_REGISTERED_MODEL_NAME}= test minst +${MR_REGISTERED_MODEL_VERSION}= 2.0.0 +${MR_REGISTERED_MODEL_AUTHOR}= Tony +${MR_TABLE_XPATH}= //table[@data-testid="registered-model-table"] +${MR_VERSION_TABLE_XPATH}= //table[@data-testid="model-versions-table"] *** Test Cases *** @@ -52,11 +57,11 @@ Verify Model Registry Integration With Secured-DB Workbench Should Be Listed workbench_title=${WORKBENCH_TITLE} Open Data Science Project Details Page project_title=${PRJ_TITLE} ${workbenches}= Create List ${WORKBENCH_TITLE} - # Create S3 Data Connection project_title=${PRJ_TITLE} dc_name=${DC_S3_NAME} - # ... aws_access_key=${S3.AWS_ACCESS_KEY_ID} aws_secret_access=${S3.AWS_SECRET_ACCESS_KEY} - # ... aws_bucket_name=${AWS_BUCKET} connected_workbench=${workbenches} - # Data Connection Should Be Listed name=${DC_S3_NAME} type=${DC_S3_TYPE} connected_workbench=${workbenches} - # Open Data Science Project Details Page project_title=${prj_title} tab_id=workbenches + Create S3 Data Connection project_title=${PRJ_TITLE} dc_name=${DC_S3_NAME} + ... aws_access_key=${S3.AWS_ACCESS_KEY_ID} aws_secret_access=${S3.AWS_SECRET_ACCESS_KEY} + ... aws_bucket_name=${AWS_BUCKET} connected_workbench=${workbenches} + Data Connection Should Be Listed name=${DC_S3_NAME} type=${DC_S3_TYPE} connected_workbench=${workbenches} + Open Data Science Project Details Page project_title=${prj_title} tab_id=workbenches Wait Until Workbench Is Started workbench_title=${WORKBENCH_TITLE} Upload File In The Workbench filepath=${SAMPLE_ONNX_MODEL} workbench_title=${WORKBENCH_TITLE} ... workbench_namespace=${PRJ_TITLE} @@ -64,12 +69,21 @@ Verify Model Registry Integration With Secured-DB ... workbench_namespace=${PRJ_TITLE} Download Python Client Dependencies ${MR_PYTHON_CLIENT_FILES} ${MR_PYTHON_CLIENT_WHL_VERSION} Upload Python Client Files In The Workbench ${MR_PYTHON_CLIENT_FILES} - Launch And Access Workbench workbench_title=${WORKBENCH_TITLE} + ${handle}= Launch And Access Workbench workbench_title=${WORKBENCH_TITLE} ... username=${TEST_USER.USERNAME} password=${TEST_USER.PASSWORD} ... auth_type=${TEST_USER.AUTH_TYPE} Upload Certificate To Jupyter Notebook ${CERTS_DIRECTORY}/domain.crt Upload Certificate To Jupyter Notebook openshift_ca.crt Jupyter Notebook Can Query Model Registry ${JUPYTER_NOTEBOOK} + SeleniumLibrary.Switch Window ${handle} + Open Model Registry Dashboard Page + SeleniumLibrary.Page Should Contain Element xpath:${MR_TABLE_XPATH}/tbody/tr/td[@data-label="Model name"]//a[.="${MR_REGISTERED_MODEL_NAME}"] + SeleniumLibrary.Page Should Contain Element xpath:${MR_TABLE_XPATH}/tbody/tr/td[@data-label="Owner"]//p[.="${MR_REGISTERED_MODEL_AUTHOR}"] + SeleniumLibrary.Page Should Contain Element xpath:${MR_TABLE_XPATH}/tbody/tr/td[@data-label="Labels" and .="-"] + SeleniumLibrary.Click Element xpath:${MR_TABLE_XPATH}/tbody/tr/td[@data-label="Model name"]//a[.="${MR_REGISTERED_MODEL_NAME}"] + Maybe Wait For Dashboard Loading Spinner Page + SeleniumLibrary.Page Should Contain Element xpath:${MR_VERSION_TABLE_XPATH}/tbody/tr/td[@data-label="Version name"]//a[.="${MR_REGISTERED_MODEL_VERSION}"] + SeleniumLibrary.Page Should Contain Element xpath:${MR_VERSION_TABLE_XPATH}/tbody/tr/td[@data-label="Author" and .="${MR_REGISTERED_MODEL_AUTHOR}"] *** Keywords *** @@ -187,6 +201,7 @@ Jupyter Notebook Can Query Model Registry [Arguments] ${filepath} Open Notebook File In JupyterLab ${filepath} Open With JupyterLab Menu Run Restart Kernel and Run All Cells… + Wait Until Page Contains Element xpath=//div[contains(text(),"Restart") and @class="jp-Dialog-buttonLabel"] Click Element xpath=//div[contains(text(),"Restart") and @class="jp-Dialog-buttonLabel"] Wait Until JupyterLab Code Cell Is Not Active timeout=120s Sleep 2m msg=Waits until the jupyter notebook has completed execution of all cells @@ -287,3 +302,21 @@ Upload Python Client Files In The Workbench ... filepath=${file_location}/${file} workbench_title=${WORKBENCH_TITLE} ... workbench_namespace=${PRJ_TITLE} END + +Open Model Registry Dashboard Page + [Documentation] Opens the Model Registry page from the dashboard nav bar + SeleniumLibrary.Wait Until Page Contains Model Registry + SeleniumLibrary.Click Link Model Registry + Wait For RHODS Dashboard To Load wait_for_cards=${FALSE} expected_page=Model Registry + SeleniumLibrary.Wait Until Page Contains Select a model registry to view and manage your registered models. + Maybe Wait For Dashboard Loading Spinner Page + ${loaded}= Run Keyword And Return Status + ... SeleniumLibrary.Page Should Not Contain Request access to model registries + WHILE ${loaded}!=${TRUE} limit=5 + SeleniumLibrary.Reload Page + SeleniumLibrary.Wait Until Page Contains Model Registry + SeleniumLibrary.Wait Until Page Contains Select a model registry to view and manage your registered models. + ${loaded}= Run Keyword And Return Status + ... SeleniumLibrary.Page Should Not Contain Request access to model registries + END + Maybe Wait For Dashboard Loading Spinner Page From e6a101379c527d6ffd76c388ceef49e18f052a20 Mon Sep 17 00:00:00 2001 From: lugi0 Date: Tue, 1 Oct 2024 18:25:35 +0200 Subject: [PATCH 02/10] uncomment code block Signed-off-by: lugi0 --- .../ODHDataScienceProject/Projects.resource | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource index 2dabfdbd3..d8085440c 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/Projects.resource @@ -143,12 +143,12 @@ Create Data Science Project Run Keyword And Warn On Failure Element Should Be Disabled ${GENERIC_CREATE_BTN_XP} Input Text ${TITLE_INPUT_XP} ${title} Input Text ${DESCR_INPUT_XP} ${description} - #Run Keyword And Continue On Failure Validate Generated Resource Name project_title=${title} - #IF "${resource_name}" == "${NONE}" - # ${resource_name}= Get Value ${RESOURCE_INPUT_XP} - # ${resource_name}= Generate Unique Namespace ${resource_name} - #END - #Input Text ${RESOURCE_INPUT_XP} ${resource_name} + Run Keyword And Continue On Failure Validate Generated Resource Name project_title=${title} + IF "${resource_name}" == "${NONE}" + ${resource_name}= Get Value ${RESOURCE_INPUT_XP} + ${resource_name}= Generate Unique Namespace ${resource_name} + END + Input Text ${RESOURCE_INPUT_XP} ${resource_name} Wait Until Element Is Enabled ${GENERIC_CREATE_BTN_XP} Click Button ${GENERIC_CREATE_BTN_XP} Wait Until Generic Modal Disappears From e5e534e6b92fb692feda82a492a9c18f3d89bee2 Mon Sep 17 00:00:00 2001 From: lugi0 Date: Wed, 2 Oct 2024 13:13:05 +0200 Subject: [PATCH 03/10] fix issues and add checks on model registry Signed-off-by: lugi0 --- ...r.yaml => serviceMeshMember_template.yaml} | 2 +- .../1302_model_registry_model_serving.robot | 48 ++++++++++++++----- 2 files changed, 38 insertions(+), 12 deletions(-) rename ods_ci/tests/Resources/CLI/ModelRegistry/{serviceMeshMember.yaml => serviceMeshMember_template.yaml} (82%) diff --git a/ods_ci/tests/Resources/CLI/ModelRegistry/serviceMeshMember.yaml b/ods_ci/tests/Resources/CLI/ModelRegistry/serviceMeshMember_template.yaml similarity index 82% rename from ods_ci/tests/Resources/CLI/ModelRegistry/serviceMeshMember.yaml rename to ods_ci/tests/Resources/CLI/ModelRegistry/serviceMeshMember_template.yaml index 46e93c5d0..323ccd5f5 100644 --- a/ods_ci/tests/Resources/CLI/ModelRegistry/serviceMeshMember.yaml +++ b/ods_ci/tests/Resources/CLI/ModelRegistry/serviceMeshMember_template.yaml @@ -2,7 +2,7 @@ apiVersion: maistra.io/v1 kind: ServiceMeshMember metadata: name: default - namespace: odh-model-registries + namespace: spec: controlPlaneRef: name: data-science-smcp diff --git a/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot b/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot index 099e2d365..d1399c873 100644 --- a/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot +++ b/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot @@ -1,3 +1,4 @@ +# robocop: off=too-many-calls-in-keyword *** Settings *** Documentation Test suite for Model Registry Integration Suite Setup Prepare Model Registry Test Setup @@ -24,7 +25,7 @@ ${ISTIO_ENV}= ${MODELREGISTRY_BASE_FOLDER}/samples/istio/ ${SAMPLE_ONNX_MODEL}= ${MODELREGISTRY_BASE_FOLDER}/mnist.onnx ${MR_PYTHON_CLIENT_FILES}= ${MODELREGISTRY_BASE_FOLDER}/Python_Dependencies ${MR_PYTHON_CLIENT_WHL_VERSION}= model_registry==0.2.6a1 -${SERVICE_MESH_MEMBER}= ${MODELREGISTRY_BASE_FOLDER}/serviceMeshMember.yaml +${SERVICE_MESH_MEMBER}= ${MODELREGISTRY_BASE_FOLDER}/serviceMeshMember_template.yaml ${ENABLE_REST_API}= ${MODELREGISTRY_BASE_FOLDER}/enable_rest_api_route.yaml ${IPYNB_UPDATE_SCRIPT}= ${MODELREGISTRY_BASE_FOLDER}/updateIPYNB.py ${CERTS_DIRECTORY}= certs @@ -33,7 +34,7 @@ ${JUPYTER_NOTEBOOK}= MRMS_UPDATED.ipynb ${JUPYTER_NOTEBOOK_FILEPATH}= ${MODELREGISTRY_BASE_FOLDER}/${JUPYTER_NOTEBOOK} ${DC_S3_TYPE}= Object storage ${NAMESPACE_ISTIO}= istio-system -${NAMESPACE_MODEL-REGISTRY}= odh-model-registries +# ${NAMESPACE_MODEL_REGISTRY}= odh-model-registries ${SECRET_PART_NAME_1}= modelregistry-sample-rest ${SECRET_PART_NAME_2}= modelregistry-sample-grpc ${SECRET_PART_NAME_3}= model-registry-db @@ -63,6 +64,7 @@ Verify Model Registry Integration With Secured-DB Data Connection Should Be Listed name=${DC_S3_NAME} type=${DC_S3_TYPE} connected_workbench=${workbenches} Open Data Science Project Details Page project_title=${prj_title} tab_id=workbenches Wait Until Workbench Is Started workbench_title=${WORKBENCH_TITLE} + Sleep 5s reason=Sometimes the pod/container is not found by oc, let's give it some time Upload File In The Workbench filepath=${SAMPLE_ONNX_MODEL} workbench_title=${WORKBENCH_TITLE} ... workbench_namespace=${PRJ_TITLE} Upload File In The Workbench filepath=${JUPYTER_NOTEBOOK_FILEPATH} workbench_title=${WORKBENCH_TITLE} @@ -91,16 +93,21 @@ Prepare Model Registry Test Setup [Documentation] Suite setup steps for testing Model Registry. Set Library Search Order SeleniumLibrary RHOSi Setup + Component Should Be Enabled modelregistry Launch Dashboard ${TEST_USER.USERNAME} ${TEST_USER.PASSWORD} ${TEST_USER.AUTH_TYPE} ... ${ODH_DASHBOARD_URL} ${BROWSER.NAME} ${BROWSER.OPTIONS} Open Data Science Projects Home Page Create Data Science Project title=${PRJ_TITLE} description=${PRJ_DESCRIPTION} - Create Namespace In Openshift ${NAMESPACE_MODEL-REGISTRY} + # This should be created by the RHOAI operator when the component is enabled in the DSC. + # We can grab the name of the NS by querying the DSC + # Create Namespace In Openshift ${NAMESPACE_MODEL_REGISTRY} + ${NAMESPACE_MODEL_REGISTRY}= Get Model Registry Namespace From DSC + Set Suite Variable ${NAMESPACE_MODEL_REGISTRY} Apply ServiceMeshMember Configuration Get Cluster Domain And Token Run Update Notebook Script Generate ModelRegistry Certificates - Apply Db Config Samples namespace=${NAMESPACE_MODEL-REGISTRY} + Apply Db Config Samples namespace=${NAMESPACE_MODEL_REGISTRY} Create Model Registry Secrets Fetch CA Certificate If RHODS Is Self-Managed @@ -151,11 +158,11 @@ Create Model Registry Secrets ... certs/${SECRET_PART_NAME_1}.domain.key certs/${SECRET_PART_NAME_1}.domain.crt certs/domain.crt Create Generic Secret ${NAMESPACE_ISTIO} ${SECRET_PART_NAME_2}-credential ... certs/${SECRET_PART_NAME_2}.domain.key certs/${SECRET_PART_NAME_2}.domain.crt certs/domain.crt - Create Generic Secret ${NAMESPACE_MODEL-REGISTRY} ${SECRET_PART_NAME_3}-credential + Create Generic Secret ${NAMESPACE_MODEL_REGISTRY} ${SECRET_PART_NAME_3}-credential ... certs/${SECRET_PART_NAME_3}.key certs/${SECRET_PART_NAME_3}.crt certs/domain.crt Secret Should Exist ${NAMESPACE_ISTIO} ${SECRET_PART_NAME_1}-credential Secret Should Exist ${NAMESPACE_ISTIO} ${SECRET_PART_NAME_2}-credential - Secret Should Exist ${NAMESPACE_MODEL-REGISTRY} ${SECRET_PART_NAME_3}-credential + Secret Should Exist ${NAMESPACE_MODEL_REGISTRY} ${SECRET_PART_NAME_3}-credential Secret Should Exist [Documentation] Check if the specified secret exists in the given namespace @@ -245,7 +252,10 @@ Get Token Apply ServiceMeshMember Configuration [Documentation] Apply a ServiceMeshMember configuration using oc. - Apply OpenShift Configuration ${SERVICE_MESH_MEMBER} + Copy File source=${SERVICE_MESH_MEMBER} destination=${MODELREGISTRY_BASE_FOLDER}/serviceMeshMember.yaml + Run sed -i'' -e 's//${NAMESPACE_MODEL_REGISTRY}/' ${MODELREGISTRY_BASE_FOLDER}/serviceMeshMember.yaml # robocop: disable + Apply OpenShift Configuration ${MODELREGISTRY_BASE_FOLDER}/serviceMeshMember.yaml + Remove File ${MODELREGISTRY_BASE_FOLDER}/serviceMeshMember.yaml Apply Rest API Configuration [Documentation] Apply a Rest API configuration using oc. @@ -273,10 +283,15 @@ Run Update Notebook Script Remove Model Registry [Documentation] Run multiple oc delete commands to remove model registry components - Run And Verify Command oc delete -k ${MODELREGISTRY_BASE_FOLDER}/samples/secure-db/mysql-tls - Run And Verify Command oc delete secret modelregistry-sample-grpc-credential -n ${NAMESPACE_ISTIO} - Run And Verify Command oc delete secret modelregistry-sample-rest-credential -n ${NAMESPACE_ISTIO} - Run And Verify Command oc delete namespace ${NAMESPACE_MODEL-REGISTRY} --force + # We don't want to stop the teardown if any of these resources are not found + Run Keyword And Continue On Failure + ... Run And Verify Command oc delete -k ${MODELREGISTRY_BASE_FOLDER}/samples/secure-db/mysql-tls + Run Keyword And Continue On Failure + ... Run And Verify Command oc delete secret modelregistry-sample-grpc-credential -n ${NAMESPACE_ISTIO} + Run Keyword And Continue On Failure + ... Run And Verify Command oc delete secret modelregistry-sample-rest-credential -n ${NAMESPACE_ISTIO} + # I don't think this NS should be removed, it's managed by the DSC + # Run And Verify Command oc delete namespace ${NAMESPACE_MODEL_REGISTRY} --force Remove Deployment Files [Documentation] Remove all files from the given directory @@ -320,3 +335,14 @@ Open Model Registry Dashboard Page ... SeleniumLibrary.Page Should Not Contain Request access to model registries END Maybe Wait For Dashboard Loading Spinner Page + +Get Model Registry Namespace From DSC + [Documentation] Fetches the namespace defined for model registry in the DSC + ${rc} ${ns}= Run And Return Rc And Output + ... oc get dsc default-dsc -o json | jq '.spec.components.modelregistry.registriesNamespace' + Should Be Equal As Integers ${rc} 0 + Log ${ns} + # Remove double quotes + ${ns}= Get Substring ${ns} 1 -1 + Log ${ns} + RETURN ${ns} From 9453eebadaa7dc6f9a7582fdc08ba6fe046c476d Mon Sep 17 00:00:00 2001 From: lugi0 Date: Wed, 2 Oct 2024 13:50:27 +0200 Subject: [PATCH 04/10] Screenshot correct element when jupyter code fails Signed-off-by: lugi0 --- .../Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot index 2a28c0382..d80858230 100644 --- a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot +++ b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot @@ -68,7 +68,10 @@ Close JupyterLab Selected Tab Maybe Close Popup JupyterLab Code Cell Error Output Should Not Be Visible - Element Should Not Be Visible xpath://div[contains(@class,"jp-OutputArea-output") and @data-mime-type="application/vnd.jupyter.stderr"] A JupyterLab code cell output returned an error + Run Keyword And Continue On Failure + ... SeleniumLibrary.Element Should Not Be Visible xpath://div[contains(@class,"jp-OutputArea-output") and @data-mime-type="application/vnd.jupyter.stderr"] A JupyterLab code cell output returned an error # robocop: disable + Capture Element Screenshot xpath://div[contains(@class,"jp-OutputArea-output") and @data-mime-type="application/vnd.jupyter.stderr"] # robocop: disable + Fail msg=A JupyterLab code cell output returned an error, see screenshot Get JupyterLab Code Cell Error Text ${error_txt} = Get Text //div[contains(@class,"jp-OutputArea-output") and @data-mime-type="application/vnd.jupyter.stderr"] From 25f064061260500e31a02cb370b9804c6b26ce3f Mon Sep 17 00:00:00 2001 From: lugi0 Date: Wed, 2 Oct 2024 13:52:00 +0200 Subject: [PATCH 05/10] remove trailing whitespace Signed-off-by: lugi0 --- .../Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot index d80858230..b54dac336 100644 --- a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot +++ b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot @@ -68,7 +68,7 @@ Close JupyterLab Selected Tab Maybe Close Popup JupyterLab Code Cell Error Output Should Not Be Visible - Run Keyword And Continue On Failure + Run Keyword And Continue On Failure ... SeleniumLibrary.Element Should Not Be Visible xpath://div[contains(@class,"jp-OutputArea-output") and @data-mime-type="application/vnd.jupyter.stderr"] A JupyterLab code cell output returned an error # robocop: disable Capture Element Screenshot xpath://div[contains(@class,"jp-OutputArea-output") and @data-mime-type="application/vnd.jupyter.stderr"] # robocop: disable Fail msg=A JupyterLab code cell output returned an error, see screenshot From 39590c093f86285810302be4e366bb6bf31fa8b7 Mon Sep 17 00:00:00 2001 From: lugi0 Date: Wed, 2 Oct 2024 13:59:10 +0200 Subject: [PATCH 06/10] specify namespace in teardown Signed-off-by: lugi0 --- .../1302_model_registry_model_serving.robot | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot b/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot index d1399c873..0f842d287 100644 --- a/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot +++ b/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot @@ -285,7 +285,8 @@ Remove Model Registry [Documentation] Run multiple oc delete commands to remove model registry components # We don't want to stop the teardown if any of these resources are not found Run Keyword And Continue On Failure - ... Run And Verify Command oc delete -k ${MODELREGISTRY_BASE_FOLDER}/samples/secure-db/mysql-tls + ... Run And Verify Command + ... oc delete -n ${NAMESPACE_MODEL_REGISTRY} -k ${MODELREGISTRY_BASE_FOLDER}/samples/secure-db/mysql-tls Run Keyword And Continue On Failure ... Run And Verify Command oc delete secret modelregistry-sample-grpc-credential -n ${NAMESPACE_ISTIO} Run Keyword And Continue On Failure From 72b9b794bc45d5a6f4416a2613f1ecbc58f6bc97 Mon Sep 17 00:00:00 2001 From: lugi0 Date: Wed, 2 Oct 2024 14:31:06 +0200 Subject: [PATCH 07/10] Grab screenshot only if failure happened Signed-off-by: lugi0 --- .../Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot index b54dac336..a0e2cf975 100644 --- a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot +++ b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot @@ -68,9 +68,11 @@ Close JupyterLab Selected Tab Maybe Close Popup JupyterLab Code Cell Error Output Should Not Be Visible - Run Keyword And Continue On Failure + ${failure}= Run Keyword And Return Status ... SeleniumLibrary.Element Should Not Be Visible xpath://div[contains(@class,"jp-OutputArea-output") and @data-mime-type="application/vnd.jupyter.stderr"] A JupyterLab code cell output returned an error # robocop: disable - Capture Element Screenshot xpath://div[contains(@class,"jp-OutputArea-output") and @data-mime-type="application/vnd.jupyter.stderr"] # robocop: disable + IF ${failure} + Capture Element Screenshot xpath://div[contains(@class,"jp-OutputArea-output") and @data-mime-type="application/vnd.jupyter.stderr"] # robocop: disable + END Fail msg=A JupyterLab code cell output returned an error, see screenshot Get JupyterLab Code Cell Error Text From 526640a1d46dcc3d73fa48441fe157abc6c17c26 Mon Sep 17 00:00:00 2001 From: lugi0 Date: Wed, 2 Oct 2024 14:39:03 +0200 Subject: [PATCH 08/10] Stop sed from creating backups Signed-off-by: lugi0 --- .../1302_model_registry_model_serving.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot b/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot index 0f842d287..e8bcd08ee 100644 --- a/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot +++ b/ods_ci/tests/Tests/1300__model_registry/1302_model_registry_model_serving.robot @@ -253,7 +253,7 @@ Get Token Apply ServiceMeshMember Configuration [Documentation] Apply a ServiceMeshMember configuration using oc. Copy File source=${SERVICE_MESH_MEMBER} destination=${MODELREGISTRY_BASE_FOLDER}/serviceMeshMember.yaml - Run sed -i'' -e 's//${NAMESPACE_MODEL_REGISTRY}/' ${MODELREGISTRY_BASE_FOLDER}/serviceMeshMember.yaml # robocop: disable + Run sed -i '' -e 's//${NAMESPACE_MODEL_REGISTRY}/' ${MODELREGISTRY_BASE_FOLDER}/serviceMeshMember.yaml # robocop: disable Apply OpenShift Configuration ${MODELREGISTRY_BASE_FOLDER}/serviceMeshMember.yaml Remove File ${MODELREGISTRY_BASE_FOLDER}/serviceMeshMember.yaml From b27473f32e79152a6e73ef0a18ce39b07d804271 Mon Sep 17 00:00:00 2001 From: lugi0 Date: Wed, 2 Oct 2024 14:55:41 +0200 Subject: [PATCH 09/10] fix screenshot logic Signed-off-by: lugi0 --- .../Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot index a0e2cf975..24f9f71c2 100644 --- a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot +++ b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot @@ -69,7 +69,7 @@ Close JupyterLab Selected Tab JupyterLab Code Cell Error Output Should Not Be Visible ${failure}= Run Keyword And Return Status - ... SeleniumLibrary.Element Should Not Be Visible xpath://div[contains(@class,"jp-OutputArea-output") and @data-mime-type="application/vnd.jupyter.stderr"] A JupyterLab code cell output returned an error # robocop: disable + ... SeleniumLibrary.Element Should Be Visible xpath://div[contains(@class,"jp-OutputArea-output") and @data-mime-type="application/vnd.jupyter.stderr"] A JupyterLab code cell output returned an error # robocop: disable IF ${failure} Capture Element Screenshot xpath://div[contains(@class,"jp-OutputArea-output") and @data-mime-type="application/vnd.jupyter.stderr"] # robocop: disable END From fbf823234b465ad4d4448999f06adda77e6024b0 Mon Sep 17 00:00:00 2001 From: lugi0 Date: Wed, 2 Oct 2024 15:07:55 +0200 Subject: [PATCH 10/10] move fail statement Signed-off-by: lugi0 --- .../Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot index 24f9f71c2..666f8a613 100644 --- a/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot +++ b/ods_ci/tests/Resources/Page/ODH/JupyterHub/JupyterLabLauncher.robot @@ -72,8 +72,8 @@ JupyterLab Code Cell Error Output Should Not Be Visible ... SeleniumLibrary.Element Should Be Visible xpath://div[contains(@class,"jp-OutputArea-output") and @data-mime-type="application/vnd.jupyter.stderr"] A JupyterLab code cell output returned an error # robocop: disable IF ${failure} Capture Element Screenshot xpath://div[contains(@class,"jp-OutputArea-output") and @data-mime-type="application/vnd.jupyter.stderr"] # robocop: disable + Fail msg=A JupyterLab code cell output returned an error, see screenshot END - Fail msg=A JupyterLab code cell output returned an error, see screenshot Get JupyterLab Code Cell Error Text ${error_txt} = Get Text //div[contains(@class,"jp-OutputArea-output") and @data-mime-type="application/vnd.jupyter.stderr"]