diff --git a/ods_ci/libs/DataSciencePipelinesKfp.py b/ods_ci/libs/DataSciencePipelinesKfp.py index 9bebc957b..b88425190 100644 --- a/ods_ci/libs/DataSciencePipelinesKfp.py +++ b/ods_ci/libs/DataSciencePipelinesKfp.py @@ -2,9 +2,11 @@ import json import os import sys +import tempfile import time from DataSciencePipelinesAPI import DataSciencePipelinesAPI +from kfp import compiler from kfp.client import Client from robotlibcore import keyword @@ -89,7 +91,17 @@ def delete_run(self, run_id): @keyword def create_run_from_pipeline_func( - self, user, pwd, project, source_code, fn, pipeline_params={}, current_path=None, route_name="ds-pipeline-dspa" + self, + user, + pwd, + project, + source_code, + fn, + pipeline_params={}, + current_path=None, + route_name="ds-pipeline-dspa", + pip_index_url=None, + pip_trusted_host=None, ): print(f"pipeline_params: {pipeline_params}") client, api = self.get_client(user, pwd, project, route_name) @@ -101,7 +113,7 @@ def create_run_from_pipeline_func( if current_path is None: current_path = os.getcwd() my_source = self.import_souce_code(f"{current_path}/tests/Resources/Files/pipeline-samples/v2/{source_code}") - pipeline = getattr(my_source, fn) + pipeline_func = getattr(my_source, fn) # pipeline_params # there are some special keys to retrieve argument values dynamically @@ -118,7 +130,27 @@ def create_run_from_pipeline_func( # create_run_from_pipeline_func will compile the code # if you need to see the yaml, for debugging purpose, call: TektonCompiler().compile(pipeline, f'{fn}.yaml') - result = client.create_run_from_pipeline_func(pipeline_func=pipeline, arguments=pipeline_params) + with tempfile.TemporaryDirectory() as tmpdir: + pipeline_package_path = os.path.join(tmpdir, "pipeline.yaml") + compiler.Compiler().compile( + pipeline_func=pipeline_func, + package_path=pipeline_package_path, + ) + + if pip_index_url is not None: + assert pip_trusted_host is not None + with open(pipeline_package_path, "r") as file: + file_content = file.read() + file_content = file_content.replace( + "python3 -m pip install", + f"python3 -m pip install --index-url {pip_index_url} --trusted-host {pip_trusted_host}", + ) + with open(pipeline_package_path, "w") as file: + file.write(file_content) + + result = client.create_run_from_pipeline_package( + pipeline_file=pipeline_package_path, arguments=pipeline_params + ) # easy to debug and double check failures print(result) return result.run_id diff --git a/ods_ci/tests/Tests/1100__data_science_pipelines/1100__data-science-pipelines-kfp.robot b/ods_ci/tests/Tests/1100__data_science_pipelines/1100__data-science-pipelines-kfp.robot index 5d9f23c87..14bd2e6bd 100644 --- a/ods_ci/tests/Tests/1100__data_science_pipelines/1100__data-science-pipelines-kfp.robot +++ b/ods_ci/tests/Tests/1100__data_science_pipelines/1100__data-science-pipelines-kfp.robot @@ -95,8 +95,14 @@ End To End Pipeline Workflow Using Kfp IF ${ray} == ${TRUE} Setup Kueue Resources ${project} cluster-queue-user resource-flavor-user local-queue-user END + # The run_robot_test.sh is sending the --variablefile ${TEST_VARIABLES_FILE} which may contain the `PIP_INDEX_URL` + # and `PIP_TRUSTED_HOST` variables, e.g. for disconnected testing. + ${pip_index_url} = Get Variable Value ${PIP_INDEX_URL} ${NONE} + ${pip_trusted_host} = Get Variable Value ${PIP_TRUSTED_HOST} ${NONE} + Log pip_index_url = ${pip_index_url} / pip_trusted_host = ${pip_trusted_host} ${run_id} Create Run From Pipeline Func ${username} ${password} ${project} - ... ${python_file} ${method_name} pipeline_params=${pipeline_params} + ... ${python_file} ${method_name} pipeline_params=${pipeline_params} pip_index_url=${pip_index_url} + ... pip_trusted_host=${pip_trusted_host} ${run_status} Check Run Status ${run_id} timeout=500 Should Be Equal As Strings ${run_status} SUCCEEDED Pipeline run doesn't have a status that means success. Check the logs Remove Pipeline Project ${project}