Add TGIS test for fetching model from Minio object storage

ods_ci/tests/Resources/CLI/Minio.resource

@@ -0,0 +1,42 @@
+*** Settings ***
+Documentation       Collcetion of keywords to deploy and remove MinIO storage
+Library             OperatingSystem
+Resource            ../Common.robot
+
+*** Variables ***
+${MINIO_RESOURCES_DIRPATH}=    ods_ci/tests/Resources/Files/minio
+${MINIO_POD_FILEPATH}=    ${MINIO_RESOURCES_DIRPATH}/minio.yaml
+${MINIO_SECRET_FILEPATH}=    ${MINIO_RESOURCES_DIRPATH}/minio_secret.yaml
+${MINIO_SA_FILEPATH}=    ${MINIO_RESOURCES_DIRPATH}/minio_serviceaccount.yaml
+
+
+*** Keywords ***
+Deploy MinIO
+    [Arguments]    ${minio_image}=quay.io/modh/ods-ci-minio-models:1.0
+    ...            ${namespace}=minio
+    # ${key}    ${pw}=    Generate Minio Random Credentials
+    ${key}=    Set Variable    user
+    ${pw}=    Set Variable    myfakepassword
+    Set Test Variable    ${key}
+    Set Test Variable    ${pw}
+    Set Test Variable    ${url}    ${minio_image}
+    Create File From Template    ${MINIO_POD_FILEPATH}    ${MINIO_RESOURCES_DIRPATH}/minio_filled.yaml
+    # Create File From Template    ${MINIO_SECRET_FILEPATH}    ${MINIO_RESOURCES_DIRPATH}/minio_secret_filled.yaml
+    ${rc}=    Run And Return Rc    oc new-project ${namespace}
+    ${rc}    ${out}=    Run And Return Rc And Output    oc apply -f ${MINIO_RESOURCES_DIRPATH}/minio_filled.yaml -n ${namespace}
+    # Oc Apply    kind=Pod    src=${MINIO_RESOURCES_DIRPATH}/minio_filled.yaml    namespace=${namespace}
+    # Oc Apply    kind=Secret    src=${MINIO_RESOURCES_DIRPATH}/minio_secret_filled.yaml    namespace=${namespace}
+    # Oc Apply    kind=ServiceAccount    src=${MINIO_SA_FILEPATH}    namespace=${namespace}
+    ${route}=    Run And Return Rc And Output    oc get route minio-route -n ${namespace} --template={{.spec.host}}
+    RETURN    ${route}
+
+Generate Minio Random Credentials
+    ${key}=    Generate Random String    chars=[NUMBERS][LETTERS]
+    ${pw}=    Generate Random String    chars=[NUMBERS][LETTERS]
+    RETURN    ${key}    ${pw}
+
+Get Minio Credentials
+    [Arguments]    ${namespace}    ${podname}=ods-ci-minio
+    ${key}=    Set Variable    user
+    ${pw}=    Set Variable    myfakepassword
+    RETURN    ${key}    ${pw}

ods_ci/tests/Resources/CLI/ModelServing/llm.resource

@@ -55,9 +55,13 @@
     [Documentation]    Creates the DS Project (if not exists), creates the data connection for the models,
     ...                creates caikit runtime. This can be used as test setup
     [Arguments]    ${namespace}    ${enable_metrics}=${FALSE}    ${runtime}=caikit-tgis-runtime    ${protocol}=grpc
+    ...            ${access_key_id}=${S3.AWS_ACCESS_KEY_ID}    ${access_key}=${S3.AWS_SECRET_ACCESS_KEY} 
+    ...            ${endpoint}=${MODELS_BUCKET.ENDPOINT}    ${verify_ssl}=${TRUE}
     Set Up Test OpenShift Project    test_ns=${namespace}
-    Create Secret For S3-Like Buckets    endpoint=${MODELS_BUCKET.ENDPOINT}
+    Create Secret For S3-Like Buckets    endpoint=${endpoint}
     ...    region=${MODELS_BUCKET.REGION}    namespace=${namespace}
+    ...    access_key_id=${access_key_id}    access_key=${access_key}
+    ...    verify_ssl=${verify_ssl}
     Deploy Serving Runtime    namespace=${namespace}    runtime=${runtime}    protocol=${protocol}
     IF   ${enable_metrics} == ${TRUE}
         Enable User Workload Monitoring
@@ -72,6 +76,7 @@
     ...            ${namespace}=${TEST_NS}    ${endpoint}=${S3.AWS_DEFAULT_ENDPOINT}
     ...            ${region}=${S3.AWS_DEFAULT_REGION}    ${access_key_id}=${S3.AWS_ACCESS_KEY_ID}
     ...            ${access_key}=${S3.AWS_SECRET_ACCESS_KEY}    ${use_https}=${USE_BUCKET_HTTPS}
+    ...            ${verify_ssl}=${TRUE}
     ${rc}    ${out}=    Run And Return Rc And Output    oc get secret ${name} -n ${namespace}
     IF    "${rc}" == "${0}"
         Log    message=Secret ${name} in ${namespace} NS already present. Skipping secret setup...
@@ -102,6 +107,12 @@
     Should Be Equal As Integers    ${rc}    ${0}
     Run Keyword And Ignore Error    Run    oc create -f ${LLM_RESOURCES_DIRPATH}/bucket_sa_filled.yaml -n ${namespace}
     Add Secret To Service Account    sa_name=${sa_name}    secret_name=${name}    namespace=${namespace}
+    IF    ${verify_ssl}
+        ${rc}    ${out}=    Run And Return Rc And Output
+        ...    oc annotate secret ${name} -n ${namespace} serving.kserve.io/s3-verifyssl='false' --overwrite
+        Should Be Equal As Integers    ${rc}    ${0}
+    END
+
 
 Compile Inference Service YAML
     [Documentation]    Prepare the Inference Service YAML file in order to deploy a model

ods_ci/tests/Resources/Files/minio/minio.yaml

@@ -0,0 +1,48 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio-service
+spec:
+  ports:
+    - name: minio-client-port
+      port: 9000
+      protocol: TCP
+      targetPort: 9000
+  selector:
+    app: minio
+---
+kind: Route
+apiVersion: route.openshift.io/v1
+metadata:
+  name: minio-route
+spec:
+  to:
+    kind: Service
+    name: minio-service
+    weight: 100
+  port:
+    targetPort: ui
+  wildcardPolicy: None
+  tls:
+    termination: edge
+    insecureEdgeTerminationPolicy: Redirect
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: minio
+  name: ods-ci-minio
+spec:
+  containers:
+    - args:
+        - server
+        - /data1
+      env:
+        - name: MINIO_ACCESS_KEY
+          value:  ${key}
+        - name: MINIO_SECRET_KEY
+          value: ${pw}
+      image: ${url}
+      imagePullPolicy: Always
+      name: minio

ods_ci/tests/Resources/RHOSi.resource

@@ -38,6 +38,7 @@ Resource            Common.robot
 ...                         Login To OCP Using API
 ...                         Generate Thanos Token
 ...                         Set Thanos Credentials Variables
+...                         Generate Minio Random Credentials
 
 
 *** Keywords ***

ods_ci/tests/Tests/400__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis_object_storages.robot

@@ -0,0 +1,62 @@
+*** Settings ***
+Documentation     Collection of CLI tests to validate the model serving stack for Large Language Models (LLM).
+...               These tests leverage on TGIS Standalone Serving Runtime
+Resource          ../../../../Resources/OCP.resource
+Resource          ../../../../Resources/CLI/ModelServing/llm.resource
+Resource          ../../../../Resources/CLI/Minio.resource
+Library           OpenShiftLibrary
+Suite Setup       Suite Setup
+Suite Teardown    RHOSi Teardown
+Test Tags         KServe
+
+
+*** Variables ***
+${MODEL_S3_DIR}=    flan-t5-small-hf
+${TEST_NS}=    tgis-storages
+${TGIS_RUNTIME_NAME}=    tgis-runtime
+@{SEARCH_METRICS}=    tgi_    istio_
+
+
+*** Test Cases ***
+Verify User Can Serve And Query A Model From Minio
+    [Documentation]    Basic tests for preparing, deploying and querying a LLM model
+    ...                using Kserve and Caikit+TGIS runtime
+    [Tags]    Tier1    RHOAIENG-3490
+    ${minio_namespace}=    Set Variable    minio-models
+    Deploy MinIO    namespace=${minio_namespace}
+    Wait For Pods To Be Ready    label_selector=app=minio
+    ...    namespace=${minio_namespace}
+    ${key}    ${pw}=    Get Minio Credentials    namespace=${minio_namespace}
+    Set Project And Runtime    runtime=${TGIS_RUNTIME_NAME}     namespace=${TEST_NS}-minio
+    ...    access_key_id=${key}    access_key=${pw}
+    ...    endpoint=minio-service.${minio_namespace}.svc.cluster.local:9000
+    ...    verify_ssl=${FALSE}    # temporary
+    ${test_namespace}=    Set Variable     ${TEST_NS}-minio
+    ${model_name}=    Set Variable    flan-t5-small-hf
+    ${models_names}=    Create List    ${model_name}
+    ${storage_uri}=    Set Variable    s3://models/${MODEL_S3_DIR}/
+    Compile Inference Service YAML    isvc_name=${model_name}
+    ...    sa_name=${DEFAULT_BUCKET_SA_NAME}
+    ...    model_storage_uri=${storage_uri}
+    ...    model_format=pytorch    serving_runtime=${TGIS_RUNTIME_NAME}
+    Deploy Model Via CLI    isvc_filepath=${INFERENCESERVICE_FILLED_FILEPATH}
+    ...    namespace=${test_namespace}
+    Wait For Pods To Be Ready    label_selector=serving.kserve.io/inferenceservice=${model_name}
+    ...    namespace=${test_namespace}
+    Query Model Multiple Times    model_name=${model_name}    runtime=${TGIS_RUNTIME_NAME}
+    ...    inference_type=all-tokens    n_times=1
+    ...    namespace=${test_namespace}
+    # Query Model Multiple Times    model_name=${model_name}    runtime=${TGIS_RUNTIME_NAME}
+    # ...    inference_type=streaming    n_times=1
+    # ...    namespace=${test_namespace}    validate_response=${FALSE}
+    # [Teardown]    Clean Up Test Project    test_ns=${test_namespace}
+    # ...    isvc_names=${models_names}    wait_prj_deletion=${FALSE}
+
+
+*** Keywords ***
+Suite Setup
+    [Documentation]
+    Skip If Component Is Not Enabled    kserve
+    # RHOSi Setup
+    Load Expected Responses
+    Run    git clone https://github.com/IBM/text-generation-inference/