diff --git a/image/cli/app-root/src/mobile-version-finder.py b/image/cli/app-root/src/mobile-version-finder.py new file mode 100644 index 0000000000..53d533e2f1 --- /dev/null +++ b/image/cli/app-root/src/mobile-version-finder.py @@ -0,0 +1,288 @@ +import os +import threading +import zipfile +import sys +import json +import yaml +import requests +from pathlib import Path +from collections import OrderedDict +from subprocess import PIPE, Popen, TimeoutExpired +from kubernetes.client import Configuration +from openshift.dynamic import DynamicClient +from kubernetes import client, config + + +instanceId = os.getenv("INSTANCE_ID") +workspaceId = os.getenv("WORKSPACE_ID") +uploadFile = os.getenv("UPLOAD_FILE") +artKey = os.getenv("ARTIFACTORY_TOKEN") +artDir = os.getenv("ARTIFACTORY_UPLOAD_DIR") +output_filename = "mobile-is-versions.json" + +class RunCmdResult(object): + def __init__(self, returnCode, output, error): + self.rc = returnCode + self.out = output + self.err = error + + def successful(self): + return self.rc == 0 + + def failed(self): + return self.rc != 0 + +def run_cmd(cmdArray, timeout=630): + """ + Run a command on the local host. This drives all the helm operations, + as there is no python Helm client available. + # Parameters + cmdArray (list): Command to execute + timeout (int): How long to allow for the command to complete + # Returns + [int, string, string]: `returnCode`, `stdOut`, `stdErr` + """ + + lock = threading.Lock() + + with lock: + p = Popen(cmdArray, stdin=PIPE, stdout=PIPE, stderr=PIPE, bufsize=-1) + try: + output, error = p.communicate(timeout=timeout) + return RunCmdResult(p.returncode, output, error) + except TimeoutExpired as e: + return RunCmdResult(127, 'TimeoutExpired', str(e)) + +def get_graphite_versions(): + # This list will contains all files found in the maxinst pod + apps_list = [] + + # Downloads all zip files from maxinst container + try: + pods = dynClient.resources.get(api_version="v1", kind="Pod") + podList = pods.get(namespace=f"mas-{instanceId}-manage", label_selector='mas.ibm.com/appType=maxinstudb') + + if podList is None or podList.items is None or len(podList.items) == 0: + pass + else: + podName = podList.items[0].metadata.name + + # list all graphite zip packages in maxinst pod + ocExecCommand = ["oc", "exec", "-n", f"mas-{instanceId}-manage", podName, "--", "ls", "/opt/IBM/SMP/maximo/tools/maximo/en/graphite/apps"] + result = run_cmd(ocExecCommand) + ls_result = result.out.decode('utf-8') + apps_list = ls_result.split("\n") + # removes last empty value from the list + apps_list.pop() + + # Download all packages that were found + for a in apps_list: + print("Downloading:", a) + ocExecCommand = [ + "oc", "cp", "-n", f"mas-{instanceId}-manage", + f"{podName}:/opt/IBM/SMP/maximo/tools/maximo/en/graphite/apps/{a}", + f"./{a}", + "--retries=10" + ] + run_cmd(ocExecCommand) + + except Exception as e: + print(f"Unable to download mobile packages from maxinst pod: {e}") + sys.exit(1) + + + # Downloading navigator from mobileapi pod + try: + pods = dynClient.resources.get(api_version="v1", kind="Pod") + podList = pods.get(namespace=f"mas-{instanceId}-core", label_selector=f'app={instanceId}-mobileapi') + + if podList is None or podList.items is None or len(podList.items) == 0: + pass + else: + podName = podList.items[0].metadata.name + + # list navigator zip packages in mobileapi pod + ocExecCommand = ["oc", "exec", "-n", f"mas-{instanceId}-core", podName, "--", "ls", "/etc/mobile/packages"] + result = run_cmd(ocExecCommand) + ls_result = result.out.decode('utf-8') + apps_list = ls_result.split("\n") + # removes last empty value from the list + apps_list.pop() + + for a in apps_list: + print("Downloading:", a) + ocExecCommand = [ + "oc", "cp", "-n", f"mas-{instanceId}-core", + f"{podName}:/etc/mobile/packages/{a}", + f"./{a}", + "--retries=10" + ] + run_cmd(ocExecCommand) + except Exception as e: + print(f"Unable to download mobileapi navigator package from mobileapi pod: {e}") + sys.exit(1) + + + # Extracting build.json from each file and deleting zip + pathlist = Path(".").glob('*.zip') + for app_zip_file in pathlist: + zip_file_path = f"./{str(app_zip_file)}" + with zipfile.ZipFile(zip_file_path, 'r') as zip_ref: + zip_ref.extract('build.json', '.') + zip_ref.close() + zip_file_prefix = zip_file_path.split(".zip") + os.rename('./build.json', f'{zip_file_prefix[0]}.json') + os.remove(zip_file_path) + + # dictionary that will contain all graphite versions for all found zips + graphite_json = {} + + pathlist = Path(".").glob('*.json') + for path in pathlist: + path_in_str = str(path) + + with open(path_in_str, 'r', encoding="utf-8") as openfile: + json_object = json.load(openfile) + + graphite_json.update( + { + json_object.get("applicationId"): { + "version": json_object.get("version"), + "applicationId": json_object.get("applicationId"), + "applicationTitle": json_object.get("applicationTitle"), + "mobileVersion": json_object.get("mobileVersion"), + "buildToolsVersion": json_object.get("buildToolsVersion"), + "appProcessorVersion": json_object.get("appProcessorVersion") + } + } + ) + + # removing empty mobile version or non mobile apps + if json_object.get("mobileVersion") is None: + del graphite_json[json_object.get("applicationId")]["mobileVersion"] + + # removing empty title from apps with no title + if json_object.get("applicationTitle") is None: + del graphite_json[json_object.get("applicationId")]["applicationTitle"] + + # delete build.json file + os.remove(path_in_str) + + # ordering json file + graphite_json_sorted = OrderedDict(sorted(graphite_json.items())) + + return graphite_json_sorted + +def get_mobile_and_is_image_tags(): + + images_json = {} + + # getting mobileapi image version from entitymgr-suite pod + try: + pods = dynClient.resources.get(api_version="v1", kind="Pod") + podList = pods.get(namespace=f"mas-{instanceId}-core", label_selector=f'app={instanceId}-entitymgr-suite') + + if podList is None or podList.items is None or len(podList.items) == 0: + pass + else: + podName = podList.items[0].metadata.name + + # list navigator zip packages in mobileapi pod + ocExecCommand = ["oc", "exec", "-n", f"mas-{instanceId}-core", podName, "--", "cat", "/opt/ansible/roles/suite/vars/images.yml"] + result = run_cmd(ocExecCommand) + cat_result = result.out.decode('utf-8') + images = yaml.safe_load(cat_result) + images_json.update({"mobileapi": images['defaultTags']['mobileapi']}) + except Exception as e: + print(f"Unable to catch images file from entitymgr pod: {e}") + sys.exit(1) + + # getting industry solutions images version from entitymgr-ws + try: + pods = dynClient.resources.get(api_version="v1", kind="Pod") + podList = pods.get(namespace=f"mas-{instanceId}-manage", label_selector='mas.ibm.com/appType=entitymgr-ws-operator') + + if podList is None or podList.items is None or len(podList.items) == 0: + pass + else: + podName = podList.items[0].metadata.name + + # list navigator zip packages in mobileapi pod + ocExecCommand = ["oc", "exec", "-n", f"mas-{instanceId}-manage", podName, "--", "cat", "/opt/ansible/roles/workspace/vars/images.yml"] + result = run_cmd(ocExecCommand) + cat_result = result.out.decode('utf-8') + images = yaml.safe_load(cat_result) + images_json.update(images['defaultTags']) + + except Exception as e: + print(f"Unable to catch images file from entitymgr pod: {e}") + sys.exit(1) + + images_json_sorted = OrderedDict(sorted(images_json.items())) + + return images_json_sorted + +def artifactory_upload(): + + url = artDir + '/' + output_filename + bearer = f"Bearer {artKey}" + headers = { + 'content-type': 'application/json', + 'Authorization': bearer + } + + with open(output_filename, 'rb') as f: + r = requests.put(url, data=f, headers=headers, timeout=10) + + if r.status_code != 201: + print("Upload failed.") + else: + print("Upload successful") + print("Download URL:", r.json()['downloadUri']) + + +if __name__ == "__main__": + + if "KUBERNETES_SERVICE_HOST" in os.environ: + config.load_incluster_config() + k8s_config = Configuration.get_default_copy() + k8s_client = client.api_client.ApiClient(configuration=k8s_config) + dynClient = DynamicClient(k8s_client) + else: + k8s_client = config.new_client_from_config() + dynClient = DynamicClient(k8s_client) + + if os.path.isfile(output_filename): + print("Found an existing output file. Deleting...") + os.remove(output_filename) + + print("Retrieving Graphite versions for Manage apps") + graphite_versions = get_graphite_versions() + + print("Retrieving image versions for Manage IS and Add-ons ") + img_versions = get_mobile_and_is_image_tags() + + print("Generating output file with versions") + mobile_is_versions = {} + mobile_is_versions.update( + { + "graphite_versions": graphite_versions, + "images_versions": img_versions + } + ) + + with open(output_filename, "w", encoding="utf-8") as outfile: + json.dump(mobile_is_versions, outfile, indent=4) + + print("Printing gererated file:") + print("************************************************") + with open(output_filename, "r", encoding="utf-8") as readfile: + print(readfile.read()) + print("************************************************") + + # Upload logs conditionally based o env var + if uploadFile: + print("Uploading file to artifactory") + artifactory_upload() + + print("Done") diff --git a/tekton/generate-tekton-tasks.yml b/tekton/generate-tekton-tasks.yml index f9d4877ce8..d95db6a0df 100644 --- a/tekton/generate-tekton-tasks.yml +++ b/tekton/generate-tekton-tasks.yml @@ -57,6 +57,7 @@ - fvt-iot - fvt-manage - fvt-mobile + - fvt-mobile-pytest - fvt-manage-directprint - fvt-manage-adhoc-report - fvt-manage-birt-report diff --git a/tekton/src/pipelines/taskdefs/fvt-mobile/phase1-setup.yml.j2 b/tekton/src/pipelines/taskdefs/fvt-mobile/phase1-setup.yml.j2 index 794335d810..1de8b4aa34 100644 --- a/tekton/src/pipelines/taskdefs/fvt-mobile/phase1-setup.yml.j2 +++ b/tekton/src/pipelines/taskdefs/fvt-mobile/phase1-setup.yml.j2 @@ -2,19 +2,13 @@ - name: fvt-mobile-setup-data taskRef: kind: Task - name: mas-fvt-manage-pytest + name: mas-fvt-mobile-pytest workspaces: - name: configs workspace: shared-configs params: - name: fvt_image_registry value: $(params.fvt_image_registry) - - name: artifactory_token - value: $(params.fvt_artifactory_token) - - name: fvt_image_namespace - value: fvt-mobile - - name: fvt_image_name - value: fvt-mobile-pytest - name: fvt_image_digest value: $(params.fvt_digest_mobile_pytest) - name: fvt_test_suite @@ -25,8 +19,6 @@ value: $(params.mas_workspace_id) - name: product_channel value: $(params.mas_app_channel_manage) - - name: product_id - value: ibm-mas-mobile when: - input: "$(params.mas_app_channel_manage)" operator: notin diff --git a/tekton/src/tasks/fvt/fvt-mobile-pytest.yml.j2 b/tekton/src/tasks/fvt/fvt-mobile-pytest.yml.j2 new file mode 100644 index 0000000000..3a0a141144 --- /dev/null +++ b/tekton/src/tasks/fvt/fvt-mobile-pytest.yml.j2 @@ -0,0 +1,210 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: mas-fvt-mobile-pytest +spec: + params: + # Control the image pull policy for the FVT container image + - name: image_pull_policy + type: string + default: IfNotPresent + + # Test Container Information + # ------------------------------------------------------------------------- + - name: fvt_image_registry + type: string + description: FVT Container Image Registry (required) + - name: fvt_image_digest + type: string + description: FVT Container Image Digest + + # Test Framework Information + # ------------------------------------------------------------------------- + - name: fvt_enable_debug + type: string + description: Turn on debug logging (verbose mode) + default: "true" + - name: fvt_test_suite + type: string + description: If the FVT container image contains multiple suites, use this to control the suite that will be executed + default: "" + - name: fvt_mas_appws_component + type: string + description: Manage product that will be tested by this task. Use official names as expected by Manage operators + default: "base" + - name: product_id + type: string + description: Product ID under test + default: ibm-mas-mobile + - name: product_channel + type: string + description: "Subscription channel for the product (todo: have the test code look this up instead)" + default: "ibm-mas-mobile" + - name: mas_instance_id + type: string + description: Instance ID of the target test environment + - name: mas_workspace_id + type: string + description: Workspace ID in MAS to use for running the tests + default: "masdev" + + # Test-specific Information (all optional) + # ------------------------------------------------------------------------- + - name: upload_file + type: string + description: Used only by fvt-mobile-version step to upload version file to artifactory + default: "true" + + stepTemplate: + name: 'fvt-mobile' + env: + - name: PRODUCT_ID + value: $(params.product_id) + - name: PRODUCT_CHANNEL + value: $(params.product_channel) + + - name: DEVOPS_MONGO_URI + valueFrom: + secretKeyRef: + name: mas-devops + key: DEVOPS_MONGO_URI + optional: true + - name: BUILD_NUM + valueFrom: + secretKeyRef: + name: mas-devops + key: DEVOPS_BUILD_NUMBER + optional: true + + - name: IBMADMIN_ENABLED + value: "true" + + - name: NAMESPACE + value: "mas-$(params.mas_instance_id)-core" + - name: INSTANCE_ID + value: "$(params.mas_instance_id)" + - name: WORKSPACE_ID + value: "$(params.mas_workspace_id)" + + - name: FVT_TEST_SUITE + value: $(params.fvt_test_suite) + - name: FVT_MAS_APPWS_COMPONENT + value: $(params.fvt_mas_appws_component) + + - name: FVT_ENABLE_DEBUG + value: "$(params.fvt_enable_debug)" + + - name: UPLOAD_FILE + value: "$(params.upload_file)" + + - name: ARTIFACTORY_TOKEN + valueFrom: + secretKeyRef: + name: mas-devops + key: ARTIFACTORY_TOKEN + optional: true + + - name: ARTIFACTORY_UPLOAD_DIR + valueFrom: + secretKeyRef: + name: mas-devops + key: ARTIFACTORY_UPLOAD_DIR + optional: true + + # Black and white listing + - name: FVT_BLACKLIST + valueFrom: + secretKeyRef: + name: mas-fvt-manage + key: FVT_BLACKLIST + optional: false + - name: FVT_WHITELIST + valueFrom: + secretKeyRef: + name: mas-fvt-manage + key: FVT_WHITELIST + optional: false + - name: FVT_BLACKLIST_IS + valueFrom: + secretKeyRef: + name: mas-fvt-manage + key: FVT_BLACKLIST_IS + optional: false + - name: FVT_WHITELIST_IS + valueFrom: + secretKeyRef: + name: mas-fvt-manage + key: FVT_WHITELIST_IS + optional: false + + # Test Data + - name: LDAP_URL + valueFrom: + secretKeyRef: + name: mas-fvt-manage + key: LDAP_URL + optional: false + - name: LDAP_BASE_DN + valueFrom: + secretKeyRef: + name: mas-fvt-manage + key: LDAP_BASE_DN + optional: false + - name: LDAP_BIND_DN + valueFrom: + secretKeyRef: + name: mas-fvt-manage + key: LDAP_BIND_DN + optional: false + - name: LDAP_BIND_PASSWORD + valueFrom: + secretKeyRef: + name: mas-fvt-manage + key: LDAP_BIND_PASSWORD + optional: false + - name: LDAP_USER_MAP + valueFrom: + secretKeyRef: + name: mas-fvt-manage + key: LDAP_USER_MAP + optional: false + - name: LDAP_CERT_ALIAS + valueFrom: + secretKeyRef: + name: mas-fvt-manage + key: LDAP_CERT_ALIAS + optional: false + - name: LDAP_CRT + valueFrom: + secretKeyRef: + name: mas-fvt-manage + key: LDAP_CRT + optional: false + + steps: + - name: fvt-mobile-version + command: + - python3 + - /opt/app-root/src/mobile-version-finder.py + image: quay.io/ibmmas/cli:latest + imagePullPolicy: $(params.image_pull_policy) + onError: continue + + - name: fvt-mobile-pytest + image: '$(params.fvt_image_registry)/fvt-mobile/fvt-mobile-pytest@$(params.fvt_image_digest)' + imagePullPolicy: $(params.image_pull_policy) + timeout: 120m # Ensure bad FVTs don't run forever + onError: continue # Ensure bad FVTs don't break the pipeline + resources: {} + workingDir: /opt/ibm/test/src + volumeMounts: + - mountPath: /dev/shm + name: dshm + volumes: + - name: dshm + emptyDir: + medium: Memory + + workspaces: + - name: configs