[patch] Improve handling for failed FVT

image/cli/masfvt/finally.yml

@@ -0,0 +1,29 @@
+---
+- hosts: localhost
+  any_errors_fatal: true
+  vars:
+    image_pull_policy: "{{ lookup('env', 'IMAGE_PULL_POLICY') }}"
+    deprovision: "{{ lookup('env', 'DEPROVISION') }}"
+    finalize: "{{ lookup('env', 'FINALIZE') }}"
+
+    # Pipeline Run Info
+    devops_build_number: "{{ lookup('env', 'DEVOPS_BUILD_NUMBER') | default('0', True) }}"
+    pipelinerun_name: "{{ lookup('env', 'PIPELINERUN_NAME') | default('mas-fvt-finally', True) }}-{{ devops_build_number }}"
+    mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}"
+    pipelinerun_namespace: "{{ lookup('env', 'PIPELINERUN_NAMESPACE') | default('mas-' ~ mas_instance_id ~ '-pipelines', True) }}"
+  tasks:
+    - name: "Debug"
+      debug:
+        msg:
+          - "pipelinerun_name ............ {{ pipelinerun_name }}"
+          - "pipelinerun_namespace ....... {{ pipelinerun_namespace }}"
+          - ""
+          - "image_pull_policy ........... {{ image_pull_policy }}"
+          - "deprovision ................. {{ deprovision }}"
+          - "finalize .................... {{ finalize }}"
+          - ""
+
+    - name: "Start FVT-finally pipeline"
+      kubernetes.core.k8s:
+        apply: true
+        template: templates/finally.yml.j2

image/cli/masfvt/templates/finally.yml.j2

@@ -0,0 +1,32 @@
+---
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: "{{ pipelinerun_name }}"
+  namespace: "{{ pipelinerun_namespace }}"
+  labels:
+    tekton.dev/pipeline: mas-fvt-finally
+spec:
+  pipelineRef:
+    name: mas-fvt-finally
+
+  serviceAccountName: pipeline
+  timeouts:
+    pipeline: "0"
+
+  params:
+    - name: image_pull_policy
+      value: "{{ image_pull_policy }}"
+    - name: finalize
+      value: "{{ finalize }}"
+    - name: deprovision
+      value: "{{ deprovision }}"
+
+  workspaces:
+    # The generated configuration files
+    - name: shared-configs
+      persistentVolumeClaim:
+        claimName: config-pvc
+    - name: shared-mustgather
+      persistentVolumeClaim:
+        claimName: shared-mustgather-storage

tekton/generate-tekton-pipelines.yml

@@ -39,6 +39,7 @@
         - fvt-core
         - fvt-deprovision-after
         - fvt-iot
+        - fvt-finally
         - fvt-manage
         - fvt-manage-is
         - fvt-manage-regr
@@ -73,7 +74,7 @@
 
     # 3. Generate Gitops Pipelines
     # -------------------------------------------------------------------------
-    - name: Generate GitOps Pipelines 
+    - name: Generate GitOps Pipelines
       vars:
         wait_for_provision: false
         wait_for_deprovision: false

tekton/generate-tekton-tasks.yml

@@ -79,6 +79,7 @@
         - launchfvt-assist
         - launchfvt-core
         - launchfvt-sls
+        - launchfvt-finally
         - launchfvt-iot
         - launchfvt-manage
         - launchfvt-manage-is

tekton/src/pipelines/fvt-finally.yml.j2

@@ -0,0 +1,89 @@
+---
+apiVersion: tekton.dev/v1beta1
+kind: Pipeline
+metadata:
+  name: mas-fvt-finally
+spec:
+  params:
+    - name: image_pull_policy
+      type: string
+
+    - name: finalize
+      type: string
+      default: "true"
+      description: "Finalize the record in the FVT database"
+
+    - name: deprovision
+      type: string
+      default: "true"
+      description: "Enable deprovision of the target cluster & associated resources following the completion of this pipeline"
+
+  tasks:
+    # Collect must-gather
+    # -------------------------------------------------------------------------
+    - name: must-gather
+      timeout: "0"
+      taskRef:
+        kind: Task
+        name: mas-devops-must-gather
+      params:
+        - name: base_output_dir
+          value: "/mustgather/$(context.pipelineRun.name)"
+      workspaces:
+        - name: mustgather
+          workspace: shared-mustgather
+
+    # Finalize the record in the FVT database
+    # -------------------------------------------------------------------------
+    - name: finalize
+      timeout: "0"
+      taskRef:
+        kind: Task
+        name: mas-fvt-finalize
+      runAfter:
+        - must-gather
+      when:
+        - input: $(params.finalize)
+          operator: in
+          values: ["true", "True"]
+
+    # Delete IBM Cloud services
+    # -------------------------------------------------------------------------
+    # COS
+    {{ lookup('template', 'taskdefs/dependencies/fvt-deprovision-cos.yml.j2') | indent(4) }}
+      when:
+        - input: $(params.deprovision)
+          operator: in
+          values: ["true", "True"]
+      runAfter:
+        - finalize
+
+    # Event Streams
+    {{ lookup('template', 'taskdefs/dependencies/fvt-deprovision-kafka.yml.j2') | indent(4) }}
+      when:
+        - input: $(params.deprovision)
+          operator: in
+          values: ["true", "True"]
+      runAfter:
+        - finalize
+
+    # Delete the named cluster
+    # -------------------------------------------------------------------------
+    - name: deprovision-cluster
+      timeout: "0"
+      taskRef:
+        kind: Task
+        name: mas-fvt-deprovision-ocp
+      when:
+        - input: $(params.deprovision)
+          operator: in
+          values: ["true", "True"]
+      runAfter:
+        - cos-deprovision
+        - kafka-deprovision
+
+  workspaces:
+    # Shared storage to hold mustgather output for tasks
+    - name: shared-mustgather
+    # The generated configuration files
+    - name: shared-configs

tekton/src/pipelines/fvt-launcher.yml.j2

@@ -921,7 +921,7 @@ spec:
         - name: max_retries
           value: "25"
         - name: ignore_failure
-          value: "True"
+          value: "False"
       when:
         - input: $(params.sync_with_install)
           operator: in
@@ -934,70 +934,21 @@ spec:
         - approval-visualinspection
         - launchivt-manage
 
-    # Collect must-gather
-    # -------------------------------------------------------------------------
-    - name: must-gather
+  finally:
+    - name: launchfvt-finally
       timeout: "0"
       taskRef:
         kind: Task
-        name: mas-devops-must-gather
+        name: mas-launchfvt-finally
       params:
-        - name: base_output_dir
-          value: "/mustgather/$(context.pipelineRun.name)"
-      workspaces:
-        - name: mustgather
-          workspace: shared-mustgather
-      runAfter:
-        - waitfor-installation
-
-    # Finalize the record in the FVT database
-    # -------------------------------------------------------------------------
-    - name: finalize
-      timeout: "0"
-      taskRef:
-        kind: Task
-        name: mas-fvt-finalize
-      runAfter:
-        - must-gather
-      when:
-        - input: $(params.finalize)
-          operator: in
-          values: ["true", "True"]
-
-    # Delete IBM Cloud services
-    # -------------------------------------------------------------------------
-    # COS
-    {{ lookup('template', 'taskdefs/dependencies/fvt-deprovision-cos.yml.j2') | indent(4) }}
-      when:
-        - input: $(params.deprovision)
-          operator: in
-          values: ["true", "True"]
-      runAfter:
-        - finalize
-
-    # Event Streams
-    {{ lookup('template', 'taskdefs/dependencies/fvt-deprovision-kafka.yml.j2') | indent(4) }}
-      when:
-        - input: $(params.deprovision)
-          operator: in
-          values: ["true", "True"]
-      runAfter:
-        - finalize
-
-    # Delete the named cluster
-    # -------------------------------------------------------------------------
-    - name: deprovision-cluster
-      timeout: "0"
-      taskRef:
-        kind: Task
-        name: mas-fvt-deprovision-ocp
-      when:
-        - input: $(params.deprovision)
-          operator: in
-          values: ["true", "True"]
-      runAfter:
-        - cos-deprovision
-        - kafka-deprovision
+        - name: deprovision
+          value: "$(params.deprovision)"
+        - name: finalize
+          value: "$(params.finalize)"
+        - name: image_pull_policy
+          value: $(params.image_pull_policy)
+        - name: pipelinerun_name
+          value: "$(params.mas_instance_id)-fvt-finally"
 
   workspaces:
     # Shared storage to hold mustgather output for tasks

tekton/src/tasks/fvt-launcher/launchfvt-finally.yml.j2

@@ -0,0 +1,47 @@
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: mas-launchfvt-finally
+spec:
+  params:
+    - name: finalize
+      type: string
+    - name: deprovision
+      type: string
+    - name: image_pull_policy
+      type: string
+      default: IfNotPresent
+    - name: pipelinerun_name
+      type: string
+
+  steps:
+    - name: start-pipelinerun
+      image: quay.io/ibmmas/cli:latest
+      imagePullPolicy: $(params.image_pull_policy)
+      command:
+        - ansible-playbook
+        - /masfvt/finally.yml
+      env:
+        - name: FINALIZE
+          value: $(params.finalize)
+        - name: DEPROVISION
+          value: $(params.deprovision)
+        - name: IMAGE_PULL_POLICY
+          value: $(params.image_pull_policy)
+        - name: PIPELINERUN_NAME
+          value: $(params.pipelinerun_name)
+        - name: PIPELINERUN_NAMESPACE
+          value: $(context.taskRun.namespace)
+        - name: DEVOPS_BUILD_NUMBER
+          valueFrom:
+            secretKeyRef:
+              name: mas-devops
+              key: DEVOPS_BUILD_NUMBER
+              optional: true
+        - name: MAS_INSTANCE_ID
+          valueFrom:
+            secretKeyRef:
+              name: mas-devops
+              key: MAS_INSTANCE_ID
+              optional: false