Skip to content

Commit

Permalink
[patch] Improve handling for failed FVT (#1143)
Browse files Browse the repository at this point in the history
  • Loading branch information
durera authored Jul 26, 2024
1 parent 03a9499 commit 7f7f861
Show file tree
Hide file tree
Showing 7 changed files with 212 additions and 62 deletions.
29 changes: 29 additions & 0 deletions image/cli/masfvt/finally.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
---
- hosts: localhost
any_errors_fatal: true
vars:
image_pull_policy: "{{ lookup('env', 'IMAGE_PULL_POLICY') }}"
deprovision: "{{ lookup('env', 'DEPROVISION') }}"
finalize: "{{ lookup('env', 'FINALIZE') }}"

# Pipeline Run Info
devops_build_number: "{{ lookup('env', 'DEVOPS_BUILD_NUMBER') | default('0', True) }}"
pipelinerun_name: "{{ lookup('env', 'PIPELINERUN_NAME') | default('mas-fvt-finally', True) }}-{{ devops_build_number }}"
mas_instance_id: "{{ lookup('env', 'MAS_INSTANCE_ID') }}"
pipelinerun_namespace: "{{ lookup('env', 'PIPELINERUN_NAMESPACE') | default('mas-' ~ mas_instance_id ~ '-pipelines', True) }}"
tasks:
- name: "Debug"
debug:
msg:
- "pipelinerun_name ............ {{ pipelinerun_name }}"
- "pipelinerun_namespace ....... {{ pipelinerun_namespace }}"
- ""
- "image_pull_policy ........... {{ image_pull_policy }}"
- "deprovision ................. {{ deprovision }}"
- "finalize .................... {{ finalize }}"
- ""

- name: "Start FVT-finally pipeline"
kubernetes.core.k8s:
apply: true
template: templates/finally.yml.j2
32 changes: 32 additions & 0 deletions image/cli/masfvt/templates/finally.yml.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
---
apiVersion: tekton.dev/v1beta1
kind: PipelineRun
metadata:
name: "{{ pipelinerun_name }}"
namespace: "{{ pipelinerun_namespace }}"
labels:
tekton.dev/pipeline: mas-fvt-finally
spec:
pipelineRef:
name: mas-fvt-finally

serviceAccountName: pipeline
timeouts:
pipeline: "0"

params:
- name: image_pull_policy
value: "{{ image_pull_policy }}"
- name: finalize
value: "{{ finalize }}"
- name: deprovision
value: "{{ deprovision }}"

workspaces:
# The generated configuration files
- name: shared-configs
persistentVolumeClaim:
claimName: config-pvc
- name: shared-mustgather
persistentVolumeClaim:
claimName: shared-mustgather-storage
3 changes: 2 additions & 1 deletion tekton/generate-tekton-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
- fvt-core
- fvt-deprovision-after
- fvt-iot
- fvt-finally
- fvt-manage
- fvt-manage-is
- fvt-manage-regr
Expand Down Expand Up @@ -73,7 +74,7 @@

# 3. Generate Gitops Pipelines
# -------------------------------------------------------------------------
- name: Generate GitOps Pipelines
- name: Generate GitOps Pipelines
vars:
wait_for_provision: false
wait_for_deprovision: false
Expand Down
1 change: 1 addition & 0 deletions tekton/generate-tekton-tasks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@
- launchfvt-assist
- launchfvt-core
- launchfvt-sls
- launchfvt-finally
- launchfvt-iot
- launchfvt-manage
- launchfvt-manage-is
Expand Down
89 changes: 89 additions & 0 deletions tekton/src/pipelines/fvt-finally.yml.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
---
apiVersion: tekton.dev/v1beta1
kind: Pipeline
metadata:
name: mas-fvt-finally
spec:
params:
- name: image_pull_policy
type: string

- name: finalize
type: string
default: "true"
description: "Finalize the record in the FVT database"

- name: deprovision
type: string
default: "true"
description: "Enable deprovision of the target cluster & associated resources following the completion of this pipeline"

tasks:
# Collect must-gather
# -------------------------------------------------------------------------
- name: must-gather
timeout: "0"
taskRef:
kind: Task
name: mas-devops-must-gather
params:
- name: base_output_dir
value: "/mustgather/$(context.pipelineRun.name)"
workspaces:
- name: mustgather
workspace: shared-mustgather

# Finalize the record in the FVT database
# -------------------------------------------------------------------------
- name: finalize
timeout: "0"
taskRef:
kind: Task
name: mas-fvt-finalize
runAfter:
- must-gather
when:
- input: $(params.finalize)
operator: in
values: ["true", "True"]

# Delete IBM Cloud services
# -------------------------------------------------------------------------
# COS
{{ lookup('template', 'taskdefs/dependencies/fvt-deprovision-cos.yml.j2') | indent(4) }}
when:
- input: $(params.deprovision)
operator: in
values: ["true", "True"]
runAfter:
- finalize

# Event Streams
{{ lookup('template', 'taskdefs/dependencies/fvt-deprovision-kafka.yml.j2') | indent(4) }}
when:
- input: $(params.deprovision)
operator: in
values: ["true", "True"]
runAfter:
- finalize

# Delete the named cluster
# -------------------------------------------------------------------------
- name: deprovision-cluster
timeout: "0"
taskRef:
kind: Task
name: mas-fvt-deprovision-ocp
when:
- input: $(params.deprovision)
operator: in
values: ["true", "True"]
runAfter:
- cos-deprovision
- kafka-deprovision

workspaces:
# Shared storage to hold mustgather output for tasks
- name: shared-mustgather
# The generated configuration files
- name: shared-configs
73 changes: 12 additions & 61 deletions tekton/src/pipelines/fvt-launcher.yml.j2
Original file line number Diff line number Diff line change
Expand Up @@ -921,7 +921,7 @@ spec:
- name: max_retries
value: "25"
- name: ignore_failure
value: "True"
value: "False"
when:
- input: $(params.sync_with_install)
operator: in
Expand All @@ -934,70 +934,21 @@ spec:
- approval-visualinspection
- launchivt-manage

# Collect must-gather
# -------------------------------------------------------------------------
- name: must-gather
finally:
- name: launchfvt-finally
timeout: "0"
taskRef:
kind: Task
name: mas-devops-must-gather
name: mas-launchfvt-finally
params:
- name: base_output_dir
value: "/mustgather/$(context.pipelineRun.name)"
workspaces:
- name: mustgather
workspace: shared-mustgather
runAfter:
- waitfor-installation

# Finalize the record in the FVT database
# -------------------------------------------------------------------------
- name: finalize
timeout: "0"
taskRef:
kind: Task
name: mas-fvt-finalize
runAfter:
- must-gather
when:
- input: $(params.finalize)
operator: in
values: ["true", "True"]

# Delete IBM Cloud services
# -------------------------------------------------------------------------
# COS
{{ lookup('template', 'taskdefs/dependencies/fvt-deprovision-cos.yml.j2') | indent(4) }}
when:
- input: $(params.deprovision)
operator: in
values: ["true", "True"]
runAfter:
- finalize

# Event Streams
{{ lookup('template', 'taskdefs/dependencies/fvt-deprovision-kafka.yml.j2') | indent(4) }}
when:
- input: $(params.deprovision)
operator: in
values: ["true", "True"]
runAfter:
- finalize

# Delete the named cluster
# -------------------------------------------------------------------------
- name: deprovision-cluster
timeout: "0"
taskRef:
kind: Task
name: mas-fvt-deprovision-ocp
when:
- input: $(params.deprovision)
operator: in
values: ["true", "True"]
runAfter:
- cos-deprovision
- kafka-deprovision
- name: deprovision
value: "$(params.deprovision)"
- name: finalize
value: "$(params.finalize)"
- name: image_pull_policy
value: $(params.image_pull_policy)
- name: pipelinerun_name
value: "$(params.mas_instance_id)-fvt-finally"

workspaces:
# Shared storage to hold mustgather output for tasks
Expand Down
47 changes: 47 additions & 0 deletions tekton/src/tasks/fvt-launcher/launchfvt-finally.yml.j2
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
---
apiVersion: tekton.dev/v1beta1
kind: Task
metadata:
name: mas-launchfvt-finally
spec:
params:
- name: finalize
type: string
- name: deprovision
type: string
- name: image_pull_policy
type: string
default: IfNotPresent
- name: pipelinerun_name
type: string

steps:
- name: start-pipelinerun
image: quay.io/ibmmas/cli:latest
imagePullPolicy: $(params.image_pull_policy)
command:
- ansible-playbook
- /masfvt/finally.yml
env:
- name: FINALIZE
value: $(params.finalize)
- name: DEPROVISION
value: $(params.deprovision)
- name: IMAGE_PULL_POLICY
value: $(params.image_pull_policy)
- name: PIPELINERUN_NAME
value: $(params.pipelinerun_name)
- name: PIPELINERUN_NAMESPACE
value: $(context.taskRun.namespace)
- name: DEVOPS_BUILD_NUMBER
valueFrom:
secretKeyRef:
name: mas-devops
key: DEVOPS_BUILD_NUMBER
optional: true
- name: MAS_INSTANCE_ID
valueFrom:
secretKeyRef:
name: mas-devops
key: MAS_INSTANCE_ID
optional: false

0 comments on commit 7f7f861

Please sign in to comment.