[minor] Setup Db2 Backup job for Disaster recovery scenario and Compliance/SLA in gitops

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2024-11-27T10:44:24Z",
+  "generated_at": "2024-11-28T11:01:25Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -182,7 +182,7 @@
         "hashed_secret": "1459943ba5fd876f7ef6e48f566a40b448a2bf08",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 449,
+        "line_number": 478,
         "type": "Secret Keyword",
         "verified_result": null
       }

image/cli/mascli/functions/gitops_cos

@@ -229,6 +229,11 @@ function gitops_cos() {
       sensitive = false
     }
 
+    output "s3db2" {
+      value     = module.s3db2
+      sensitive = false
+    }
+
     module "s3c" {
       name_prefix = local.name_prefix
       source = "git::https://git:[email protected]/maximoappsuite/mas-iac-aws-s3.git//module?ref=1.0.1"
@@ -256,6 +261,15 @@ function gitops_cos() {
       force_bucket_destroy    = true
     }
 
+    module "s3db2" {
+      name_prefix = local.name_prefix
+      source = "git::https://git:[email protected]/maximoappsuite/mas-iac-aws-s3.git//module?ref=1.0.1"
+      s3_bucket               = "${CLUSTER_ID}-${MAS_INSTANCE_ID}-s3db2"
+      s3_encryption           = true
+      s3_encryption_algorithm = "AES256"
+      force_bucket_destroy    = true
+    }
+
     output "s3c_access_point" {
       value     = module.s3c_access_point
       sensitive = false
@@ -271,6 +285,11 @@ function gitops_cos() {
       sensitive = false
     }
 
+    output "s3db2_access_point" {
+      value     = module.s3db2_access_point
+      sensitive = false
+    }
+
     module "s3c_access_point" {
       source = "git::https://git:[email protected]/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.10"
       name_prefix = local.name_prefix
@@ -318,6 +337,22 @@ function gitops_cos() {
       secret_suffix                  = "manage_logging/s3secretkey"
       manage_secret_format           = true
     }
+
+    module "s3db2_access_point" {
+      source = "git::https://git:[email protected]/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.10"
+      name_prefix = local.name_prefix
+      s3_access_point_bucket_id      = module.s3db2.s3_bucket_id
+      s3_bucket_arn                  = module.s3db2.s3_bucket_arn
+      s3_bucket_region               = module.s3db2.s3_bucket_region
+      s3_access_point_name           = "${CLUSTER_ID}-${MAS_INSTANCE_ID}-s3db2-access"
+      s3_access_point_policy_actions = [ "s3:DeleteObject", "s3:GetObject", "s3:GetObjectAcl", "s3:PutObject", "s3:PutObjectAcl", "s3:ListBucket" ]
+      s3_access_point_user           = "${CLUSTER_ID}-${MAS_INSTANCE_ID}-s3db2-user"
+      mas_cluster_id                 = "$CLUSTER_ID"
+      mas_instance_id                = "$MAS_INSTANCE_ID"
+      secret_recovery_days           = 0
+      secret_suffix                  = "db2_backup"
+      manage_secret_format           = true
+    }
 EOF
 
     cat > $TEMP_DIR/provider.tf <<EOF

image/cli/mascli/functions/gitops_db2u_database

@@ -61,6 +61,12 @@ IBM DB2U:
       --db2-addons-audit-config-yaml ${COLOR_YELLOW}DB2_ADDONS_AUDIT_CONFIG_YAML${TEXT_RESET}                 Yaml file containing the db2ucluster addons audit config section.
       --jdbc-route ${COLOR_YELLOW}JDBC_ROUTE${TEXT_RESET}                                                     By default routes are not exposed to public. To expose route, set this to public.
       --db2-timezone ${COLOR_YELLOW}DB2_TIMEZONE${TEXT_RESET}                                                 DB2 DB Timezone (optional, DB2 default of "UTC" will be used if not specified)
+      --db2-backup-bucket-name ${COLOR_YELLOW}DB2_BACKUP_BUCKET_NAME${TEXT_RESET}                             COS Bucket name where DB2 backup is stored
+      --db2-backup-bucket-endpoint ${COLOR_YELLOW}DB2_BACKUP_BUCKET_ENDPOINT${TEXT_RESET}                     COS Bucket endpoint of the bucket where DB2 backup is stored
+      --db2-backup-bucket-access-key-secret ${COLOR_YELLOW}SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY${TEXT_RESET} Secret name of DB2 COS bucket's access key
+      --db2-backup-bucket-secret-key-secret ${COLOR_YELLOW}SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY${TEXT_RESET} Secret name of DB2 COS bucket's secret key
+      --db2-backup-notify-slack-url ${COLOR_YELLOW}DB2_BACKUP_NOTIFY_SLACK_URL${TEXT_RESET}                   Slack URL to notify DB2 backup failures
+      --db2-backup-icd-auth-key ${COLOR_YELLOW}SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY${TEXT_RESET}                ICD auth key used to create ICD incidents when DB2 backup fails
 
 Secrets Manager:
       --secrets-path ${COLOR_YELLOW}SECRETS_PATH${TEXT_RESET}                     Secrets Manager path
@@ -272,6 +278,24 @@ function gitops_db2u_database_noninteractive() {
       --db2-timezone)
         export DB2_TIMEZONE=$1 && shift
         ;;
+      --db2-backup-bucket-name)
+        export DB2_BACKUP_BUCKET_NAME$1 && shift
+        ;;
+      --db2-backup-bucket-endpoint)
+        export DB2_BACKUP_BUCKET_ENDPOINT=$1 && shift
+        ;;
+      --db2-backup-bucket-access-key-secret)
+        export SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY=$1 && shift
+        ;;
+      --db2-backup-bucket-secret-key-secret)
+        export SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY=$1 && shift
+        ;;
+      --db2-backup-notify-slack-url)
+        export DB2_BACKUP_NOTIFY_SLACK_URL=$1 && shift
+        ;;
+      --db2-backup-icd-auth-key)
+        export SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY=$1 && shift
+        ;;
 
 
       # Automatic GitHub Push
@@ -328,6 +352,8 @@ function gitops_db2u_database_noninteractive() {
   [[ -z "$DB2_LOGS_STORAGE_CLASS" ]] && gitops_db2u_database_help "DB2_LOGS_STORAGE_CLASS is not set"
   [[ -z "$MAS_INSTANCE_ID" ]] && gitops_db2u_database_help "MAS_INSTANCE_ID is not set"
   [[ -z "$MAS_APP_ID" ]] && gitops_db2u_database_help "MAS_APP_ID is not set"
+  [[ -z "$DB2_BACKUP_NOTIFY_SLACK_URL" ]] && gitops_db2u_database_help "DB2_BACKUP_NOTIFY_SLACK_URL is not set"
+  [[ -z "$ICD_AUTH_KEY" ]] && gitops_db2u_database_help "ICD_AUTH_KEY is not set"
 
   #MAS
   [[ -z "$MAS_INSTANCE_ID" ]] && gitops_db2u_database_help "MAS_INSTANCE_ID is not set"
@@ -366,6 +392,9 @@ function gitops_db2u_database() {
   # as both of these scrtipts modify the same file
   GIT_LOCK_BRANCH=$(git_lock_branch_name "gitops-db2u-database" "${ACCOUNT_ID}" "${CLUSTER_ID}" "${MAS_INSTANCE_ID}")
 
+  export SECRET_NAME_DB2_BACKUP=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}${MAS_INSTANCE_ID}${SECRETS_KEY_SEPERATOR}db2_backup
+  export SECRET_NAME_ICD_AUTH_KEY=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}${MAS_INSTANCE_ID}${SECRETS_KEY_SEPERATOR}icd_auth_key
+
   #Defaults
   if [[ -z $DB2_TLS_VERSION ]]; then
     export DB2_TLS_VERSION=TLSv1.2
@@ -454,6 +483,21 @@ DB2_WORKLOAD: '${DB2_WORKLOAD}'"
   if [[ -z $DB2_INSTANCE_NAME ]]; then
     export DB2_INSTANCE_NAME=db2wh-${MAS_INSTANCE_ID}-${MAS_APP_ID}
   fi
+  if [[ -z $DB2_BACKUP_BUCKET_NAME ]]; then
+    export DB2_BACKUP_BUCKET_NAME=${SECRET_NAME_DB2_BACKUP}#bucketName
+  fi
+  if [[ -z $SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY ]]; then
+    export SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY=${SECRET_NAME_DB2_BACKUP}#accessKey
+  fi
+  if [[ -z $SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY ]]; then
+    export SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY=${SECRET_NAME_DB2_BACKUP}#access_secret_key
+  fi
+  if [[ -z $DB2_BACKUP_BUCKET_ENDPOINT ]]; then
+    export DB2_BACKUP_BUCKET_ENDPOINT=${SECRET_NAME_DB2_BACKUP}#endpointURL
+  fi
+  if [[ -z $SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY ]]; then
+    export SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY=${SECRET_NAME_ICD_AUTH_KEY}#icd_auth_key
+  fi
   export DB2_NAMESPACE="db2u-${MAS_INSTANCE_ID}"
   export DB2_DBNAME=${DB2_DBNAME:-"BLUDB"}
   export JDBC_ROUTE=${JDBC_ROUTE:-"default"}
@@ -547,13 +591,22 @@ DB2_WORKLOAD: '${DB2_WORKLOAD}'"
   echo_reset_dim "db2-tolerate-effect ........................... ${COLOR_MAGENTA}${DB2_TOLERATE_EFFECT}"
   echo_reset_dim "JDBC_ROUTE .................................... ${COLOR_MAGENTA}${JDBC_ROUTE}"
   echo_reset_dim "db2-timezone .................................. ${COLOR_MAGENTA}${DB2_TIMEZONE}"
+  echo_reset_dim "db2-backup-bucket-name ........................ ${COLOR_MAGENTA}${DB2_BACKUP_BUCKET_NAME}"
+  echo_reset_dim "db2-backup-bucket-endpoint .................... ${COLOR_MAGENTA}${DB2_BACKUP_BUCKET_ENDPOINT}"
+  echo_reset_dim "db2-backup-bucket-access-key-secret............ ${COLOR_MAGENTA}${SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY}"
+  echo_reset_dim "db2-backup-bucket-secret-key-secret............ ${COLOR_MAGENTA}${SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY}"
+  echo_reset_dim "db2-backup-notify-slack-url.................... ${COLOR_MAGENTA}${DB2_BACKUP_NOTIFY_SLACK_URL}"
+  echo_reset_dim "db2-backup-icd-auth-key........................ ${COLOR_MAGENTA}${SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY}"
   reset_colors
 
 
   AVP_TYPE=aws  # Support for IBM will be added later
   sm_login
 
   export SECRET_KEY_CLUSTER_DOMAIN=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}cluster_domain#cluster_domain
+  TAGS="[{\"Key\": \"source\", \"Value\": \"gitops_db2u_database\"}, {\"Key\": \"account\", \"Value\": \"${ACCOUNT_ID}\"}, {\"Key\": \"cluster\", \"Value\": \"${CLUSTER_ID}\"}]"
+
+  sm_update_secret ${SECRET_NAME_ICD_AUTH_KEY} "{\"icd_auth_key\": \"${ICD_AUTH_KEY}\"}" "${TAGS}"
 
   TEMP_DIR=$GITOPS_WORKING_DIR/tmp-db2u-database
   mkdir -p $TEMP_DIR

image/cli/mascli/functions/gitops_deprovision_cos

@@ -233,6 +233,11 @@ function gitops_deprovision_cos() {
       sensitive = false
     }
 
+    output "s3db2" {
+      value     = module.s3db2
+      sensitive = false
+    }
+
     module "s3c" {
       name_prefix = local.name_prefix
       source = "git::https://git:[email protected]/maximoappsuite/mas-iac-aws-s3.git//module?ref=1.0.1"
@@ -260,6 +265,15 @@ function gitops_deprovision_cos() {
       force_bucket_destroy    = true
     }
 
+    module "s3db2" {
+      name_prefix = local.name_prefix
+      source = "git::https://git:[email protected]/maximoappsuite/mas-iac-aws-s3.git//module?ref=1.0.1"
+      s3_bucket               = "${CLUSTER_ID}-${MAS_INSTANCE_ID}-s3db2"
+      s3_encryption           = true
+      s3_encryption_algorithm = "AES256"
+      force_bucket_destroy    = true
+    }
+
     output "s3c_access_point" {
       value     = module.s3c_access_point
       sensitive = false
@@ -275,8 +289,13 @@ function gitops_deprovision_cos() {
       sensitive = false
     }
 
+    output "s3db2_access_point" {
+      value     = module.s3db2_access_point
+      sensitive = false
+    }
+
     module "s3c_access_point" {
-      source = "git::https://git:[email protected]/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.8"
+      source = "git::https://git:[email protected]/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.10"
       name_prefix = local.name_prefix
       s3_access_point_bucket_id      = module.s3c.s3_bucket_id
       s3_bucket_arn                  = module.s3c.s3_bucket_arn
@@ -292,7 +311,7 @@ function gitops_deprovision_cos() {
     }
 
     module "s3a_access_point" {
-      source = "git::https://git:[email protected]/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.8"
+      source = "git::https://git:[email protected]/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.10"
       name_prefix = local.name_prefix
       s3_access_point_bucket_id      = module.s3a.s3_bucket_id
       s3_bucket_arn                  = module.s3a.s3_bucket_arn
@@ -308,7 +327,7 @@ function gitops_deprovision_cos() {
     }
 
     module "s3l_access_point" {
-      source = "git::https://git:[email protected]/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.8"
+      source = "git::https://git:[email protected]/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.10"
       name_prefix = local.name_prefix
       s3_access_point_bucket_id      = module.s3l.s3_bucket_id
       s3_bucket_arn                  = module.s3l.s3_bucket_arn
@@ -322,6 +341,22 @@ function gitops_deprovision_cos() {
       secret_suffix                  = "manage_logging/s3secretkey"
       manage_secret_format           = true
     }
+
+    module "s3db2_access_point" {
+      source = "git::https://git:[email protected]/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.10"
+      name_prefix = local.name_prefix
+      s3_access_point_bucket_id      = module.s3db2.s3_bucket_id
+      s3_bucket_arn                  = module.s3db2.s3_bucket_arn
+      s3_bucket_region               = module.s3db2.s3_bucket_region
+      s3_access_point_name           = "${CLUSTER_ID}-${MAS_INSTANCE_ID}-s3db2-access"
+      s3_access_point_policy_actions = [ "s3:DeleteObject", "s3:GetObject", "s3:GetObjectAcl", "s3:PutObject", "s3:PutObjectAcl", "s3:ListBucket" ]
+      s3_access_point_user           = "${CLUSTER_ID}-${MAS_INSTANCE_ID}-s3db2-user"
+      mas_cluster_id                 = "$CLUSTER_ID"
+      mas_instance_id                = "$MAS_INSTANCE_ID"
+      secret_recovery_days           = 0
+      secret_suffix                  = "db2_backup"
+      manage_secret_format           = true
+    }
 EOF
 
     cat > $TEMP_DIR/provider.tf <<EOF

image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2

@@ -69,4 +69,11 @@ storage_class_definitions:
   {% filter indent(width=2) -%}
   {{STORAGE_CLASS_DEFINITIONS}}
   {%- endfilter %}
-{% endif %}
+{% endif %}
+
+db2_backup_bucket_name: <path:{{ SECRETS_PATH }}:{{ DB2_BACKUP_BUCKET_NAME }}>
+db2_backup_bucket_endpoint: <path:{{ SECRETS_PATH }}:{{ DB2_BACKUP_BUCKET_ENDPOINT }}>
+db2_backup_bucket_access_key: <path:{{ SECRETS_PATH }}:{{ SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY }}>
+db2_backup_bucket_secret_key: <path:{{ SECRETS_PATH }}:{{ SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY }}>
+db2_backup_notify_slack_url: {{DB2_BACKUP_NOTIFY_SLACK_URL}}
+db2_backup_icd_auth_key: <path:{{ SECRETS_PATH }}:{{ SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY }}>

tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2

@@ -209,6 +209,24 @@ spec:
     - name: db2_timezone
       type: string
       default: ""
+    - name: db2_backup_bucket_name
+      type: string
+      default: ""
+    - name: db2_backup_bucket_endpoint
+      type: string
+      default: ""
+    - name: db2_backup_bucket_access_key
+      type: string
+      default: ""
+    - name: db2_backup_bucket_secret_key
+      type: string
+      default: ""
+    - name: db2_backup_notify_slack_url
+      type: string
+      default: ""
+    - name: db2_backup_icd_auth_key
+      type: string
+      default: ""
 
     - name: custom_labels
       type: string
@@ -651,6 +669,18 @@ spec:
           value: $(params.jdbc_route_iot)
         - name: db2_timezone
           value: $(params.db2_timezone)
+        - name: db2_backup_bucket_name
+          value: $(params.db2_backup_bucket_name)
+        - name: db2_backup_bucket_endpoint
+          value: $(params.db2_backup_bucket_endpoint)
+        - name: db2_backup_bucket_access_key
+          value: $(params.db2_backup_bucket_access_key)
+        - name: db2_backup_bucket_secret_key
+          value: $(params.db2_backup_bucket_secret_key)
+        - name: db2_backup_notify_slack_url
+          value: $(params.db2_backup_notify_slack_url)
+        - name: db2_backup_icd_auth_key
+          value: $(params.db2_backup_icd_auth_key)
 
       workspaces:
         - name: configs
@@ -809,6 +839,18 @@ spec:
           value: $(params.jdbc_route_manage)
         - name: db2_timezone
           value: $(params.db2_timezone)
+        - name: db2_backup_bucket_name
+          value: $(params.db2_backup_bucket_name)
+        - name: db2_backup_bucket_endpoint
+          value: $(params.db2_backup_bucket_endpoint)
+        - name: db2_backup_bucket_access_key
+          value: $(params.db2_backup_bucket_access_key)
+        - name: db2_backup_bucket_secret_key
+          value: $(params.db2_backup_bucket_secret_key)
+        - name: db2_backup_notify_slack_url
+          value: $(params.db2_backup_notify_slack_url)
+        - name: db2_backup_icd_auth_key
+          value: $(params.db2_backup_icd_auth_key)
 
       workspaces:
         - name: configs

tekton/src/tasks/gitops/gitops-db2u-database.yml.j2

@@ -109,6 +109,24 @@ spec:
     - name: jdbc_route
       type: string
       default: ""
+    - name: db2_backup_bucket_name
+      type: string
+      default: ""
+    - name: db2_backup_bucket_endpoint
+      type: string
+      default: ""
+    - name: db2_backup_bucket_access_key
+      type: string
+      default: ""
+    - name: db2_backup_bucket_secret_key
+      type: string
+      default: ""
+    - name: db2_backup_notify_slack_url
+      type: string
+      default: ""
+    - name: db2_backup_icd_auth_key
+      type: string
+      default: ""
   stepTemplate:
     name: gitops-db2u-database
     env:
@@ -214,6 +232,18 @@ spec:
         value: $(params.mas_app_id)
       - name: JDBC_ROUTE
         value: $(params.jdbc_route)
+      - name: DB2_BACKUP_BUCKET_NAME
+        value: $(params.db2_backup_bucket_name)
+      - name: DB2_BACKUP_BUCKET_ENDPOINT
+        value: $(params.db2_backup_bucket_endpoint)
+      - name: SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY
+        value: $(params.db2_backup_bucket_access_key)
+      - name: SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY
+        value: $(params.db2_backup_bucket_secret_key)
+      - name: DB2_BACKUP_NOTIFY_SLACK_URL
+        value: $(params.db2_backup_notify_slack_url)
+      - name: SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY
+        value: $(params.db2_backup_icd_auth_key)
     envFrom:
       - configMapRef:
           name: environment-properties