From 44fbce89944b19a9ab3094fa8629dc5cfef199ec Mon Sep 17 00:00:00 2001 From: Nivedithaa Mahendran Date: Fri, 22 Nov 2024 15:55:11 +0530 Subject: [PATCH 01/14] Setup Db2 Backup job for Disaster recovery scenario and Compliance/SLA Updated function and configurations for DB2 backup Issue: #MASCORE-4470 --- .../cli/mascli/functions/gitops_db2u_database | 47 +++++++++++++++++++ .../db2-databases/ibm-db2u-database.yaml.j2 | 9 +++- .../pipelines/gitops/gitops-mas-apps.yml.j2 | 42 +++++++++++++++++ .../tasks/gitops/gitops-db2u-database.yml.j2 | 30 ++++++++++++ 4 files changed, 127 insertions(+), 1 deletion(-) diff --git a/image/cli/mascli/functions/gitops_db2u_database b/image/cli/mascli/functions/gitops_db2u_database index 3783c86b6f..a76b8fa0de 100644 --- a/image/cli/mascli/functions/gitops_db2u_database +++ b/image/cli/mascli/functions/gitops_db2u_database @@ -61,6 +61,12 @@ IBM DB2U: --db2-addons-audit-config-yaml ${COLOR_YELLOW}DB2_ADDONS_AUDIT_CONFIG_YAML${TEXT_RESET} Yaml file containing the db2ucluster addons audit config section. --jdbc-route ${COLOR_YELLOW}JDBC_ROUTE${TEXT_RESET} By default routes are not exposed to public. To expose route, set this to public. --db2-timezone ${COLOR_YELLOW}DB2_TIMEZONE${TEXT_RESET} DB2 DB Timezone (optional, DB2 default of "UTC" will be used if not specified) + --db2-backup-bucket-name ${COLOR_YELLOW}DB2_BACKUP_BUCKET_NAME${TEXT_RESET} COS Bucket name where DB2 backup is stored + --db2-backup-bucket-endpoint ${COLOR_YELLOW}DB2_BACKUP_BUCKET_ENDPOINT${TEXT_RESET} COS Bucket endpoint of the bucket where DB2 backup is stored + --db2-backup-bucket-access-key-secret ${COLOR_YELLOW}SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY${TEXT_RESET} Secret name of DB2 COS bucket's access key + --db2-backup-bucket-secret-key-secret ${COLOR_YELLOW}SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY${TEXT_RESET} Secret name of DB2 COS bucket's secret key + --db2-backup-notify-slack-url ${COLOR_YELLOW}DB2_BACKUP_NOTIFY_SLACK_URL${TEXT_RESET} Slack URL to notify DB2 backup failures + --db2-backup-icd-auth-key ${COLOR_YELLOW}SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY${TEXT_RESET} ICD auth key used to create ICD incidents when DB2 backup fails Secrets Manager: --secrets-path ${COLOR_YELLOW}SECRETS_PATH${TEXT_RESET} Secrets Manager path @@ -272,6 +278,24 @@ function gitops_db2u_database_noninteractive() { --db2-timezone) export DB2_TIMEZONE=$1 && shift ;; + --db2-backup-bucket-name) + export DB2_BACKUP_BUCKET_NAME$1 && shift + ;; + --db2-backup-bucket-endpoint) + export DB2_BACKUP_BUCKET_ENDPOINT=$1 && shift + ;; + --db2-backup-bucket-access-key-secret) + export SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY=$1 && shift + ;; + --db2-backup-bucket-secret-key-secret) + export SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY=$1 && shift + ;; + --db2-backup-notify-slack-url) + export DB2_BACKUP_NOTIFY_SLACK_URL=$1 && shift + ;; + --db2-backup-icd-auth-key) + export SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY=$1 && shift + ;; # Automatic GitHub Push @@ -329,6 +353,12 @@ function gitops_db2u_database_noninteractive() { [[ -z "$DB2_LOGS_STORAGE_CLASS" ]] && gitops_db2u_database_help "DB2_LOGS_STORAGE_CLASS is not set" [[ -z "$MAS_INSTANCE_ID" ]] && gitops_db2u_database_help "MAS_INSTANCE_ID is not set" [[ -z "$MAS_APP_ID" ]] && gitops_db2u_database_help "MAS_APP_ID is not set" + [[ -z "$DB2_BACKUP_BUCKET_NAME" ]] && gitops_db2u_database_help "DB2_BACKUP_BUCKET_NAME is not set" + [[ -z "$DB2_BACKUP_BUCKET_ENDPOINT" ]] && gitops_db2u_database_help "DB2_BACKUP_BUCKET_ENDPOINT is not set" + [[ -z "$DB2_BACKUP_NOTIFY_SLACK_URL" ]] && gitops_db2u_database_help "DB2_BACKUP_NOTIFY_SLACK_URL is not set" + # [[ -z "$SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY" ]] && gitops_db2u_database_help "SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY is not set" + # [[ -z "$SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY" ]] && gitops_db2u_database_help "SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY is not set" + # [[ -z "$SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY" ]] && gitops_db2u_database_help "SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY is not set" #MAS [[ -z "$MAS_INSTANCE_ID" ]] && gitops_db2u_database_help "MAS_INSTANCE_ID is not set" @@ -367,6 +397,8 @@ function gitops_db2u_database() { # as both of these scrtipts modify the same file GIT_LOCK_BRANCH=$(git_lock_branch_name "gitops-db2u-database" "${ACCOUNT_ID}" "${CLUSTER_ID}" "${MAS_INSTANCE_ID}") + export SECRET_NAME_DB2_BACKUP=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}${MAS_INSTANCE_ID}${SECRETS_KEY_SEPERATOR}db2_backup + #Defaults if [[ -z $DB2_TLS_VERSION ]]; then export DB2_TLS_VERSION=TLSv1.2 @@ -455,6 +487,15 @@ DB2_WORKLOAD: '${DB2_WORKLOAD}'" if [[ -z $DB2_INSTANCE_NAME ]]; then export DB2_INSTANCE_NAME=db2wh-${MAS_INSTANCE_ID}-${MAS_APP_ID} fi + if [[ -z $SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY ]]; then + export SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY=${SECRET_NAME_DB2_BACKUP}#cos_access_key + fi + if [[ -z $SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY ]]; then + export SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY=${SECRET_NAME_DB2_BACKUP}#cos_secret_key + fi + if [[ -z $SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY ]]; then + export SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY=${SECRET_NAME_DB2_BACKUP}#icd_auth_key + fi export DB2_NAMESPACE="db2u-${MAS_INSTANCE_ID}" export DB2_DBNAME=${DB2_DBNAME:-"BLUDB"} export JDBC_ROUTE=${JDBC_ROUTE:-"default"} @@ -548,6 +589,12 @@ DB2_WORKLOAD: '${DB2_WORKLOAD}'" echo_reset_dim "db2-tolerate-effect ........................... ${COLOR_MAGENTA}${DB2_TOLERATE_EFFECT}" echo_reset_dim "JDBC_ROUTE .................................... ${COLOR_MAGENTA}${JDBC_ROUTE}" echo_reset_dim "db2-timezone .................................. ${COLOR_MAGENTA}${DB2_TIMEZONE}" + echo_reset_dim "db2-backup-bucket-name ........................ ${COLOR_MAGENTA}${DB2_BACKUP_BUCKET_NAME}" + echo_reset_dim "db2-backup-bucket-endpoint .................... ${COLOR_MAGENTA}${DB2_BACKUP_BUCKET_ENDPOINT}" + echo_reset_dim "db2-backup-bucket-access-key-secret............ ${COLOR_MAGENTA}${SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY}" + echo_reset_dim "db2-backup-bucket-secret-key-secret............ ${COLOR_MAGENTA}${SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY}" + echo_reset_dim "db2-backup-notify-slack-url.................... ${COLOR_MAGENTA}${DB2_BACKUP_NOTIFY_SLACK_URL}" + echo_reset_dim "db2-backup-icd-auth-key........................ ${COLOR_MAGENTA}${SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY}" reset_colors diff --git a/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 b/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 index 9317a87958..903d1f64d1 100644 --- a/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 +++ b/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 @@ -69,4 +69,11 @@ storage_class_definitions: {% filter indent(width=2) -%} {{STORAGE_CLASS_DEFINITIONS}} {%- endfilter %} -{% endif %} \ No newline at end of file +{% endif %} + +db2_backup_bucket_name: {{DB2_BACKUP_BUCKET_NAME}} +db2_backup_bucket_endpoint: {{DB2_BUCKET_ENDPOINT}} +db2_backup_bucket_access_key: +db2_backup_bucket_secret_key: +db2_backup_notify_slack_url: {{DB2_BACKUP_NOTIFY_SLACK_URL}} +db2_backup_icd_auth_key: \ No newline at end of file diff --git a/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 b/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 index 29416d4bab..73d5dfda9c 100644 --- a/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 +++ b/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 @@ -209,6 +209,24 @@ spec: - name: db2_timezone type: string default: "" + - name: db2_backup_bucket_name + type: string + default: "" + - name: db2_backup_bucket_endpoint + type: string + default: "" + - name: db2_backup_bucket_access_key + type: string + default: "" + - name: db2_backup_bucket_secret_key + type: string + default: "" + - name: db2_backup_notify_slack_url + type: string + default: "" + - name: db2-backup-icd-auth-key + type: string + default: "" - name: custom_labels type: string @@ -651,6 +669,18 @@ spec: value: $(params.jdbc_route_iot) - name: db2_timezone value: $(params.db2_timezone) + - name: db2_backup_bucket_name + value: $(params.db2_backup_bucket_name) + - name: db2_backup_bucket_endpoint + value: $(params.db2_backup_bucket_endpoint) + - name: db2_backup_bucket_access_key + value: $(params.db2_backup_bucket_access_key) + - name: db2_backup_bucket_secret_key + value: $(params.db2_backup_bucket_secret_key) + - name: db2_backup_notify_slack_url + value: $(params.db2_backup_notify_slack_url) + - name: db2-backup-icd-auth-key + value: $(params.db2_backup_icd_auth_key) workspaces: - name: configs @@ -809,6 +839,18 @@ spec: value: $(params.jdbc_route_manage) - name: db2_timezone value: $(params.db2_timezone) + - name: db2_backup_bucket_name + value: $(params.db2_backup_bucket_name) + - name: db2_backup_bucket_endpoint + value: $(params.db2_backup_bucket_endpoint) + - name: db2_backup_bucket_access_key + value: $(params.db2_backup_bucket_access_key) + - name: db2_backup_bucket_secret_key + value: $(params.db2_backup_bucket_secret_key) + - name: db2_backup_notify_slack_url + value: $(params.db2_backup_notify_slack_url) + - name: db2_backup_icd_auth_key + value: $(params.db2_backup_icd_auth_key) workspaces: - name: configs diff --git a/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 b/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 index 956d48d50b..be94840fe1 100644 --- a/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 +++ b/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 @@ -109,6 +109,24 @@ spec: - name: jdbc_route type: string default: "" + - name: db2_backup_bucket_name + type: string + default: "" + - name: db2_backup_bucket_endpoint + type: string + default: "" + - name: db2_backup_bucket_access_key + type: string + default: "" + - name: db2_backup_bucket_secret_key + type: string + default: "" + - name: db2_backup_notify_slack_url + type: string + default: "" + - name: db2_backup_icd_auth_key + type: string + default: "" stepTemplate: name: gitops-db2u-database env: @@ -214,6 +232,18 @@ spec: value: $(params.mas_app_id) - name: JDBC_ROUTE value: $(params.jdbc_route) + - name: db2_backup_bucket_name + value: $(params.db2_backup_bucket_name) + - name: db2_backup_bucket_endpoint + value: $(params.db2_backup_bucket_endpoint) + - name: db2_backup_bucket_access_key + value: $(params.db2_backup_bucket_access_key) + - name: db2_backup_bucket_secret_key + value: $(params.db2_backup_bucket_secret_key) + - name: db2_backup_notify_slack_url + value: $(params.db2_backup_notify_slack_url) + - name: db2_backup_icd_auth_key + value: $(params.db2_backup_icd_auth_key) envFrom: - configMapRef: name: environment-properties From 0749d01247638c0d5101d4615394e3bf48ad59ad Mon Sep 17 00:00:00 2001 From: Nivedithaa Mahendran Date: Fri, 22 Nov 2024 20:36:02 +0530 Subject: [PATCH 02/14] Fixed detect-secrets issue --- .secrets.baseline | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 6fa752f9fe..762f54d068 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2024-11-16T12:02:00Z", + "generated_at": "2024-11-22T15:00:11Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -182,7 +182,7 @@ "hashed_secret": "1459943ba5fd876f7ef6e48f566a40b448a2bf08", "is_secret": false, "is_verified": false, - "line_number": 450, + "line_number": 482, "type": "Secret Keyword", "verified_result": null } From fecdb65db10a5920a111c9c1a9a34454d25b9db0 Mon Sep 17 00:00:00 2001 From: Nivedithaa Mahendran Date: Mon, 25 Nov 2024 14:52:32 +0530 Subject: [PATCH 03/14] Fixed icd_auth_key param --- tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 b/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 index 73d5dfda9c..9597dc3fac 100644 --- a/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 +++ b/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 @@ -679,7 +679,7 @@ spec: value: $(params.db2_backup_bucket_secret_key) - name: db2_backup_notify_slack_url value: $(params.db2_backup_notify_slack_url) - - name: db2-backup-icd-auth-key + - name: db2_backup_icd_auth_key value: $(params.db2_backup_icd_auth_key) workspaces: From 374f8fe26dca8c44ef08a65c70ac8dcc35d2a144 Mon Sep 17 00:00:00 2001 From: Nivedithaa Mahendran Date: Mon, 25 Nov 2024 15:15:53 +0530 Subject: [PATCH 04/14] updated icd_auth_key var --- tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 b/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 index 9597dc3fac..73ada0fe5d 100644 --- a/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 +++ b/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 @@ -224,7 +224,7 @@ spec: - name: db2_backup_notify_slack_url type: string default: "" - - name: db2-backup-icd-auth-key + - name: db2_backup_icd_auth_key type: string default: "" From 46ac454d4cbd479973ff8daa39251dfcf7b5e32f Mon Sep 17 00:00:00 2001 From: Nivedithaa Mahendran Date: Mon, 25 Nov 2024 15:55:25 +0530 Subject: [PATCH 05/14] updated env vars in pipeline --- tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 b/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 index be94840fe1..3361b9935f 100644 --- a/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 +++ b/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 @@ -232,17 +232,17 @@ spec: value: $(params.mas_app_id) - name: JDBC_ROUTE value: $(params.jdbc_route) - - name: db2_backup_bucket_name + - name: DB2_BACKUP_BUCKET_NAME value: $(params.db2_backup_bucket_name) - - name: db2_backup_bucket_endpoint + - name: DB2_BACKUP_BUCKET_ENDPOINT value: $(params.db2_backup_bucket_endpoint) - - name: db2_backup_bucket_access_key + - name: SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY value: $(params.db2_backup_bucket_access_key) - - name: db2_backup_bucket_secret_key + - name: SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY value: $(params.db2_backup_bucket_secret_key) - - name: db2_backup_notify_slack_url + - name: DB2_BACKUP_NOTIFY_SLACK_URL value: $(params.db2_backup_notify_slack_url) - - name: db2_backup_icd_auth_key + - name: SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY value: $(params.db2_backup_icd_auth_key) envFrom: - configMapRef: From 7bfc97da211c9d079925255b2224dee958b0b0e3 Mon Sep 17 00:00:00 2001 From: Nivedithaa Mahendran Date: Mon, 25 Nov 2024 16:55:29 +0530 Subject: [PATCH 06/14] update endpoint --- .../cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 b/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 index 903d1f64d1..5c578b86a7 100644 --- a/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 +++ b/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 @@ -72,7 +72,7 @@ storage_class_definitions: {% endif %} db2_backup_bucket_name: {{DB2_BACKUP_BUCKET_NAME}} -db2_backup_bucket_endpoint: {{DB2_BUCKET_ENDPOINT}} +db2_backup_bucket_endpoint: {{DB2_BACKUP_BUCKET_ENDPOINT}} db2_backup_bucket_access_key: db2_backup_bucket_secret_key: db2_backup_notify_slack_url: {{DB2_BACKUP_NOTIFY_SLACK_URL}} From 8251f84fd79c73a55a5fcc1e37520d96c1c393a6 Mon Sep 17 00:00:00 2001 From: Nivedithaa Mahendran Date: Wed, 27 Nov 2024 12:23:39 +0530 Subject: [PATCH 07/14] Updates --- .secrets.baseline | 4 +-- image/cli/mascli/functions/gitops_cos | 35 +++++++++++++++++++ .../cli/mascli/functions/gitops_db2u_database | 20 ++++++++--- .../db2-databases/ibm-db2u-database.yaml.j2 | 4 +-- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index fff8a0220e..f015b5aaf3 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2024-11-22T12:20:31Z", + "generated_at": "2024-11-27T06:47:49Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -182,7 +182,7 @@ "hashed_secret": "1459943ba5fd876f7ef6e48f566a40b448a2bf08", "is_secret": false, "is_verified": false, - "line_number": 482, + "line_number": 483, "type": "Secret Keyword", "verified_result": null } diff --git a/image/cli/mascli/functions/gitops_cos b/image/cli/mascli/functions/gitops_cos index bb2d8ce837..7bc2c4390b 100644 --- a/image/cli/mascli/functions/gitops_cos +++ b/image/cli/mascli/functions/gitops_cos @@ -229,6 +229,11 @@ function gitops_cos() { sensitive = false } + output "s3db2" { + value = module.s3db2 + sensitive = false + } + module "s3c" { name_prefix = local.name_prefix source = "git::https://git:$GITHUB_PAT@github.ibm.com/maximoappsuite/mas-iac-aws-s3.git//module?ref=1.0.1" @@ -256,6 +261,15 @@ function gitops_cos() { force_bucket_destroy = true } + module "s3db2" { + name_prefix = local.name_prefix + source = "git::https://git:$GITHUB_PAT@github.ibm.com/maximoappsuite/mas-iac-aws-s3.git//module?ref=1.0.1" + s3_bucket = "${CLUSTER_ID}-${MAS_INSTANCE_ID}-s3db2" + s3_encryption = true + s3_encryption_algorithm = "AES256" + force_bucket_destroy = true + } + output "s3c_access_point" { value = module.s3c_access_point sensitive = false @@ -271,6 +285,11 @@ function gitops_cos() { sensitive = false } + output "s3db2_access_point" { + value = module.s3db2_access_point + sensitive = false + } + module "s3c_access_point" { source = "git::https://git:$GITHUB_PAT@github.ibm.com/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.10" name_prefix = local.name_prefix @@ -318,6 +337,22 @@ function gitops_cos() { secret_suffix = "manage_logging/s3secretkey" manage_secret_format = true } + + module "s3db2_access_point" { + source = "git::https://git:$GITHUB_PAT@github.ibm.com/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.10" + name_prefix = local.name_prefix + s3_access_point_bucket_id = module.s3db2.s3_bucket_id + s3_bucket_arn = module.s3db2.s3_bucket_arn + s3_bucket_region = module.s3db2.s3_bucket_region + s3_access_point_name = "${CLUSTER_ID}-${MAS_INSTANCE_ID}-s3db2-access" + s3_access_point_policy_actions = [ "s3:DeleteObject", "s3:GetObject", "s3:GetObjectAcl", "s3:PutObject", "s3:PutObjectAcl", "s3:ListBucket" ] + s3_access_point_user = "${CLUSTER_ID}-${MAS_INSTANCE_ID}-s3db2-user" + mas_cluster_id = "$CLUSTER_ID" + mas_instance_id = "$MAS_INSTANCE_ID" + secret_recovery_days = 0 + secret_suffix = "db2_backup" + manage_secret_format = true + } EOF cat > $TEMP_DIR/provider.tf < +db2_backup_bucket_endpoint: db2_backup_bucket_access_key: db2_backup_bucket_secret_key: db2_backup_notify_slack_url: {{DB2_BACKUP_NOTIFY_SLACK_URL}} From 9deed65d9cc690d3afcbad6e5d6d5c699b2478e1 Mon Sep 17 00:00:00 2001 From: Nivedithaa Mahendran Date: Wed, 27 Nov 2024 16:23:59 +0530 Subject: [PATCH 08/14] Update key of icd_auth_key --- image/cli/mascli/functions/gitops_db2u_database | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/image/cli/mascli/functions/gitops_db2u_database b/image/cli/mascli/functions/gitops_db2u_database index bee9805f0e..55a08dc7ce 100644 --- a/image/cli/mascli/functions/gitops_db2u_database +++ b/image/cli/mascli/functions/gitops_db2u_database @@ -359,7 +359,7 @@ function gitops_db2u_database_noninteractive() { # [[ -z "$SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY" ]] && gitops_db2u_database_help "SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY is not set" # [[ -z "$SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY" ]] && gitops_db2u_database_help "SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY is not set" # [[ -z "$SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY" ]] && gitops_db2u_database_help "SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY is not set" - [[ -z "$DB2_BACKUP_ICD_AUTH_KEY" ]] && gitops_db2u_database_help "DB2_BACKUP_ICD_AUTH_KEY is not set" + [[ -z "$ICD_AUTH_KEY" ]] && gitops_db2u_database_help "ICD_AUTH_KEY is not set" #MAS [[ -z "$MAS_INSTANCE_ID" ]] && gitops_db2u_database_help "MAS_INSTANCE_ID is not set" @@ -501,7 +501,7 @@ DB2_WORKLOAD: '${DB2_WORKLOAD}'" export DB2_BACKUP_BUCKET_ENDPOINT=${SECRET_NAME_DB2_BACKUP}#endpointURL fi if [[ -z $SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY ]]; then - export SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}${MAS_INSTANCE_ID}${SECRETS_KEY_SEPERATOR}icd_auth_key + export SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}${MAS_INSTANCE_ID}${SECRETS_KEY_SEPERATOR}icd_auth_key#icd_auth_key fi export DB2_NAMESPACE="db2u-${MAS_INSTANCE_ID}" export DB2_DBNAME=${DB2_DBNAME:-"BLUDB"} @@ -611,7 +611,7 @@ DB2_WORKLOAD: '${DB2_WORKLOAD}'" export SECRET_KEY_CLUSTER_DOMAIN=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}cluster_domain#cluster_domain TAGS="[{\"Key\": \"source\", \"Value\": \"gitops_db2u_database\"}, {\"Key\": \"account\", \"Value\": \"${ACCOUNT_ID}\"}, {\"Key\": \"cluster\", \"Value\": \"${CLUSTER_ID}\"}]" - sm_update_secret ${SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY} "{\"icd_auth_key\": \"${DB2_BACKUP_ICD_AUTH_KEY}\"}" "${TAGS}" + sm_update_secret ${SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY} "{\"icd_auth_key\": \"${ICD_AUTH_KEY}\"}" "${TAGS}" TEMP_DIR=$GITOPS_WORKING_DIR/tmp-db2u-database mkdir -p $TEMP_DIR From bd711f79afc749a0f38764f95ae0d254e885eed8 Mon Sep 17 00:00:00 2001 From: Nivedithaa Mahendran Date: Wed, 27 Nov 2024 17:25:19 +0530 Subject: [PATCH 09/14] Updates to icd auth key secret format --- .secrets.baseline | 4 ++-- image/cli/mascli/functions/gitops_db2u_database | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 4c31cb79c6..9e10a5a37e 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2024-11-27T06:47:49Z", + "generated_at": "2024-11-27T11:55:06Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -182,7 +182,7 @@ "hashed_secret": "1459943ba5fd876f7ef6e48f566a40b448a2bf08", "is_secret": false, "is_verified": false, - "line_number": 483, + "line_number": 484, "type": "Secret Keyword", "verified_result": null } diff --git a/image/cli/mascli/functions/gitops_db2u_database b/image/cli/mascli/functions/gitops_db2u_database index 55a08dc7ce..918f1c840b 100644 --- a/image/cli/mascli/functions/gitops_db2u_database +++ b/image/cli/mascli/functions/gitops_db2u_database @@ -399,6 +399,7 @@ function gitops_db2u_database() { GIT_LOCK_BRANCH=$(git_lock_branch_name "gitops-db2u-database" "${ACCOUNT_ID}" "${CLUSTER_ID}" "${MAS_INSTANCE_ID}") export SECRET_NAME_DB2_BACKUP=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}${MAS_INSTANCE_ID}${SECRETS_KEY_SEPERATOR}db2_backup + export SECRET_NAME_ICD_AUTH_KEY=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}${MAS_INSTANCE_ID}${SECRETS_KEY_SEPERATOR}icd_auth_key #Defaults if [[ -z $DB2_TLS_VERSION ]]; then @@ -501,7 +502,7 @@ DB2_WORKLOAD: '${DB2_WORKLOAD}'" export DB2_BACKUP_BUCKET_ENDPOINT=${SECRET_NAME_DB2_BACKUP}#endpointURL fi if [[ -z $SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY ]]; then - export SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}${MAS_INSTANCE_ID}${SECRETS_KEY_SEPERATOR}icd_auth_key#icd_auth_key + export SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY=${SECRET_NAME_ICD_AUTH_KEY}#icd_auth_key fi export DB2_NAMESPACE="db2u-${MAS_INSTANCE_ID}" export DB2_DBNAME=${DB2_DBNAME:-"BLUDB"} @@ -611,7 +612,7 @@ DB2_WORKLOAD: '${DB2_WORKLOAD}'" export SECRET_KEY_CLUSTER_DOMAIN=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}cluster_domain#cluster_domain TAGS="[{\"Key\": \"source\", \"Value\": \"gitops_db2u_database\"}, {\"Key\": \"account\", \"Value\": \"${ACCOUNT_ID}\"}, {\"Key\": \"cluster\", \"Value\": \"${CLUSTER_ID}\"}]" - sm_update_secret ${SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY} "{\"icd_auth_key\": \"${ICD_AUTH_KEY}\"}" "${TAGS}" + sm_update_secret ${SECRET_NAME_ICD_AUTH_KEY} "{\"icd_auth_key\": \"${ICD_AUTH_KEY}\"}" "${TAGS}" TEMP_DIR=$GITOPS_WORKING_DIR/tmp-db2u-database mkdir -p $TEMP_DIR From 800af043e71d9e188deec580b6d4a7e7b4b0e41a Mon Sep 17 00:00:00 2001 From: Nivedithaa Mahendran Date: Wed, 27 Nov 2024 22:27:25 +0530 Subject: [PATCH 10/14] updates gitops deprovision cos --- .../mascli/functions/gitops_deprovision_cos | 41 +++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/image/cli/mascli/functions/gitops_deprovision_cos b/image/cli/mascli/functions/gitops_deprovision_cos index 7d4441f2e6..c0c6a2ace2 100644 --- a/image/cli/mascli/functions/gitops_deprovision_cos +++ b/image/cli/mascli/functions/gitops_deprovision_cos @@ -233,6 +233,11 @@ function gitops_deprovision_cos() { sensitive = false } + output "s3db2" { + value = module.s3db2 + sensitive = false + } + module "s3c" { name_prefix = local.name_prefix source = "git::https://git:$GITHUB_PAT@github.ibm.com/maximoappsuite/mas-iac-aws-s3.git//module?ref=1.0.1" @@ -260,6 +265,15 @@ function gitops_deprovision_cos() { force_bucket_destroy = true } + module "s3db2" { + name_prefix = local.name_prefix + source = "git::https://git:$GITHUB_PAT@github.ibm.com/maximoappsuite/mas-iac-aws-s3.git//module?ref=1.0.1" + s3_bucket = "${CLUSTER_ID}-${MAS_INSTANCE_ID}-s3db2" + s3_encryption = true + s3_encryption_algorithm = "AES256" + force_bucket_destroy = true + } + output "s3c_access_point" { value = module.s3c_access_point sensitive = false @@ -275,8 +289,13 @@ function gitops_deprovision_cos() { sensitive = false } + output "s3db2_access_point" { + value = module.s3db2_access_point + sensitive = false + } + module "s3c_access_point" { - source = "git::https://git:$GITHUB_PAT@github.ibm.com/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.8" + source = "git::https://git:$GITHUB_PAT@github.ibm.com/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.10" name_prefix = local.name_prefix s3_access_point_bucket_id = module.s3c.s3_bucket_id s3_bucket_arn = module.s3c.s3_bucket_arn @@ -292,7 +311,7 @@ function gitops_deprovision_cos() { } module "s3a_access_point" { - source = "git::https://git:$GITHUB_PAT@github.ibm.com/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.8" + source = "git::https://git:$GITHUB_PAT@github.ibm.com/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.10" name_prefix = local.name_prefix s3_access_point_bucket_id = module.s3a.s3_bucket_id s3_bucket_arn = module.s3a.s3_bucket_arn @@ -308,7 +327,7 @@ function gitops_deprovision_cos() { } module "s3l_access_point" { - source = "git::https://git:$GITHUB_PAT@github.ibm.com/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.8" + source = "git::https://git:$GITHUB_PAT@github.ibm.com/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.10" name_prefix = local.name_prefix s3_access_point_bucket_id = module.s3l.s3_bucket_id s3_bucket_arn = module.s3l.s3_bucket_arn @@ -322,6 +341,22 @@ function gitops_deprovision_cos() { secret_suffix = "manage_logging/s3secretkey" manage_secret_format = true } + + module "s3db2_access_point" { + source = "git::https://git:$GITHUB_PAT@github.ibm.com/maximoappsuite/mas-iac-aws-s3-access-point.git//module?ref=1.0.10" + name_prefix = local.name_prefix + s3_access_point_bucket_id = module.s3db2.s3_bucket_id + s3_bucket_arn = module.s3db2.s3_bucket_arn + s3_bucket_region = module.s3db2.s3_bucket_region + s3_access_point_name = "${CLUSTER_ID}-${MAS_INSTANCE_ID}-s3db2-access" + s3_access_point_policy_actions = [ "s3:DeleteObject", "s3:GetObject", "s3:GetObjectAcl", "s3:PutObject", "s3:PutObjectAcl", "s3:ListBucket" ] + s3_access_point_user = "${CLUSTER_ID}-${MAS_INSTANCE_ID}-s3db2-user" + mas_cluster_id = "$CLUSTER_ID" + mas_instance_id = "$MAS_INSTANCE_ID" + secret_recovery_days = 0 + secret_suffix = "db2_backup" + manage_secret_format = true + } EOF cat > $TEMP_DIR/provider.tf < Date: Thu, 28 Nov 2024 16:31:38 +0530 Subject: [PATCH 11/14] updates --- .secrets.baseline | 4 ++-- image/cli/mascli/functions/gitops_db2u_database | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 85df52da70..af05e138e7 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2024-11-28T10:56:22Z", + "generated_at": "2024-11-28T11:01:25Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -182,7 +182,7 @@ "hashed_secret": "1459943ba5fd876f7ef6e48f566a40b448a2bf08", "is_secret": false, "is_verified": false, - "line_number": 483, + "line_number": 478, "type": "Secret Keyword", "verified_result": null } diff --git a/image/cli/mascli/functions/gitops_db2u_database b/image/cli/mascli/functions/gitops_db2u_database index 6553dbf1f4..25e5b47cd4 100644 --- a/image/cli/mascli/functions/gitops_db2u_database +++ b/image/cli/mascli/functions/gitops_db2u_database @@ -352,12 +352,7 @@ function gitops_db2u_database_noninteractive() { [[ -z "$DB2_LOGS_STORAGE_CLASS" ]] && gitops_db2u_database_help "DB2_LOGS_STORAGE_CLASS is not set" [[ -z "$MAS_INSTANCE_ID" ]] && gitops_db2u_database_help "MAS_INSTANCE_ID is not set" [[ -z "$MAS_APP_ID" ]] && gitops_db2u_database_help "MAS_APP_ID is not set" - # [[ -z "$DB2_BACKUP_BUCKET_NAME" ]] && gitops_db2u_database_help "DB2_BACKUP_BUCKET_NAME is not set" - # [[ -z "$DB2_BACKUP_BUCKET_ENDPOINT" ]] && gitops_db2u_database_help "DB2_BACKUP_BUCKET_ENDPOINT is not set" [[ -z "$DB2_BACKUP_NOTIFY_SLACK_URL" ]] && gitops_db2u_database_help "DB2_BACKUP_NOTIFY_SLACK_URL is not set" - # [[ -z "$SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY" ]] && gitops_db2u_database_help "SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY is not set" - # [[ -z "$SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY" ]] && gitops_db2u_database_help "SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY is not set" - # [[ -z "$SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY" ]] && gitops_db2u_database_help "SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY is not set" [[ -z "$ICD_AUTH_KEY" ]] && gitops_db2u_database_help "ICD_AUTH_KEY is not set" #MAS From 37647675ba8919c462b8107deb534fe92a8795d5 Mon Sep 17 00:00:00 2001 From: Nivedithaa Mahendran Date: Fri, 29 Nov 2024 10:55:46 +0530 Subject: [PATCH 12/14] Review comments --- .secrets.baseline | 4 +-- .../cli/mascli/functions/gitops_db2u_database | 25 +++++++++++-------- .../db2-databases/ibm-db2u-database.yaml.j2 | 4 ++- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index af05e138e7..290bf40fcf 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2024-11-28T11:01:25Z", + "generated_at": "2024-11-29T05:25:41Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -182,7 +182,7 @@ "hashed_secret": "1459943ba5fd876f7ef6e48f566a40b448a2bf08", "is_secret": false, "is_verified": false, - "line_number": 478, + "line_number": 477, "type": "Secret Keyword", "verified_result": null } diff --git a/image/cli/mascli/functions/gitops_db2u_database b/image/cli/mascli/functions/gitops_db2u_database index 25e5b47cd4..0fce31df2b 100644 --- a/image/cli/mascli/functions/gitops_db2u_database +++ b/image/cli/mascli/functions/gitops_db2u_database @@ -278,6 +278,7 @@ function gitops_db2u_database_noninteractive() { --db2-timezone) export DB2_TIMEZONE=$1 && shift ;; + # Optional DB2 Backup details --db2-backup-bucket-name) export DB2_BACKUP_BUCKET_NAME$1 && shift ;; @@ -352,8 +353,6 @@ function gitops_db2u_database_noninteractive() { [[ -z "$DB2_LOGS_STORAGE_CLASS" ]] && gitops_db2u_database_help "DB2_LOGS_STORAGE_CLASS is not set" [[ -z "$MAS_INSTANCE_ID" ]] && gitops_db2u_database_help "MAS_INSTANCE_ID is not set" [[ -z "$MAS_APP_ID" ]] && gitops_db2u_database_help "MAS_APP_ID is not set" - [[ -z "$DB2_BACKUP_NOTIFY_SLACK_URL" ]] && gitops_db2u_database_help "DB2_BACKUP_NOTIFY_SLACK_URL is not set" - [[ -z "$ICD_AUTH_KEY" ]] && gitops_db2u_database_help "ICD_AUTH_KEY is not set" #MAS [[ -z "$MAS_INSTANCE_ID" ]] && gitops_db2u_database_help "MAS_INSTANCE_ID is not set" @@ -393,7 +392,7 @@ function gitops_db2u_database() { GIT_LOCK_BRANCH=$(git_lock_branch_name "gitops-db2u-database" "${ACCOUNT_ID}" "${CLUSTER_ID}" "${MAS_INSTANCE_ID}") export SECRET_NAME_DB2_BACKUP=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}${MAS_INSTANCE_ID}${SECRETS_KEY_SEPERATOR}db2_backup - export SECRET_NAME_ICD_AUTH_KEY=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}${MAS_INSTANCE_ID}${SECRETS_KEY_SEPERATOR}icd_auth_key + export SECRET_NAME_ICD_AUTH_KEY=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}${MAS_INSTANCE_ID}${SECRETS_KEY_SEPERATOR}icd #Defaults if [[ -z $DB2_TLS_VERSION ]]; then @@ -591,12 +590,6 @@ DB2_WORKLOAD: '${DB2_WORKLOAD}'" echo_reset_dim "db2-tolerate-effect ........................... ${COLOR_MAGENTA}${DB2_TOLERATE_EFFECT}" echo_reset_dim "JDBC_ROUTE .................................... ${COLOR_MAGENTA}${JDBC_ROUTE}" echo_reset_dim "db2-timezone .................................. ${COLOR_MAGENTA}${DB2_TIMEZONE}" - echo_reset_dim "db2-backup-bucket-name ........................ ${COLOR_MAGENTA}${DB2_BACKUP_BUCKET_NAME}" - echo_reset_dim "db2-backup-bucket-endpoint .................... ${COLOR_MAGENTA}${DB2_BACKUP_BUCKET_ENDPOINT}" - echo_reset_dim "db2-backup-bucket-access-key-secret............ ${COLOR_MAGENTA}${SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY}" - echo_reset_dim "db2-backup-bucket-secret-key-secret............ ${COLOR_MAGENTA}${SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY}" - echo_reset_dim "db2-backup-notify-slack-url.................... ${COLOR_MAGENTA}${DB2_BACKUP_NOTIFY_SLACK_URL}" - echo_reset_dim "db2-backup-icd-auth-key........................ ${COLOR_MAGENTA}${SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY}" reset_colors @@ -604,9 +597,11 @@ DB2_WORKLOAD: '${DB2_WORKLOAD}'" sm_login export SECRET_KEY_CLUSTER_DOMAIN=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}cluster_domain#cluster_domain - TAGS="[{\"Key\": \"source\", \"Value\": \"gitops_db2u_database\"}, {\"Key\": \"account\", \"Value\": \"${ACCOUNT_ID}\"}, {\"Key\": \"cluster\", \"Value\": \"${CLUSTER_ID}\"}]" - sm_update_secret ${SECRET_NAME_ICD_AUTH_KEY} "{\"icd_auth_key\": \"${ICD_AUTH_KEY}\"}" "${TAGS}" + if [[ -n "$DB2_BACKUP_NOTIFY_SLACK_URL" && -n "$ICD_AUTH_KEY" ]]; then + TAGS="[{\"Key\": \"source\", \"Value\": \"gitops_db2u_database\"}, {\"Key\": \"account\", \"Value\": \"${ACCOUNT_ID}\"}, {\"Key\": \"cluster\", \"Value\": \"${CLUSTER_ID}\"}]" + sm_update_secret ${SECRET_NAME_ICD_AUTH_KEY} "{\"icd_auth_key\": \"${ICD_AUTH_KEY}\"}" "${TAGS}" + fi TEMP_DIR=$GITOPS_WORKING_DIR/tmp-db2u-database mkdir -p $TEMP_DIR @@ -634,6 +629,14 @@ DB2_WORKLOAD: '${DB2_WORKLOAD}'" export DB2_INSTANCE_REGISTRY=$(cat ${DB2_INSTANCE_REGISTRY_YAML}) fi + # Optional DB backup configs + # --------------------------------------------------------------------------- + if [[ -n "$DB2_BACKUP_NOTIFY_SLACK_URL" && -n "$ICD_AUTH_KEY" ]]; then + echo + echo_h2 "Optional DB backup configurations are provided" + export BACKUP=true + fi + if [ -z $GIT_SSH ]; then export GIT_SSH=false fi diff --git a/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 b/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 index f5956ed3be..a58f29e7f1 100644 --- a/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 +++ b/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 @@ -71,9 +71,11 @@ storage_class_definitions: {%- endfilter %} {% endif %} +{% if BACKUP %} db2_backup_bucket_name: db2_backup_bucket_endpoint: db2_backup_bucket_access_key: db2_backup_bucket_secret_key: db2_backup_notify_slack_url: {{DB2_BACKUP_NOTIFY_SLACK_URL}} -db2_backup_icd_auth_key: \ No newline at end of file +db2_backup_icd_auth_key: +{% endif %} \ No newline at end of file From ec134a96c2f23fbb6386f37436edb2157a2bc9b6 Mon Sep 17 00:00:00 2001 From: Nivedithaa Mahendran Date: Fri, 29 Nov 2024 16:08:26 +0530 Subject: [PATCH 13/14] Review comments --- .secrets.baseline | 4 +-- .../cli/mascli/functions/gitops_db2u_database | 20 ----------- .../pipelines/gitops/gitops-mas-apps.yml.j2 | 35 ------------------- .../tasks/gitops/gitops-db2u-database.yml.j2 | 25 ------------- 4 files changed, 2 insertions(+), 82 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index 290bf40fcf..53a819efb8 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2024-11-29T05:25:41Z", + "generated_at": "2024-11-29T10:38:22Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -182,7 +182,7 @@ "hashed_secret": "1459943ba5fd876f7ef6e48f566a40b448a2bf08", "is_secret": false, "is_verified": false, - "line_number": 477, + "line_number": 457, "type": "Secret Keyword", "verified_result": null } diff --git a/image/cli/mascli/functions/gitops_db2u_database b/image/cli/mascli/functions/gitops_db2u_database index 0fce31df2b..0f39d6dfce 100644 --- a/image/cli/mascli/functions/gitops_db2u_database +++ b/image/cli/mascli/functions/gitops_db2u_database @@ -61,12 +61,7 @@ IBM DB2U: --db2-addons-audit-config-yaml ${COLOR_YELLOW}DB2_ADDONS_AUDIT_CONFIG_YAML${TEXT_RESET} Yaml file containing the db2ucluster addons audit config section. --jdbc-route ${COLOR_YELLOW}JDBC_ROUTE${TEXT_RESET} By default routes are not exposed to public. To expose route, set this to public. --db2-timezone ${COLOR_YELLOW}DB2_TIMEZONE${TEXT_RESET} DB2 DB Timezone (optional, DB2 default of "UTC" will be used if not specified) - --db2-backup-bucket-name ${COLOR_YELLOW}DB2_BACKUP_BUCKET_NAME${TEXT_RESET} COS Bucket name where DB2 backup is stored - --db2-backup-bucket-endpoint ${COLOR_YELLOW}DB2_BACKUP_BUCKET_ENDPOINT${TEXT_RESET} COS Bucket endpoint of the bucket where DB2 backup is stored - --db2-backup-bucket-access-key-secret ${COLOR_YELLOW}SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY${TEXT_RESET} Secret name of DB2 COS bucket's access key - --db2-backup-bucket-secret-key-secret ${COLOR_YELLOW}SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY${TEXT_RESET} Secret name of DB2 COS bucket's secret key --db2-backup-notify-slack-url ${COLOR_YELLOW}DB2_BACKUP_NOTIFY_SLACK_URL${TEXT_RESET} Slack URL to notify DB2 backup failures - --db2-backup-icd-auth-key ${COLOR_YELLOW}SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY${TEXT_RESET} ICD auth key used to create ICD incidents when DB2 backup fails Secrets Manager: --secrets-path ${COLOR_YELLOW}SECRETS_PATH${TEXT_RESET} Secrets Manager path @@ -279,24 +274,9 @@ function gitops_db2u_database_noninteractive() { export DB2_TIMEZONE=$1 && shift ;; # Optional DB2 Backup details - --db2-backup-bucket-name) - export DB2_BACKUP_BUCKET_NAME$1 && shift - ;; - --db2-backup-bucket-endpoint) - export DB2_BACKUP_BUCKET_ENDPOINT=$1 && shift - ;; - --db2-backup-bucket-access-key-secret) - export SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY=$1 && shift - ;; - --db2-backup-bucket-secret-key-secret) - export SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY=$1 && shift - ;; --db2-backup-notify-slack-url) export DB2_BACKUP_NOTIFY_SLACK_URL=$1 && shift ;; - --db2-backup-icd-auth-key) - export SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY=$1 && shift - ;; # Automatic GitHub Push diff --git a/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 b/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 index 73ada0fe5d..c6a32259f9 100644 --- a/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 +++ b/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 @@ -209,24 +209,9 @@ spec: - name: db2_timezone type: string default: "" - - name: db2_backup_bucket_name - type: string - default: "" - - name: db2_backup_bucket_endpoint - type: string - default: "" - - name: db2_backup_bucket_access_key - type: string - default: "" - - name: db2_backup_bucket_secret_key - type: string - default: "" - name: db2_backup_notify_slack_url type: string default: "" - - name: db2_backup_icd_auth_key - type: string - default: "" - name: custom_labels type: string @@ -669,18 +654,8 @@ spec: value: $(params.jdbc_route_iot) - name: db2_timezone value: $(params.db2_timezone) - - name: db2_backup_bucket_name - value: $(params.db2_backup_bucket_name) - - name: db2_backup_bucket_endpoint - value: $(params.db2_backup_bucket_endpoint) - - name: db2_backup_bucket_access_key - value: $(params.db2_backup_bucket_access_key) - - name: db2_backup_bucket_secret_key - value: $(params.db2_backup_bucket_secret_key) - name: db2_backup_notify_slack_url value: $(params.db2_backup_notify_slack_url) - - name: db2_backup_icd_auth_key - value: $(params.db2_backup_icd_auth_key) workspaces: - name: configs @@ -839,18 +814,8 @@ spec: value: $(params.jdbc_route_manage) - name: db2_timezone value: $(params.db2_timezone) - - name: db2_backup_bucket_name - value: $(params.db2_backup_bucket_name) - - name: db2_backup_bucket_endpoint - value: $(params.db2_backup_bucket_endpoint) - - name: db2_backup_bucket_access_key - value: $(params.db2_backup_bucket_access_key) - - name: db2_backup_bucket_secret_key - value: $(params.db2_backup_bucket_secret_key) - name: db2_backup_notify_slack_url value: $(params.db2_backup_notify_slack_url) - - name: db2_backup_icd_auth_key - value: $(params.db2_backup_icd_auth_key) workspaces: - name: configs diff --git a/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 b/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 index e6fe332651..ecef563478 100644 --- a/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 +++ b/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 @@ -111,24 +111,9 @@ spec: - name: jdbc_route type: string default: "" - - name: db2_backup_bucket_name - type: string - default: "" - - name: db2_backup_bucket_endpoint - type: string - default: "" - - name: db2_backup_bucket_access_key - type: string - default: "" - - name: db2_backup_bucket_secret_key - type: string - default: "" - name: db2_backup_notify_slack_url type: string default: "" - - name: db2_backup_icd_auth_key - type: string - default: "" stepTemplate: name: gitops-db2u-database env: @@ -236,18 +221,8 @@ spec: value: $(params.mas_app_id) - name: JDBC_ROUTE value: $(params.jdbc_route) - - name: DB2_BACKUP_BUCKET_NAME - value: $(params.db2_backup_bucket_name) - - name: DB2_BACKUP_BUCKET_ENDPOINT - value: $(params.db2_backup_bucket_endpoint) - - name: SECRET_KEY_DB2_BACKUP_BUCKET_ACCESS_KEY - value: $(params.db2_backup_bucket_access_key) - - name: SECRET_KEY_DB2_BACKUP_BUCKET_SECRET_KEY - value: $(params.db2_backup_bucket_secret_key) - name: DB2_BACKUP_NOTIFY_SLACK_URL value: $(params.db2_backup_notify_slack_url) - - name: SECRET_KEY_DB2_BACKUP_ICD_AUTH_KEY - value: $(params.db2_backup_icd_auth_key) envFrom: - configMapRef: name: environment-properties From aa45727a93ffe8f5bb53b5e3bb53f6ae37182507 Mon Sep 17 00:00:00 2001 From: Nivedithaa Mahendran Date: Wed, 4 Dec 2024 16:10:57 +0530 Subject: [PATCH 14/14] Making ICD auth key as cluster level secret --- image/cli/mascli/functions/gitops_db2u_database | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/image/cli/mascli/functions/gitops_db2u_database b/image/cli/mascli/functions/gitops_db2u_database index 0f39d6dfce..9d5f53c1db 100644 --- a/image/cli/mascli/functions/gitops_db2u_database +++ b/image/cli/mascli/functions/gitops_db2u_database @@ -372,7 +372,7 @@ function gitops_db2u_database() { GIT_LOCK_BRANCH=$(git_lock_branch_name "gitops-db2u-database" "${ACCOUNT_ID}" "${CLUSTER_ID}" "${MAS_INSTANCE_ID}") export SECRET_NAME_DB2_BACKUP=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}${MAS_INSTANCE_ID}${SECRETS_KEY_SEPERATOR}db2_backup - export SECRET_NAME_ICD_AUTH_KEY=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}${MAS_INSTANCE_ID}${SECRETS_KEY_SEPERATOR}icd + export SECRET_NAME_ICD_AUTH_KEY=${ACCOUNT_ID}${SECRETS_KEY_SEPERATOR}${CLUSTER_ID}${SECRETS_KEY_SEPERATOR}icd #Defaults if [[ -z $DB2_TLS_VERSION ]]; then