From 3a6609b75cc9bc9b151f9ae9cdc6e992a61da763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20=27Perk=27=20Sto=C5=BCek?= Date: Tue, 19 May 2020 19:54:09 +0200 Subject: [PATCH 1/8] Add grep, sed and bash versions checks --- deploy/helm/sumologic/upgrade-1.0.0.sh | 65 ++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/deploy/helm/sumologic/upgrade-1.0.0.sh b/deploy/helm/sumologic/upgrade-1.0.0.sh index c7789d0645..7d7c6be8ca 100755 --- a/deploy/helm/sumologic/upgrade-1.0.0.sh +++ b/deploy/helm/sumologic/upgrade-1.0.0.sh @@ -10,6 +10,10 @@ readonly PREVIOUS_VERSION=0.17 readonly TEMP_FILE=upgrade-1.0.0-temp-file +readonly MIN_SED_VERSION=4 +readonly MIN_GREP_VERSION=2.5 +readonly MIN_BASH_VERSION=4.4 + readonly KEY_MAPPINGS=" eventsDeployment.nodeSelector:fluentd.events.statefulset.nodeSelector eventsDeployment.resources.limits.cpu:fluentd.events.statefulset.resources.limits.cpu @@ -168,10 +172,62 @@ function check_required_command() { command -v "${command_to_check}" >/dev/null 2>&1 || { error "Required command is missing: ${command_to_check}"; fatal "Please consult --help and install missing commands before continue. Aborting."; } } +function compare_versions() { + local no_lower_than="${1}" + local app_version="${2}" + + if [[ "$(printf '%s\n' "${app_version}" "$no_lower_than" | sort -V | head -n 1)" == "${no_lower_than}" ]]; then + echo "pass" + else + echo "fail" + fi +} + +function check_app_version() { + local app_name="${1}" + local no_lower_than="${2}" + local app_version="${3}" + + if [[ -z ${app_version} ]] || [[ $(compare_versions "${no_lower_than}" "${app_version}") == "fail" ]]; then + error "${app_name} version is invalid - it should be no lower than ${no_lower_than}" + fatal "Please update your ${app_name} version and retry." + fi +} + function check_yq_version() { yq --version | grep 3.2.1 >/dev/null 2>&1 || { error "yq version is invalid. It should be exactly 3.2.1"; fatal "Please install it from: https://github.com/mikefarah/yq/releases/tag/3.2.1"; } } +function check_grep_version() { + local sed_version + + # do not fail on missing or incorrect sed + set +e + grep_version=$(grep -V 2>&1 | head -n 1 | grep -oE "\d.*$") + set -e + + local stripped_grep_version=${grep_version%%-*} + + check_app_version "grep" "${MIN_GREP_VERSION}" "${stripped_grep_version}" +} + +function check_sed_version() { + local sed_version + + # do not fail on missing or incorrect sed + set +e + sed_version=$(sed --version 2>&1 | head -n 1 | grep -oE "\d.*$") + set -e + + local stripped_sed_version=${sed_version%%-*} + + check_app_version "sed" "${MIN_SED_VERSION}" "${stripped_sed_version}" +} + +function check_bash_version() { + check_app_version "bash" "${MIN_BASH_VERSION}" "${BASH_VERSION}" +} + function create_temp_file() { echo -n > ${TEMP_FILE} } @@ -228,7 +284,7 @@ function migrate_customer_keys() { info "Casting ${key} to str" # As yq doesn't cast `on` and `off` from bool to cast, we use sed based casts yq w -i ${TEMP_FILE} -- "${key}" "$(yq r "${OLD_VALUES_YAML}" "${key}")__YQ_REPLACEMENT_CAST" - sed -i '' 's/\(^.*: \)\(.*\)__YQ_REPLACEMENT_CAST/\1"\2"/g' ${TEMP_FILE} + sed -i 's/\(^.*: \)\(.*\)__YQ_REPLACEMENT_CAST/\1"\2"/g' ${TEMP_FILE} fi done echo @@ -431,12 +487,12 @@ function migrate_fluentbit_db_path() { info 'Replacing tail-db/tail-containers-state.db to tail-db/tail-containers-state-sumo.db' warning 'Please ensure that new fluent-bit configuration is correct' - sed -i '' 's?tail-db/tail-containers-state.db?tail-db/tail-containers-state-sumo.db?g' ${TEMP_FILE} + sed -i 's?tail-db/tail-containers-state.db?tail-db/tail-containers-state-sumo.db?g' ${TEMP_FILE} } function fix_yq() { # account for yq bug that stringifies empty maps - sed -i '' "s/'{}'/{}/g" ${TEMP_FILE} + sed -i "s/'{}'/{}/g" ${TEMP_FILE} } function rename_temp_file() { @@ -449,10 +505,13 @@ function echo_footer() { } check_if_print_help_and_exit "${OLD_VALUES_YAML}" +check_bash_version check_required_command yq check_yq_version check_required_command grep +check_grep_version check_required_command sed +check_sed_version create_temp_file From cbc1fc924760f441e5030b7c7cf7f65cd85e300a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20=27Perk=27=20Sto=C5=BCek?= Date: Tue, 19 May 2020 20:35:46 +0200 Subject: [PATCH 2/8] Change upgrade script to work with both GNU and BSD versions of sed --- deploy/helm/sumologic/upgrade-1.0.0.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/deploy/helm/sumologic/upgrade-1.0.0.sh b/deploy/helm/sumologic/upgrade-1.0.0.sh index 7d7c6be8ca..fd5e9998ee 100755 --- a/deploy/helm/sumologic/upgrade-1.0.0.sh +++ b/deploy/helm/sumologic/upgrade-1.0.0.sh @@ -284,7 +284,7 @@ function migrate_customer_keys() { info "Casting ${key} to str" # As yq doesn't cast `on` and `off` from bool to cast, we use sed based casts yq w -i ${TEMP_FILE} -- "${key}" "$(yq r "${OLD_VALUES_YAML}" "${key}")__YQ_REPLACEMENT_CAST" - sed -i 's/\(^.*: \)\(.*\)__YQ_REPLACEMENT_CAST/\1"\2"/g' ${TEMP_FILE} + sed -i.bak 's/\(^.*: \)\(.*\)__YQ_REPLACEMENT_CAST/\1"\2"/g' ${TEMP_FILE} fi done echo @@ -487,18 +487,22 @@ function migrate_fluentbit_db_path() { info 'Replacing tail-db/tail-containers-state.db to tail-db/tail-containers-state-sumo.db' warning 'Please ensure that new fluent-bit configuration is correct' - sed -i 's?tail-db/tail-containers-state.db?tail-db/tail-containers-state-sumo.db?g' ${TEMP_FILE} + sed -i.bak 's?tail-db/tail-containers-state.db?tail-db/tail-containers-state-sumo.db?g' ${TEMP_FILE} } function fix_yq() { # account for yq bug that stringifies empty maps - sed -i "s/'{}'/{}/g" ${TEMP_FILE} + sed -i.bak "s/'{}'/{}/g" ${TEMP_FILE} } function rename_temp_file() { mv ${TEMP_FILE} new_values.yaml } +function cleanup_bak_file() { + rm ${TEMP_FILE}.bak +} + function echo_footer() { DONE="\nThank you for upgrading to v1.0.0 of the Sumo Logic Kubernetes Collection Helm chart.\nA new yaml file has been generated for you. Please check the current directory for new_values.yaml." echo -e "$DONE" @@ -511,7 +515,8 @@ check_yq_version check_required_command grep check_grep_version check_required_command sed -check_sed_version +# do not check sed version as it works now on both GNU and BSD versions +#check_sed_version create_temp_file @@ -532,6 +537,7 @@ migrate_fluentbit_db_path fix_yq rename_temp_file +cleanup_bak_file echo_footer From 42927ed6a2d2068ebe5313cf5b53a4256d2c6e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20=27Perk=27=20Sto=C5=BCek?= Date: Tue, 19 May 2020 20:41:12 +0200 Subject: [PATCH 3/8] Upgrade script cleanup after sed version check removal --- deploy/helm/sumologic/upgrade-1.0.0.sh | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/deploy/helm/sumologic/upgrade-1.0.0.sh b/deploy/helm/sumologic/upgrade-1.0.0.sh index fd5e9998ee..2f3e771f08 100755 --- a/deploy/helm/sumologic/upgrade-1.0.0.sh +++ b/deploy/helm/sumologic/upgrade-1.0.0.sh @@ -10,7 +10,6 @@ readonly PREVIOUS_VERSION=0.17 readonly TEMP_FILE=upgrade-1.0.0-temp-file -readonly MIN_SED_VERSION=4 readonly MIN_GREP_VERSION=2.5 readonly MIN_BASH_VERSION=4.4 @@ -199,9 +198,9 @@ function check_yq_version() { } function check_grep_version() { - local sed_version + local grep_version - # do not fail on missing or incorrect sed + # do not fail on missing or incorrect grep set +e grep_version=$(grep -V 2>&1 | head -n 1 | grep -oE "\d.*$") set -e @@ -211,19 +210,6 @@ function check_grep_version() { check_app_version "grep" "${MIN_GREP_VERSION}" "${stripped_grep_version}" } -function check_sed_version() { - local sed_version - - # do not fail on missing or incorrect sed - set +e - sed_version=$(sed --version 2>&1 | head -n 1 | grep -oE "\d.*$") - set -e - - local stripped_sed_version=${sed_version%%-*} - - check_app_version "sed" "${MIN_SED_VERSION}" "${stripped_sed_version}" -} - function check_bash_version() { check_app_version "bash" "${MIN_BASH_VERSION}" "${BASH_VERSION}" } @@ -515,8 +501,6 @@ check_yq_version check_required_command grep check_grep_version check_required_command sed -# do not check sed version as it works now on both GNU and BSD versions -#check_sed_version create_temp_file From fa77e43c4aab0a20015eda6416f857a1f7e2b464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20=27Perk=27=20Sto=C5=BCek?= Date: Tue, 19 May 2020 20:50:12 +0200 Subject: [PATCH 4/8] Upgrade script cleanup after grep version check removal --- deploy/helm/sumologic/upgrade-1.0.0.sh | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/deploy/helm/sumologic/upgrade-1.0.0.sh b/deploy/helm/sumologic/upgrade-1.0.0.sh index 2f3e771f08..8770b42016 100755 --- a/deploy/helm/sumologic/upgrade-1.0.0.sh +++ b/deploy/helm/sumologic/upgrade-1.0.0.sh @@ -10,7 +10,6 @@ readonly PREVIOUS_VERSION=0.17 readonly TEMP_FILE=upgrade-1.0.0-temp-file -readonly MIN_GREP_VERSION=2.5 readonly MIN_BASH_VERSION=4.4 readonly KEY_MAPPINGS=" @@ -197,19 +196,6 @@ function check_yq_version() { yq --version | grep 3.2.1 >/dev/null 2>&1 || { error "yq version is invalid. It should be exactly 3.2.1"; fatal "Please install it from: https://github.com/mikefarah/yq/releases/tag/3.2.1"; } } -function check_grep_version() { - local grep_version - - # do not fail on missing or incorrect grep - set +e - grep_version=$(grep -V 2>&1 | head -n 1 | grep -oE "\d.*$") - set -e - - local stripped_grep_version=${grep_version%%-*} - - check_app_version "grep" "${MIN_GREP_VERSION}" "${stripped_grep_version}" -} - function check_bash_version() { check_app_version "bash" "${MIN_BASH_VERSION}" "${BASH_VERSION}" } @@ -499,7 +485,6 @@ check_bash_version check_required_command yq check_yq_version check_required_command grep -check_grep_version check_required_command sed create_temp_file From 825394dec0e0b36d0b3383f99f9ec77596110d22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20=27Perk=27=20Sto=C5=BCek?= Date: Tue, 19 May 2020 21:02:42 +0200 Subject: [PATCH 5/8] Allow for yq version >= 3.2.1 --- deploy/helm/sumologic/upgrade-1.0.0.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/deploy/helm/sumologic/upgrade-1.0.0.sh b/deploy/helm/sumologic/upgrade-1.0.0.sh index 8770b42016..4c99e8a405 100755 --- a/deploy/helm/sumologic/upgrade-1.0.0.sh +++ b/deploy/helm/sumologic/upgrade-1.0.0.sh @@ -11,6 +11,7 @@ readonly PREVIOUS_VERSION=0.17 readonly TEMP_FILE=upgrade-1.0.0-temp-file readonly MIN_BASH_VERSION=4.4 +readonly MIN_YQ_VERSION=3.2.1 readonly KEY_MAPPINGS=" eventsDeployment.nodeSelector:fluentd.events.statefulset.nodeSelector @@ -138,7 +139,7 @@ This script will automatically take the configurations of your existing values.y and return one that is compatible with v1.0.0. Requirements: - yq (3.2.1) https://github.com/mikefarah/yq/releases/tag/3.2.1 + yq (>=3.2.1) https://github.com/mikefarah/yq/releases/tag/3.2.1 grep sed @@ -193,7 +194,10 @@ function check_app_version() { } function check_yq_version() { - yq --version | grep 3.2.1 >/dev/null 2>&1 || { error "yq version is invalid. It should be exactly 3.2.1"; fatal "Please install it from: https://github.com/mikefarah/yq/releases/tag/3.2.1"; } + local yq_version + yq_version=$(yq --version | grep -oE '[^[:space:]]+$') + + check_app_version "grep" "${MIN_YQ_VERSION}" "${yq_version}" } function check_bash_version() { From cbb2c98d86e5b4fdc135e916372db7d13ffd6f68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20=27Perk=27=20Sto=C5=BCek?= Date: Wed, 20 May 2020 10:19:05 +0200 Subject: [PATCH 6/8] Add both yq 3.2.1 and 3.3.0 to vagrant env; use 3.3.0 by default --- vagrant/provision.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vagrant/provision.sh b/vagrant/provision.sh index e31e998a14..9e45b7299e 100644 --- a/vagrant/provision.sh +++ b/vagrant/provision.sh @@ -42,8 +42,12 @@ sudo -H -u vagrant -i helm init --wait usermod -a -G microk8s vagrant -# Install yq -sudo curl https://github.com/mikefarah/yq/releases/download/3.2.1/yq_linux_amd64 -L -o /usr/local/bin/yq && sudo chmod +x /usr/local/bin/yq +# install yq with access to file structure +curl https://github.com/mikefarah/yq/releases/download/3.2.1/yq_linux_amd64 -L -o /usr/local/bin/yq-3.2.1 +chmod +x /usr/local/bin/yq-3.2.1 +curl https://github.com/mikefarah/yq/releases/download/3.3.0/yq_linux_amd64 -L -o /usr/local/bin/yq-3.3.0 +chmod +x /usr/local/bin/yq-3.3.0 +ln -s /usr/local/bin/yq-3.3.0 /usr/local/bin/yq set +x echo Dashboard local in-vagrant IP: From 0496f2de8bd6d93bdcc04ee584c434d5c9e6499f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20=27Perk=27=20Sto=C5=BCek?= Date: Wed, 20 May 2020 10:22:14 +0200 Subject: [PATCH 7/8] Add info about used app version when throwing error --- deploy/helm/sumologic/upgrade-1.0.0.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/helm/sumologic/upgrade-1.0.0.sh b/deploy/helm/sumologic/upgrade-1.0.0.sh index 4c99e8a405..3e9a666722 100755 --- a/deploy/helm/sumologic/upgrade-1.0.0.sh +++ b/deploy/helm/sumologic/upgrade-1.0.0.sh @@ -188,8 +188,8 @@ function check_app_version() { local app_version="${3}" if [[ -z ${app_version} ]] || [[ $(compare_versions "${no_lower_than}" "${app_version}") == "fail" ]]; then - error "${app_name} version is invalid - it should be no lower than ${no_lower_than}" - fatal "Please update your ${app_name} version and retry." + error "${app_name} version: '${app_version}' is invalid - it should be no lower than ${no_lower_than}" + fatal "Please update your ${app_name} and retry." fi } From c3a1ebe2d71c80a1e04cc78cdc01b4415172ce3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20=27Perk=27=20Sto=C5=BCek?= Date: Wed, 20 May 2020 10:25:15 +0200 Subject: [PATCH 8/8] Add min bash version to --help --- deploy/helm/sumologic/upgrade-1.0.0.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deploy/helm/sumologic/upgrade-1.0.0.sh b/deploy/helm/sumologic/upgrade-1.0.0.sh index 3e9a666722..a639bf5491 100755 --- a/deploy/helm/sumologic/upgrade-1.0.0.sh +++ b/deploy/helm/sumologic/upgrade-1.0.0.sh @@ -139,9 +139,10 @@ This script will automatically take the configurations of your existing values.y and return one that is compatible with v1.0.0. Requirements: - yq (>=3.2.1) https://github.com/mikefarah/yq/releases/tag/3.2.1 + yq (>= ${MIN_YQ_VERSION}) https://github.com/mikefarah/yq/releases/tag/3.2.1 grep sed + bash (>= ${MIN_BASH_VERSION}) Usage: # for default helm release name 'collection' and namespace 'sumologic'