From fefb5ed1e021eff595c9ef06fea38c4a0768134e Mon Sep 17 00:00:00 2001 From: jvoravong Date: Thu, 30 Jan 2025 08:33:36 -0700 Subject: [PATCH] Refactor `make render` to support rendering TLS certificates in such a way that doesn't break CI/CD checks. Certificates would generate a unique values per `make render` execution, we updated the unique values to be redacted and link rules to ignore checking files where TLS certificates exists. --- .pre-commit-config.yaml | 2 +- Makefile | 2 +- ci_scripts/base_util.sh | 51 +++++++++++++++++++++ {examples => ci_scripts}/render-examples.sh | 11 +++-- tools/splunk_kubernetes_debug_info.sh | 27 +---------- 5 files changed, 63 insertions(+), 30 deletions(-) rename {examples => ci_scripts}/render-examples.sh (89%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 99ca8731ed..5958b3a260 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,6 +17,6 @@ repos: exclude: "^examples|^test" - id: check-yaml # Can't check source yaml since it has go templates in it. - exclude: "^helm-charts" + exclude: "^helm-charts|operator-webhook.yaml" args: [ --allow-multiple-documents ] - id: check-added-large-files diff --git a/Makefile b/Makefile index 4378a8f021..03f494fa7c 100644 --- a/Makefile +++ b/Makefile @@ -72,7 +72,7 @@ dep-update: ## Fetch Helm chart dependency repositories, build the Helm chart wi # make render VALUES="values1.yaml values2.yaml" .PHONY: render render: dep-update ## Render the Helm chart with the examples as input. Users can also provide value overrides. - @examples/render-examples.sh $(VALUES) || exit 1 + @ci_scripts/render-examples.sh $(VALUES) || exit 1 ##@ Test # Tasks related to testing the Helm chart diff --git a/ci_scripts/base_util.sh b/ci_scripts/base_util.sh index da925ad425..a6f8127be9 100755 --- a/ci_scripts/base_util.sh +++ b/ci_scripts/base_util.sh @@ -447,3 +447,54 @@ maybe_update_version() { fi echo "Image update process completed successfully for '$yaml_file_path'." } + +# Function: redact_sensitive_info +# Description: Redacts sensitive information from a given input string and returns the redacted content as a string. +# The function uses `awk` to redact specific patterns such as certificates, sensitive data, tokens, and passwords. +# Usage: redact_sensitive_info "$input_string" +redact_sensitive_info() { + local input="$1" + + # Redact sensitive information from the input string using awk and return the result + echo "$input" | awk ' + # Redact certificate sections + /BEGIN CERTIFICATE/,/END CERTIFICATE/ { + if (/BEGIN CERTIFICATE/) print; + else if (/END CERTIFICATE/) print; + else print " [CERTIFICATE REDACTED]"; + next; + } + # Redact sensitive data patterns like caBundle, certificates, keys + /caBundle|ca\.crt|client\.crt|client\.key|tls\.crt|tls\.key/ { + print " [SENSITIVE DATA REDACTED]"; + next; + } + # Redact tokens + /[Tt][Oo][Kk][Ee][Nn]/ { + print " [TOKEN REDACTED]"; + next; + } + # Redact passwords + /[Pp][Aa][Ss][Ss][Ww][Oo][Rr][Dd]/ { + print " [PASSWORD REDACTED]"; + next; + } + # Print other content unchanged + {print} + ' +} + +# Function: redact_files +# Description: Redacts sensitive information from all files matching the provided file pattern in the specified directory using a for loop. +# Usage: redact_files "path/to/directory" "*.yaml" +redact_files() { + local dir="$1" + local file_pattern="$2" + + # Use find to search for files matching the pattern in the specified directory + for file in $(find "$dir" -type f -name "$file_pattern"); do + # Redact the content of the file and save it back to the original file + redacted_content=$(redact_sensitive_info "$(cat "$file")") + echo "$redacted_content" > "$file" + done +} diff --git a/examples/render-examples.sh b/ci_scripts/render-examples.sh similarity index 89% rename from examples/render-examples.sh rename to ci_scripts/render-examples.sh index 464148af59..b7d242ff5e 100755 --- a/examples/render-examples.sh +++ b/ci_scripts/render-examples.sh @@ -13,7 +13,9 @@ # ./render-examples.sh extra-values.yaml # ./render-examples.sh values1.yaml values2.yaml -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +EXAMPLES_DIR="$SCRIPT_DIR/../examples" +source "$SCRIPT_DIR/base_util.sh" render_task() { example_dir=$1 @@ -47,6 +49,9 @@ render_task() { exit 1 fi + # Redact data that has a unique value per run such as certificate data for the operator webhook + redact_files "${rendered_manifests_dir}" "**webhook.yaml" + # Move the chart renders cp -rp "${rendered_manifests_dir}/splunk-otel-collector/templates/"* "$rendered_manifests_dir" if [ $? -ne 0 ]; then @@ -74,12 +79,12 @@ render_task() { # Collect additional values files passed as arguments values_files=("$@") -for example_dir in $SCRIPT_DIR/*/; do +for example_dir in $EXAMPLES_DIR/*/; do render_task "${example_dir}" & done wait # Let all the render tasks finish -for example_dir in $SCRIPT_DIR/*/; do +for example_dir in $EXAMPLES_DIR/*/; do rendered_manifests_dir="${example_dir}rendered_manifests" if [ ! -d "${rendered_manifests_dir}" ]; then echo "Examples were rendered, failure occurred" diff --git a/tools/splunk_kubernetes_debug_info.sh b/tools/splunk_kubernetes_debug_info.sh index 826ca55f17..972533797b 100755 --- a/tools/splunk_kubernetes_debug_info.sh +++ b/tools/splunk_kubernetes_debug_info.sh @@ -56,31 +56,8 @@ write_output() { fi fi - # Redact sensitive information - output=$(echo "$output" | awk ' - /BEGIN CERTIFICATE/,/END CERTIFICATE/ { - if (/BEGIN CERTIFICATE/) print; - else if (/END CERTIFICATE/) print; - else print " [CERTIFICATE REDACTED]"; - next; - } - /ca\.crt|client\.crt|client\.key|tls\.crt|tls\.key/ { - print " [SENSITIVE DATA REDACTED]"; - next; - } - /[Tt][Oo][Kk][Ee][Nn]/ { - print " [TOKEN REDACTED]"; - next; - } - /[Pp][Aa][Ss][Ss][Ww][Oo][Rr][Dd]/ { - print " [PASSWORD REDACTED]"; - next; - } - {print}') - - # Write command and output to file - echo "# Command: $cmd" > "$file_name" - echo "$output" >> "$file_name" + # Redact sensitive information from output + redact_sensitive_info "$output" "$file_name" } # Function to collect data for a given namespace