-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1 from bcgov/feature/DBC22-2113
DBC22-2113: Update logging process
- Loading branch information
Showing
16 changed files
with
621 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| FROM alpine:3 | ||
| RUN apk update && apk upgrade | ||
| #Need goaccess 1.92 for a timezone fix. Once that version is in the regular branch, we can pull it from there. | ||
| RUN apk add goaccess --repository=https://dl-cdn.alpinelinux.org/alpine/edge/main | ||
|
|
||
| RUN apk add --no-cache \ | ||
| aws-cli \ | ||
| bash \ | ||
| coreutils \ | ||
| tzdata | ||
|
|
||
| COPY ./compose/openshiftjobs/entrypoint.sh / | ||
| COPY ./compose/openshiftjobs/scripts/analyzeexportlogs.sh /scripts/ | ||
| COPY ./compose/openshiftjobs/scripts/ziplogs.sh /scripts/ | ||
|
|
||
| RUN sed -i 's/\r$//g' /entrypoint.sh && chmod +x /entrypoint.sh | ||
| RUN sed -i 's/\r$//g' /scripts/analyzeexportlogs.sh && chmod +x /scripts/analyzeexportlogs.sh | ||
| RUN sed -i 's/\r$//g' /scripts/ziplogs.sh && chmod +x /scripts/ziplogs.sh | ||
|
|
||
|
|
||
| ENTRYPOINT ["/entrypoint.sh"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Check the number of arguments | ||
| if [ "$#" -lt 1 ]; then | ||
| echo "Usage: $0 <script_name> [args...]" | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Determine which script to run based on the first argument | ||
| case "$1" in | ||
| ziplogs) | ||
| # Run ziplogs.sh which will zip all files that were created in the previous hour or older in the nginx log storage pvc | ||
| /scripts/ziplogs.sh | ||
| ;; | ||
| analyzeexportlogs) | ||
| # Run analyzeexportlogs with additional arguments which will send the specified days logs through goaccess and then upload to s3. | ||
| shift # Remove the first argument (script number) | ||
| /scripts/analyzeexportlogs.sh "$@" | ||
| ;; | ||
| *) | ||
| echo "Invalid script" | ||
| exit 1 | ||
| ;; | ||
| esac |
107 changes: 107 additions & 0 deletions
107
compose/caching/openshiftjobs/scripts/analyzeexportlogs.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Directory where the logs are stored | ||
| log_dir="/logs" | ||
| cd $log_dir || { echo "Failed to change directory to $log_dir. Exiting..."; exit 1; } | ||
|
|
||
| # Initialize an empty array | ||
| zipped_files=() | ||
|
|
||
| # Get the number of days ago as a positional parameter | ||
| days_ago="$1" | ||
|
|
||
| # If no argument is provided, default to 1 day ago | ||
| if [ -z "$days_ago" ]; then | ||
| days_ago=1 | ||
| fi | ||
|
|
||
| # Get the start and end time for $days_ago days ago in the America/Vancouver timezone | ||
| start_time=$(TZ=America/Vancouver date -d "$days_ago days ago 00:00") | ||
| end_time=$(TZ=America/Vancouver date -d "$days_ago days ago 23:59") | ||
|
|
||
| # Convert the start and end time to UTC | ||
| start_time_utc=$(date -u -d "$start_time" +%Y%m%d%H) | ||
| end_time_utc=$(date -u -d "$end_time" +%Y%m%d%H) | ||
|
|
||
| echo "Will analyze and archive logs between $start_time_utc and $end_time_utc" | ||
|
|
||
| # Loop through all files in the current directory ending with ".gz" | ||
| for file in *.gz; do | ||
| # Check if the file exists and is a regular file | ||
| if [[ -f $file ]]; then | ||
| # Extract the date and time part from the filename (assuming UTC timezone) | ||
| file_datetime_utc=$(echo "$file" | grep -oE '[0-9]{10}') | ||
| # Check if the file date and time in UTC falls within days_ago's start and end time in UTC | ||
| if [[ $file_datetime_utc -ge $start_time_utc && $file_datetime_utc -le $end_time_utc ]]; then | ||
| zipped_files+=("$file") | ||
| fi | ||
| fi | ||
| done | ||
|
|
||
| # Print the elements of the array | ||
| echo "Log files that will be processed:" | ||
| printf '%s\n' "${zipped_files[@]}" | ||
|
|
||
| # Get the start dates formatted for the goaccess report name | ||
| start_time_formatted=$(date -d "$start_time" +%Y%m%d) | ||
|
|
||
| # Run the following command only if zipped_files array is not empty | ||
| if [ ${#zipped_files[@]} -gt 0 ]; then | ||
| # Define file URL and destination directory to pull down the latest geoip data | ||
| FILE_URL="https://download.db-ip.com/free/dbip-city-lite-$(date -u +'%Y-%m').mmdb.gz" | ||
| OLD_FILE="dbip-city-lite-$(date -d 'last month' +'%Y-%m').mmdb" | ||
|
|
||
| # Check if the file exists | ||
| if [ ! -f "dbip-city-lite-$(date -u +'%Y-%m').mmdb" ]; then | ||
| # If the file doesn't exist, download it | ||
| echo "Downloading file..." | ||
| if wget -q --spider "$FILE_URL"; then | ||
| wget "$FILE_URL" -P "$log_dir" | ||
| echo "Download complete." | ||
| gzip -d "dbip-city-lite-$(date -u +'%Y-%m').mmdb.gz" | ||
| # Delete the old file if it exists | ||
| if [ -f "$OLD_FILE" ]; then | ||
| echo "Deleting old file ($OLD_FILE)..." | ||
| rm "$OLD_FILE" | ||
| echo "Old file deleted." | ||
| fi | ||
| else | ||
| echo "Failed to download file. URL is unreachable." | ||
| fi | ||
| else | ||
| echo "MMDB file already exists." | ||
| fi | ||
| mmdb_file=$(find . -maxdepth 1 -type f -name "*.mmdb") | ||
|
|
||
| #Run goaccess on all the log files from the date entered | ||
| goaccess_report_name=$start_time_formatted-goaccess_report.html | ||
| zcat "${zipped_files[@]}" | goaccess - -o "$goaccess_report_name" --log-format='%h %e %^[%x] "%r" %s %b "%R" "%u" %C "%M" %T' --datetime-format='%d/%b/%Y:%H:%M:%S %z' --ignore-panel=REMOTE_USER --ignore-panel=ASN --tz=America/Vancouver --jobs=2 --geoip-database=$mmdb_file | ||
| echo "GoAccess report generated successfully at $goaccess_report_name" | ||
|
|
||
| # Get the start date formated in YYYY/MM/DD format | ||
| start_time_formatted_s3=$(date -d "$start_time" +"%Y/%m/%d") | ||
|
|
||
| # Create folder structure in S3 bucket | ||
| s3_path="s3://$AWS_BUCKET/$ENVIRONMENT/logs/$start_time_formatted_s3/" | ||
|
|
||
| # Upload zipped files to S3 | ||
| for file in "${zipped_files[@]}"; do | ||
| aws --endpoint-url "$AWS_ENDPOINT" s3 cp "$file" "$s3_path" || { echo "Failed to upload $file to S3. Exiting..."; exit 1; } | ||
| echo "File $file copied to S3 bucket under $s3_path" | ||
| done | ||
|
|
||
| echo "All files copied to S3 bucket under $s3_path" | ||
|
|
||
|
|
||
| # Upload HTML report to S3 | ||
| aws --endpoint-url $AWS_ENDPOINT s3 cp "$goaccess_report_name" "$s3_path" || { echo "Failed to upload HTML report to S3. Exiting..."; exit 1; } | ||
| echo "HTML report copied to S3 bucket under $s3_path" | ||
|
|
||
| # Delete the zipped files and HTML report | ||
| rm "${zipped_files[@]}" "$goaccess_report_name" | ||
|
|
||
| echo "Zipped Files and HTML report deleted from PVC successfully" | ||
|
|
||
| else | ||
| echo "No files to process for $start_time_formatted" | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| #!/bin/bash | ||
|
|
||
| # Calculate the hour one hour ago in UTC | ||
| previous_hour=$(TZ=UTC date -d '1 hour ago' +"%Y%m%d%H") | ||
| echo "Previous Hour in UTC: $previous_hour" | ||
|
|
||
| # Directory where the logs are stored | ||
| log_dir="./logs" | ||
|
|
||
| # Iterate over log files | ||
| find "$log_dir" -type f -name '*.log' | while read -r file; do | ||
| # Extract timestamp from filename | ||
| timestamp=$(echo "$file" | grep -oE '[0-9]{10}') | ||
|
|
||
| # Check if timestamp is less than or equal to previous hour | ||
| if [[ $timestamp -le $previous_hour ]]; then | ||
| # gzip the file | ||
| gzip "$file" | ||
| echo "File $file with timestamp $timestamp gzipped." | ||
| fi | ||
| done |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| {{- $deploymentTag := .Values.image.tag | default .Chart.AppVersion -}} | ||
| {{- $deploymentTime := now | date "2006-01-02 15:04:05.000000" -}} | ||
|
|
||
| apiVersion: batch/v1 | ||
| kind: CronJob | ||
| metadata: | ||
| name: {{ template "app.fullname" . }}-{{ .Values.cronjobs.analyzeuploadlogs.name }} | ||
| labels: {{ include "app.labels" . | nindent 4 }} | ||
|
|
||
| spec: | ||
| schedule: {{ .Values.cronjobs.analyzeuploadlogs.schedule }} | ||
| concurrencyPolicy: Replace | ||
| suspend: false | ||
| jobTemplate: | ||
| spec: | ||
| template: | ||
| spec: | ||
| volumes: | ||
| - name: log-storage | ||
| persistentVolumeClaim: | ||
| claimName: {{ .Values.cronjobs.analyzeuploadlogs.deployment.volumes.logs }} | ||
| containers: | ||
| - resources: | ||
| limits: | ||
| cpu: {{ .Values.cronjobs.analyzeuploadlogs.deployment.resources.limits.cpu }} | ||
| memory: {{ .Values.cronjobs.analyzeuploadlogs.deployment.resources.limits.memory }} | ||
| requests: | ||
| cpu: {{ .Values.cronjobs.analyzeuploadlogs.deployment.resources.requests.cpu }} | ||
| memory: {{ .Values.cronjobs.analyzeuploadlogs.deployment.resources.requests.memory }} | ||
| name: {{ include "app.fullname" . }}-{{ .Values.cronjobs.analyzeuploadlogs.name }} | ||
| args: | ||
| - "analyzeexportlogs" | ||
| - "1" | ||
| volumeMounts: | ||
| - name: log-storage | ||
| mountPath: /logs | ||
| env: | ||
| - name: DEPLOYMENT_TAG | ||
| value: {{ $deploymentTag | quote }} | ||
| - name: DEPLOYMENT_TIME | ||
| value: {{ $deploymentTime | quote }} | ||
| - name: ENVIRONMENT | ||
| value: {{ .Values.cronjobs.analyzeuploadlogs.deployment.env.environment }} | ||
| envFrom: | ||
| - secretRef: | ||
| name: {{ .Values.cronjobs.analyzeuploadlogs.deployment.env.s3Secret }} | ||
| imagePullPolicy: IfNotPresent | ||
| image: {{ .Values.image.repository }}:{{ .Values.image.tag }} | ||
| securityContext: | ||
| seccompProfile: | ||
| type: 'RuntimeDefault' | ||
| capabilities: | ||
| drop: | ||
| - all | ||
| add: | ||
| - NET_BIND_SERVICE | ||
| restartPolicy: Never |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| {{- if not (lookup "v1" "Secret" .Release.Namespace .Values.cronjobs.analyzeuploadlogs.s3secret.name) }} | ||
| apiVersion: v1 | ||
| kind: Secret | ||
| metadata: | ||
| name: {{ .Values.cronjobs.analyzeuploadlogs.s3secret.name }} | ||
| annotations: | ||
| "helm.sh/resource-policy": "keep" | ||
| type: Opaque | ||
| data: | ||
| AWS_ACCESS_KEY_ID: {{ .Values.cronjobs.analyzeuploadlogs.s3secret.access_key_id | b64enc }} | ||
| AWS_BUCKET: {{ .Values.cronjobs.analyzeuploadlogs.s3secret.bucket | b64enc }} | ||
| AWS_ENDPOINT: {{ .Values.cronjobs.analyzeuploadlogs.s3secret.endpoint | b64enc }} | ||
| AWS_SECRET_ACCESS_KEY: {{ .Values.cronjobs.analyzeuploadlogs.s3secret.secret_access_key | b64enc }} | ||
| {{- end }} |
Oops, something went wrong.