prevent issues with restarting of pods and spam mailing in case of er…

…rors
willemvd · May 1, 2017 · ffccb8b · ffccb8b
1 parent 356dd6e
commit ffccb8b
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 8 deletions.
diff --git a/README.md b/README.md
@@ -41,4 +41,6 @@ SMTP_AUTH_USER=<username to login to the SMTP server>
 SMTP_AUTH_PASSWORD=<password for the SMTP user>
 SMTP_FROM=<name and email of the sender in the format: Name <full email adres> >
 SMTP_TO=<email address to send to, multiple addresses can be used with a comma seperated list>
+
+ALWAYS_EXIT_ZERO=<when true, always return 0 as exit code to prevent failure detection by other scripts (prevent restarts and spam mailing)>
 ```
diff --git a/backup.sh b/backup.sh
@@ -2,9 +2,20 @@
 
 set -e
 
+handleFailureExitCode() {
+    if [  "x$ALWAYS_EXIT_ZERO" = "xtrue" ]; then
+        exit 0
+    fi
+
+    # abort
+    exit 1
+}
+
 handleError() {
     MSG="Failed to run backup of database '$PGDATABASE' from '$PGHOST:$PGPORT/$PGDATABASE' with user '$PGUSER' to 's3://$S3_BUCKET_NAME/$BACKUP_TYPE/$BACKUP_FILE_NAME'"
 
+    echo ${MSG}
+
     if [  "x$ENABLE_ERROR_MAIL" = "xtrue" ]; then
 
         OPTS=""
@@ -13,21 +24,17 @@ handleError() {
         fi
 
         echo ${MSG} | \
-
         mailx -v -s "$ERROR_MAIL_SUBJECT" \
         -S smtp="$SMTP_HOST:$SMTP_PORT" \
         ${OPTS} \
         -S smtp-auth=$SMTP_AUTH \
         -S smtp-auth-user="$SMTP_AUTH_USER" \
         -S smtp-auth-password="$SMTP_AUTH_PASSWORD" \
         -r "$SMTP_FROM" \
-        $SMTP_TO
-    else
-        echo ${MSG}
+        $SMTP_TO || handleFailureExitCode
     fi
 
-    # abort
-    exit 1
+    handleFailureExitCode
 }
 
 DATE=$(date +%Y-%m-%d-%H-%M-%S)

diff --git a/openshift-postgresql-s3-backup-scheduledJob.yaml b/openshift-postgresql-s3-backup-scheduledJob.yaml
@@ -37,6 +37,7 @@ items:
           spec:
             # 30 minutes timeout
             activeDeadlineSeconds: 1800
+            restartPolicy: OnFailure
             containers:
             - name: postgresql-s3-hourly-backup
               image: willemvd/postgresql-client-side-encrypted-s3-backup:1.1.0
@@ -100,7 +101,8 @@ items:
                 value: ""
               - name: SMTP_TO
                 value: ""
-            restartPolicy: OnFailure
+              - name: ALWAYS_EXIT_ZERO
+                value: "true"
 - apiVersion: batch/v2alpha1
   kind: ScheduledJob
   metadata:
@@ -114,6 +116,7 @@ items:
           spec:
             # 30 minutes timeout
             activeDeadlineSeconds: 1800
+            restartPolicy: OnFailure
             containers:
             - name: postgresql-s3-daily-backup
               image: willemvd/postgresql-client-side-encrypted-s3-backup:1.1.0
@@ -177,4 +180,5 @@ items:
                 value: ""
               - name: SMTP_TO
                 value: ""
-            restartPolicy: OnFailure
+              - name: ALWAYS_EXIT_ZERO
+                value: "true"