From 874e4cb913da086e66ec0601ff4733dfa254cd38 Mon Sep 17 00:00:00 2001
From: Jonathan Budzenski <jon@elastic.co>
Date: Tue, 6 Sep 2022 13:43:40 -0400
Subject: [PATCH] [artifacts] Improve cloud deployment error handling (#137775)

* [artifacts] Improve cloud deployment error handling

* Update .buildkite/scripts/steps/artifacts/cloud.sh

Co-authored-by: Spencer <spencer@elastic.co>

* update retry codes

Co-authored-by: Spencer <spencer@elastic.co>
---
 .buildkite/pipelines/artifacts.yml          | 12 +++++++-
 .buildkite/scripts/steps/artifacts/cloud.sh | 33 +++++++++++----------
 2 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml
index 8e08c736694e8..b6d3cc9fc9b14 100644
--- a/.buildkite/pipelines/artifacts.yml
+++ b/.buildkite/pipelines/artifacts.yml
@@ -73,13 +73,23 @@ steps:
 
   - command: .buildkite/scripts/steps/artifacts/cloud.sh
     label: 'Cloud Deployment'
-    soft_fail: true
+    soft_fail:
+      - exit_status: 255
     agents:
       queue: n2-2
     timeout_in_minutes: 30
     if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'"
     retry:
       automatic:
+        # Timeout and graceful shutdown | ecctl deployment create falure
+        - exit_status: 255
+          limit: 0
+
+        # Timeout and forced shutdown
+        - exit_status: '-1'
+          limit: 0
+
+        # Test failures
         - exit_status: '*'
           limit: 1
 
diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh
index 5bf2285ab162a..4d2317ce0b6c7 100644
--- a/.buildkite/scripts/steps/artifacts/cloud.sh
+++ b/.buildkite/scripts/steps/artifacts/cloud.sh
@@ -12,28 +12,28 @@ mkdir -p target
 
 download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
-node scripts/build \
-  --skip-initialize \
-  --skip-generic-folders \
-  --skip-platform-folders \
-  --skip-archives \
-  --docker-images \
-  --skip-docker-ubi \
-  --skip-docker-ubuntu \
-  --skip-docker-contexts
-
-docker load --input target/kibana-cloud-$FULL_VERSION-docker-image.tar.gz
-
 TAG="$FULL_VERSION-$GIT_COMMIT"
-KIBANA_BASE_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$FULL_VERSION"
 KIBANA_TEST_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$TAG"
 
-docker tag "$KIBANA_BASE_IMAGE" "$KIBANA_TEST_IMAGE"
-
 echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
 trap 'docker logout docker.elastic.co' EXIT
 
-docker push "$KIBANA_TEST_IMAGE"
+if  docker manifest inspect $KIBANA_TEST_IMAGE &> /dev/null; then
+  echo "Distribution already exists, skipping build"
+else
+  node scripts/build \
+    --skip-initialize \
+    --skip-generic-folders \
+    --skip-platform-folders \
+    --skip-archives \
+    --docker-images \
+    --docker-tag-qualifier="$GIT_COMMIT" \
+    --docker-push \
+    --skip-docker-ubi \
+    --skip-docker-ubuntu \
+    --skip-docker-contexts
+fi
+
 docker logout docker.elastic.co
 
 echo "--- Create deployment"
@@ -62,6 +62,7 @@ function shutdown {
 trap "shutdown" EXIT
 
 ecctl deployment create --track --output json --file "$DEPLOYMENT_SPEC" > "$LOGS"
+
 CLOUD_DEPLOYMENT_USERNAME=$(jq -r --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$LOGS")
 CLOUD_DEPLOYMENT_PASSWORD=$(jq -r --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$LOGS")
 CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$LOGS")