[artifacts] Improve cloud deployment error handling (#137775)

* [artifacts] Improve cloud deployment error handling * Update .buildkite/scripts/steps/artifacts/cloud.sh Co-authored-by: Spencer <[email protected]> * update retry codes Co-authored-by: Spencer <[email protected]>
elastic · Sep 6, 2022 · 874e4cb · 874e4cb
1 parent cfe8ec2
commit 874e4cb
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 17 deletions.
diff --git a/.buildkite/pipelines/artifacts.yml b/.buildkite/pipelines/artifacts.yml
@@ -73,13 +73,23 @@ steps:
 
   - command: .buildkite/scripts/steps/artifacts/cloud.sh
     label: 'Cloud Deployment'
-    soft_fail: true
+    soft_fail:
+      - exit_status: 255
     agents:
       queue: n2-2
     timeout_in_minutes: 30
     if: "build.env('RELEASE_BUILD') == null || build.env('RELEASE_BUILD') == '' || build.env('RELEASE_BUILD') == 'false'"
     retry:
       automatic:
+        # Timeout and graceful shutdown | ecctl deployment create falure
+        - exit_status: 255
+          limit: 0
+
+        # Timeout and forced shutdown
+        - exit_status: '-1'
+          limit: 0
+
+        # Test failures
         - exit_status: '*'
           limit: 1
 

diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh
@@ -12,28 +12,28 @@ mkdir -p target
 
 download_artifact "kibana-$FULL_VERSION-linux-x86_64.tar.gz" ./target --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}"
 
-node scripts/build \
-  --skip-initialize \
-  --skip-generic-folders \
-  --skip-platform-folders \
-  --skip-archives \
-  --docker-images \
-  --skip-docker-ubi \
-  --skip-docker-ubuntu \
-  --skip-docker-contexts
-
-docker load --input target/kibana-cloud-$FULL_VERSION-docker-image.tar.gz
-
 TAG="$FULL_VERSION-$GIT_COMMIT"
-KIBANA_BASE_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$FULL_VERSION"
 KIBANA_TEST_IMAGE="docker.elastic.co/kibana-ci/kibana-cloud:$TAG"
 
-docker tag "$KIBANA_BASE_IMAGE" "$KIBANA_TEST_IMAGE"
-
 echo "$KIBANA_DOCKER_PASSWORD" | docker login -u "$KIBANA_DOCKER_USERNAME" --password-stdin docker.elastic.co
 trap 'docker logout docker.elastic.co' EXIT
 
-docker push "$KIBANA_TEST_IMAGE"
+if  docker manifest inspect $KIBANA_TEST_IMAGE &> /dev/null; then
+  echo "Distribution already exists, skipping build"
+else
+  node scripts/build \
+    --skip-initialize \
+    --skip-generic-folders \
+    --skip-platform-folders \
+    --skip-archives \
+    --docker-images \
+    --docker-tag-qualifier="$GIT_COMMIT" \
+    --docker-push \
+    --skip-docker-ubi \
+    --skip-docker-ubuntu \
+    --skip-docker-contexts
+fi
+
 docker logout docker.elastic.co
 
 echo "--- Create deployment"
@@ -62,6 +62,7 @@ function shutdown {
 trap "shutdown" EXIT
 
 ecctl deployment create --track --output json --file "$DEPLOYMENT_SPEC" > "$LOGS"
+
 CLOUD_DEPLOYMENT_USERNAME=$(jq -r --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.username' "$LOGS")
 CLOUD_DEPLOYMENT_PASSWORD=$(jq -r --slurp '.[]|select(.resources).resources[] | select(.credentials).credentials.password' "$LOGS")
 CLOUD_DEPLOYMENT_ID=$(jq -r --slurp '.[0].id' "$LOGS")