frontend: ping frontend on docker image release (#2101)

Stacked on top of #2138 ## Why The frontend is now part of quickstart-vdk. On changes to the docker image, the helm chart needs to be redeployed. We need to make sure that the frontend is operational once the helm chart is deployed. ## What Trigger deployment of test control service on frontend changes merged into main Ping the frontend in the test environment using heartbeat once control service is deployed Tag the frontend image as stable if the test is successful ## How has this been tested [Test pipeline](https://gitlab.com/vmware-analytics/versatile-data-kit/-/pipelines/894015566) ## What kind of change is this feature, non-breaking Signed-off-by: Dilyan Marinov <[email protected]>
vmware · Jun 9, 2023 · 3788fc9 · 3788fc9
1 parent e96d729
commit 3788fc9
Show file tree

Hide file tree

Showing 5 changed files with 84 additions and 6 deletions.
diff --git a/...service/projects/helm_charts/pipelines-control-service/templates/frontend/deployment.yaml b/...service/projects/helm_charts/pipelines-control-service/templates/frontend/deployment.yaml
@@ -9,6 +9,7 @@ metadata:
   name: {{ .Release.Name }}-ui
   labels:
     app: {{ .Release.Name }}-ui
+    date: "{{ now | unixEpoch }}"
 spec:
   replicas: {{ .Values.operationsUi.replicaCount }}
   selector:
@@ -18,6 +19,7 @@ spec:
     metadata:
       labels:
         app: {{ .Release.Name }}-ui
+        date: "{{ now | unixEpoch }}"
     spec:
       {{- if and (.Values.operationsUi.image.username) (.Values.operationsUi.image.password) }}
       imagePullSecrets:

diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml b/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml
@@ -14,7 +14,7 @@ global: {}
 image:
   registry: registry.hub.docker.com/versatiledatakit
   repository: pipelines-control-service
-  tag: "99a4957"
+  tag: "stable"
   ## Specify a imagePullPolicy
   ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
   ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
@@ -32,7 +32,7 @@ operationsUi:
   image:
     registry: registry.hub.docker.com/versatiledatakit
     repository: vdk-operations-ui
-    tag: "latest"
+    tag: "stable"
     username:
     password:
   replicaCount: 1

diff --git a/projects/frontend/cicd/.gitlab-ci.yml b/projects/frontend/cicd/.gitlab-ci.yml
@@ -23,6 +23,12 @@
   - "projects/frontend/data-pipelines/gui/Dockerfile"
   - "projects/frontend/data-pipelines/gui/config/nginx.conf"
 
+.frontend_retry:
+  retry_options:
+    max: 1
+    when:
+      - always
+
 frontend-data-pipelines-build:
   image: "versatiledatakit/vdk-cicd-base-gui:2.0.0"
   stage: build
@@ -102,13 +108,31 @@ frontend-data-pipelines-release:
     expire_in: 1 week
 
 frontend_publish_ui_image:
+  stage: pre_release_image
+  before_script:
+    - cd projects/frontend/data-pipelines/gui
+  script:
+    - apk --no-cache add bash git
+    - docker login --username "${VDK_DOCKER_REGISTRY_USERNAME}" --password "${VDK_DOCKER_REGISTRY_PASSWORD}" "${VDK_DOCKER_REGISTRY_URL}"
+    - ../../cicd/publish_image_dockerhub.sh vdk-operations-ui . $CI_PIPELINE_ID
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      when: never
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      changes: *frontend_shared_components_locations
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      changes: *frontend_data_pipelines_locations
+  extends: .frontend_publish_docker_image
+
+frontend_tag_ui_image_stable:
   stage: release
   before_script:
     - cd projects/frontend
   script:
-    - apk --no-cache add bash
+    - apk --no-cache add bash git
+    - export IMAGE_TAG="$(git rev-parse --short HEAD)"
     - docker login --username "${VDK_DOCKER_REGISTRY_USERNAME}" --password "${VDK_DOCKER_REGISTRY_PASSWORD}" "${VDK_DOCKER_REGISTRY_URL}"
-    - ./cicd/publish_image_dockerhub.sh vdk-operations-ui ./data-pipelines/gui $CI_PIPELINE_ID
+    - bash -ex ../../projects/control-service/cicd/tag_image_dockerhub.sh vdk-operations-ui $IMAGE_TAG stable
   rules:
     - if: '$CI_PIPELINE_SOURCE == "schedule"'
       when: never
@@ -118,8 +142,53 @@ frontend_publish_ui_image:
       changes: *frontend_data_pipelines_locations
   extends: .frontend_publish_docker_image
 
+frontend_deploy_testing_data_pipelines:
+  stage: pre_release_test
+  image: docker:23.0.1
+  script:
+    - apk --no-cache add bash openssl curl git gettext zip py-pip
+    - pip install --upgrade pip && pip install awscli
+    - export DESIRED_VERSION=v3.11.3 # helm version 3.11.3
+    - export TAG=latest
+    - export FRONTEND_TAG=$(git rev-parse --short HEAD)
+    - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+    - bash -ex ./projects/control-service/cicd/deploy-testing-pipelines-service.sh
+  retry: !reference [.frontend_retry, retry_options]
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      when: never
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      changes: *frontend_shared_components_locations
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      changes: *frontend_data_pipelines_locations
+
+frontend_heartbeat_test:
+  stage: pre_release_test
+  needs:
+    - job: frontend_deploy_testing_data_pipelines
+      optional: false
+      artifacts: false
+  image: python:3.7
+  script:
+    - set -x
+    - pip install quickstart-vdk
+    - pip install vdk-heartbeat[trino]
+    - vdk version
+    - export IMAGE_TAG="$(git rev-parse --short HEAD)"
+    - export VDKCLI_OAUTH2_REFRESH_TOKEN=$VDK_API_TOKEN
+    - export VDK_HEARTBEAT_OP_ID="vdkcs-$CI_PIPELINE_ID"
+    - vdk-heartbeat -f projects/frontend/cicd/ping_frontend_heartbeat_config.ini
+  retry: !reference [.frontend_retry, retry_options]
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "schedule"'
+      when: never
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      changes: *frontend_shared_components_locations
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      changes: *frontend_data_pipelines_locations
+
 frontend_publish_test_image:
-  stage: release
+  stage: release_image
   script:
     - docker login --username "${VDK_DOCKER_REGISTRY_USERNAME}" --password "${VDK_DOCKER_REGISTRY_PASSWORD}" "${VDK_DOCKER_REGISTRY_URL}"
     - cd projects/frontend/cicd

diff --git a/projects/frontend/cicd/ping_frontend_heartbeat_config.ini b/projects/frontend/cicd/ping_frontend_heartbeat_config.ini
@@ -0,0 +1,5 @@
+[DEFAULT]
+JOB_RUN_TEST_MODULE_NAME=vdk.internal.heartbeat.ping_frontend_test
+JOB_RUN_TEST_CLASS_NAME=PingFrontendTest
+
+CONTROL_API_URL=http://cicd-control-service-ui:80
diff --git a/projects/frontend/cicd/publish_image_dockerhub.sh b/projects/frontend/cicd/publish_image_dockerhub.sh
@@ -19,7 +19,9 @@ version_tag=$(awk -v id=$patch_version 'BEGIN { FS="."; OFS = "."; ORS = "" } {
 
 image_repo="$VDK_DOCKER_REGISTRY_URL/$name"
 image_tag="$image_repo:$version_tag"
+commit_sha=$(git rev-parse --short HEAD)
 
-docker build -t "$image_tag" -t "$image_repo:latest" $ui_path
+docker build -t "$image_tag" -t "$image_repo:latest" -t "$image_repo:$commit_sha" $ui_path
 docker push "$image_tag"
 docker push "$image_repo:latest"
+docker push "$image_repo:$commit_sha"