frontend: use versatiledatakit/shared rc-builds for data-pipelines (#…

…2312)

## Why?

Linking shared-components to data-pipelines using npm fails to produce a
working UI docker image. This is a CI-only issue.

## What?

Introduce rc-versions for @versatiledatakit/shared
On a successful build of shared, and rc-version is pushed to npm
registry
The data-pipelines build pulls the latest rc-version when running,
instead of linking to the local installation of shared

Versioning schema follows semver

{major}.{minor}.{patch}-rc.{rc-number}

The rc-number is incremented based on the latest rc-number for the
pipeline-id. This helps avoid conflicts between different pipelines
running builds for shared

Regular shared versions are tagged with latest in npm registry
Rc versions are tagged with rc, so they don't get pulled accidentally
when running npm install

## How was this tested?

Ran a build in CI and tested that the docker image starts locally as
part of quickstart-vdk by changing the deployment tag


https://gitlab.com/vmware-analytics/versatile-data-kit/-/pipelines/909682172

Docker image tag pushed by pipeline `5eb2c00`
To check it locally, run quickstart-vdk and edit the deployment to use
this tag.
<img width="1654" alt="Screenshot 2023-06-26 at 12 17 49"
src="https://github.com/vmware/versatile-data-kit/assets/91800778/1cf6ba0c-b899-46f1-8827-abd29c4fcbf9">


## What kind of change is this?

Feature/non-breaking

Signed-off-by: Dilyan Marinov <[email protected]>

projects/frontend/cicd/.gitlab-ci.yml

@@ -38,7 +38,10 @@ frontend-data-pipelines-build:
   before_script:
     - cd projects/frontend/
   script:
-    - ./cicd/install_data_pipelines.sh
+    # Note: if frontend-shared-components-build is not part of the pipeline and no rc-version exists, $SHARED_RC_VERSION will be empty.
+    # If $SHARED_RC_VERSION is empty, calling install_data_pipelines.sh will install the latest @versatiledatakit/shared package from registry.
+    - export SHARED_RC_VERSION=$(../../cicd/get_latest_shared_rc_version.sh $CI_PIPELINE_ID)
+    - ./cicd/install_data_pipelines.sh $SHARED_RC_VERSION
   coverage: /^TOTAL\s+\d+\s+\d+\s+(\d+\%)$/
   retry: !reference [.retry, retry_options]
   rules:
@@ -93,7 +96,7 @@ frontend-data-pipelines-release:
   before_script:
     - cd projects/frontend/
   script:
-    - ./cicd/publish_package_npm.sh data-pipelines $CI_PIPELINE_ID $NPM_TOKEN $NPM_REGISTRY
+    - ./cicd/publish_package_npm.sh data-pipelines $CI_PIPELINE_ID $NPM_TOKEN $NPM_REGISTRY latest
   retry: !reference [.retry, retry_options]
   rules:
     - if: '$CI_PIPELINE_SOURCE == "schedule"'
@@ -124,6 +127,7 @@ frontend_publish_ui_image:
       changes: *frontend_data_pipelines_locations
   extends: .frontend_publish_docker_image
 
+# TODO: Enable this when https://github.com/vmware/versatile-data-kit/issues/2332 is resolved
 # frontend_tag_ui_image_stable:
 #   stage: release
 #   before_script:
@@ -211,6 +215,9 @@ frontend-shared-components-build:
     - cd projects/frontend/shared-components/gui
   script:
     - ../../cicd/install_shared.sh
+    # Note: if this is the first time the pipeline runs and no rc-version exists, $SHARED_RC_INCREMENT will equal 1.
+    - export SHARED_RC_INCREMENT=$(../../cicd/get_next_shared_rc_version_number.sh $CI_PIPELINE_ID)
+    - ../../cicd/publish_package_npm.sh shared-components $CI_PIPELINE_ID-rc.$SHARED_RC_INCREMENT $NPM_TOKEN $NPM_REGISTRY rc
   retry: !reference [.retry, retry_options]
   rules:
     - if: '$CI_COMMIT_BRANCH == "main" || $CI_PIPELINE_SOURCE == "external_pull_request_event"'
@@ -234,7 +241,7 @@ frontend-shared-components-release:
   before_script:
     - cd projects/frontend/
   script:
-    - ./cicd/publish_package_npm.sh shared-components $CI_PIPELINE_ID $NPM_TOKEN $NPM_REGISTRY
+    - ./cicd/publish_package_npm.sh shared-components $CI_PIPELINE_ID $NPM_TOKEN $NPM_REGISTRY latest
   retry: !reference [.retry, retry_options]
   rules:
     - if: '$CI_PIPELINE_SOURCE == "schedule"'

projects/frontend/cicd/get_latest_shared_rc_version.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -e
+
+# Copyright 2021-2023 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+
+# Extracts the latest rc-version for the pipeline id that's passed
+if [ $# -eq 0 ]
+  then
+    echo "ERROR: No argument for pipeline id supplied"
+    exit 3
+fi
+npm view @versatiledatakit/shared versions --json | grep $1-rc | cut -d \" -f 2 | tail -1

projects/frontend/cicd/get_next_shared_rc_version_number.sh

@@ -0,0 +1,17 @@
+#!/bin/bash -e
+
+# Copyright 2021-2023 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+# Extracts the latest rc-version number for the pipeline id that's passed
+# and returns the incremented version number
+# e.g. if the latest rc-version is 1.3.${pipeline_id}-rc.3, the script will return 4
+if [ $# -eq 0 ]
+  then
+    echo "ERROR: No argument for pipeline id supplied"
+    exit 3
+fi
+
+out=$(npm view @versatiledatakit/shared versions --json | grep $1-rc | cut -d \" -f 2 | tail -1 | awk -F '.' '{print $4}')
+out=$((out+1))
+echo $out

projects/frontend/cicd/install_data_pipelines.sh

@@ -9,6 +9,8 @@
 # lints them to verify format, builds the UI application and runs the unit tests.
 ###
 
+shared_version=$1
+
 if ! which npm >/dev/null 2>&1 ; then
   echo "ERROR:"
   echo "Please install npm 8.5.5+. Install cannot continue without it."
@@ -26,8 +28,14 @@ rm -f "package-lock.json"
 shared_dist_dir="../../shared-components/gui/dist/shared"
 if [ -d "$shared_dist_dir" ]
 then
-  echo "Linking the shared-components dist rebuild found..."
-  npm link "$shared_dist_dir"
+  if [ -n "$shared_version"]
+  then
+    echo "Installing the latest shared-components rc-version"
+    npm install --no-save "@versatiledatakit/shared@$shared_version"
+  else
+    echo "Linking the shared-components dist rebuild found..."
+    npm link "$shared_dist_dir"
+  fi
 else
   echo "No shared-components dist rebuild found."
 fi

projects/frontend/cicd/publish_package_npm.sh

@@ -54,14 +54,19 @@ if [ $# -eq 2 ]
 fi
 npm_token=$3
 
-# optional; defaults to registry.npmjs.org public repo
 if [ $# -eq 3 ]
   then
-    npm_registry="registry.npmjs.org"
-  else
-    npm_registry=$4
+    echo "ERROR: No argument for npm registry url provided."
+    exit 3
 fi
+npm_registry=$4
 
+if [ $# -eq 4 ]
+  then
+    echo "ERROR: No argument for npm tag provided."
+    exit 3
+fi
+npm_tag=$5
 
 # Locate the project dist directory
 if [ -d "./dist/$project_type/" ]
@@ -87,4 +92,4 @@ npm set //$npm_registry/:_authToken $npm_token
 
 # Publish
 echo "Publishing @vdk/$(basename "$PWD"):${package_version}..."
-npm publish --ignore-scripts --access public
+npm publish --ignore-scripts --access public --tag $npm_tag