From c3fa77e4fadaaf7ad9c32444ee72e76b09fa0231 Mon Sep 17 00:00:00 2001 From: Dilyan Marinov <91800778+DeltaMichael@users.noreply.github.com> Date: Wed, 28 Jun 2023 15:21:01 +0300 Subject: [PATCH] frontend: use versatiledatakit/shared rc-builds for data-pipelines (#2312) ## Why? Linking shared-components to data-pipelines using npm fails to produce a working UI docker image. This is a CI-only issue. ## What? Introduce rc-versions for @versatiledatakit/shared On a successful build of shared, and rc-version is pushed to npm registry The data-pipelines build pulls the latest rc-version when running, instead of linking to the local installation of shared Versioning schema follows semver {major}.{minor}.{patch}-rc.{rc-number} The rc-number is incremented based on the latest rc-number for the pipeline-id. This helps avoid conflicts between different pipelines running builds for shared Regular shared versions are tagged with latest in npm registry Rc versions are tagged with rc, so they don't get pulled accidentally when running npm install ## How was this tested? Ran a build in CI and tested that the docker image starts locally as part of quickstart-vdk by changing the deployment tag https://gitlab.com/vmware-analytics/versatile-data-kit/-/pipelines/909682172 Docker image tag pushed by pipeline `5eb2c00` To check it locally, run quickstart-vdk and edit the deployment to use this tag. Screenshot 2023-06-26 at 12 17 49 ## What kind of change is this? Feature/non-breaking Signed-off-by: Dilyan Marinov --- projects/frontend/cicd/.gitlab-ci.yml | 13 ++++++++++--- .../cicd/get_latest_shared_rc_version.sh | 13 +++++++++++++ .../cicd/get_next_shared_rc_version_number.sh | 17 +++++++++++++++++ .../frontend/cicd/install_data_pipelines.sh | 12 ++++++++++-- projects/frontend/cicd/publish_package_npm.sh | 15 ++++++++++----- 5 files changed, 60 insertions(+), 10 deletions(-) create mode 100755 projects/frontend/cicd/get_latest_shared_rc_version.sh create mode 100755 projects/frontend/cicd/get_next_shared_rc_version_number.sh diff --git a/projects/frontend/cicd/.gitlab-ci.yml b/projects/frontend/cicd/.gitlab-ci.yml index 842a8d3281..930c734b61 100644 --- a/projects/frontend/cicd/.gitlab-ci.yml +++ b/projects/frontend/cicd/.gitlab-ci.yml @@ -38,7 +38,10 @@ frontend-data-pipelines-build: before_script: - cd projects/frontend/ script: - - ./cicd/install_data_pipelines.sh + # Note: if frontend-shared-components-build is not part of the pipeline and no rc-version exists, $SHARED_RC_VERSION will be empty. + # If $SHARED_RC_VERSION is empty, calling install_data_pipelines.sh will install the latest @versatiledatakit/shared package from registry. + - export SHARED_RC_VERSION=$(../../cicd/get_latest_shared_rc_version.sh $CI_PIPELINE_ID) + - ./cicd/install_data_pipelines.sh $SHARED_RC_VERSION coverage: /^TOTAL\s+\d+\s+\d+\s+(\d+\%)$/ retry: !reference [.retry, retry_options] rules: @@ -93,7 +96,7 @@ frontend-data-pipelines-release: before_script: - cd projects/frontend/ script: - - ./cicd/publish_package_npm.sh data-pipelines $CI_PIPELINE_ID $NPM_TOKEN $NPM_REGISTRY + - ./cicd/publish_package_npm.sh data-pipelines $CI_PIPELINE_ID $NPM_TOKEN $NPM_REGISTRY latest retry: !reference [.retry, retry_options] rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' @@ -124,6 +127,7 @@ frontend_publish_ui_image: changes: *frontend_data_pipelines_locations extends: .frontend_publish_docker_image +# TODO: Enable this when https://github.com/vmware/versatile-data-kit/issues/2332 is resolved # frontend_tag_ui_image_stable: # stage: release # before_script: @@ -211,6 +215,9 @@ frontend-shared-components-build: - cd projects/frontend/shared-components/gui script: - ../../cicd/install_shared.sh + # Note: if this is the first time the pipeline runs and no rc-version exists, $SHARED_RC_INCREMENT will equal 1. + - export SHARED_RC_INCREMENT=$(../../cicd/get_next_shared_rc_version_number.sh $CI_PIPELINE_ID) + - ../../cicd/publish_package_npm.sh shared-components $CI_PIPELINE_ID-rc.$SHARED_RC_INCREMENT $NPM_TOKEN $NPM_REGISTRY rc retry: !reference [.retry, retry_options] rules: - if: '$CI_COMMIT_BRANCH == "main" || $CI_PIPELINE_SOURCE == "external_pull_request_event"' @@ -234,7 +241,7 @@ frontend-shared-components-release: before_script: - cd projects/frontend/ script: - - ./cicd/publish_package_npm.sh shared-components $CI_PIPELINE_ID $NPM_TOKEN $NPM_REGISTRY + - ./cicd/publish_package_npm.sh shared-components $CI_PIPELINE_ID $NPM_TOKEN $NPM_REGISTRY latest retry: !reference [.retry, retry_options] rules: - if: '$CI_PIPELINE_SOURCE == "schedule"' diff --git a/projects/frontend/cicd/get_latest_shared_rc_version.sh b/projects/frontend/cicd/get_latest_shared_rc_version.sh new file mode 100755 index 0000000000..175a6013fd --- /dev/null +++ b/projects/frontend/cicd/get_latest_shared_rc_version.sh @@ -0,0 +1,13 @@ +#!/bin/bash -e + +# Copyright 2021-2023 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 + + +# Extracts the latest rc-version for the pipeline id that's passed +if [ $# -eq 0 ] + then + echo "ERROR: No argument for pipeline id supplied" + exit 3 +fi +npm view @versatiledatakit/shared versions --json | grep $1-rc | cut -d \" -f 2 | tail -1 diff --git a/projects/frontend/cicd/get_next_shared_rc_version_number.sh b/projects/frontend/cicd/get_next_shared_rc_version_number.sh new file mode 100755 index 0000000000..c5892aa8fc --- /dev/null +++ b/projects/frontend/cicd/get_next_shared_rc_version_number.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e + +# Copyright 2021-2023 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 + +# Extracts the latest rc-version number for the pipeline id that's passed +# and returns the incremented version number +# e.g. if the latest rc-version is 1.3.${pipeline_id}-rc.3, the script will return 4 +if [ $# -eq 0 ] + then + echo "ERROR: No argument for pipeline id supplied" + exit 3 +fi + +out=$(npm view @versatiledatakit/shared versions --json | grep $1-rc | cut -d \" -f 2 | tail -1 | awk -F '.' '{print $4}') +out=$((out+1)) +echo $out diff --git a/projects/frontend/cicd/install_data_pipelines.sh b/projects/frontend/cicd/install_data_pipelines.sh index 3f59a722b6..8eb2b5b88d 100755 --- a/projects/frontend/cicd/install_data_pipelines.sh +++ b/projects/frontend/cicd/install_data_pipelines.sh @@ -9,6 +9,8 @@ # lints them to verify format, builds the UI application and runs the unit tests. ### +shared_version=$1 + if ! which npm >/dev/null 2>&1 ; then echo "ERROR:" echo "Please install npm 8.5.5+. Install cannot continue without it." @@ -26,8 +28,14 @@ rm -f "package-lock.json" shared_dist_dir="../../shared-components/gui/dist/shared" if [ -d "$shared_dist_dir" ] then - echo "Linking the shared-components dist rebuild found..." - npm link "$shared_dist_dir" + if [ -n "$shared_version"] + then + echo "Installing the latest shared-components rc-version" + npm install --no-save "@versatiledatakit/shared@$shared_version" + else + echo "Linking the shared-components dist rebuild found..." + npm link "$shared_dist_dir" + fi else echo "No shared-components dist rebuild found." fi diff --git a/projects/frontend/cicd/publish_package_npm.sh b/projects/frontend/cicd/publish_package_npm.sh index 9d903738d4..7b83f35c14 100755 --- a/projects/frontend/cicd/publish_package_npm.sh +++ b/projects/frontend/cicd/publish_package_npm.sh @@ -54,14 +54,19 @@ if [ $# -eq 2 ] fi npm_token=$3 -# optional; defaults to registry.npmjs.org public repo if [ $# -eq 3 ] then - npm_registry="registry.npmjs.org" - else - npm_registry=$4 + echo "ERROR: No argument for npm registry url provided." + exit 3 fi +npm_registry=$4 +if [ $# -eq 4 ] + then + echo "ERROR: No argument for npm tag provided." + exit 3 +fi +npm_tag=$5 # Locate the project dist directory if [ -d "./dist/$project_type/" ] @@ -87,4 +92,4 @@ npm set //$npm_registry/:_authToken $npm_token # Publish echo "Publishing @vdk/$(basename "$PWD"):${package_version}..." -npm publish --ignore-scripts --access public +npm publish --ignore-scripts --access public --tag $npm_tag