[patch] Convert upgrade & uninstall functions to Python

.github/workflows/build-cli.yml

@@ -11,8 +11,10 @@ jobs:
   build-cli:
     runs-on: ubuntu-latest
     steps:
+      # 1. Initialize the build
+      # -------------------------------------------------------------------------------------------
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         # Without this option, we don't get the tag information
         with:
           fetch-depth: 0
@@ -23,21 +25,21 @@ jobs:
           $GITHUB_WORKSPACE/build/bin/initbuild.sh
           source $GITHUB_WORKSPACE/build/bin/.functions.sh
 
-      # 1. Download Ansible collection from Artifactory
+      # 2. Download Ansible collection from Artifactory
       # -------------------------------------------------------------------------------------------
       - name: Download Ansible collection from Artifactory
         env:
           ARTIFACTORY_GENERIC_RELEASE_URL: ${{ secrets.ARTIFACTORY_GENERIC_RELEASE_URL }}
           ARTIFACTORY_TOKEN: ${{ secrets.ARTIFACTORY_TOKEN }}
         run: |
-          if [[ -e $GITHUB_WORKSPACE/image/cli/install-ansible/ibm-mas_devops.tar.gz ]]; then
-            echo "Found a local Ansible collection to be used in $GITHUB_WORKSPACE/image/cli/install-ansible/ibm-mas_devops.tar.gz! Skip download from Artifactory..."
+          if [[ -e $GITHUB_WORKSPACE/image/cli/install/ibm-mas_devops.tar.gz ]]; then
+            echo "Found a local Ansible collection to be used in $GITHUB_WORKSPACE/image/cli/install/ibm-mas_devops.tar.gz! Skip download from Artifactory..."
           else
             echo "Downloading from ***/ibm-mas/ansible-devops/latest/ibm-mas_devops-latest.tar.gz"
-            wget --header="Authorization:Bearer $ARTIFACTORY_TOKEN" $ARTIFACTORY_GENERIC_RELEASE_URL/ibm-mas/ansible-devops/latest/ibm-mas_devops-latest.tar.gz -O $GITHUB_WORKSPACE/image/cli/install-ansible/ibm-mas_devops.tar.gz
+            wget --header="Authorization:Bearer $ARTIFACTORY_TOKEN" $ARTIFACTORY_GENERIC_RELEASE_URL/ibm-mas/ansible-devops/latest/ibm-mas_devops-latest.tar.gz -O $GITHUB_WORKSPACE/image/cli/install/ibm-mas_devops.tar.gz
           fi
 
-      # 2. Tekton
+      # 3. Tekton
       # -------------------------------------------------------------------------------------------
       - name: Build the Tekton definitions
         run: $GITHUB_WORKSPACE/build/bin/build-tekton.sh
@@ -57,7 +59,23 @@ jobs:
           $GITHUB_WORKSPACE/build/bin/artifactory-release.sh $GITHUB_WORKSPACE/tekton/target/ibm-mas-tekton.yaml
           $GITHUB_WORKSPACE/build/bin/artifactory-release.sh $GITHUB_WORKSPACE/tekton/target/ibm-mas-tekton-fvt.yaml
 
-      # 3. CLI installer
+      # 4. Python Package
+      # -------------------------------------------------------------------------------------------
+      - name: Build the Python package
+        run: |
+          sed -i "s#__version__ = \"100.0.0\"#__version__ = \"${{ env.VERSION_NOPREREL }}\"#g" ${GITHUB_WORKSPACE}/python/src/mas/cli/__init__.py
+          cat ${GITHUB_WORKSPACE}/python/src/mas/cli/__init__.py
+          mkdir -p $GITHUB_WORKSPACE/python/src/mas/cli/templates
+          cp $GITHUB_WORKSPACE/tekton/target/ibm-mas-tekton.yaml $GITHUB_WORKSPACE/python/src/mas/cli/templates/ibm-mas-tekton.yaml
+          cd $GITHUB_WORKSPACE/python
+          python -m pip install --upgrade pip
+          pip install .[dev]
+          flake8 src --count --select=E9,F63,F7,F82 --show-source --statistics
+          flake8 src --count --exit-zero --max-complexity=10 --max-line-length=200 --statistics
+          python -m build
+          cp $GITHUB_WORKSPACE/python/dist/mas_cli-${{ env.VERSION_NOPREREL }}.tar.gz $GITHUB_WORKSPACE/image/cli/install/mas_cli.tar.gz
+
+      # 5. CLI installer
       # -------------------------------------------------------------------------------------------
       - name: Build the cli package
         run: |
@@ -70,7 +88,7 @@ jobs:
           path: ${{ github.workspace }}/ibm-mas-cli-${{ env.VERSION }}.tgz
           retention-days: 30
 
-      # 4. CLI container image
+      # 6. CLI container image
       # -------------------------------------------------------------------------------------------
       - name: Build the docker image
         run: |
@@ -91,7 +109,7 @@ jobs:
         #   username: ${{ secrets.QUAYIO_USERNAME }}
         #   password: ${{ secrets.QUAYIO_PASSWORD }}
 
-      # 5. OWASP Dependency Check
+      # 7. OWASP Dependency Check
       # -------------------------------------------------------------------------------------------
       - name: Perform dependency check
         uses: dependency-check/Dependency-Check_Action@main

.github/workflows/docs.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2.3.1
+        uses: actions/checkout@v4
 
       - name: Install and Build
         run: |

.github/workflows/release-cli.yml

@@ -1,24 +1,30 @@
 name: Release CLI
 on:
-  push:
-    branches-ignore:
-      - '**'
-    tags:
-      - '**'
+  release:
+    types: [ published ]
+
+permissions:
+  contents: read
+
 jobs:
   build-cli:
     runs-on: ubuntu-latest
     steps:
+      # 1. Initialize the build
+      # -------------------------------------------------------------------------------------------
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
+        # Without this option, we don't get the tag information
+        with:
+          fetch-depth: 0
 
       - name: Initialise the build system
         run: |
           chmod u+x $GITHUB_WORKSPACE/build/bin/*.sh
           $GITHUB_WORKSPACE/build/bin/initbuild.sh
           source $GITHUB_WORKSPACE/build/bin/.functions.sh
 
-      # 1. Tekton
+      # 2. Tekton
       # -------------------------------------------------------------------------------------------
       - name: Build the Tekton definitions
         run: $GITHUB_WORKSPACE/build/bin/build-tekton.sh
@@ -40,8 +46,30 @@ jobs:
           $GITHUB_WORKSPACE/build/bin/artifactory-release.sh $GITHUB_WORKSPACE/tekton/target/ibm-mas-tekton.yaml
           $GITHUB_WORKSPACE/build/bin/artifactory-release.sh $GITHUB_WORKSPACE/tekton/target/ibm-mas-tekton-fvt.yaml
 
+      # 3. Python Package
+      # -------------------------------------------------------------------------------------------
+      - name: Build the Python package
+        run: |
+          sed -i "s#__version__ = \"100.0.0\"#__version__ = \"${{ env.VERSION_NOPREREL }}\"#g" ${GITHUB_WORKSPACE}/python/src/mas/cli/__init__.py
+          cat ${GITHUB_WORKSPACE}/python/src/mas/cli/__init__.py
+          mkdir -p $GITHUB_WORKSPACE/python/src/mas/cli/templates
+          cp $GITHUB_WORKSPACE/tekton/target/ibm-mas-tekton.yaml $GITHUB_WORKSPACE/python/src/mas/cli/templates/ibm-mas-tekton.yaml
+          cd $GITHUB_WORKSPACE/python
+          python -m pip install --upgrade pip
+          pip install .[dev]
+          flake8 src --count --select=E9,F63,F7,F82 --show-source --statistics
+          flake8 src --count --exit-zero --max-complexity=10 --max-line-length=200 --statistics
+          python -m build
+          cp $GITHUB_WORKSPACE/python/dist/mas_cli-${{ env.VERSION_NOPREREL }}.tar.gz $GITHUB_WORKSPACE/image/cli/install/mas_cli.tar.gz
+
+      - name: Publish package
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          packages-dir: python/dist
 
-      # 2. CLI installer
+      # 4. CLI installer
       # -------------------------------------------------------------------------------------------
       - name: Build the cli package
         run: |
@@ -56,8 +84,7 @@ jobs:
           tag: ${{ github.ref }}
           overwrite: true
 
-
-      # 3. CLI container image
+      # 5. CLI container image
       # -------------------------------------------------------------------------------------------
       - name: Build the docker image
         run: |
@@ -80,7 +107,7 @@ jobs:
         #   username: ${{ secrets.QUAYIO_USERNAME }}
         #   password: ${{ secrets.QUAYIO_PASSWORD }}
 
-      # 4. OWASP Dependency Check
+      # 6. OWASP Dependency Check
       # -------------------------------------------------------------------------------------------
       - name: Perform dependency check
         uses: dependency-check/Dependency-Check_Action@main

.github/workflows/verify-cli-pr.yml

@@ -9,11 +9,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Fail if local Ansible collection is present in PR to master
         run: |
-          if [[ -e $GITHUB_WORKSPACE/image/cli/install-ansible/ibm-mas_devops.tar.gz ]]; then
-            echo "Found a local Ansible collection($GITHUB_WORKSPACE/image/cli/install-ansible/ibm-mas_devops.tar.gz) in this PR, Local ansible collection is not allowed in master branch. Failing build."
+          if [[ -e $GITHUB_WORKSPACE/image/cli/install/ibm-mas_devops.tar.gz ]]; then
+            echo "Found a local Ansible collection($GITHUB_WORKSPACE/image/cli/install/ibm-mas_devops.tar.gz) in this PR, Local ansible collection is not allowed in master branch. Failing build."
             exit 1
           fi

.gitignore

@@ -1,7 +1,5 @@
-
 *.DS_Store*
 .vscode
-
 *.pyc
 
 image/cli/mascli/templates/ibm-mas-tekton.yaml
@@ -12,3 +10,15 @@ tmp/
 tekton/test-*.yaml
 tekton/target
 tekton/tmp
+
+kubectl.exe
+mas.log
+
+python/pandoc-*-amd64.deb
+python/dist
+python/venv
+python/src/mas_cli.egg-info
+python/README.rst
+python/src/mas/cli/templates/ibm-mas-tekton.yaml
+/python/build
+python/mas-upgrade.spec

CONTRIBUTING.md

@@ -124,4 +124,4 @@ The MAS Ansible Devops collection contains the ansible roles that are used to au
 When building a MAS CLI pre-release image version, the build system will embed the MAS Ansible Devops `tar.gz` following the rule:
 
 - By default, MAS CLI will build its pre-release image using [the latest MAS Ansible Devops released version](https://github.com/ibm-mas/ansible-devops/releases).
-- **To use a custom MAS Ansible Devops collection within MAS CLI:**  If you are developing a custom MAS Ansible Devops collection and you want to build a MAS CLI image that will make use of this ansible collection, from the `cli` root folder, you can run `make ansible-build` to build and place your MAS Ansible Devops `tar.gz` into [`cli/image/cli/install-ansible`](image/cli/install-ansible/), that's the folder that CLI uses to install the MAS Ansible Devops collection that will be used within the MAS CLI container during the image build process.
+- **To use a custom MAS Ansible Devops collection within MAS CLI:**  If you are developing a custom MAS Ansible Devops collection and you want to build a MAS CLI image that will make use of this ansible collection, from the `cli` root folder, you can run `make ansible-build` to build and place your MAS Ansible Devops `tar.gz` into [`cli/image/cli/install`](image/cli/install/), that's the folder that CLI uses to install the MAS Ansible Devops collection that will be used within the MAS CLI container during the image build process.

Makefile

@@ -5,10 +5,10 @@
 .DEFAULT_GOAL := all
 
 ansible-build:
-	ansible-galaxy collection build --output-path image/cli/install-ansible ../ansible-devops/ibm/mas_devops --force
-	mv image/cli/install-ansible/ibm-mas_devops-100.0.0.tar.gz image/cli/install-ansible/ibm-mas_devops.tar.gz
+	ansible-galaxy collection build --output-path image/cli/install ../ansible-devops/ibm/mas_devops --force
+	mv image/cli/install/ibm-mas_devops-100.0.0.tar.gz image/cli/install/ibm-mas_devops.tar.gz
 ansible-install:
-	ansible-galaxy collection install image/cli/install-ansible/ibm-mas_devops.tar.gz --force --no-deps
+	ansible-galaxy collection install image/cli/install/ibm-mas_devops.tar.gz --force --no-deps
 ansible: ansible-build ansible-install
 
 tekton:
@@ -23,7 +23,7 @@ run:
 	docker run -ti cli:local
 
 clean:
-	rm image/cli/install-ansible/ibm-mas_devops.tar.gz
+	rm image/cli/install/ibm-mas_devops.tar.gz
 	rm image/cli/bin/templates/ibm-mas-tekton.yaml
 
 create:

build/bin/.env.sh

@@ -12,6 +12,11 @@ if [ -f "$VERSION_FILE" ]; then
   export VERSION=$(cat ${VERSION_FILE})
 fi
 
+export VERSION_FILE_NOPREREL=${GITHUB_WORKSPACE}/.version-noprerel
+if [ -f "$VERSION_FILE_NOPREREL" ]; then
+  export VERSION_NOPREREL=$(cat ${VERSION_FILE_NOPREREL})
+fi
+
 # During initbuild we record the release level (aka the version bump from the last release)
 export SEMVER_RELEASE_LEVEL_FILE=${GITHUB_WORKSPACE}/.releaselevel
 if [ -f "$SEMVER_RELEASE_LEVEL_FILE" ]; then
@@ -27,14 +32,18 @@ if [ -z $BUILD_SYSTEM_ENV_LOADED ]; then
   export BUILD_SYSTEM_ENV_LOADED=1
 
   if [ ! -z $GITHUB_ENV ]; then
-    echo "GITHUB_ENV is defined, exporting properties to $GITHUB_ENV"
     # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#environment-files
-    echo "VERSION=$VERSION" >> $GITHUB_ENV
+    echo "GITHUB_ENV is defined, exporting properties to $GITHUB_ENV"
+
     echo "VERSION_FILE=$VERSION_FILE" >> $GITHUB_ENV
+    echo "VERSION_FILE_NOPREREL=$VERSION_FILE_NOPREREL" >> $GITHUB_ENV
     echo "VERSION=$VERSION" >> $GITHUB_ENV
+    echo "VERSION_NOPREREL=$VERSION_NOPREREL" >> $GITHUB_ENV
     echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
+
     echo "SEMVER_RELEASE_LEVEL_FILE=$SEMVER_RELEASE_LEVEL_FILE" >> $GITHUB_ENV
     echo "SEMVER_RELEASE_LEVEL=$SEMVER_RELEASE_LEVEL" >> $GITHUB_ENV
+
     echo "BUILD_SYSTEM_ENV_LOADED=1" >> $GITHUB_ENV
   else
     echo "GITHUB_ENV is not defined"
@@ -45,7 +54,9 @@ if [ -z $BUILD_SYSTEM_ENV_LOADED ]; then
   echo_highlight "PATH ....................... $PATH"
   echo_highlight ""
   echo_highlight "VERSION_FILE ............... $VERSION_FILE"
+  echo_highlight "VERSION_FILE_NOPREREL ...... $VERSION_FILE_NOPREREL"
   echo_highlight "VERSION .................... $VERSION"
+  echo_highlight "VERSION_NOPREREL ........... $VERSION_NOPREREL"
   echo_highlight "DOCKER_TAG ................. $DOCKER_TAG"
   echo_highlight ""
   echo_highlight "SEMVER_RELEASE_LEVEL_FILE .. $SEMVER_RELEASE_LEVEL_FILE"

build/bin/initbuild.sh

@@ -9,6 +9,7 @@ pip install --quiet pyyaml yamllint
 # 1. Set up semantic versioning
 # -----------------------------------------------------------------------------
 VERSION_FILE=$GITHUB_WORKSPACE/.version
+VERSION_FILE_NOPREREL=$GITHUB_WORKSPACE/.version-noprerel
 
 if [[ "${GITHUB_REF_TYPE}" == "tag" ]]; then
   echo "Note: non-branch build for a tag named '${GITHUB_REF_NAME}'"
@@ -91,6 +92,7 @@ fi
 
 # 2. Tweak version string for pre-release builds
 # -----------------------------------------------------------------------------
+cp $VERSION_FILE $VERSION_FILE_NOPREREL
 if [[ "${GITHUB_REF_TYPE}" == "branch" ]]; then
     semver bump prerel pre.$GITHUB_REF_NAME $(cat $VERSION_FILE) > $VERSION_FILE
     echo "Pre-release build: $(cat $VERSION_FILE)"

image/cli/Dockerfile

@@ -1,6 +1,9 @@
 FROM quay.io/ibmmas/cli-base:latest
 ARG VERSION_LABEL
 
+# Auto-expire the image in quay.io after 3 weeks
+LABEL quay.expires-after=3w
+
 # 1. Copy basics
 COPY app-root/src/ /opt/app-root/src/
 COPY mascli/ /mascli/
@@ -15,21 +18,13 @@ ENV ANSIBLE_COLLECTIONS_PATH=/opt/app-root/lib64/python3.9/site-packages/ansible
     VERSION=${VERSION_LABEL:-x.y.z} \
     VIRTUAL_ENV_DISABLE_PROMPT=1
 
-# 3. Install Ansible collections -- do this last for efficient dev builds
-# 4. Set file permissions to be developer (hack) friendly
-COPY install-ansible /tmp/install-ansible
+# 3. Install Python packages
+# 4. Install Ansible collections
+# 5. Set file permissions to be developer (hack) friendly
+COPY install /tmp/install
 RUN umask 0002 && \
-    bash /tmp/install-ansible/install-ansible-collections.sh && \
-    rm -rf /tmp/install-ansible && \
-    chmod -R ug+rwx /opt/app-root/src/env.sh && \
-    chmod -R ug+rwx /opt/app-root/src/.ansible && \
-    chmod +x /opt/app-root/src/*.sh && \
-    chmod +x /opt/app-root/src/.bashrc && \
-    chmod -R ug+w /mascli && \
-    chmod +x /mascli/mas && \
-    chmod +x /mascli/must-gather/* && \
-    chmod +x /mascli/backup-restore/* && \
-    chmod -R ug+w /masfvt && \
-    chmod +x /usr/bin/gather && \
-    chmod -R g+w $ANSIBLE_COLLECTIONS_PATH/ibm/mas_devops && \
-    ln -s $ANSIBLE_COLLECTIONS_PATH/ibm/mas_devops /mascli/ansible-devops
+    ls /tmp/install && \
+    bash /tmp/install/install-python-packages.sh && \
+    bash /tmp/install/install-ansible-collections.sh && \
+    bash /tmp/install/permissions-updates.sh && \
+    rm -rf /tmp/install

image/cli/install/install-ansible-collections.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+ansible-galaxy collection install -r /tmp/install/requirements.yml -p $ANSIBLE_COLLECTIONS_PATH
+
+if [[ -e /tmp/install/ibm-mas_devops.tar.gz ]]
+then ansible-galaxy collection install /tmp/install/ibm-mas_devops.tar.gz -p $ANSIBLE_COLLECTIONS_PATH
+else ansible-galaxy collection install ibm.mas_devops
+fi

image/cli/install/install-python-packages.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+python3 -m pip install pip --upgrade
+python3 -m pip install /tmp/install/mas_cli.tar.gz

image/cli/install/permissions-updates.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -x
+set -e
+
+chmod -R ug+rwx /opt/app-root/src/env.sh
+chmod -R ug+rwx /opt/app-root/src/.ansible
+chmod +x /opt/app-root/src/*.sh
+chmod +x /opt/app-root/src/.bashrc
+chmod -R ug+w /mascli
+chmod +x /mascli/mas
+chmod +x /mascli/must-gather/*
+chmod +x /mascli/backup-restore/*
+chmod -R ug+w /masfvt
+chmod +x /usr/bin/gather
+chmod -R g+w $ANSIBLE_COLLECTIONS_PATH/ibm/mas_devops
+ln -s $ANSIBLE_COLLECTIONS_PATH/ibm/mas_devops /mascli/ansible-devops