[minor] Improvements to Red Hat mirroring (#651)

build/bin/.functions.sh

@@ -8,7 +8,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 # COLOR_BLUE=`tput setaf 4`
 # COLOR_MAGENTA=`tput setaf 5`
 # COLOR_CYAN=`tput setaf 6`
-# COLOR_RESET=`tput sgr0`
+# TEXT_RESET=`tput sgr0`
 
 # tput doesn't work in GitHub actions
 # TODO: Integrate properly with GitHub actions to annotate the output as errors etc
@@ -18,7 +18,7 @@ COLOR_YELLOW=""
 COLOR_BLUE=""
 COLOR_MAGENTA=""
 COLOR_CYAN=""
-COLOR_RESET=""
+TEXT_RESET=""
 
 
 function echo_h1() {

docs/catalogs/v8-230926-amd64.md

@@ -76,7 +76,6 @@ For more information about the OCP lifecycle refer to the [Red Hat OpenShift Con
 
 The following packages from this catalog are used in the Maximo Application Suite install:
 
-- **strimzi-kafka-operator** required by `ibm.mas_devops.kafka` role (if using Strimzi as Kakfa provider)
 - **crunchy-postgres-operator** required by `ibm.mas_devops.uds` role
 - **gpu-operator-certified** required by `ibm.mas_devops.nvidia_gpu` role
 - **kubeturbo-certified** required by `ibm.mas_devops.kubeturbo` role
@@ -90,6 +89,7 @@ The following packages from this catalog are used in the Maximo Application Suit
 
 - **grafana-operator** required by `ibm.mas_devops.cluster_monitoring` role
 - **opentelemetry-operator** required by `ibm.mas_devops.cluster_monitoring` role
+- **strimzi-kafka-operator** required by `ibm.mas_devops.kafka` role (if using Strimzi as Kakfa provider)
 
 
 ### Red Hat Operators

image/cli/app-root/src/.bashrc

@@ -9,7 +9,6 @@ COLOR_YELLOW=`tput setaf 3`
 COLOR_BLUE=`tput setaf 4`
 COLOR_MAGENTA=`tput setaf 5`
 COLOR_CYAN=`tput setaf 6`
-COLOR_RESET=`tput sgr0`
 
 TEXT_BOLD=$(tput bold)
 TEXT_DIM=$(tput dim)

image/cli/mascli/functions/configtool_oidc

@@ -9,27 +9,27 @@ Usage:
 Where ${COLOR_YELLOW}specified${TEXT_RESET} each option may also be defined by setting the appropriate environment variable.
 When no options are specified on the command line, interactive-mode will be enabled by default.
 
-${COLOR_YELLOW}1. Cluster credentials${TEXT_RESET}  
-Log in to your cluster with your IBMid by using the following method, browse to the OpenShift web console. 
+${COLOR_YELLOW}1. Cluster credentials${TEXT_RESET}
+Log in to your cluster with your IBMid by using the following method, browse to the OpenShift web console.
 From the dropdown menu in the upper right of the page, click Copy Login Command.
-- CLUSTER_TOKEN=sha256~COA8-2Hd6G45rUN0HZLLh47sFByoX8QCC8j92jWB3to  
+- CLUSTER_TOKEN=sha256~COA8-2Hd6G45rUN0HZLLh47sFByoX8QCC8j92jWB3to
 - CLUSTER_SERVER=https://c130-e.us-south.containers.cloud.ibm.com:32250
 example: oc login --token=sha256~COA8-2Hd6G45rUN0HZLLh47sFByoX8QCC8j92jWB3to --server=https://c130-e.us-south.containers.cloud.ibm.com:32250
 
 ${COLOR_YELLOW}2. MAS home URL and trust ui prefix${TEXT_RESET}
-- MAS_HOME=masdev.home.mobfound1.masdev.suite.maximo.com  
-- TRUST_UI_PREFIX="http://localhost:3000, http://localhost:3001"          
+- MAS_HOME=masdev.home.mobfound1.masdev.suite.maximo.com
+- TRUST_UI_PREFIX="http://localhost:3000, http://localhost:3001"
 
 ${COLOR_YELLOW}3. Command operation${TEXT_RESET}
 ### mas oidc register
 Register oidc client for config tool.
-If client ever registered, it will be deleted firstly. 
+If client ever registered, it will be deleted firstly.
 
 ${COLOR_YELLOW}### mas oidc unregister${TEXT_RESET}
 Unregister oidc client for config tool previously registered.
 
 ${COLOR_YELLOW}### mas oidc update${TEXT_RESET}
-So far only trust ui prefix is supported to update. Same as register command. 
+So far only trust ui prefix is supported to update. Same as register command.
 
 ${COLOR_YELLOW}### mas oidc [-h|--help]${TEXT_RESET}
 Show this help message
@@ -66,7 +66,7 @@ function configtool_oidc_noninteractive() {
         ;;
       *)
         # unknown option
-        echo -e "${COLOR_RED}Usage Error: Unsupported option \"${key}\"${COLOR_RESET}\n"
+        echo -e "${COLOR_RED}Usage Error: Unsupported option \"${key}\"${TEXT_RESET}\n"
         configtool_oidc_help
         exit 1
         ;;
@@ -83,7 +83,7 @@ function configtool_oidc_noninteractive() {
 function configtool_oidc_interactive() {
 
   echo_h2 "Cluster Credentials"
-  echo "Log in to your cluster with your IBMid by using the following method, browse to the OpenShift web console." 
+  echo "Log in to your cluster with your IBMid by using the following method, browse to the OpenShift web console."
   echo "From the dropdown menu in the upper right of the page, click Copy Login Command."
   echo ""
   prompt_for_input "Cluster Token" CLUSTER_TOKEN && export CLUSTER_TOKEN
@@ -115,13 +115,13 @@ function configtool_oidc() {
         ;;
       *)
         # unknown option
-        echo -e "${COLOR_RED}Usage Error: Unsupported operation \"${OIDC_OP}\"${COLOR_RESET}\n"
+        echo -e "${COLOR_RED}Usage Error: Unsupported operation \"${OIDC_OP}\"${TEXT_RESET}\n"
         configtool_oidc_help
         exit 1
         ;;
       esac
   fi
-  
+
   # noneinteractive if more parameters
   if [[ $# -gt 0 ]]; then
     configtool_oidc_noninteractive "$@"
@@ -140,7 +140,7 @@ function configtool_oidc() {
   # instance name and domain
   echo preparing for $MAS_HOME...
   if [[ -z $MAS_HOME ]]; then
-    echo "${COLOR_RED}MAS_HOME must be provided and not empty. sample: export MAS_HOME=\"masdev.home.mobfound1.masdev.suite.maximo.com\"${COLOR_RESET}"
+    echo "${COLOR_RED}MAS_HOME must be provided and not empty. sample: export MAS_HOME=\"masdev.home.mobfound1.masdev.suite.maximo.com\"${TEXT_RESET}"
     exit 0
   fi
   MAS_PARTS=(`echo $MAS_HOME | tr "." " "`)
@@ -160,7 +160,7 @@ function configtool_oidc() {
     i=$((i + 1))
   done
   if [[ $i -lt 4 ]]; then
-    echo "${COLOR_RED}MAS_HOME is incorrect. sample: \"masdev.home.mobfound1.masdev.suite.maximo.com\"${COLOR_RESET}"
+    echo "${COLOR_RED}MAS_HOME is incorrect. sample: \"masdev.home.mobfound1.masdev.suite.maximo.com\"${TEXT_RESET}"
     exit 0
   fi
 
@@ -199,12 +199,12 @@ function configtool_oidc() {
   # trust ui prefix
   echo TRUST_UI_PREFIX: $TRUST_UI_PREFIX
   if [[ -z $TRUST_UI_PREFIX ]]; then
-    echo "${COLOR_RED}TRUST_UI_PREFIX must be provided and not empty. sample: export TRUST_UI_PREFIX=\"http://localhost:3000,http://localhost:3001\"${COLOR_RESET}"
+    echo "${COLOR_RED}TRUST_UI_PREFIX must be provided and not empty. sample: export TRUST_UI_PREFIX=\"http://localhost:3000,http://localhost:3001\"${TEXT_RESET}"
     exit 0
   fi
   TRUST_UI_PARTS=(`echo $TRUST_UI_PREFIX | tr "," " "`)
   if [[ ${#TRUST_UI_PARTS[@]} -eq 0 ]]; then
-    echo "${COLOR_RED}TRUST_UI_PREFIX is empty, at least define one URL. \"http://localhost:3000\"${COLOR_RESET}"
+    echo "${COLOR_RED}TRUST_UI_PREFIX is empty, at least define one URL. \"http://localhost:3000\"${TEXT_RESET}"
     exit 0
   fi
   CALLBACK="/auth/callback"
@@ -221,7 +221,7 @@ function configtool_oidc() {
       REDIRECT_UIS+="\""$ELE$CALLBACK"\""
     fi
     j=$((j + 1))
-  done 
+  done
   TRUST_UIS+="]"
   REDIRECT_UIS+="]"
 

image/cli/mascli/functions/configure_mirror

@@ -14,11 +14,12 @@ Private Registry Details (Required):
   -H, --host ${COLOR_YELLOW}REGISTRY_PRIVATE_HOST${TEXT_RESET}        Hostname of the private registry
   -P, --port ${COLOR_YELLOW}REGISTRY_PRIVATE_PORT${TEXT_RESET}        Port number for the private registry
   -c, --ca-file ${COLOR_YELLOW}REGISTRY_PRIVATE_CA_FILE${TEXT_RESET}  Location of the CA file for the private registry
-
-Private Registry Authentication (Optional):
   -u, --username ${COLOR_YELLOW}REGISTRY_USERNAME${TEXT_RESET}    Username to authenticate to the private registry
   -p, --password ${COLOR_YELLOW}REGISTRY_PASSWORD${TEXT_RESET}    Password to authenticate to the private registry
 
+Setup Red Hat Operator Catalogs (Optional):
+  --setup-redhat-catalogs                 Set up Red Hat Operator, Community, and Certified catalogs with ICSP
+
 Other Commands:
       --no-confirm                        Provision the cluster without prompting for confirmation
   -h, --help                              Show this help message
@@ -29,6 +30,10 @@ EOM
 
 
 function configure_ocp_for_mirror_noninteractive() {
+  confirm_connection
+  OCP_VERSION=$(oc get clusterversion version -o jsonpath='{.status.desired.version}')
+  OCP_RELEASE=$(echo "$OCP_VERSION" | cut -d "." -f 1,2)
+
   while [[ $# -gt 0 ]]
   do
     key="$1"
@@ -49,6 +54,12 @@ function configure_ocp_for_mirror_noninteractive() {
       -p|--password)
         REGISTRY_PASSWORD=$1 && shift
         ;;
+
+      --setup-redhat-catalogs)
+        SETUP_REDHAT_CATALOGS=true
+        OCP_OPERATORHUB_DISABLE_REDHAT_SOURCES=true
+        ;;
+
       --no-confirm)
         NO_CONFIRM=true
         ;;
@@ -57,7 +68,7 @@ function configure_ocp_for_mirror_noninteractive() {
         ;;
       *)
         # unknown option
-        echo -e "${COLOR_RED}Usage Error: Unsupported option \"${key}\"${COLOR_RESET}\n"
+        echo -e "${COLOR_RED}Usage Error: Unsupported option \"${key}\"${TEXT_RESET}\n"
         configure_ocp_for_mirror_help
         exit 1
         ;;
@@ -68,16 +79,20 @@ function configure_ocp_for_mirror_noninteractive() {
   [[ -z "$REGISTRY_PRIVATE_HOST" ]] && configure_ocp_for_mirror_help "REGISTRY_PRIVATE_HOST is not set"
   [[ -z "$REGISTRY_PRIVATE_PORT" ]] && configure_ocp_for_mirror_help "REGISTRY_PRIVATE_PORT is not set"
   [[ -z "$REGISTRY_PRIVATE_CA_FILE" ]] && configure_ocp_for_mirror_help "REGISTRY_PRIVATE_CA_FILE is not set"
+  [[ -z "$REGISTRY_USERNAME" ]] && configure_ocp_for_mirror_help "REGISTRY_USERNAME is not set"
+  [[ -z "$REGISTRY_PASSWORD" ]] && configure_ocp_for_mirror_help "REGISTRY_PASSWORD is not set"
   if [[ ! -e $REGISTRY_PRIVATE_CA_FILE ]]; then
     echo_warning "Certificate file '$REGISTRY_PRIVATE_CA_FILE' does not exist"
     exit 1
   fi
 
-  confirm_connection
 }
 
 function configure_ocp_for_mirror_interactive() {
+  load_config
   connect
+  OCP_VERSION=$(oc get clusterversion version -o jsonpath='{.status.desired.version}')
+  OCP_RELEASE=$(echo "$OCP_VERSION" | cut -d "." -f 1,2)
 
   echo
   echo_h2 "Configure Target Mirror"
@@ -92,7 +107,22 @@ function configure_ocp_for_mirror_interactive() {
   echo
   echo_h2 "Configure Authentication"
   prompt_for_input "Mirror Registry Username" REGISTRY_USERNAME && export REGISTRY_USERNAME
-  prompt_for_input "Mirror Registry Password" REGISTRY_PASSWORD && export REGISTRY_PASSWORD
+  prompt_for_secret "Mirror Registry Password" REGISTRY_PASSWORD "Re-use saved registry password?" && export REGISTRY_PASSWORD
+
+  echo
+  echo_h2 "Red Hat Catalog Management"
+  echo -n "${COLOR_CYAN}NEW!${TEXT_RESET} ${TEXT_DIM}From release 7.9 of the MAS CLI it is now possible to mirror a curated version of the Red Hat Operator catalogs containing only "
+  echo "the operators that IBM Maximo Application Suite requires using ${COLOR_CYAN}mas mirror-redhat${TEXT_RESET}${TEXT_DIM} and automatically configure OpenShift Container Platform to use these catalogs:"
+  echo
+  echo "1. OperatorHub will be re-configured to disable the default online catalog sources"
+  echo "2. Three offline catalog sources will be created/updated in the openshift-marketplace namespace:"
+  echo "  - certified-operator-index -> $REGISTRY_PRIVATE_HOST:$REGISTRY_PRIVATE_PORT/redhat/certified-operator-index:v$OCP_RELEASE"
+  echo "  - community-operator-index -> $REGISTRY_PRIVATE_HOST:$REGISTRY_PRIVATE_PORT/redhat/community-operator-index:v$OCP_RELEASE"
+  echo "  - redhat-operator-index -> $REGISTRY_PRIVATE_HOST:$REGISTRY_PRIVATE_PORT/redhat/redhat-operator-index:v$OCP_RELEASE"
+  echo
+  reset_colors
+
+  prompt_for_confirm "Enable IBM managed Red Hat catalogs?" SETUP_REDHAT_CATALOGS
 
 }
 
@@ -113,29 +143,49 @@ function configure_ocp_for_mirror() {
   export REGISTRY_USERNAME
   export REGISTRY_PASSWORD
 
+  export SETUP_REDHAT_CATALOGS
+  export OCP_OPERATORHUB_DISABLE_REDHAT_SOURCES
+
   echo
   reset_colors
-  echo_h2 "Review Settings"
+  echo_h4 "Review Settings"
+
   echo "${TEXT_DIM}"
-  echo_h2 "Private Registry Connection" "    "
+  echo_h4 "Private Registry Connection" "    "
   echo_reset_dim "Host ...................... ${COLOR_MAGENTA}${REGISTRY_PRIVATE_HOST}"
   echo_reset_dim "Port ...................... ${COLOR_MAGENTA}${REGISTRY_PRIVATE_PORT}"
   echo_reset_dim "CA File ................... ${COLOR_MAGENTA}${REGISTRY_PRIVATE_CA_FILE}"
-  reset_colors
-  echo "${TEXT_DIM}"
-  echo_h2 "Private Registry Authentication" "    "
-  echo_reset_dim "Username .................. ${COLOR_MAGENTA}${REGISTRY_USERNAME}"
-  echo_reset_dim "Password .................. ${COLOR_MAGENTA}${REGISTRY_PASSWORD:0:8}<snip>"
+  echo "${TEXT_RESET}${TEXT_DIM}"
+
+  echo_h4 "Private Registry Authentication" "    "
+  echo_reset_dim "Username .................. ${COLOR_MAGENTA}${REGISTRY_USERNAME:0:2}********"
+  echo_reset_dim "Password .................. ${COLOR_MAGENTA}${REGISTRY_PASSWORD:0:2}********"
+  echo "${TEXT_RESET}${TEXT_DIM}"
+
+  echo_h4 "Red Hat Catalog Management" "    "
+  if [[ "${SETUP_REDHAT_CATALOGS}" == "true" ]]; then
+    echo_reset_dim "Management Mode ........... ${COLOR_MAGENTA}IBM Managed/Configured"
+  else
+    echo_reset_dim "Management Mode ........... ${COLOR_MAGENTA}User Managed/Configured"
+  fi
   reset_colors
 
   echo
-  reset_colors
   if [[ "$NO_CONFIRM" != "true" ]]; then
     prompt_for_confirm "Proceed with these settings" || exit 0
   fi
 
   echo
-  echo_h2 "Configure Image Content Source Policy"
-  ROLE_NAME=ocp_contentsourcepolicy ansible-playbook ibm.mas_devops.run_role || exit 1
+  if [[ "${SETUP_REDHAT_CATALOGS}" == "true" ]]; then
+    echo_h2 "Disable Online Red Hat Catalog Sources"
+    ROLE_NAME=ocp_config ansible-playbook ibm.mas_devops.run_role || exit 1
+
+    echo_h2 "Configure Image Content Source Policy (with Red Hat catalogs)"
+    ROLE_NAME=ocp_contentsourcepolicy ansible-playbook ibm.mas_devops.run_role || exit 1
+  else
+    echo_h2 "Configure Image Content Source Policy"
+    ROLE_NAME=ocp_contentsourcepolicy ansible-playbook ibm.mas_devops.run_role || exit 1
+  fi
+
 
 }

image/cli/mascli/functions/help/mirror_redhat_help

@@ -0,0 +1,37 @@
+function mirror_redhat_help() {
+  [[ -n "$1" ]] && echo_warning "$1"
+  reset_colors
+  cat << EOM
+Usage:
+  mas mirror-images [options]
+Where ${COLOR_YELLOW}specified${TEXT_RESET} each option may also be defined by setting the appropriate environment variable.
+When no options are specified on the command line, interactive-mode will be enabled by default.
+
+Mirror Mode (Required):
+  -m, --mode ${COLOR_YELLOW}MIRROR_MODE${TEXT_RESET}                         Operation mode (direct, to-filesystem, from-filesystem)
+  -d, --dir ${COLOR_YELLOW}MIRROR_WORKING_DIR${TEXT_RESET}                   Working directory for the mirror process
+  --mirror-platform                               Mirror Red Hat Platform images
+  --mirror-operators                              Mirror selected content from Red Hat Operator catalogs
+
+Registry Details (Required when mode is direct and from-filesystem only):
+  -H, --host ${COLOR_YELLOW}REGISTRY_PUBLIC_HOST${TEXT_RESET}                 Hostname of the target registry
+  -P, --port ${COLOR_YELLOW}REGISTRY_PUBLIC_PORT${TEXT_RESET}                 Port number for the target registry
+  -u, --username ${COLOR_YELLOW}REGISTRY_USERNAME${TEXT_RESET}                Username to authenticate to the target registry
+  -p, --password ${COLOR_YELLOW}REGISTRY_PASSWORD${TEXT_RESET}                Password to authenticate to the target registry
+
+Red Hat Image Pull Secret (Required):
+      --pullsecret ${COLOR_YELLOW}REDHAT_PULLSECRET${TEXT_RESET}              Red Hat OpenShift Pull Secret (https://console.redhat.com/openshift/install/pull-secret)
+
+Content Selection (Required):
+      --release ${COLOR_YELLOW}OCP_RELEASE${TEXT_RESET}                       OCP Release to mirror content for (e.g. 4.10, 4.12)
+
+Platform Version Range (Required when mirroring the platform):
+      --min-version ${COLOR_YELLOW}OCP_MIN_VERSION${TEXT_RESET}               Minimum version of the OCP release to mirror
+      --max-version ${COLOR_YELLOW}OCP_MAX_VERSION${TEXT_RESET}               Maximium version of the OCP release to mirror
+
+Other Commands:
+      --no-confirm                                Mirror images without prompting for confirmation
+  -h, --help                                      Show this help message
+EOM
+  [[ -n "$1" ]] && exit 1 || exit 0
+}

image/cli/mascli/functions/install

@@ -534,7 +534,7 @@ function install_noninteractive() {
         ;;
       *)
         # unknown option
-        echo -e "${COLOR_RED}Usage Error: Unsupported option \"${key}\"${COLOR_RESET}\n"
+        echo -e "${COLOR_RED}Usage Error: Unsupported option \"${key}\"${TEXT_RESET}\n"
         install_help
         exit 1
         ;;
@@ -790,7 +790,7 @@ function ocp_version_check() {
     echo_warning " - The installation of IBM Maximo Application Suite may not proceed, you must upgrade to at least OCP v4.10 before installing MAS"
     echo
     echo_warning "For more information refer to the Red Hat OpenShift Container Platform Life Cycle Policy:"
-    echo "  ${COLOR_CYAN}${TEXT_UNDERLINE}https://access.redhat.com/support/policy/updates/openshift/${COLOR_RESET}${TEXT_RESET}"
+    echo "  ${COLOR_CYAN}${TEXT_UNDERLINE}https://access.redhat.com/support/policy/updates/openshift/${TEXT_RESET}${TEXT_RESET}"
     reset_colors
     exit 1
   elif [[ "$OCP_VERSION" =~ ^4\.10\.* ]]; then
@@ -800,7 +800,7 @@ function ocp_version_check() {
     echo_warning " - The installation of IBM Maximo Application Suite may proceed, but we strongly encourage you to upgrade to OCP v4.12 first"
     echo
     echo_warning "For more information refer to the Red Hat OpenShift Container Platform Life Cycle Policy:"
-    echo "  ${COLOR_CYAN}${TEXT_UNDERLINE}https://access.redhat.com/support/policy/updates/openshift/${COLOR_RESET}${TEXT_RESET}"
+    echo "  ${COLOR_CYAN}${TEXT_UNDERLINE}https://access.redhat.com/support/policy/updates/openshift/${TEXT_RESET}${TEXT_RESET}"
     reset_colors
     echo
     prompt_for_confirm "Continue anyway?" || exit 0