[minor] Add must-gather summarizer (#295)

Co-authored-by: David Parker <[email protected]>

.gitignore

@@ -10,3 +10,4 @@ tmp/
 tekton/test-*.yaml
 tekton/target
 tekton/tmp
+*.pyc

image/cli/install/requirements.txt

@@ -6,3 +6,4 @@ kubernetes==12.0.1
 openshift==0.12.1
 jmespath==1.0.0
 click==8.1.3
+prettytable==3.7.0

image/cli/mascli/functions/must_gather

@@ -94,12 +94,19 @@ function mustgather() {
 
 
   echo "Must gather will be saved to: $OUTPUT_FILE"
+  echo ""
 
   # Generate OCP Report
   # -----------------------------------------------------------------------------
   echo_h2 "Generate OCP Report"
+
   echo "- Cluster Resources"
-  for RESOURCE in namespaces packagemanifests clusterversions storageclasses nodes clusterroles clusterrolebindings
+  for RESOURCE in nodes storageclasses clusterversions
+  do
+    $MG_SCRIPT_DIR/mg-collect-resources -r $RESOURCE -d $OUTPUT_DIR/resources
+  done
+
+  for RESOURCE in namespaces packagemanifests clusterroles clusterrolebindings
   do
     $MG_SCRIPT_DIR/mg-collect-resources -r $RESOURCE -d $OUTPUT_DIR/resources --no-detail
   done
@@ -110,8 +117,14 @@ function mustgather() {
     $MG_SCRIPT_DIR/mg-collect-resources -n openshift-marketplace -r $RESOURCE -d $OUTPUT_DIR/resources
   done
 
-  echo "- Operator Conditions"
+  echo "- Operators"
+  echo "  - Collecting debug information: operatorconditions"
   oc get operatorconditions --all-namespaces &> ${OUTPUT_DIR}/operatorconditions.txt
+  for RESOURCE in subscriptions installplans
+  do
+    $MG_SCRIPT_DIR/mg-collect-resources -r $RESOURCE -d $OUTPUT_DIR/resources -a
+  done
+
 
   # Generate Dependency Report
   # -----------------------------------------------------------------------------
@@ -126,6 +139,7 @@ function mustgather() {
   echo "- IBM Db2 Universal Operator"
   $MG_SCRIPT_DIR/mg-summary-db2u &> ${OUTPUT_DIR}/db2u.txt
 
+
   # Find MAS instances
   # -----------------------------------------------------------------------------
   MAS_INSTANCE_IDS=$(oc get suite --all-namespaces --ignore-not-found -o jsonpath='{.items[*].metadata.name}')
@@ -162,6 +176,8 @@ function mustgather() {
     done
   done
 
+  python3 $MG_SCRIPT_DIR/summarizer/mg-print-summary.py $OUTPUT_DIR | tee $OUTPUT_DIR/summary.txt
+
   tar -czf $OUTPUT_FILE -C $MG_DIR $TIMESTAMP
   if [[ -n "$ARTIFACTORY_TOKEN" && -n "$ARTIFACTORY_UPLOAD_DIR" ]]; then
     set -e
@@ -182,6 +198,8 @@ function mustgather() {
   if [[ "$KEEP_FILES" != "true" ]]; then
     rm -rf "$OUTPUT_DIR"
   fi
+
+  echo
   echo_green "Must gather successfully saved to: $OUTPUT_FILE"
 
   true

image/cli/mascli/must-gather/mg-collect-mas-core

@@ -27,3 +27,10 @@ $DIR/mg-collect-reconcile-logs $NAMESPACE ibm-mas-cfg-mongo $OUTPUT_DIR
 $DIR/mg-collect-reconcile-logs $NAMESPACE ibm-mas-cfg-kafka $OUTPUT_DIR
 $DIR/mg-collect-reconcile-logs $NAMESPACE ibm-mas-cfg-objectstorage $OUTPUT_DIR
 $DIR/mg-collect-reconcile-logs $NAMESPACE ibm-mas-cfg-smtp $OUTPUT_DIR
+
+# Collect Custom Resources
+# -----------------------------------------------------------------------------
+  for RESOURCE in Suite Workspace Truststore CoreIDP AppConnect Humai MVIEdge BasCfg SlsCfg IDPCfg ScimCfg JdbcCfg MongoCfg KafkaCfg ObjectStorageCfg SmtpCfg
+  do
+    $DIR/mg-collect-resources -n $NAMESPACE -r $RESOURCE -d $OUTPUT_DIR/resources
+  done

image/cli/mascli/must-gather/mg-collect-resources

@@ -5,6 +5,7 @@ set -e
 NAMESPACE_FLAG=""
 NAMESPACE=""
 NO_DETAIL=false
+ALL_NAMESPACES=""
 
 while [[ $# -gt 0 ]]
 do
@@ -24,6 +25,9 @@ do
     --no-detail)
         NO_DETAIL=true
         ;;
+    -a)
+        ALL_NAMESPACES="--all-namespaces"
+        ;;
     esac
 done
 
@@ -44,17 +48,21 @@ echo "  - Collecting debug information: ${RESOURCE_TYPE}"
 
 # Generate Summary
 # -----------------------------------------------------------------------------
-oc ${NAMESPACE_FLAG} get ${RESOURCE_TYPE} -o wide > ${NAMESPACE_DIR}/${RESOURCE_TYPE}.txt
+oc ${NAMESPACE_FLAG} get ${RESOURCE_TYPE} ${ALL_NAMESPACES} -o wide > ${NAMESPACE_DIR}/${RESOURCE_TYPE}.txt
 
 # Generate Detailed Report
 # -----------------------------------------------------------------------------
 if [[ "$NO_DETAIL" == "false" ]]; then
-    RESOURCES=$(oc ${NAMESPACE_FLAG} get ${RESOURCE_TYPE} -o name)
-    for RESOURCE in ${RESOURCES[@]}
-    do
-        # echo "    - Collecting $RESOURCE"
-        RESOURCE_NAME=$(echo ${RESOURCE} | cut -d '/' -f 2)
-
-        oc ${NAMESPACE_FLAG} get ${RESOURCE} -o yaml > ${RESOURCE_DIR}/${RESOURCE_NAME}.yaml
-    done
+    if [[ -z "$ALL_NAMESPACES" ]]; then
+        RESOURCES=$(oc ${NAMESPACE_FLAG} get ${RESOURCE_TYPE} -o name)
+        for RESOURCE in ${RESOURCES[@]}
+        do
+            # echo "    - Collecting $RESOURCE"
+            RESOURCE_NAME=$(echo ${RESOURCE} | cut -d '/' -f 2)
+
+            oc ${NAMESPACE_FLAG} get ${RESOURCE} -o yaml > ${RESOURCE_DIR}/${RESOURCE_NAME}.yaml
+        done
+    else
+        oc get ${RESOURCE_TYPE} ${ALL_NAMESPACES} -o yaml > ${RESOURCE_DIR}/all-namespaces.yaml
+    fi
 fi

image/cli/mascli/must-gather/summarizer/mg-print-summary.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+
+import sys
+
+import mg.cluster
+import mg.catalogs
+import mg.subscriptions
+
+if __name__ == "__main__":
+  output_dir=sys.argv[1]
+
+  mg.cluster.summarize(output_dir)
+  mg.catalogs.summarize(output_dir)
+  mg.subscriptions.summarize(output_dir)

image/cli/mascli/must-gather/summarizer/mg/catalogs.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+import yaml
+from prettytable import PrettyTable
+
+from mg.utils import printHeader
+
+# process the catalog source information and show any catalogsources that have problems
+def process_catalogsource( output_dir, catalogsource, catalogsource_table ):
+  catalogsourcename = catalogsource.rsplit()[0]
+  catalogsource_yaml_file = output_dir + "/resources/openshift-marketplace/catalogsources/" + catalogsourcename + ".yaml"
+
+  with open(catalogsource_yaml_file, 'r') as file:
+    catalogsource_yaml=yaml.safe_load( file )
+
+  catalogsource_table.add_row([catalogsourcename,
+                               catalogsource_yaml['spec']['displayName'],
+                               catalogsource_yaml['spec']['publisher'],
+                               catalogsource_yaml['status']['connectionState']['lastObservedState']])
+
+
+# Create a report summarizing the catalog source status
+def summarize(output_dir):
+  catalogsource_file=output_dir + "/resources/openshift-marketplace/catalogsources.txt"
+  catalogsource_table=PrettyTable()
+  catalogsource_table.field_names = [ "Name", "Display Name", "Publisher", "Status" ]
+  header=True
+  with open( catalogsource_file ) as file:
+    for catalogsource in file:
+      if header == True:
+        header=False
+      else:
+         process_catalogsource( output_dir, catalogsource, catalogsource_table )
+
+  printHeader("Catalog Sources")
+  catalogsource_table.align = "l"
+  print(catalogsource_table)

image/cli/mascli/must-gather/summarizer/mg/cluster.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+import yaml
+from prettytable import PrettyTable
+
+from mg.utils import printHeader
+
+# Create a report summarizing the cluster verison of the cluster
+def processClusterVersion(output_dir):
+  clusterversion_file=output_dir + "/resources/_cluster/clusterversions.txt"
+  with open(clusterversion_file , 'r') as file:
+    clusterversion = file.read()
+  print(clusterversion)
+
+
+# process the node information and show any nodes that have problems
+def processNode( output_dir, node, node_table ):
+  nodename = node.rsplit()[0]
+  node_yaml_file = output_dir + "/resources/_cluster/nodes/" + nodename + ".yaml"
+
+  with open(node_yaml_file, 'r') as file:
+    node_yaml=yaml.safe_load( file )
+
+  # Extract the conditions we are interested in
+  condition_types=['MemoryPressure', 'DiskPressure', 'PIDPressure', 'Ready']
+  condition_statuses= {x['type']:{
+    "status": x[ 'status']} for x in node_yaml['status']['conditions'] if x['type'] in condition_types}
+  condition_statuses
+
+  node_table.add_row([nodename,
+                      node_yaml['status']['capacity']['cpu'],
+                      node_yaml['status']['capacity']['memory'],
+                      condition_statuses['Ready']['status'],
+                      condition_statuses['MemoryPressure']['status'],
+                      condition_statuses['DiskPressure']['status'],
+                      condition_statuses['PIDPressure']['status'] ])
+
+def processNodes(output_dir):
+  nodes_file=output_dir + "/resources/_cluster/nodes.txt"
+  node_table=PrettyTable()
+  node_table.field_names = [ "Node", "CPU", "Memory", "Ready" , "Memory Pressure", "Disk Pressure", "PID Pressure" ]
+  header=True
+  with open( nodes_file ) as nodes_file:
+    for node in nodes_file:
+      if header == True:
+        header=False
+      else:
+         processNode( output_dir, node, node_table )
+
+  node_table.align = "l"
+  print( node_table )
+
+
+def summarize(output_dir):
+  printHeader("Cluster Information")
+  processClusterVersion(output_dir)
+  processNodes(output_dir)

image/cli/mascli/must-gather/summarizer/mg/subscriptions.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+import yaml
+from prettytable import PrettyTable
+
+from mg.utils import printHeader
+
+def process_subscription( output_dir, subscriptionYaml, ipYaml, subscriptions_table ):
+  # Extract the conditions we are interested in
+  sub_condition_types=["CatalogSourcesUnhealthy"]
+  sub_condition_statuses= {x["type"]:{
+    "status": x[ "status"]} for x in subscriptionYaml["status"]["conditions"] if x["type"] in sub_condition_types}
+
+  if ipYaml is not None:
+    ip_status = ipYaml["status"]["phase"]
+  else:
+    ip_status="<unknown>"
+
+  if "installPlanApproval" in subscriptionYaml["spec"]:
+    approval = subscriptionYaml["spec"]["installPlanApproval"]
+  else:
+    approval = "<undefined>"
+
+  if "installplan" in subscriptionYaml["status"]:
+    installPlan = subscriptionYaml["status"]["installplan"]["name"]
+  else:
+    installPlan = "<undefined>"
+
+  if "installedCSV" in subscriptionYaml["status"]:
+    installedCSV = subscriptionYaml["status"]["installedCSV"]
+  else:
+    installedCSV = "<undefined>"
+
+  subscriptions_table.add_row([
+    subscriptionYaml["metadata"]["namespace"],
+    subscriptionYaml["metadata"]["name"],
+    subscriptionYaml["spec"]["channel"],
+    subscriptionYaml["spec"]["source"],
+    installedCSV,
+    approval,
+    sub_condition_statuses["CatalogSourcesUnhealthy"]["status"],
+    installPlan,
+    ip_status
+  ])
+
+
+def summarize( output_dir):
+  subsFile = output_dir + "/resources/_cluster/subscriptions/all-namespaces.yaml"
+  ipsFile = output_dir + "/resources/_cluster/installplans/all-namespaces.yaml"
+
+  with open(subsFile, "r") as file:
+    subs=yaml.safe_load( file )
+
+  with open(ipsFile, "r") as file:
+    ips=yaml.safe_load( file )
+
+  installPlansByName={}
+  for ip in ips["items"]:
+    installPlansByName[ip["metadata"]["name"]]=ip
+
+  subscriptions_table=PrettyTable()
+  subscriptions_table.field_names = [ "Namespace", "Name", "Channel", "Source", "Installed CSV" ,"Approval", "CS Health", "Install Plan", "Install Phase"]
+
+  for sub in subs["items"]:
+    if "installplan" in sub["status"] and sub["status"]["installplan"]["name"] in installPlansByName:
+      ip = installPlansByName[sub["status"]["installplan"]["name"]]
+    else:
+      ip = None
+    process_subscription( output_dir, sub, ip, subscriptions_table )
+
+  printHeader( "Subscriptions" )
+  subscriptions_table.align = "l"
+  print( subscriptions_table )

image/cli/mascli/must-gather/summarizer/mg/utils.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+
+def printHeader(header):
+  print("")
+  print(header)
+  print("====================================================================================================")
+  print("")