Merge pull request #144 from weka/cst_jack_add_recommended_resolutions

Added recommend resolutions for the "easier to fix" situations

scripts.d/ta/290_check_traces_free_space.sh

@@ -52,7 +52,10 @@ if (( ${WEKA_ENSURE_FREE} > ${TRACES_FS_SIZE})) ; then
     echo "Weka is currently set to ensure that ${WEKA_ENSURE_FREE} bytes are free"
     echo "on ${WEKA_TRACES_DIR}, but this filesystem is only ${TRACES_FS_SIZE} bytes" 
     echo "in size. These conditions cannot co-exist, so the outcome is that no"
-    echo "traces will be stored"
+    echo "traces will be stored."
+    echo "Recommended options:"
+    echo "   . Increase the size of ${WEKA_TRACES_DIR}"
+    echo "   . Reduce the size of traces with \"weka debug traces retention set --server-ensure-free XXXX\""
     RETURN_CODE=1
 fi 
 if [[ ${RETURN_CODE} -eq 0 ]]; then

scripts.d/ta/390_data_folder.sh

@@ -29,6 +29,7 @@ if [ -d "/data" ] ; then
         else
             echo "to ${JIRA_REFERENCE}, SFDC ${KB_REFERENCE}"                                                                                                   
         fi
+        echo "The recommend fix is to upgrade your version of Weka"
         RETURN_CODE=1
     fi
 fi

scripts.d/ta/400_s3_using_etcd.sh

@@ -40,12 +40,14 @@ if [ ${WEKA_S3_RUNNING} -ge 1 ] ; then
     if verlte ${MIN_VERSION} ${WEKA_VERSION} && verlte ${WEKA_VERSION} ${MAX_VERSION} ; then
         WEKA_ETCD_HOSTS=$(weka s3 cluster --json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(len(data["etcd_cluster_hosts"]))')
         if [ ${WEKA_ETCD_HOSTS} -gt 0 ] ; then
-            echo "S3 cluster is running, and this version of Weka requires migration"
+            echo "S3 cluster is running, and this version of Weka requires a configuration change."
             if [[ ! -z "${WTA_REFERENCE}" ]]; then
-                echo "to ${JIRA_REFERENCE}, discussed in ${WTA_REFERENCE}, SFDC ${KB_REFERENCE}"
+                echo "Refer to ${JIRA_REFERENCE}, discussed in ${WTA_REFERENCE}, SFDC ${KB_REFERENCE}"
             else
-                echo "to ${JIRA_REFERENCE}, SFDC ${KB_REFERENCE}"                                                                                                   
+                echo "Refer to ${JIRA_REFERENCE}, SFDC ${KB_REFERENCE}"                                                                                                   
             fi
+            echo "If you require the S3 service, please contact Customer Success indicating"
+            echo " you need to move the S3 service from ETCD to KWAS, as indicated in KB 1181"
             RETURN_CODE=254
         fi
     fi

scripts.d/ta/420_check_cross-numa_zone_memory_balance.sh

@@ -37,6 +37,8 @@ if [[ ${RATIO_SEEN} -gt ${MAX_ALLOWED_RATIO} ]]; then
     echo "from starting due to lack of NUMA zone-local memory"
     echo "The ratio is ${RATIO_SEEN}% and the maximum allowed ratio is ${MAX_ALLOWED_RATIO}%"
     echo "The memory in the highest zone is ${MAX_MEMORY_SEEN_KB} and in the lowest zone is ${MIN_MEMORY_SEEN_KB}"
+    echo "One recommend resolution is to balance the memory between NUMA zones by physically"
+    echo "moving memory, or by adding more to the smaller NUMA zone"
 fi
 
 if [[ ${RETURN_CODE} -eq 0 ]]; then

scripts.d/ta/430_nvme_used_capacity_vs_maximum.sh

@@ -27,6 +27,13 @@ echo ${WEKA_SSD_USED_BYTES}
 
 # if we've allocated more than half the maximum theoretical SSD space, warn
 if [[ $((${WEKA_SSD_USED_BYTES}*2)) -gt ${WEKA_THEORETICAL_MAX_SSD_BYTES} ]] ; then 
+    echo "You have used a significant proportion of the theoretical maximum"
+    echo "NVME capacity of the cluster which is decided at first install time."
+    echo "Please contact customer success to discuss options. Possible actions include:"
+    echo " . Adding an Object Store to expand data storage while keeping NVME capacity down"
+    echo " . In-place cluster resizing and migration (perhaps via snap2obj for fast backup/restore)"
+    echo " . Migrating to a different, larger cluster"
+    echo " . Pruning unnecessary data"
     RETURN_CODE=254
 fi
 if [[ ${RETURN_CODE} -eq 0 ]]; then

scripts.d/ta/440_hostnames_rfc952.sh

@@ -23,6 +23,7 @@ GREP_RESULT=$(echo ${SHORT_HOSTNAME} | grep "[^-a-z0-9.]")
 if [[ $? -eq 0 ]]; then
     echo "The hostname ${SHORT_HOSTNAME} appears to contain a character other than [a-z], -, and [0-9]."
     echo "Refer to RFC 952 for more information"
+    echo "Recommended resolution: change the hostname to include only alphanumerics and underscores"
     RETURN_CODE=254
 fi
 

scripts.d/ta/450_custom_ca_certs.sh

@@ -15,6 +15,8 @@ grep -q SSL_CERT_FILE /opt/weka/dist/release/${WEKA_VERSION}.spec 2>/dev/null
 
 if [[ $? -eq 0 ]] ; then 
         echo "This version of weka appears to use custom CA certificates. Care will be needed for upgrading"
+        echo "Recommended resolution: remove custom CA specification, and upgrade to a more recent"
+        echo "version that natively supports additional CA bundles"
         RETURN_CODE=254
 fi
 if [[ ${RETURN_CODE} -eq 0 ]]; then

scripts.d/ta/460_ip_source-based_routing.sh

@@ -78,6 +78,9 @@ if [[ ${SOURCE_BASED_ROUTING_RECOMMENDED} -ge "1" ]] ; then
         echo "Warning: Not every interface appears to have arp_filter=1 set. This could lead to communication problems"
         RETURN_CODE="254"
     fi
+    echo "Recommended resolution: Although networking is typically site- and hardware-dependent,"
+    echo " some example configurations for the common dual NIC setup are noted on the WEKA"
+    echo " documentation site: https://docs.weka.io/planning-and-installation/bare-metal/setting-up-the-hosts#configure-the-ha-networking"
 fi
 
 if [[ ${RETURN_CODE} -eq 0 ]]; then

scripts.d/ta/470_number_of_numa_domains.sh

@@ -36,6 +36,8 @@ echo -n "Detected $NUMBER_OF_NUMA_DOMAINS NUMA domains - "
 if [[ $NUMBER_OF_NUMA_DOMAINS -gt $MAXIMUM_NUMA_DOMAINS ]]; then
     RETURN_CODE=254
     echo "Weka currenty only supports a maximum of 32 NUMA domains (4.2.11+)."
+    echo " Recommended resolution: reduce the number of NUMA domains, perhaps by reducing"
+    echo " the NUMAs per socket setting in the machine's firmware"
 
 # 8 or fewer NUMAs is always supported
 elif [[ $NUMBER_OF_NUMA_DOMAINS -le 8 ]]; then
@@ -45,16 +47,25 @@ elif [[ $NUMBER_OF_NUMA_DOMAINS -le 8 ]]; then
 elif vergte $WEKA_VERSION "4.3.0" && verlt $WEKA_VERSION "4.3.2" && [[ $NUMBER_OF_NUMA_DOMAINS -gt 16 ]]; then
     RETURN_CODE=254
     echo "Weka only supports more than 16 NUMA domains in 4.3.2 and higher."
+    echo " Recommended resolutions: either"
+    echo "  . Reduce the number of NUMA domains, perhaps by reducing the NUMAs per socket setting in the machine's firmware"
+    echo "  . Upgrade Weka to a more recent version"
 
 # More than 16 NUMAs only supported in 4.2.11+
 elif vergt $WEKA_VERSION "4.2.6" && verlt $WEKA_VERSION "4.2.11" && [[ $NUMBER_OF_NUMA_DOMAINS -gt 16 ]]; then
     RETURN_CODE=254
     echo "Weka only supports more than 16 NUMA domains in (4.2.11+, 4.3.2+)."
+    echo " Recommended resolutions: either"
+    echo "  . Reduce the number of NUMA domains, perhaps by reducing the NUMAs per socket setting in the machine's firmware"
+    echo "  . Upgrade Weka to a more recent version"
 
 # More than 8 NUMAs only supported in 4.2.7+
 elif verlt $WEKA_VERSION "4.2.7" && [[ $NUMBER_OF_NUMA_DOMAINS -gt 8 ]]; then
     RETURN_CODE=254
     echo "Weka only supports more than 8 NUMA domains in 4.2.7 and higher."
+    echo " Recommended resolutions: either"
+    echo "  . Reduce the number of NUMA domains, perhaps by reducing the NUMAs per socket setting in the machine's firmware"
+    echo "  . Upgrade Weka to a more recent version"
 else
     echo "Number of NUMA domains is within supported limits."
 fi

scripts.d/ta/480_check_weka_agent.sh

@@ -17,11 +17,15 @@ if [[ $? -ne "0" ]] ; then
     RETURN_CODE=254
     echo "The service weka-agent is not reported as enabled by systemd"
     echo "This may cause weka to fail to start"
+    echo " Recommended Resolution: enable the service with systemctl enable weka-agent"
 
-
     if [[ ! -L /etc/init.d ]]; then
         echo "/etc/init.d is expected to be a symlink to /etc/rc.d/init.d"
         echo "Without this systemd is unable to find and thus start the weka-agent sysV init script"
+        echo " Recommended Resolution: on RHEL-based OSes move any scripts to /etc/rc.d/init.d, remove"
+        echo " the /etc/init.d directory, and re-create it as a link. The following commands are"
+        echo " one way to achieve this"
+        echo " mv /etc/init.d/* /etc/rc.d/init.d/ && rmdir /etc/init.d && ln -s /etc/rc.d/init.d /etc/init.d"
     fi
 fi
 

scripts.d/ta/490_ip_route_metrics.sh

@@ -46,6 +46,8 @@ if [[ ${NUMBER_OF_OVERLAPPING_ROUTES_WITH_METRICS} -gt "1" ]]; then
     echo "that these entries will negatively affect the performance of e.g. floating IP"
     echo "addresses. In any case it is unlikely that preferential IP routes are of"
     echo "benefit in a high-performance local network"
+    echo "Recommended Resolution: review the output of \"ip route\" and rationalize the routes,"
+    echo " likely by removing or coalescing the overlapping routes into larger ranges"
 fi
 
 if [[ ${RETURN_CODE} -eq 0 ]]; then

scripts.d/ta/500_sysctl_rp_filter.sh

@@ -35,16 +35,18 @@ if [[ $RP_FILTER_VALUE_ALL != "2" ]]; then
       echo "The value for net.ipv4.conf.${INTERFACE}.rp_filter is set to ${RP_FILTER_VALUE}."
       echo "This can disrupt floating IP addresses for protocols."
       echo "It is recommended to set net.ipv4.conf.${INTERFACE}.rp_filter to 2."
+      echo "Recommended resolution: set this value in e.g. /etc/sysctl.d/99-weka-nics.conf"
     elif [[ $RP_FILTER_VALUE_ALL == "1" && $RP_FILTER_VALUE == "0" ]]; then
       RETURN_CODE="254"
       echo "The value for net.ipv4.conf.${INTERFACE}.rp_filter is set to ${RP_FILTER_VALUE}."
       echo "The value for net.ipv4.conf.all.rp_filter is set to ${RP_FILTER_VALUE_ALL} and takes precedence."
       echo "This can disrupt floating IP addresses for protocols."
       echo "It is recommended to set net.ipv4.conf.${INTERFACE}.rp_filter or net.ipv4.conf.all.rp_filter to 2."
+      echo "Recommended resolution: set this value in e.g. /etc/sysctl.d/99-weka-nics.conf"
     fi
   done
 else
   echo "net.ipv4.conf.all.rp_filter is set to 2, no further testing necessary."
 fi
 
-exit ${RETURN_CODE}
+exit ${RETURN_CODE}

scripts.d/ta/510_check_for_noprefixroute.sh

@@ -28,7 +28,10 @@ if [[ "${NOPREFIXROUTE_COUNT}" != "0" ]]; then
     echo "Certain IP addresses are configured with noprefixroute. This will inhibit the ability"
     echo "of certain cluster floating ips to accurately determine which link should be preferred"
     echo "The command \"ip -o -f inet route list match xxx.xxx.xxx.xxx/32 scope link\" needs to"
-    echo "Be able to return a device for each floating IP configured"
+    echo "be able to return a device for each floating IP configured"
+    echo "Recommended Resolution: remove the noprefixroute flag or otherwise ensure the"
+    echo " ip route list command given above can resolve the link on which you wish the"
+    echo " floating IP to be configured"
 fi
 
 if [[ ${RETURN_CODE} -eq 0 ]]; then

scripts.d/ta/520_bucket_and_process_uptime.sh

@@ -36,12 +36,20 @@ CURRENT_TIME_EPOCH=$(                 date +%s)
 if [[ $((${CURRENT_TIME_EPOCH}-${MOST_RECENT_BUCKET_STARTTIME_EPOCH})) -lt 3600 ]]; then
     RETURN_CODE="254"
     echo "Weka buckets have been restarted within the last hour, or have never started. This may not be a problem on a new cluster"
-    echo "but could be indicative of problems (e.g. network flapping"
+    echo "but could be indicative of problems (e.g. network flapping)"
+    echo "Recommended Resolutions:"
+    echo " . If this is a new cluster, or hosts have been upgraded/reboot, this is likely expected"
+    echo " . Otherwise the most likely cause is network problems, such as link flapping or congestion."
+    echo " . Review hardware and network stability, then contact customer success"
 fi
 if [[ $((${CURRENT_TIME_EPOCH}-${MOST_RECENT_PROCESS_STARTTIME_EPOCH})) -lt 3600 ]]; then
     RETURN_CODE="254"
     echo "Weka processes have been restarted within the last hour, or have never started. This may not be a problem on a new cluster"
     echo "but could be indicative of problems (e.g. network flapping"
+    echo "Recommended Resolutions:"
+    echo " . If this is a new cluster, or hosts have been upgraded/reboot, this is likely expected"
+    echo " . Otherwise the most likely cause is network problems, such as link flapping or congestion."
+    echo " . Review hardware and network stability, then contact customer success"
 fi
 
 if [[ ${RETURN_CODE} -eq 0 ]]; then

scripts.d/ta/530_high_drive_read_ssd_ratio.sh

@@ -34,6 +34,9 @@ if [[ ${HIGHER_THAN_EXPECTED} == "YES" ]]; then
     echo "The ratio of NVMe read requests vs DRIVE node read operations is higher than expected over the last ${TIME_TO_EXAMINE}"
     echo "This could indicate a number of things, such as splitting of read requests or perhaps read amplification"
     echo "Review ${JIRA_REFERENCE} for details"
+    echo "Recommended Resolutions:"
+    echo " . This may be expected behavior for your workload"
+    echo " . The data may be read using much larger blocksizes than those in which it was written, and matching those may help"
 fi
 
 if [[ ${RETURN_CODE} -eq 0 ]]; then

scripts.d/ta/550_iptables_nats_local_traffic.sh

@@ -21,6 +21,7 @@ for IP_ADDRESS in $(hostname --all-ip-addresses) ; do
     if [[ $? -eq 0 ]] ; then
         echo "Warning: it is possible that traffic to or from local IP address ${IP_ADDRESS} will be subject to NAT"
         echo "This can cause intra-WEKA communication errors"
+        echo "Recommended Resolution: Do not NAT WEKA traffic"
         RETURN_CODE="254"
     fi
 done
@@ -29,6 +30,7 @@ for IP_ROUTE in $(ip -4 --json route list  | python3 -c 'import sys, json, colle
     if [[ $? -eq 0 ]] ; then
         echo "Warning: it is possible that traffic to or from subnet ${IP_ROUTE} will be subject to NAT"
         echo "This can cause intra-WEKA communication errors"
+        echo "Recommended Resolution: Do not NAT WEKA traffic"
         RETURN_CODE="254"
     fi
 done

scripts.d/ta/560_check_for_swap.sh

@@ -14,6 +14,7 @@ SWAPTOTAL=$(grep SwapTotal /proc/meminfo  | awk '{print $2}')
 if [[ ${SWAPTOTAL} -ne "0" ]] ; then
     echo "This host has swap configured - this is unlikely to be"
     echo "helpful in a large memory system"
+    echo "Recommended Resolution: if the host has enough RAM, disable swap with swapoff then disable swap at boot time (likely in /etc/fstab)"
     RETURN_CODE="254"
 fi
 

scripts.d/ta/570_does_weka_use_swap.sh

@@ -17,6 +17,10 @@ for WEKAPID in $(ps -eo pid,comm | grep weka_init | awk '{print $1}') ; do
     if [[ ${NUM_PROCS_USING_SWAP} -gt "0" ]] ; then
         echo "There are Weka processes using swap - this is likely to be"
         echo "detrimental to performance"
+        echo "Recommended Resolutions:"
+        echo " . Add more RAM if the host is truly constrained"
+        echo " . Review if the host has not correctly released RAM"
+        echo " . Reduce the amount of RAM allocated to WEKA (a last resort)"
         RETURN_CODE="254"
     fi
 done

scripts.d/ta/580_weka_version_available_everywhere.sh

@@ -27,6 +27,8 @@ CURRENT_AGENT_VERSION=$(weka local status | awk 'NR==1{print $5}' | tr -d ')')
 if [[ ${WEKA_CLUSTER_VERSION} != ${CURRENT_AGENT_VERSION} ]] ; then
     echo "The currently running cluster version ${WEKA_CLUSTER_VERSION} does not match the"
     echo "default installed local agent version ${CURRENT_AGENT_VERSION}"
+    echo "Recommended Resolution: update this host to the cluster version, either by"
+    echo " unmounting and re-mounting filesystems or using the weka local upgrade utility"
     RETURN_CODE="254"
 fi
 

scripts.d/ta/590_single_dns_entry.sh

@@ -43,6 +43,7 @@ fi
 if [[ ${NUMBER_OF_A_RECORDS} != "1" ]] ; then
     echo "There are ${NUMBER_OF_A_RECORDS} A records in DNS for ${HOSTNAME}"
     echo "This is very likely to cause problems with (at least) SMB-W clustering"
+    echo "Recommended Resolution: add a DNS record of type A for ${HOSTNAME} pointing to the IPv4 address"
     RETURN_CODE=254
 else
     echo "There is exactly one A record in DNS for ${HOSTNAME}"

scripts.d/ta/610_nfs_aliases_sbr.sh

@@ -71,6 +71,8 @@ main() {
         done < <(ip -4 rule | awk '{print $3}' | grep -v "all")
         if [[ $found_rule -eq 0 ]]; then
             echo "WARNING: No ip rule for address $NFS_IP! It is possible source-based routing should be configured."
+            echo "Recommended Resolution: configure source-based routing. Examples are mentioned in the WEKA docs:"
+            echo "https://docs.weka.io/planning-and-installation/bare-metal/setting-up-the-hosts#configure-the-ha-networking"
             RETURN_CODE=254
         fi
         done < <(weka nfs interface-group assignment --no-header | awk '$3 == '$weka_host_id'' | awk '{print $1}')

scripts.d/ta/620_same_mtu_across_nics.sh

@@ -33,6 +33,10 @@ for CONTAINER in $(weka local ps --no-header | awk '{print $1}' | grep -vw -e en
             echo "has an MTU of ${MTU}, which is less than the MTU ${SMALLEST_MTU_REQUIRED} seen elsewhere in this host"
             echo "This can lead to cluster communication problems"
             echo "Please see ${JIRA_REFERENCE} for more information"
+            echo "Recommended Resolution: Increase the MTUs of all NICs in the cluster to at least ${SMALLEST_MTU_REQUIRED}"
+            echo "Review your OS documentation for how to set this permanently, but NetworkManager-based OSes will use"
+            echo "something like \"nmcli connection modify eno1 802-3-ethernet.mtu ${SMALLEST_MTU_REQUIRED}\" and then"
+            echo "\"nmcli connection apply eno1\", but connection names will vary"
             RETURN_CODE=254
         fi
     done

scripts.d/ta/630_opt_weka_exists_but_not_mounted.sh

@@ -29,6 +29,11 @@ main() {
         echo
         echo "This means that changes made to the live system as it is now"
         echo "are unlikely to be present on the system post-reboot"
+        echo "Recommended Resolution:"
+        echo " . Do NOT reboot the host"
+        echo " . You need to verify that the on-boot configuration for /opt/weka"
+        echo "   matches and uses the currently used layout. This may involve"
+        echo "   editing filesystem layouts in /etc/fstab or systemd"
         RETURN_CODE=254
     else
         echo "No immediate directory/mount overlaps found"

scripts.d/ta/640_opt_weka_is_not_symlink.sh

@@ -13,6 +13,9 @@ main() {
     if [[ -L /opt/weka ]] ; then
         echo "/opt/weka is a symlink. This is not supported and"
         echo "is very unlikely to work due to chroot-style container behaviour"
+        echo "Recommended Resolution: Do not install Weka in a symlink"
+        echo "Resolving this can involve a rolling deactivation and re-installation"
+        echo "of Weka, depending on how and why this was done"
         RETURN_CODE=254
     else
         echo "/opt/weka is not a symlink. This is ok"

scripts.d/ta/650_firewall_check_quick.sh

@@ -54,6 +54,9 @@ for ip in ${!BACKEND_IPS[@]}; do
         for port in ${ports[@]}; do
             if (! echo -n 2>/dev/null < /dev/tcp/$ip/$port); then
                 echo "WARN: Unable to connect to $ip tcp/$port"
+                echo "Recommended Resolution: There is likely something blocking network communication between"
+                echo "this host and ${ip} tcp/${port}. Please review network connectivity and/or firewalls"
+                echo "In particular DDOS-style protection on switches may prevent communication"
                 RETURN_CODE=254
             fi
         done
@@ -67,4 +70,4 @@ if [[ ${RETURN_CODE} -eq 0 ]]; then
     echo "No backend management ports blocked."
 fi
 
-exit ${RETURN_CODE}
+exit ${RETURN_CODE}

scripts.d/ta/660_hugepages_check.sh

@@ -28,6 +28,8 @@ if [[ -n $WEKA_HUGE_1G ]]; then
     if [[ $DIFF_1G != 0 ]]; then
       RETURN_CODE=254
       echo "Discrepancy of $DIFF_1G 1GiB hugepage(s) between Weka and OS."
+      echo "Recommended Resolution: Review if other applications (such as hypervisors) are"
+      echo "using hugepages. If they are, this may be expected."
     fi
 fi
 
@@ -37,6 +39,8 @@ if [[ -n $WEKA_HUGE_2M ]]; then
     if [[ $DIFF_2M != 0 ]]; then
         RETURN_CODE=254
         echo "Discrepancy of $DIFF_2M 2MiB hugepage(s) between Weka and OS."
+        echo "Recommended Resolution: Review if other applications (such as hypervisors) are"
+        echo "using hugepages. If they are, this may be expected."
     fi
 fi
 
@@ -45,4 +49,4 @@ if [[ $RETURN_CODE -eq 0 ]]; then
     echo "No hugepages allocation discrepancy."
 fi
 
-exit ${RETURN_CODE}
+exit ${RETURN_CODE}

scripts.d/ta/670_crowdstrike_check.sh

@@ -14,9 +14,13 @@ RETURN_CODE=0
 
 if systemctl status falcon-sensor &> /dev/null; then
     echo "Warning: CrowdStrike Falcon Sensor is running"
+    echo "Recommended Resolution: we do not recommend using this software in conjunction with WEKA as"
+    echo "it has been shown to cause problems unloading kernel modules"
     exit 254
 elif lsmod | grep -q -m 1 falcon_lsm; then
     echo "Warning: Crowdstrike Falcon kernel module loaded"
+    echo "Recommended Resolution: we do not recommend using this software in conjunction with WEKA as"
+    echo "it has been shown to cause problems unloading kernel modules"
     exit 254
 fi
 echo "CrowdStrike Falcon Sensor is not running"