compute-heap-opts

#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Specifically, this script is intended SOLELY to support the Confluent
# Quick Start offering in Amazon Web Services. It is not recommended
# for use in any other production environment.
#
#
# Simple script to divide up available memory into 
# logica options for launching confluent services.
#
# usage
#	Source the script (eg ' . $THIS_SCRIPT ')
#
# outputs
#	Heap settings for different services in the form "-Xms <MB> -Xmx <MB"
#		BROKER_HEAP_OPTS
#		ZOOKEEPER_HEAP_OPTS
#		SR_HEAP_OPTS
#		REST_HEAP_OPTS
#		CONNECT_HEAP_OPTS
#		CC_HEAP_OPTS

THIS_HOST=`hostname -s`

# set -x

#
# We need to know what services will be provisioned on this
# node.  Long term, there are better ways.   For now, we look
# for the package or the presence of a role file (usually
# created as part of our automated install process).
#	NOTE: This is really squirrelly, since our packages
#	don't align cleanly with roles, and there is no
#	explicit zookeeper package.
#

isBroker() {
	if [ -f /tmp/brokers ] ; then
		return $(grep -q -w $THIS_HOST /tmp/brokers)
	elif `which dpkg &> /dev/null` ; then
		return $(dpkg --get-selections confluent-kafka-2* 2> /dev/null | grep -q install)
	elif `which rpm &> /dev/null` ; then
		return $(rpm -qa | grep -q confluent-kafka-2)
	fi
	
	return `/bin/false`
}

isZookeeper() {
	if [ -f /tmp/zookeepers ] ; then
		return $(grep -q -w $THIS_HOST /tmp/zookeepers)
	elif `which dpkg &> /dev/null` ; then
		return $(dpkg --get-selections confluent-kafka-2* 2> /dev/null | grep -q install)
	elif `which rpm &> /dev/null` ; then
		return $(rpm -qa | grep -q confluent-kafka-2)
	fi
	
	return `/bin/false`
}

isWorker() {
	if [ -f /tmp/workers ] ; then
		return $(grep -q -w $THIS_HOST /tmp/workers)
	elif `which dpkg &> /dev/null` ; then
		return $(dpkg --get-selections confluent-kafka-[A-z]* 2> /dev/null | grep -q install)
	elif `which rpm &> /dev/null` ; then
		return $(rpm -qa | grep -q confluent-kafka-[A-z])
	fi
	
	return `/bin/false`
}

isFirstWorker() {
	if [ -f /tmp/workers ] ; then
		return $(grep -n -w $THIS_HOST /tmp/workers | grep -q ^1:)
	fi
	
	return `/bin/false`
}

# $1 : available memory for all Confluent services
# $2 : target pct
# $3 : absolute min
# $4 : absolute max
calc_target_heap() {
	tgt=$(( $1 * $2 / 100 ))
	tgt=$(( $tgt < $3 ? $3 : $tgt ))
	tgt=$(( $tgt > $4 ? $4 : $tgt ))

	echo $tgt
}


freeMB=$(free -m | grep ^Mem | awk '{print $4}')
(>&2 echo "Free memory: ${freeMB}M")

# Ratchet down our total pool; we could be more agressive
# about this.  Want to make sure we avoid swapping.
if [ $freeMB -gt 10240 ] ; then
	availMB=$(($freeMB / 512 * 512))
	(>&2 echo " rounded to 512M: ${availMB}M")
else
	availMB=$(($freeMB / 256 * 256))
	(>&2 echo " rounded to 256M: ${availMB}M")
fi

# Set up some targets and boundaries for our memory allocation.
# Assumes reasonable isolation of services (adjustments made later).
#	MIN values are from hard-coded defaults in Confluent 3.1.1
BROKER_TARGET_PCT=25
BROKER_HEAP_MAX=4608
BROKER_HEAP_MIN=1024

ZOOKEEPER_TARGET_PCT=10
ZOOKEEPER_HEAP_MAX=2048
ZOOKEEPER_HEAP_MIN=512

CONNECT_TARGET_PCT=15
CONNECT_HEAP_MAX=4096
CONNECT_HEAP_MIN=256

REST_TARGET_PCT=15
REST_HEAP_MAX=2048
REST_HEAP_MIN=256

SR_TARGET_PCT=15
SR_HEAP_MAX=2048
SR_HEAP_MIN=512

CC_TARGET_PCT=40
CC_HEAP_MAX=8192
CC_HEAP_MIN=3072

# The above defaults are for "broker-0" and "worker-0" in our
# compacted cluster 
#	broker and zk on all broker-* nodes
#	worker-0 will have CC and SchemaReg along with Connect and REST
#	all other workers will have Connect and REST only
#
# Adjust the settings if necessary (based on the roles 
# identified with gen-cluster-hosts.sh or installed packages).
#
# Standalone ZK nodes can take more memory
isZookeeper
if [ $? -eq 0 ] ; then
	isBroker
	if [ $? -ne 0 ] ; then
		ZOOKEEPER_TARGET_PCT=25
	fi
fi

isWorker
if [ $? -eq 0 ] ; then
	isBroker
	if [ $? -eq 0 ] ; then
			# Sandbox environment (all services on one node)
		BROKER_TARGET_PCT=25
		ZOOKEEPER_TARGET_PCT=5
		SR_TARGET_PCT=10
		REST_TARGET_PCT=10
		CONNECT_TARGET_PCT=10
		CC_TARGET_PCT=25
	else
		isFirstWorker
		if [ $? -ne 0 ] ; then
			SR_TARGET_PCT=10		# irrelevant since service not deployed
			REST_TARGET_PCT=20
			CONNECT_TARGET_PCT=40
			CC_TARGET_PCT=10		# irrelevant since service not deployed
		fi
	fi
fi

BROKER_HEAP=$( calc_target_heap $availMB $BROKER_TARGET_PCT $BROKER_HEAP_MIN $BROKER_HEAP_MAX )
ZOOKEEPER_HEAP=$( calc_target_heap $availMB $ZOOKEEPER_TARGET_PCT $ZOOKEEPER_HEAP_MIN $ZOOKEEPER_HEAP_MAX )
SR_HEAP=$( calc_target_heap $availMB $SR_TARGET_PCT $SR_HEAP_MIN $SR_HEAP_MAX )
REST_HEAP=$( calc_target_heap $availMB $REST_TARGET_PCT $REST_HEAP_MIN $REST_HEAP_MAX )
CONNECT_HEAP=$( calc_target_heap $availMB $CONNECT_TARGET_PCT $CONNECT_HEAP_MIN $CONNECT_HEAP_MAX )
CC_HEAP=$( calc_target_heap $availMB $CC_TARGET_PCT $CC_HEAP_MIN $CC_HEAP_MAX )

# For now, we align our new opts with the java options used in
# Confluent 3.1.1 (not every service sets -Xms)

# Design decision :
#	We could avoid setting ANYTHING here if the computed HEAP is the same
#	as the target HEAP_MIN.   We aren't doing that now so that the start
#	script will always have an explicit setting.
#
export BROKER_HEAP_OPTS="-Xms${BROKER_HEAP}M -Xmx${BROKER_HEAP}M"
export ZOOKEEPER_HEAP_OPTS="-Xms${ZOOKEEPER_HEAP}M -Xmx${ZOOKEEPER_HEAP}M"
export SR_HEAP_OPTS="-Xmx${SR_HEAP}M"
export REST_HEAP_OPTS="-Xmx${REST_HEAP}M"
export CONNECT_HEAP_OPTS="-Xmx${CONNECT_HEAP}M"
export CC_HEAP_OPTS="-Xmx${CC_HEAP}M"

# set +x