Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Print metastatus free resources alongside total resource capacity #315

Merged
merged 3 commits into from
Mar 11, 2016
Merged

Print metastatus free resources alongside total resource capacity #315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ae163ca
Print metastatus resource usage alongside total resource availability
Mar 9, 2016
9d58f8d
Copy calculated totals, add another host to test
Mar 11, 2016
b7123f4
Fix tests
Mar 11, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 18 additions & 10 deletions paasta_tools/paasta_metastatus.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import argparse
import copy
import sys
from collections import Counter
from collections import defaultdict
Expand Down Expand Up @@ -98,18 +99,21 @@ def get_extra_mesos_attribute_data(mesos_state):
attributes = set().union(*(slave['attributes'].keys() for slave in mesos_state['slaves']))

for attribute in attributes:
resource_dict = defaultdict(Counter)
resource_free_dict = defaultdict(Counter)
resource_availability_dict = dict()
slave_attribute_mapping = {}
for slave in mesos_state['slaves']:
slave_attribute_name = slave['attributes'].get(attribute, 'UNDEFINED')
slave_attribute_mapping[slave['id']] = slave_attribute_name
resource_dict[slave_attribute_name].update(filter_mesos_state_metrics(slave['resources']))
filtered_resources = filter_mesos_state_metrics(slave['resources'])
resource_free_dict[slave_attribute_name].update(filtered_resources)
resource_availability_dict = copy.deepcopy(resource_free_dict)
for framework in mesos_state.get('frameworks', []):
for task in framework.get('tasks', []):
task_resources = task['resources']
attribute_value = slave_attribute_mapping[task['slave_id']]
resource_dict[attribute_value].subtract(filter_mesos_state_metrics(task_resources))
yield (attribute, resource_dict)
resource_free_dict[attribute_value].subtract(filter_mesos_state_metrics(task_resources))
yield (attribute, {"free": resource_free_dict, "availability": resource_availability_dict})


def quorum_ok(masters, quorum):
Expand Down Expand Up @@ -251,14 +255,18 @@ def assert_extra_attribute_data(mesos_state):
extra_attribute_data = list(get_extra_mesos_attribute_data(mesos_state))
rows = []
for attribute, resource_dict in extra_attribute_data:
if len(resource_dict.keys()) >= 2: # filter out attributes that apply to every slave in the cluster
rows.append((attribute.capitalize(), 'CPU free', 'RAM free', 'Disk free'))
for attribute_location, resources_remaining in resource_dict.items():
resource_free_dict = resource_dict["free"]
resource_availability_dict = resource_dict["availability"]
if len(resource_free_dict.keys()) >= 2: # filter out attributes that apply to every slave in the cluster
rows.append((attribute.capitalize(), 'CPU (free/total)', 'RAM (free/total)', 'Disk (free/total)'))
for attribute_location, resources_remaining in resource_free_dict.items():
resources_available = resource_availability_dict[attribute_location]

rows.append((
attribute_location,
'%.2f' % resources_remaining['cpus'],
'%.2f' % resources_remaining['mem'],
'%.2f' % resources_remaining['disk'],
'%.2f/%.2f' % (resources_remaining['cpus'], resources_available['cpus']),
'%.2f/%.2f' % (resources_remaining['mem'], resources_available['mem']),
'%.2f/%.2f' % (resources_remaining['disk'], resources_available['disk'])
))
if len(rows) == 0:
result = (" No slave attributes that apply to more than one slave were detected.", True)
Expand Down
53 changes: 41 additions & 12 deletions tests/test_paasta_metastatus.py
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -584,8 +584,8 @@ def test_get_mesos_habitat_data():
'id': 'test-slave',
'hostname': 'test.somewhere.www',
'resources': {
'cpus': 50,
'disk': 200,
'cpus': 75,
'disk': 250,
'mem': 1000,
},
'attributes': {
Expand All @@ -598,7 +598,19 @@ def test_get_mesos_habitat_data():
'resources': {
'cpus': 50,
'disk': 200,
'mem': 1000,
'mem': 750,
},
'attributes': {
'habitat': 'test-habitat-2',
},
},
{
'id': 'test-slave3',
'hostname': 'test3.somewhere.www',
'resources': {
'cpus': 22,
'disk': 201,
'mem': 920,
},
'attributes': {
'habitat': 'test-habitat-2',
Expand Down Expand Up @@ -626,18 +638,35 @@ def test_get_mesos_habitat_data():
(
'habitat',
{
'test-habitat': {
'cpus': 0,
'disk': 100,
'mem': 1000,
},
'test-habitat-2': {
'cpus': 50,
'disk': 200,
'mem': 1000,
'free':
{
'test-habitat': {
'cpus': 25,
'disk': 150,
'mem': 1000,
},
'test-habitat-2': {
'cpus': 72,
'disk': 401,
'mem': 1670,
},
},
'availability':
{
'test-habitat': {
'cpus': 75,
'disk': 250,
'mem': 1000,
},
'test-habitat-2': {
'cpus': 72,
'disk': 401,
'mem': 1670,
},
}
}
),
)
extra_mesos_habitat_data = paasta_metastatus.get_extra_mesos_attribute_data(mesos_state)

assert (tuple(extra_mesos_habitat_data) == expected_free_resources)