From f8007195bcf69f0dfef648efab6c5c627dc660a7 Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 5 Feb 2016 11:41:29 -0800 Subject: [PATCH 1/2] Move get_container_id_for_mesos_id into mesos_tools --- paasta_tools/mesos_tools.py | 15 +++++++ paasta_tools/paasta_execute_docker_command.py | 16 +------- tests/test_mesos_tools.py | 39 +++++++++++++++++++ tests/test_paasta_execute_docker_command.py | 39 ------------------- 4 files changed, 55 insertions(+), 54 deletions(-) diff --git a/paasta_tools/mesos_tools.py b/paasta_tools/mesos_tools.py index 81f6b60126..6c48579686 100644 --- a/paasta_tools/mesos_tools.py +++ b/paasta_tools/mesos_tools.py @@ -376,3 +376,18 @@ def slave_passes_blacklist(slave, blacklist): if attributes.get(location_type) == location: return False return True + + +def get_container_id_for_mesos_id(client, mesos_task_id): + running_containers = client.containers() + + container_id = None + for container in running_containers: + info = client.inspect_container(container) + if info['Config']['Env']: + for env_var in info['Config']['Env']: + if ('MESOS_TASK_ID=%s' % mesos_task_id) in env_var: + container_id = info['Id'] + break + + return container_id diff --git a/paasta_tools/paasta_execute_docker_command.py b/paasta_tools/paasta_execute_docker_command.py index 1d3c2a8306..6f4ce8416a 100755 --- a/paasta_tools/paasta_execute_docker_command.py +++ b/paasta_tools/paasta_execute_docker_command.py @@ -32,6 +32,7 @@ from docker import Client +from paasta_tools.mesos_tools import get_container_id_for_mesos_id from paasta_tools.utils import get_docker_host @@ -44,21 +45,6 @@ def parse_args(): return args -def get_container_id_for_mesos_id(client, mesos_task_id): - running_containers = client.containers() - - container_id = None - for container in running_containers: - info = client.inspect_container(container) - if info['Config']['Env']: - for env_var in info['Config']['Env']: - if ('MESOS_TASK_ID=%s' % mesos_task_id) in env_var: - container_id = info['Id'] - break - - return container_id - - class TimeoutException(Exception): pass diff --git a/tests/test_mesos_tools.py b/tests/test_mesos_tools.py index ca83f4ca3d..6c480c5d3c 100644 --- a/tests/test_mesos_tools.py +++ b/tests/test_mesos_tools.py @@ -14,6 +14,7 @@ import contextlib import datetime +import docker import mesos import mock import requests @@ -383,3 +384,41 @@ def test_get_mesos_state_from_leader_raises_on_non_elected_leader(): mesos.cli.master.CURRENT.state = un_elected_fake_state with raises(mesos_tools.MasterNotAvailableException): assert mesos_tools.get_mesos_state_from_leader() == un_elected_fake_state + + +def test_get_paasta_execute_docker_healthcheck(): + mock_docker_client = mock.MagicMock(spec_set=docker.Client) + fake_container_id = 'fake_container_id' + fake_mesos_id = 'fake_mesos_id' + fake_container_info = [ + {'Config': {'Env': None}}, + {'Config': {'Env': ['fake_key1=fake_value1', 'MESOS_TASK_ID=fake_other_mesos_id']}, 'Id': '11111'}, + {'Config': {'Env': ['fake_key2=fake_value2', 'MESOS_TASK_ID=%s' % fake_mesos_id]}, 'Id': fake_container_id}, + ] + mock_docker_client.containers = mock.MagicMock( + spec_set=docker.Client, + return_value=['fake_container_1', 'fake_container_2', 'fake_container_3'], + ) + mock_docker_client.inspect_container = mock.MagicMock( + spec_set=docker.Client, + side_effect=fake_container_info, + ) + assert mesos_tools.get_container_id_for_mesos_id(mock_docker_client, fake_mesos_id) == fake_container_id + + +def test_get_paasta_execute_docker_healthcheck_when_not_found(): + mock_docker_client = mock.MagicMock(spec_set=docker.Client) + fake_mesos_id = 'fake_mesos_id' + fake_container_info = [ + {'Config': {'Env': ['fake_key1=fake_value1', 'MESOS_TASK_ID=fake_other_mesos_id']}, 'Id': '11111'}, + {'Config': {'Env': ['fake_key2=fake_value2', 'MESOS_TASK_ID=fake_other_mesos_id2']}, 'Id': '2222'}, + ] + mock_docker_client.containers = mock.MagicMock( + spec_set=docker.Client, + return_value=['fake_container_1', 'fake_container_2'], + ) + mock_docker_client.inspect_container = mock.MagicMock( + spec_set=docker.Client, + side_effect=fake_container_info, + ) + assert mesos_tools.get_container_id_for_mesos_id(mock_docker_client, fake_mesos_id) is None diff --git a/tests/test_paasta_execute_docker_command.py b/tests/test_paasta_execute_docker_command.py index 2a60354c5e..959f9e10bd 100644 --- a/tests/test_paasta_execute_docker_command.py +++ b/tests/test_paasta_execute_docker_command.py @@ -18,49 +18,10 @@ import pytest from paasta_tools.paasta_execute_docker_command import execute_in_container -from paasta_tools.paasta_execute_docker_command import get_container_id_for_mesos_id from paasta_tools.paasta_execute_docker_command import main from paasta_tools.paasta_execute_docker_command import TimeoutException -def test_get_paasta_execute_docker_healthcheck(): - mock_docker_client = mock.MagicMock(spec_set=docker.Client) - fake_container_id = 'fake_container_id' - fake_mesos_id = 'fake_mesos_id' - fake_container_info = [ - {'Config': {'Env': None}}, - {'Config': {'Env': ['fake_key1=fake_value1', 'MESOS_TASK_ID=fake_other_mesos_id']}, 'Id': '11111'}, - {'Config': {'Env': ['fake_key2=fake_value2', 'MESOS_TASK_ID=%s' % fake_mesos_id]}, 'Id': fake_container_id}, - ] - mock_docker_client.containers = mock.MagicMock( - spec_set=docker.Client, - return_value=['fake_container_1', 'fake_container_2', 'fake_container_3'], - ) - mock_docker_client.inspect_container = mock.MagicMock( - spec_set=docker.Client, - side_effect=fake_container_info, - ) - assert get_container_id_for_mesos_id(mock_docker_client, fake_mesos_id) == fake_container_id - - -def test_get_paasta_execute_docker_healthcheck_when_not_found(): - mock_docker_client = mock.MagicMock(spec_set=docker.Client) - fake_mesos_id = 'fake_mesos_id' - fake_container_info = [ - {'Config': {'Env': ['fake_key1=fake_value1', 'MESOS_TASK_ID=fake_other_mesos_id']}, 'Id': '11111'}, - {'Config': {'Env': ['fake_key2=fake_value2', 'MESOS_TASK_ID=fake_other_mesos_id2']}, 'Id': '2222'}, - ] - mock_docker_client.containers = mock.MagicMock( - spec_set=docker.Client, - return_value=['fake_container_1', 'fake_container_2'], - ) - mock_docker_client.inspect_container = mock.MagicMock( - spec_set=docker.Client, - side_effect=fake_container_info, - ) - assert get_container_id_for_mesos_id(mock_docker_client, fake_mesos_id) is None - - def test_execute_in_container(): fake_container_id = 'fake_container_id' fake_return_code = 0 From 2939420b5f441551da9048203341db974067662d Mon Sep 17 00:00:00 2001 From: Kyle Anderson Date: Fri, 5 Feb 2016 13:15:04 -0800 Subject: [PATCH 2/2] Added experimental orphaned task killing script --- .../kill_orphaned_docker_containers.py | 61 +++++++++++++++++++ paasta_tools/mesos_tools.py | 16 +++++ 2 files changed, 77 insertions(+) create mode 100755 paasta_tools/contrib/kill_orphaned_docker_containers.py diff --git a/paasta_tools/contrib/kill_orphaned_docker_containers.py b/paasta_tools/contrib/kill_orphaned_docker_containers.py new file mode 100755 index 0000000000..405d875c39 --- /dev/null +++ b/paasta_tools/contrib/kill_orphaned_docker_containers.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +import argparse + +from docker import Client + +from paasta_tools import mesos_tools +from paasta_tools.utils import get_docker_host + + +def parse_args(): + parser = argparse.ArgumentParser( + description=( + 'Cross references running containers with task ids from the mesos slave', + ' and optionally kills them.' + ) + ) + parser.add_argument('-f', '--force', help="Actually kill the containers. (defaults to dry-run)") + args = parser.parse_args() + return args + + +def get_running_task_ids_from_mesos_slave(): + state = mesos_tools.get_local_slave_state() + frameworks = state.get('frameworks') + executors = [ex for fw in frameworks for ex in fw.get('executors', []) + if u'TASK_RUNNING' in [t[u'state'] for t in ex.get('tasks', [])]] + return [e["id"] for e in executors] + + +def get_running_mesos_docker_containers(client): + running_containers = client.containers() + return [container for container in running_containers if "mesos-" in container["Names"][0]] + + +def get_docker_client(): + base_docker_url = get_docker_host() + return Client(base_url=base_docker_url) + + +def main(): + args = parse_args() + docker_client = get_docker_client() + running_mesos_task_ids = get_running_task_ids_from_mesos_slave() + running_mesos_docker_containers = get_running_mesos_docker_containers(docker_client) + print running_mesos_task_ids + + for container in running_mesos_docker_containers: + mesos_task_id = mesos_tools.get_mesos_id_from_container(container=container, client=docker_client) + print mesos_task_id + if mesos_task_id not in running_mesos_task_ids: + if args.force: + print "Killing %s. (%s)" % (container["Names"][0], mesos_task_id) + docker_client.kill(container) + else: + print "Would kill %s. (%s)" % (container["Names"][0], mesos_task_id) + else: + print "Not killing %s. (%s)" % (container["Names"][0], mesos_task_id) + + +if __name__ == "__main__": + main() diff --git a/paasta_tools/mesos_tools.py b/paasta_tools/mesos_tools.py index 6c48579686..09d5459a3f 100644 --- a/paasta_tools/mesos_tools.py +++ b/paasta_tools/mesos_tools.py @@ -391,3 +391,19 @@ def get_container_id_for_mesos_id(client, mesos_task_id): break return container_id + + +def get_mesos_id_from_container(container, client): + mesos_id = None + info = client.inspect_container(container) + if info['Config']['Env']: + for env_var in info['Config']['Env']: + # In marathon it is like this + if 'MESOS_TASK_ID=' in env_var: + mesos_id = re.match("MESOS_TASK_ID=(.*)", env_var).group(1) + break + # Chronos it is like this? + if 'mesos_task_id=' in env_var: + mesos_id = re.match("mesos_task_id=(.*)", env_var).group(1) + break + return mesos_id