From c3f722ed32cf36e7529c98be44ea4e1b93c40178 Mon Sep 17 00:00:00 2001 From: Seth Pellegrino Date: Thu, 3 Dec 2015 12:45:44 -0800 Subject: [PATCH] Remove linux import-time dependency on /proc/stat The very excellent work that went in under giampaolo/psutil#558 gave us a psutil that can be used on systems with a moved /proc. However, on some systems (e.g. test harnesses, containers) may not have any files mounted at /proc at all. In these cases, psutil fails at import time with an IOError trying to access /proc/stat. This patch removes the import-time dependency on a live procfs mounted at /proc, and validates the behavior of some of the details we were hoping to read at import time (i.e. cpu % stats). --- psutil/__init__.py | 14 ++++++---- psutil/_pslinux.py | 54 +++++++++++++++++++++----------------- test/_linux.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 28 deletions(-) diff --git a/psutil/__init__.py b/psutil/__init__.py index 1a229b6ba5..cc967115c2 100644 --- a/psutil/__init__.py +++ b/psutil/__init__.py @@ -1411,9 +1411,13 @@ def cpu_times(percpu=False): else: return _psplatform.per_cpu_times() - -_last_cpu_times = cpu_times() -_last_per_cpu_times = cpu_times(percpu=True) +try: + _last_cpu_times = cpu_times() + _last_per_cpu_times = cpu_times(percpu=True) +except IOError: + from collections import namedtuple + _last_cpu_times = namedtuple('emptycpu', 'idle')(0) + _last_per_cpu_times = [] def cpu_percent(interval=None, percpu=False): @@ -1521,8 +1525,8 @@ def cpu_times_percent(interval=None, percpu=False): def calculate(t1, t2): nums = [] all_delta = sum(t2) - sum(t1) - for field in t1._fields: - field_delta = getattr(t2, field) - getattr(t1, field) + for field in t2._fields: + field_delta = getattr(t2, field) - getattr(t1, field, 0) try: field_perc = (100 * field_delta) / all_delta except ZeroDivisionError: diff --git a/psutil/_pslinux.py b/psutil/_pslinux.py index 61bfe5f1e1..927e9f045b 100644 --- a/psutil/_pslinux.py +++ b/psutil/_pslinux.py @@ -149,31 +149,41 @@ def get_procfs_path(): # --- named tuples -def set_scputimes_ntuple(procfs_path): - """Return a namedtuple of variable fields depending on the +class LazyScputimes(object): + """Acts as a namedtuple of variable fields depending on the CPU times available on this Linux kernel version which may be: (user, nice, system, idle, iowait, irq, softirq, [steal, [guest, [guest_nice]]]) + + We require a lazy wrapper so as to not have a depdenceny on /proc/stat at + import time. """ - global scputimes - with open_binary('%s/stat' % procfs_path) as f: - values = f.readline().split()[1:] - fields = ['user', 'nice', 'system', 'idle', 'iowait', 'irq', 'softirq'] - vlen = len(values) - if vlen >= 8: - # Linux >= 2.6.11 - fields.append('steal') - if vlen >= 9: - # Linux >= 2.6.24 - fields.append('guest') - if vlen >= 10: - # Linux >= 3.2.0 - fields.append('guest_nice') - scputimes = namedtuple('scputimes', fields) - return scputimes - - -scputimes = set_scputimes_ntuple('/proc') + + @staticmethod + def get_tuple(procfs_path): + with open('%s/stat' % procfs_path, 'rb') as f: + values = f.readline().split()[1:] + fields = ['user', 'nice', 'system', 'idle', 'iowait', 'irq', 'softirq'] + vlen = len(values) + if vlen >= 8: + # Linux >= 2.6.11 + fields.append('steal') + if vlen >= 9: + # Linux >= 2.6.24 + fields.append('guest') + if vlen >= 10: + # Linux >= 3.2.0 + fields.append('guest_nice') + return namedtuple('scputimes', fields) + + def __call__(self, *args, **kwargs): + return LazyScputimes.get_tuple(get_procfs_path())(*args, **kwargs) + + def __getattr__(self, item): + return getattr(LazyScputimes.get_tuple(get_procfs_path()), item) + + +scputimes = LazyScputimes() svmem = namedtuple( 'svmem', ['total', 'available', 'percent', 'used', 'free', @@ -256,7 +266,6 @@ def cpu_times(): Last 3 fields may not be available on all Linux kernel versions. """ procfs_path = get_procfs_path() - set_scputimes_ntuple(procfs_path) with open_binary('%s/stat' % procfs_path) as f: values = f.readline().split() fields = values[1:len(scputimes._fields) + 1] @@ -269,7 +278,6 @@ def per_cpu_times(): for every CPU available on the system. """ procfs_path = get_procfs_path() - set_scputimes_ntuple(procfs_path) cpus = [] with open_binary('%s/stat' % procfs_path) as f: # get rid of the first line which refers to system wide CPU stats diff --git a/test/_linux.py b/test/_linux.py index 76e8c81456..8ef6da4d9d 100644 --- a/test/_linux.py +++ b/test/_linux.py @@ -15,6 +15,7 @@ import os import pprint import re +import shutil import socket import struct import sys @@ -478,6 +479,69 @@ def test_procfs_path(self): def test_psutil_is_reloadable(self): imp.reload(psutil) + def test_no_procfs_for_import(self): + my_procfs = tempfile.mkdtemp() + + with open(os.path.join(my_procfs, 'stat'), 'w') as f: + f.write('cpu 0 0 0 0 0 0 0 0 0 0\n') + f.write('cpu0 0 0 0 0 0 0 0 0 0 0\n') + f.write('cpu1 0 0 0 0 0 0 0 0 0 0\n') + + self.assertNotAlmostEqual(psutil.cpu_percent(), 0) + self.assertNotAlmostEqual(sum(psutil.cpu_times_percent()), 0) + try: + orig_open = open + + def open_mock(name, *args): + if name.startswith('/proc'): + # simulate an ENOENT + raise IOError('rejecting access to /proc') + return orig_open(name, *args) + with mock.patch('__builtin__.open', side_effect=open_mock): + imp.reload(psutil) + + self.assertRaises(IOError, psutil.cpu_times) + self.assertRaises(IOError, psutil.cpu_times, percpu=True) + self.assertRaises(IOError, psutil.cpu_percent) + self.assertRaises(IOError, psutil.cpu_percent, percpu=True) + self.assertRaises(IOError, psutil.cpu_times_percent) + self.assertRaises( + IOError, psutil.cpu_times_percent, percpu=True) + + psutil.PROCFS_PATH = my_procfs + + self.assertAlmostEqual(psutil.cpu_percent(), 0) + self.assertAlmostEqual(sum(psutil.cpu_times_percent()), 0) + + # since we don't know the number of CPUs at import time, + # we awkwardly say there are none until the second call + per_cpu_percent = psutil.cpu_percent(percpu=True) + self.assertEqual(len(per_cpu_percent), 0) + self.assertAlmostEqual(sum(per_cpu_percent), 0) + + # ditto awkward length + per_cpu_times_percent = psutil.cpu_times_percent(percpu=True) + self.assertEqual(len(per_cpu_times_percent), 0) + self.assertAlmostEqual(sum(map(sum, per_cpu_percent)), 0) + + # much user, very busy + with open(os.path.join(my_procfs, 'stat'), 'w') as f: + f.write('cpu 1 0 0 0 0 0 0 0 0 0\n') + f.write('cpu0 1 0 0 0 0 0 0 0 0 0\n') + f.write('cpu1 1 0 0 0 0 0 0 0 0 0\n') + + self.assertNotAlmostEqual(psutil.cpu_percent(), 0) + self.assertNotAlmostEqual( + sum(psutil.cpu_percent(percpu=True)), 0) + self.assertNotAlmostEqual(sum(psutil.cpu_times_percent()), 0) + self.assertNotAlmostEqual( + sum(map(sum, psutil.cpu_times_percent(percpu=True))), 0) + finally: + shutil.rmtree(my_procfs) + imp.reload(psutil) + + assert psutil.PROCFS_PATH == '/proc' + # --- tests for specific kernel versions @unittest.skipUnless(