python 3.6 -> 3.7: collections.Callable to collections.abc.Callable

squashlater: deprecated imp replaced with importlib.machinery

squashlater: added @todo to skipped tests

squashlater: fixed rose-app-run

squashlater: mod travis.yml

fixlater: pycodestyle fix

.travis.yml

@@ -4,7 +4,7 @@
 ---
 language: python
 python:
-    - 3.6
+    - 3.7
 dist: xenial
 
 

lib/python/rose/apps/rose_ana_v1.py

@@ -36,7 +36,7 @@
 from rose.reporter import Reporter, Event
 from rose.resource import ResourceLocator
 from rose.app_run import BuiltinApp
-import collections
+import collections.abc
 
 WARN = -1
 PASS = 0
@@ -467,7 +467,7 @@ def load_user_comparison_modules(self, files):
                 att_name = "run"
                 if hasattr(
                     obj, att_name) and isinstance(getattr(obj, att_name),
-                                                  collections.Callable):
+                                                  collections.abc.Callable):
                     doc_string = obj.__doc__
                     user_methods.append((comparison_name, obj_name, att_name,
                                         doc_string))

lib/python/rose/cmp_source_vc.py

@@ -30,7 +30,7 @@
 from rose.reporter import Reporter
 from rose.run_source_vc import write_source_vc_info
 from rose.suite_engine_proc import SuiteEngineProcessor
-import collections
+import collections.abc
 
 
 class SuiteVCComparator(object):
@@ -69,7 +69,7 @@ def cmp_source_vc_info(self, suite_name):
 
     def handle_event(self, *args, **kwargs):
         """Handle event."""
-        if isinstance(self.event_handler, collections.Callable):
+        if isinstance(self.event_handler, collections.abc.Callable):
             self.event_handler(*args, **kwargs)
 
 

lib/python/rose/config_processor.py

@@ -25,7 +25,7 @@
 from rose.popen import RosePopener
 from rose.scheme_handler import SchemeHandlersManager
 import sys
-import collections
+import collections.abc
 
 
 class UnknownContentError(Exception):
@@ -115,7 +115,7 @@ def __init__(self, event_handler=None, popen=None, fs_util=None):
 
     def handle_event(self, *args, **kwargs):
         """Report an event."""
-        if isinstance(self.event_handler, collections.Callable):
+        if isinstance(self.event_handler, collections.abc.Callable):
             return self.event_handler(*args, **kwargs)
 
     def process(self, conf_tree, item, orig_keys=None, orig_value=None,

lib/python/rose/config_processors/fileinstall.py

@@ -38,7 +38,7 @@
 import sys
 from tempfile import mkdtemp
 from urllib.parse import urlparse
-import collections
+import collections.abc
 
 
 class ConfigProcessorForFile(ConfigProcessorBase):
@@ -741,7 +741,7 @@ def __init__(self, event_handler=None, popen=None, fs_util=None):
 
     def handle_event(self, *args, **kwargs):
         """Call self.event_handler with given arguments if possible."""
-        if isinstance(self.event_handler, collections.Callable):
+        if isinstance(self.event_handler, collections.abc.Callable):
             return self.event_handler(*args, **kwargs)
 
     def parse(self, loc, conf_tree):

lib/python/rose/env.py

@@ -27,7 +27,7 @@
 import os
 import re
 from rose.reporter import Event
-import collections
+import collections.abc
 
 
 # _RE_DEFAULT = re.compile(r"""
@@ -116,7 +116,7 @@ def env_export(key, value, event_handler=None):
         #      normally quite small.
         _EXPORTED_ENVS[key] = value
         os.environb[key.encode('UTF-8')] = value.encode('UTF-8')
-        if isinstance(event_handler, collections.Callable):
+        if isinstance(event_handler, collections.abc.Callable):
             event_handler(EnvExportEvent(key, value))
 
 

lib/python/rose/fs_util.py

@@ -23,7 +23,7 @@
 import os
 from rose.reporter import Event
 import shutil
-import collections
+import collections.abc
 
 
 class FileSystemEvent(Event):
@@ -65,7 +65,7 @@ def __init__(self, event_handler=None):
     def handle_event(self, *args, **kwargs):
         """Handle an event using the runner's event handler."""
 
-        if isinstance(self.event_handler, collections.Callable):
+        if isinstance(self.event_handler, collections.abc.Callable):
             return self.event_handler(*args, **kwargs)
 
     def chdir(self, path):

lib/python/rose/host_select.py

@@ -32,7 +32,7 @@
 import sys
 from time import sleep, time
 import traceback
-import collections
+import collections.abc
 
 
 class NoHostError(Exception):
@@ -188,7 +188,7 @@ def get_scorer(self, method=None):
 
     def handle_event(self, *args, **kwargs):
         """Handle an event using the runner's event handler."""
-        if isinstance(self.event_handler, collections.Callable):
+        if isinstance(self.event_handler, collections.abc.Callable):
             return self.event_handler(*args, **kwargs)
 
     def expand(self, names=None, rank_method=None, thresholds=None):

lib/python/rose/macro.py

@@ -42,13 +42,13 @@ def test_cleanup(stuff_to_remove):
 import ast
 import copy
 import glob
-import imp
 import inspect
 import os
 import re
 import sys
 import traceback
 from functools import cmp_to_key
+from importlib.machinery import SourceFileLoader
 
 import rose.config
 import rose.config_tree
@@ -57,7 +57,7 @@ def test_cleanup(stuff_to_remove):
 import rose.reporter
 import rose.resource
 import rose.variable
-import collections
+import collections.abc
 
 
 ALLOWED_MACRO_CLASS_METHODS = ["transform", "validate", "downgrade", "upgrade",
@@ -746,7 +746,7 @@ def load_meta_macro_modules(meta_files, module_prefix=None):
         else:
             as_name = module_prefix + macro_name
         try:
-            modules.append(imp.load_source(as_name, meta_file))
+            modules.append(SourceFileLoader(as_name, meta_file).load_module())
         except Exception:
             rose.reporter.Reporter()(
                 MacroLoadError(meta_file, traceback.format_exc()))
@@ -767,7 +767,7 @@ def get_macro_class_methods(macro_modules):
             for att_name in ALLOWED_MACRO_CLASS_METHODS:
                 if (hasattr(obj, att_name) and
                         isinstance(getattr(obj, att_name),
-                                   collections.Callable)):
+                                   collections.abc.Callable)):
                     doc_string = obj.__doc__
                     macro_methods.append((macro_name, obj_name, att_name,
                                           doc_string))
@@ -878,8 +878,8 @@ def report_config(app_config, meta_config, run_macro_list, modules,
                     break
             res = {}
             if not opt_non_interactive:
-                arglist = inspect.getargspec(macro_meth).args
-                defaultlist = inspect.getargspec(macro_meth).defaults
+                arglist = inspect.getfullargspec(macro_meth).args
+                defaultlist = inspect.getfullargspec(macro_meth).defaults
                 optionals = {}
                 while defaultlist is not None and len(defaultlist) > 0:
                     if arglist[-1] not in ["self", "config", "meta_config"]:
@@ -935,8 +935,8 @@ def transform_config(config, meta_config, transformer_macro, modules,
                 break
         res = {}
         if not opt_non_interactive:
-            arglist = inspect.getargspec(macro_method).args
-            defaultlist = inspect.getargspec(macro_method).defaults
+            arglist = inspect.getfullargspec(macro_method).args
+            defaultlist = inspect.getfullargspec(macro_method).defaults
             optionals = {}
             while defaultlist is not None and len(defaultlist) > 0:
                 if arglist[-1] not in ["self", "config", "meta_config"]:

lib/python/rose/popen.py

@@ -27,7 +27,7 @@
 import shlex
 from subprocess import Popen, PIPE
 import sys
-import collections
+import collections.abc
 
 
 class RosePopenError(Exception):
@@ -75,7 +75,8 @@ def __str__(self):
             try:
                 # FIXME: Is this safe?
                 pos = self.stdin.tell()
-                ret += " <<'__STDIN__'\n" + self.stdin.read() + "\n'__STDIN__'"
+                ret += " <<'__STDIN__'\n" +\
+                       self.stdin.read().decode() + "\n'__STDIN__'"
                 self.stdin.seek(pos)
             except IOError:
                 pass
@@ -112,7 +113,7 @@ def __init__(self, event_handler=None):
 
     def handle_event(self, *args, **kwargs):
         """Handle an event using the runner's event handler."""
-        if isinstance(self.event_handler, collections.Callable):
+        if isinstance(self.event_handler, collections.abc.Callable):
             return self.event_handler(*args, **kwargs)
 
     def get_cmd(self, key, *args):

lib/python/rose/reporter.py

@@ -25,7 +25,7 @@
 import sys
 
 import time
-import collections
+import collections.abc
 
 
 class Reporter(object):
@@ -136,7 +136,7 @@ def report(self, message, kind=None, level=None, prefix=None, clip=None):
         arguments and return its result instead.
 
         """
-        if isinstance(self.event_handler, collections.Callable):
+        if isinstance(self.event_handler, collections.abc.Callable):
             return self.event_handler(message, kind, level, prefix, clip)
 
         if isinstance(message, Event):
@@ -164,10 +164,10 @@ def report(self, message, kind=None, level=None, prefix=None, clip=None):
                 continue
             if prefix is None:
                 prefix = context.get_prefix(kind, level)
-            elif isinstance(prefix, collections.Callable):
+            elif isinstance(prefix, collections.abc.Callable):
                 prefix = prefix(kind, level)
             if msg is None:
-                if isinstance(message, collections.Callable):
+                if isinstance(message, collections.abc.Callable):
                     msg = message()
                 else:
                     msg = message
@@ -241,7 +241,7 @@ def get_prefix(self, kind, level):
                 return self._tty_colour_err(Reporter.PREFIX_WARN)
             else:
                 return self._tty_colour_err(Reporter.PREFIX_FAIL)
-        if isinstance(self.prefix, collections.Callable):
+        if isinstance(self.prefix, collections.abc.Callable):
             return self.prefix(kind, level)
         else:
             return self.prefix

lib/python/rose/resource.py

@@ -23,10 +23,10 @@
 
 import os
 from rose.config import ConfigLoader, ConfigNode
-import imp
 import inspect
 import string
 import sys
+from importlib.machinery import SourceFileLoader
 
 
 ERROR_LOCATE_OBJECT = "Could not locate {0}"
@@ -224,7 +224,7 @@ def import_object(import_string, from_files, error_handler,
         for filename in module_files:
             sys.path.insert(0, os.path.dirname(filename))
             try:
-                module = imp.load_source(as_name, filename)
+                module = SourceFileLoader(as_name, filename).load_module()
             except (ImportError, ModuleNotFoundError) as exc:
                 error_handler(exc)
             sys.path.pop(0)

lib/python/rose/run.py

@@ -30,7 +30,7 @@
 import shlex
 import shutil
 from uuid import uuid4
-import collections
+import collections.abc
 
 
 class RunConfigLoadEvent(Event):
@@ -125,7 +125,7 @@ def __init__(self, event_handler=None, popen=None, config_pm=None,
     def handle_event(self, *args, **kwargs):
         """Handle an event using the runner's event handler."""
 
-        if isinstance(self.event_handler, collections.Callable):
+        if isinstance(self.event_handler, collections.abc.Callable):
             return self.event_handler(*args, **kwargs)
 
     def config_load(self, opts):

lib/python/rose/scheme_handler.py

@@ -24,7 +24,7 @@
 import inspect
 import os
 import sys
-import collections
+import collections.abc
 
 
 class SchemeHandlersManager(object):
@@ -125,5 +125,5 @@ def guess_handler(self, item):
         for handler in self.handlers.values():
             can_handle = getattr(handler, self.can_handle, None)
             if (isinstance(can_handle,
-                           collections.Callable) and can_handle(item)):
+                           collections.abc.Callable) and can_handle(item)):
                 return handler

lib/python/rose/suite_engine_proc.py

@@ -34,7 +34,7 @@
 from rose.scheme_handler import SchemeHandlersManager
 import sys
 import webbrowser
-import collections
+import collections.abc
 
 
 class NoSuiteLogError(Exception):
@@ -539,7 +539,7 @@ def get_version_env_name(self):
 
     def handle_event(self, *args, **kwargs):
         """Call self.event_handler if it is callable."""
-        if isinstance(self.event_handler, collections.Callable):
+        if isinstance(self.event_handler, collections.abc.Callable):
             return self.event_handler(*args, **kwargs)
 
     def gcontrol(self, suite_name, args=None):

lib/python/rose/suite_hook.py

@@ -30,7 +30,7 @@
 from rose.suite_engine_proc import SuiteEngineProcessor
 from smtplib import SMTP, SMTPException
 import socket
-import collections
+import collections.abc
 
 
 class RoseSuiteHook(object):
@@ -49,7 +49,7 @@ def __init__(self, event_handler=None, popen=None, suite_engine_proc=None):
 
     def handle_event(self, *args, **kwargs):
         """Call self.event_handler if it is callabale."""
-        if isinstance(self.event_handler, collections.Callable):
+        if isinstance(self.event_handler, collections.abc.Callable):
             return self.event_handler(*args, **kwargs)
 
     def run(self, suite_name, task_id, hook_event, hook_message=None,

lib/python/rose/suite_restart.py

@@ -28,7 +28,7 @@
 from rose.reporter import Reporter
 from rose.suite_control import get_suite_name, SuiteNotFoundError
 from rose.suite_engine_proc import SuiteEngineProcessor
-import collections
+import collections.abc
 
 
 class SuiteRestarter(object):
@@ -42,7 +42,7 @@ def __init__(self, event_handler=None):
 
     def handle_event(self, *args, **kwargs):
         """Handle event."""
-        if isinstance(self.event_handler, collections.Callable):
+        if isinstance(self.event_handler, collections.abc.Callable):
             self.event_handler(*args, **kwargs)
 
     def restart(

lib/python/rose/upgrade.py

@@ -572,8 +572,8 @@ def transform(self, config, meta_config=None, opt_non_interactive=False,
                 func = macro.upgrade
             res = {}
             if not opt_non_interactive:
-                arglist = inspect.getargspec(func).args
-                defaultlist = inspect.getargspec(func).defaults
+                arglist = inspect.getfullargspec(func).args
+                defaultlist = inspect.getfullargspec(func).defaults
                 optionals = {}
                 while defaultlist is not None and len(defaultlist) > 0:
                     if arglist[-1] not in ["self", "config", "meta_config"]: