Merge pull request #2288 from sadielbartholomew/i2271-rosie-disco-mig…

…rations rosie & rosa: migrate to Python 3(.6-.7) & Tornado
metomi · May 13, 2019 · c379fb0 · c379fb0
2 parents d1eee0e + 2dd607f
commit c379fb0
Show file tree

Hide file tree

Showing 47 changed files with 543 additions and 354 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -25,10 +25,10 @@ install:
     - >
         sudo apt-get install -y at build-essential gfortran heirloom-mailx
         python-pip python-dev graphviz libgraphviz-dev python-jinja2
-        python-sqlalchemy libxml-parser-perl libconfig-inifiles-perl
+        python3-sqlalchemy libxml-parser-perl libconfig-inifiles-perl
         libdbi-perl libdbd-sqlite3-perl latexmk texlive
         texlive-generic-extra texlive-latex-extra texlive-fonts-recommended
-    - pip install cherrypy EmPy Jinja2 requests sqlalchemy pycodestyle python-jose pyzmq colorama pillow
+    - pip install tornado EmPy Jinja2 requests sqlalchemy pycodestyle python-jose pyzmq colorama pillow
     - pip install pygraphviz --install-option="--include-path=/usr/include/graphviz" --install-option="--library-path=/usr/lib/graphviz/"
     - sudo sh -c 'echo "deb http://opensource.wandisco.com/ubuntu `lsb_release -cs` svn19" >> /etc/apt/sources.list.d/subversion19.list'
     - sudo wget -q http://opensource.wandisco.com/wandisco-debian.gpg -O- | sudo apt-key add -

diff --git a/bin/rose-check-software b/bin/rose-check-software
@@ -290,9 +290,9 @@ def main(check=check):
     ])
 
     check_all('Rosie', [
-        check('py:cherrypy'),
+        check('py:tornado', (3, 0), attr_name='version'),
         check('py:requests', (2, 2, 1)),
-        check('py:sqlalchemy', (0, 6, 9)),
+        check('py:sqlalchemy', (0, 9)),
         check('svn', (1, 8), command_template=['--version', '--quiet']),
         check('fcm', version_template=r'FCM ([\d\.\-]+)'),
         check('cmd:perl', (5, 10, 1),
@@ -365,4 +365,4 @@ if __name__ == '__main__':
         sys.exit(0 if docs() else 1)
     else:
         # Check software dependencies, report and exit.
-        main()
+        main()
diff --git a/bin/rose-test-battery b/bin/rose-test-battery
@@ -68,7 +68,7 @@ else
         NPROC=$(grep -ic processor /proc/cpuinfo)
     else
         NPROC=$(python3 -c \
-            'import multiprocessing; print multiprocessing.cpu_count()')
+            'import multiprocessing; print(multiprocessing.cpu_count())')
     fi
     exec prove -j "$NPROC" -s -r "${@:-t}"
 fi
diff --git a/lib/bash/rose_init b/lib/bash/rose_init
@@ -31,7 +31,6 @@
 #     * load any FUNC specified in the argument list.
 #-------------------------------------------------------------------------------
 
-
 function rose_init() {
     set -eu
     ROSE_NS=$(basename $0)

diff --git a/lib/html/template/rosie-disco/index.html b/lib/html/template/rosie-disco/index.html
@@ -3,8 +3,8 @@
 <head>
   <title>{{title}} @ {{host}}</title>
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <link rel="icon" href="{{script}}/favicon.ico" />
-  <link rel="shortcut icon" href="{{script}}/favicon.ico" />
+  <link rel="icon" href="{{script}}/rosie-favicon.png" />
+  <link rel="shortcut icon" href="{{script}}/rosie-favicon.png" />
   <link type="text/css" href="{{script}}/css/bootstrap.min.css"
   rel="stylesheet" media="screen" />
 </head>

diff --git a/lib/html/template/rosie-disco/prefix-index.html b/lib/html/template/rosie-disco/prefix-index.html
@@ -4,8 +4,8 @@
 
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
-  <link rel="icon" href="{{script}}/favicon.ico" />
-  <link rel="shortcut icon" href="{{script}}/favicon.ico" />
+  <link rel="icon" href="{{script}}/rosie-favicon.png" />
+  <link rel="shortcut icon" href="{{script}}/rosie-favicon.png" />
   <link href="{{script}}/css/bootstrap.min.css" rel="stylesheet" />
   <link href="{{script}}/css/jquery.dataTables.css" rel="stylesheet">
   <!--link href="{{script}}/css/jquery.dataTables.colVis.min.css" rel="stylesheet"-->
@@ -27,7 +27,7 @@
         <span class="icon-bar"></span>
         <span class="icon-bar"></span>
       </button>
-      <a class="navbar-brand" href="{{script}}/">
+      <a class="navbar-brand" href="{{service_root}}">
         {{prefix}}: {{title}}
         <small>@ {{host}}</small>
       </a>

diff --git a/lib/python/rose/apps/rose_ana_v1.py b/lib/python/rose/apps/rose_ana_v1.py
@@ -36,7 +36,6 @@
 from rose.reporter import Reporter, Event
 from rose.resource import ResourceLocator
 from rose.app_run import BuiltinApp
-import collections.abc
 
 WARN = -1
 PASS = 0

diff --git a/lib/python/rose/cmp_source_vc.py b/lib/python/rose/cmp_source_vc.py
@@ -30,7 +30,6 @@
 from rose.reporter import Reporter
 from rose.run_source_vc import write_source_vc_info
 from rose.suite_engine_proc import SuiteEngineProcessor
-import collections.abc
 
 
 class SuiteVCComparator(object):

diff --git a/lib/python/rose/config_processor.py b/lib/python/rose/config_processor.py
@@ -25,7 +25,6 @@
 from rose.popen import RosePopener
 from rose.scheme_handler import SchemeHandlersManager
 import sys
-import collections.abc
 
 
 class UnknownContentError(Exception):

diff --git a/lib/python/rose/config_processors/fileinstall.py b/lib/python/rose/config_processors/fileinstall.py
@@ -38,7 +38,6 @@
 import sys
 from tempfile import mkdtemp
 from urllib.parse import urlparse
-import collections.abc
 
 
 class ConfigProcessorForFile(ConfigProcessorBase):

diff --git a/lib/python/rose/env.py b/lib/python/rose/env.py
@@ -27,7 +27,6 @@
 import os
 import re
 from rose.reporter import Event
-import collections.abc
 
 
 # _RE_DEFAULT = re.compile(r"""

diff --git a/lib/python/rose/fs_util.py b/lib/python/rose/fs_util.py
@@ -23,7 +23,6 @@
 import os
 from rose.reporter import Event
 import shutil
-import collections.abc
 
 
 class FileSystemEvent(Event):

diff --git a/lib/python/rose/host_select.py b/lib/python/rose/host_select.py
@@ -32,7 +32,6 @@
 import sys
 from time import sleep, time
 import traceback
-import collections.abc
 
 
 class NoHostError(Exception):

diff --git a/lib/python/rose/macro.py b/lib/python/rose/macro.py
@@ -57,7 +57,6 @@ def test_cleanup(stuff_to_remove):
 import rose.reporter
 import rose.resource
 import rose.variable
-import collections.abc
 
 
 ALLOWED_MACRO_CLASS_METHODS = ["transform", "validate", "downgrade", "upgrade",

diff --git a/lib/python/rose/metadata_graph.py b/lib/python/rose/metadata_graph.py
@@ -218,11 +218,11 @@ def output_graph(graph, debug_mode=False, filename=None, form="svg"):
     if filename is None:
         image_file_handle = tempfile.NamedTemporaryFile(suffix=("." + form))
     else:
-        image_file_handle = open(filename, "w")
+        image_file_handle = open(filename, "wb")
     graph.draw(image_file_handle.name, prog="dot")
     if debug_mode:
         image_file_handle.seek(0)
-        print(image_file_handle.read())
+        print(image_file_handle.read().decode())
         image_file_handle.close()
         return
     rose.external.launch_image_viewer(image_file_handle.name, run_fg=True)

diff --git a/lib/python/rose/popen.py b/lib/python/rose/popen.py
@@ -27,7 +27,6 @@
 import shlex
 from subprocess import Popen, PIPE
 import sys
-import collections.abc
 
 
 class RosePopenError(Exception):

diff --git a/lib/python/rose/reporter.py b/lib/python/rose/reporter.py
@@ -19,13 +19,9 @@
 # -----------------------------------------------------------------------------
 """Reporter for diagnostic messages."""
 
-import queue
 
-import multiprocessing
 import sys
-
 import time
-import collections.abc
 
 
 class Reporter(object):
@@ -241,7 +237,7 @@ def get_prefix(self, kind, level):
                 return self._tty_colour_err(Reporter.PREFIX_WARN)
             else:
                 return self._tty_colour_err(Reporter.PREFIX_FAIL)
-        if isinstance(self.prefix, collections.abc.Callable):
+        if callable(self.prefix):
             return self.prefix(kind, level)
         else:
             return self.prefix
@@ -270,57 +266,6 @@ def _tty_colour_err(self, str_):
         return str_
 
 
-class ReporterContextQueue(ReporterContext):
-
-    """A context for the reporter object.
-
-    It has the following attributes:
-    kind:
-        The message kind to report to this context.
-        (Reporter.KIND_ERR, Reporter.KIND_ERR or None.)
-    verbosity:
-        The verbosity of this context.
-    queue:
-        The multiprocessing.Queue.
-    prefix:
-        The default message prefix (str or callable).
-
-    """
-
-    def __init__(self,
-                 kind=None,
-                 verbosity=Reporter.DEFAULT,
-                 queue=None,
-                 prefix=None):
-        ReporterContext.__init__(self, kind, verbosity, None, prefix)
-        if queue is None:
-            queue = multiprocessing.Manager().Queue()
-        self.queue = queue
-        self.closed = False
-        self._messages_pending = []
-
-    def close(self):
-        self._send_pending_messages()
-        self.closed = True
-
-    def is_closed(self):
-        return self.closed
-
-    def write(self, message):
-        self._messages_pending.append(message)
-        self._send_pending_messages()
-
-    def _send_pending_messages(self):
-        while self._messages_pending:
-            message = self._messages_pending[0]
-            try:
-                self.queue.put(message, block=False)
-            except queue.Full:
-                break
-            else:
-                del self._messages_pending[0]
-
-
 class Event(object):
 
     """A base class for events suitable for feeding into a Reporter."""

diff --git a/lib/python/rose/run.py b/lib/python/rose/run.py
@@ -29,7 +29,6 @@
 import shlex
 import shutil
 from uuid import uuid4
-import collections.abc
 
 
 class RunConfigLoadEvent(Event):

diff --git a/lib/python/rose/scheme_handler.py b/lib/python/rose/scheme_handler.py
@@ -24,7 +24,6 @@
 import inspect
 import os
 import sys
-import collections.abc
 
 
 class SchemeHandlersManager(object):

diff --git a/lib/python/rose/suite_engine_proc.py b/lib/python/rose/suite_engine_proc.py
@@ -34,7 +34,6 @@
 from rose.scheme_handler import SchemeHandlersManager
 import sys
 import webbrowser
-import collections.abc
 
 
 class NoSuiteLogError(Exception):
@@ -539,7 +538,7 @@ def get_version_env_name(self):
 
     def handle_event(self, *args, **kwargs):
         """Call self.event_handler if it is callable."""
-        if isinstance(self.event_handler, collections.abc.Callable):
+        if callable(self.event_handler):
             return self.event_handler(*args, **kwargs)
 
     def gcontrol(self, suite_name, args=None):

diff --git a/lib/python/rose/suite_hook.py b/lib/python/rose/suite_hook.py
@@ -30,7 +30,6 @@
 from rose.suite_engine_proc import SuiteEngineProcessor
 from smtplib import SMTP, SMTPException
 import socket
-import collections.abc
 
 
 class RoseSuiteHook(object):
@@ -49,7 +48,7 @@ def __init__(self, event_handler=None, popen=None, suite_engine_proc=None):
 
     def handle_event(self, *args, **kwargs):
         """Call self.event_handler if it is callabale."""
-        if isinstance(self.event_handler, collections.abc.Callable):
+        if callable(self.event_handler):
             return self.event_handler(*args, **kwargs)
 
     def run(self, suite_name, task_id, hook_event, hook_message=None,

diff --git a/lib/python/rose/suite_restart.py b/lib/python/rose/suite_restart.py
@@ -28,7 +28,6 @@
 from rose.reporter import Reporter
 from rose.suite_control import get_suite_name, SuiteNotFoundError
 from rose.suite_engine_proc import SuiteEngineProcessor
-import collections.abc
 
 
 class SuiteRestarter(object):
@@ -42,7 +41,7 @@ def __init__(self, event_handler=None):
 
     def handle_event(self, *args, **kwargs):
         """Handle event."""
-        if isinstance(self.event_handler, collections.abc.Callable):
+        if callable(self.event_handler):
             self.event_handler(*args, **kwargs)
 
     def restart(

diff --git a/lib/python/rosie/db.py b/lib/python/rosie/db.py
@@ -229,6 +229,7 @@ def query(self, filters, all_revs=0):
 
         """
         self._connect()
+        all_revs = int(all_revs)  # so distinguish 0 or 1 below, else both True
         if all_revs:
             from_obj, cols = self._get_hist_join_and_columns()
         else:
@@ -357,6 +358,7 @@ def search(self, s, all_revs=0):
 
         """
         self._connect()
+        all_revs = int(all_revs)  # so distinguish 0 or 1 below, else both True
         if all_revs:
             from_obj, cols = self._get_hist_join_and_columns()
         else:

diff --git a/lib/python/rosie/db_create.py b/lib/python/rosie/db_create.py
@@ -31,7 +31,6 @@
 from rosie.db import (
     LATEST_TABLE_NAME, MAIN_TABLE_NAME, META_TABLE_NAME, OPTIONAL_TABLE_NAME)
 from rosie.svn_post_commit import RosieSvnPostCommitHook
-import collections.abc
 
 
 class RosieDatabaseCreateEvent(Event):

diff --git a/lib/python/rosie/suite_id.py b/lib/python/rosie/suite_id.py
@@ -143,9 +143,10 @@ def get_latest(cls, prefix=None):
                 if i == 0:
                     return None
                 raise SuiteIdLatestError(prefix)
-            dirs = [line for line in out.splitlines() if line.endswith(b"/")]
+            dirs = [line for line in out.decode().splitlines() if
+                    line.endswith("/")]
             # Note - 'R/O/S/I/E' sorts to top for lowercase initial idx letter
-            dir_url = dir_url + "/" + sorted(dirs)[-1].rstrip(b"/").decode()
+            dir_url = dir_url + "/" + sorted(dirs)[-1].rstrip("/")
 
         # FIXME: not sure why a closure for "state" does not work here?
         state = {"idx-sid": None, "stack": [], "try_text": False}