Merge pull request #2408 from boegel/R_exts_parallel

add support for installing R extensions in parallel

easybuild/easyblocks/generic/rpackage.py

@@ -33,6 +33,7 @@
 @author: Balazs Hajgato (Vrije Universiteit Brussel)
 """
 import os
+import re
 
 from easybuild.easyblocks.r import EXTS_FILTER_R_PACKAGES, EB_R
 from easybuild.easyblocks.generic.configuremake import check_config_guess, obtain_config_guess
@@ -85,6 +86,7 @@ def __init__(self, *args, **kwargs):
         self.configurevars = []
         self.configureargs = []
         self.ext_src = None
+        self._required_deps = None
 
     def make_r_cmd(self, prefix=None):
         """Create a command to run in R to install an R package."""
@@ -162,10 +164,16 @@ def build_step(self):
     def install_R_package(self, cmd, inp=None):
         """Install R package as specified, and check for errors."""
 
-        cmdttdouterr, _ = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False)
+        output, _ = run_cmd(cmd, log_all=True, simple=False, inp=inp, regexp=False)
+        self.check_install_output(output)
 
-        cmderrors = parse_log_for_error(cmdttdouterr, regExp="^ERROR:")
-        if cmderrors:
+    def check_install_output(self, output):
+        """
+        Check output of installation command, and clean up installation if needed.
+        """
+        errors = parse_log_for_error(output, regExp="^ERROR:")
+        if errors:
+            self.handle_installation_errors()
             cmd = "R -q --no-save"
             stdin = """
             remove.library(%s)
@@ -175,7 +183,7 @@ def install_R_package(self, cmd, inp=None):
             run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False)
             raise EasyBuildError("Errors detected during installation of R package %s!", self.name)
         else:
-            self.log.debug("R package %s installed succesfully" % self.name)
+            self.log.debug("R package %s installed succesfully", self.name)
 
     def update_config_guess(self, path):
         """Update any config.guess found in specified directory"""
@@ -197,13 +205,69 @@ def install_step(self):
         cmd, stdin = self.make_cmdline_cmd(prefix=os.path.join(self.installdir, self.cfg['exts_subdir']))
         self.install_R_package(cmd, inp=stdin)
 
-    def run(self):
-        """Install R package as an extension."""
+    @property
+    def required_deps(self):
+        """Return list of required dependencies for this extension."""
+
+        if self._required_deps is None:
+            if self.src:
+                cmd = "tar --wildcards --extract --file %s --to-stdout '*/DESCRIPTION'" % self.src
+                out, _ = run_cmd(cmd, simple=False, trace=False)
+
+                # lines that start with whitespace are merged with line above
+                lines = []
+                for line in out.splitlines():
+                    if line and line[0] in (' ', '\t'):
+                        lines[-1] = lines[-1] + line
+                    else:
+                        lines.append(line)
+                out = '\n'.join(lines)
+
+                pkg_key = 'Package:'
+                deps_map = {}
+                deps = []
+                pkg = None
+
+                for line in out.splitlines():
+                    if pkg_key in line:
+                        if pkg is not None:
+                            deps = []
+
+                        pkg_name_regex = re.compile(r'Package:\s*([^ ]+)')
+                        res = pkg_name_regex.search(line)
+                        if res:
+                            pkg = res.group(1)
+                            if pkg in deps_map:
+                                deps = deps_map[pkg]
+                        else:
+                            raise EasyBuildError("Failed to determine package name from line '%s'", line)
+
+                        deps_map[pkg] = deps
+
+                    elif any(line.startswith(x) for x in ('Depends:', 'Imports:', 'LinkingTo:')):
+                        # entries may specify version requirements between brackets (which we don't care about here)
+                        dep_names = [x.split('(')[0].strip() for x in line.split(':', 1)[1].split(',')]
+                        deps.extend([d for d in dep_names if d not in ('', 'R', self.name)])
+
+                self._required_deps = deps_map.get(self.name, [])
+                self.log.info("Required dependencies for %s: %s", self.name, self._required_deps)
+            else:
+                # no source => no required dependencies assumed
+                self._required_deps = []
+
+        return self._required_deps
+
+    def prepare_r_ext_install(self):
+        """
+        Prepare installation of R package as extension.
+
+        :return: Shell command to run + string to pass to stdin.
+        """
 
         # determine location
         if isinstance(self.master, EB_R):
             # extension is being installed as part of an R installation/module
-            (out, _) = run_cmd("R RHOME", log_all=True, simple=False)
+            (out, _) = run_cmd("R RHOME", log_all=True, simple=False, trace=False)
             rhome = out.strip()
             lib_install_prefix = os.path.join(rhome, 'library')
         else:
@@ -223,8 +287,36 @@ def run(self):
             self.log.debug("Installing most recent version of R package %s (source not found)." % self.name)
             cmd, stdin = self.make_r_cmd(prefix=lib_install_prefix)
 
+        return cmd, stdin
+
+    def run(self):
+        """
+        Install R package as an extension.
+        """
+        cmd, stdin = self.prepare_r_ext_install()
         self.install_R_package(cmd, inp=stdin)
 
+    def run_async(self):
+        """
+        Start installation of R package as an extension asynchronously.
+        """
+        cmd, stdin = self.prepare_r_ext_install()
+        self.async_cmd_start(cmd, inp=stdin)
+
+    def async_cmd_check(self):
+        """
+        Check progress of installation command that was started asynchronously.
+
+        Output is checked for errors on completion.
+
+        :return: True if command completed, False otherwise
+        """
+        done = super(RPackage, self).async_cmd_check()
+        if done:
+            self.check_install_output(self.async_cmd_output)
+
+        return done
+
     def sanity_check_step(self, *args, **kwargs):
         """
         Custom sanity check for R packages

easybuild/easyblocks/r/rmpi.py

@@ -40,8 +40,10 @@
 class EB_Rmpi(RPackage):
     """Build and install Rmpi R library."""
 
-    def run(self):
-        """Set various configure arguments prior to building."""
+    def prepare_rmpi_configureargs(self):
+        """
+        Prepare configure arguments for installing Rpmi.
+        """
 
         mpi_types = {
             toolchain.MPI_TYPE_OPENMPI: "OPENMPI",
@@ -51,16 +53,31 @@ def run(self):
         # type of MPI
         # MPI_TYPE does not distinguish between MPICH and IntelMPI, which is why we also check mpi_family()
         mpi_type = self.toolchain.mpi_family()
-        Rmpi_type = mpi_types[self.toolchain.MPI_TYPE]
+        rmpi_type = mpi_types[self.toolchain.MPI_TYPE]
         # Rmpi versions 0.6-4 and up support INTELMPI (using --with-Rmpi-type=INTELMPI)
         if ((LooseVersion(self.version) >= LooseVersion('0.6-4')) and (mpi_type == toolchain.INTELMPI)):
-            Rmpi_type = 'INTELMPI'
+            rmpi_type = 'INTELMPI'
 
         self.log.debug("Setting configure args for Rmpi")
         self.configureargs = [
             "--with-Rmpi-include=%s" % self.toolchain.get_variable('MPI_INC_DIR'),
             "--with-Rmpi-libpath=%s" % self.toolchain.get_variable('MPI_LIB_DIR'),
             "--with-mpi=%s" % self.toolchain.get_software_root(self.toolchain.MPI_MODULE_NAME)[0],
-            "--with-Rmpi-type=%s" % Rmpi_type,
+            "--with-Rmpi-type=%s" % rmpi_type,
         ]
-        super(EB_Rmpi, self).run()  # it might be needed to get the R cmd and run it with mympirun...
+
+    def run(self):
+        """
+        Install Rmpi as extension, after seting various configure arguments.
+        """
+        self.prepare_rmpi_configureargs()
+        # it might be needed to get the R cmd and run it with mympirun...
+        super(EB_Rmpi, self).run()
+
+    def run_async(self):
+        """
+        Asynchronously install Rmpi as extension, after seting various configure arguments.
+        """
+        self.prepare_rmpi_configureargs()
+        # it might be needed to get the R cmd and run it with mympirun...
+        super(EB_Rmpi, self).run_async()

easybuild/easyblocks/r/rserve.py

@@ -39,6 +39,10 @@ class EB_Rserve(RPackage):
 
     def run(self):
         """Set LIBS environment variable correctly prior to building."""
-
         self.configurevars = ['LIBS="$LIBS -lpthread"']
         super(EB_Rserve, self).run()
+
+    def run_async(self):
+        """Set LIBS environment variable correctly prior to building."""
+        self.configurevars = ['LIBS="$LIBS -lpthread"']
+        super(EB_Rserve, self).run_async()

easybuild/easyblocks/x/xml.py

@@ -39,7 +39,7 @@
 class EB_XML(RPackage):
     """Support for installing the XML R package."""
 
-    def install_R_package(self, cmd, inp=None):
+    def install_R_package(self, *args, **kwargs):
         """Customized install procedure for XML R package, add zlib lib path to LIBS."""
 
         libs = os.getenv('LIBS', '')
@@ -52,4 +52,4 @@ def install_R_package(self, cmd, inp=None):
         else:
             raise EasyBuildError("zlib module not loaded (required)")
 
-        super(EB_XML, self).install_R_package(cmd, inp)
+        return super(EB_XML, self).install_R_package(*args, **kwargs)