Adding the unit tests for the clean_workspace tool

cmake/std/PullRequestLinuxDriver-Test.sh

@@ -62,6 +62,22 @@ function test_pr_constraints_master()
 }
 
 
+function framework_tests_only()
+{
+    pkg_file=${1:?}
+    cat << EOF > ${pkg_file}
+
+MACRO(PR_ENABLE_BOOL  VAR_NAME  VAR_VAL)
+  MESSAGE("-- Setting ${VAR_NAME} = ${VAR_VAL}")
+  SET(\${VAR_NAME} \${VAR_VAL} CACHE BOOL "Set in packageEnables.cmake")
+ENDMACRO()
+
+PR_ENABLE_BOOL(Trilinos_ENABLE_TrilinosFrameworkTests ON)
+
+EOF
+
+}
+
 
 # This script expects to start out in the root level of the Jenkins workspace.
 # Let's make sure we're there.
@@ -266,6 +282,20 @@ elif [ "Trilinos_pullrequest_intel_17.0.1" == "${JOB_BASE_NAME:?}" ] ; then
         echo -e "There was an issue loading the intel environment. The error code was: $ierror"
         exit $ierror
     fi
+elif [ "Trilinos_pullrequest_python_2" == "${JOB_BASE_NAME:?}" ]; then
+    source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestPython2TestingEnv.sh
+    ierror=$?
+    if [[ $ierror != 0 ]]; then
+        echo -e "There was an issue loading the python2 environment. The error code was: $ierror"
+        exit $ierror
+    fi
+elif [ "Trilinos_pullrequest_python_3" == "${JOB_BASE_NAME:?}" ]; then
+    source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestPython3TestingEnv.sh
+    ierror=$?
+    if [[ $ierror != 0 ]]; then
+        echo -e "There was an issue loading the python3 environment. The error code was: $ierror"
+        exit $ierror
+    fi
 else
     ierror=42
     echo -e "ERROR: Unable to find matching environment for job: ${JOB_BASE_NAME:?}"
@@ -380,6 +410,12 @@ else
         CONFIG_SCRIPT=PullRequestLinuxGCC7.3.0TestingSettings.cmake
     elif [ "Trilinos_pullrequest_cuda_9.2" == "${JOB_BASE_NAME:?}" ]; then
         CONFIG_SCRIPT=PullRequestLinuxCuda9.2TestingSettings.cmake
+    elif [ "Trilinos_pullrequest_python_2" == "${JOB_BASE_NAME:?}" ]; then
+        CONFIG_SCRIPT=PullRequestLinuxPython2.cmake
+        framework_tests_only ../packageEnables.cmake
+    elif [ "Trilinos_pullrequest_python_3" == "${JOB_BASE_NAME:?}" ]; then
+        CONFIG_SCRIPT=PullRequestLinuxPython3.cmake
+        framework_tests_only ../packageEnables.cmake
     fi
 fi
 

cmake/std/PullRequestLinuxPython2.cmake

@@ -0,0 +1,18 @@
+# This file contains the options needed to both run the pull request testing
+# for Trilinos for the Linux GCC 4.8.4 pull request testing builds, and to reproduce
+# the errors reported by those builds. Prior to using this this file, the
+# appropriate set of SEMS modules must be loaded and accessible through the
+# SEMS NFS mount. (See the sems/PullRequestGCC*TestingEnv.sh files.)
+
+# Usage: cmake -C PullRequestLinuxGCC4.8.4TestingSettings.cmake
+
+# Misc options typically added by CI testing mode in TriBITS
+
+# Use the below option only when submitting to the dashboard
+#set (CTEST_USE_LAUNCHERS ON CACHE BOOL "Set by default for PR testing")
+
+set (TFW_Python2_Testing ON CACHE BOOL "Set by default for PR testing")
+set (TFW_Python_Testing ON CACHE BOOL "Set by default for PR testing")
+
+include("${CMAKE_CURRENT_LIST_DIR}/PullRequestLinuxCommonTestingSettings.cmake")
+

cmake/std/PullRequestLinuxPython3.cmake

@@ -0,0 +1,18 @@
+# This file contains the options needed to both run the pull request testing
+# for Trilinos for the Linux GCC 4.8.4 pull request testing builds, and to reproduce
+# the errors reported by those builds. Prior to using this this file, the
+# appropriate set of SEMS modules must be loaded and accessible through the
+# SEMS NFS mount. (See the sems/PullRequestGCC*TestingEnv.sh files.)
+
+# Usage: cmake -C PullRequestLinuxGCC4.8.4TestingSettings.cmake
+
+# Misc options typically added by CI testing mode in TriBITS
+
+# Use the below option only when submitting to the dashboard
+#set (CTEST_USE_LAUNCHERS ON CACHE BOOL "Set by default for PR testing")
+
+set (TFW_Python3_Testing ON CACHE BOOL "Set by default for PR testing")
+set (TFW_Python_Testing ON CACHE BOOL "Set by default for PR testing")
+
+include("${CMAKE_CURRENT_LIST_DIR}/PullRequestLinuxCommonTestingSettings.cmake")
+

cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh

@@ -31,7 +31,6 @@ module load sems-cmake/3.10.3
 # available in main-line Ninja) to significantly speed up
 # compile and link times.
 module load atdm-env
-module load atdm-cmake/3.11.1
 module load atdm-ninja_fortran/1.7.2
 
 # add the OpenMP environment variable we need

cmake/std/sems/PullRequestPython2TestingEnv.sh

@@ -0,0 +1,43 @@
+# This script can be used to load the appropriate environment for the
+# GCC 4.8.4 Pull Request testing build on a Linux machine that has access to
+# the SEMS NFS mount.
+
+# usage: $ source PullRequestGCC4.8.4TestingEnv.sh
+
+# After the environment is no longer needed, it can be purged using
+# $ module purge
+# or Trilinos/cmake/unload_sems_dev_env.sh
+
+source /projects/sems/modulefiles/utils/sems-modules-init.sh
+
+module load sems-gcc/4.8.4
+module load sems-openmpi/1.10.1
+module load sems-git/2.10.1
+module load sems-boost/1.63.0/base
+module load sems-zlib/1.2.8/base
+module load sems-hdf5/1.8.12/parallel
+module load sems-netcdf/4.4.1/exo_parallel
+module load sems-parmetis/4.0.3/parallel
+module load sems-scotch/6.0.3/nopthread_64bit_parallel
+module load sems-superlu/4.3/base
+
+# Load the SEMS CMake Module
+# - One of the SEMS modules will load CMake 3.4.x also,
+#   so this will pull in the SEMS cmake 3.10.3 version
+#   for Trilinos compatibility.
+module load sems-cmake/3.10.3
+module load sems-ninja_fortran/1.8.2
+
+# we will have implicitly gotten the sems python from
+# the boost module above for whaever reason - reset it
+# to one that has the mock packages installed
+module unload sems-python
+# module load sierra-python/2.7.15 - permissions do not allow this, but the execs are ok
+PATH=/projects/sierra/linux_rh7/install/Python/2.7.15/bin:${PATH}
+PATH=/projects/sierra/linux_rh7/install/Python/extras/bin:${PATH}
+PYTHONPATH=/projects/sierra/linux_rh7/install/Python/extras/lib/python2.7/site-packages:${PYTHONPATH}
+MANPATH=/projects/sierra/linux_rh7/install/Python/2.7.15/share/man:${MANPATH}
+unset PYTHONHOME 
+
+# add the OpenMP environment variable we need
+export OMP_NUM_THREADS=2

cmake/std/sems/PullRequestPython3TestingEnv.sh

@@ -0,0 +1,48 @@
+# This script can be used to load the appropriate environment for the
+# GCC 7.2.0 Pull Request testing build on a Linux machine that has access to
+# the SEMS NFS mount.
+
+# usage: $ source PullRequestGCC7.2.0TestingEnv.sh
+
+source /projects/sems/modulefiles/utils/sems-modules-init.sh
+
+module load sems-gcc/7.2.0
+module load sems-openmpi/1.10.1
+module load sems-git/2.10.1
+module load sems-boost/1.63.0/base
+module load sems-zlib/1.2.8/base
+module load sems-hdf5/1.8.12/parallel
+module load sems-netcdf/4.4.1/exo_parallel
+module load sems-parmetis/4.0.3/parallel
+module load sems-scotch/6.0.3/nopthread_64bit_parallel
+module load sems-superlu/4.3/base
+
+# Load the SEMS CMake Module
+# - One of the SEMS modules will load CMake 3.4.x also,
+#   so this will pull in the SEMS cmake 3.10.3 version
+#   for Trilinos compatibility.
+module load sems-cmake/3.10.3
+
+# Using CMake and Ninja modules from the ATDM project space.
+# SEMS does not yet supply a recent enough version of CMake
+# for the single configure/build/test capability. We are also
+# using a custom version of Ninja (with Fortran support not
+# available in main-line Ninja) to significantly speed up
+# compile and link times.
+module load atdm-env
+module load atdm-ninja_fortran/1.7.2
+
+# we will have implicitly gotten the sems python from
+# the boost module above for whaever reason - reset it
+# to one that has the proper sym-links from python -> python3
+module unload sems-python
+# module load sierra-python/3.6.3 - permissions do not allow this, but the execs are ok
+PATH=/projects/sierra/linux_rh7/install/Python/3.6..3/bin:${PATH}
+PATH=/projects/sierra/linux_rh7/install/Python/extras/bin:${PATH}
+PYTHONPATH=/projects/sierra/linux_rh7/install/Python/extras/lib/python3.6/site-packages:${PYTHONPATH}
+MANPATH=/projects/sierra/linux_rh7/install/Python/3.6.3/share/man:${MANPATH}
+unset PYTHONHOME 
+
+# add the OpenMP environment variable we need
+export OMP_NUM_THREADS=2
+

commonTools/framework/CMakeLists.txt

@@ -10,6 +10,46 @@ TRIBITS_ADD_ADVANCED_TEST( ProjectCiFileChangeLogic_UnitTests
   )
 
 
+TRIBITS_ADD_ADVANCED_TEST( clean_all_jobs_UnitTests
+  OVERALL_WORKING_DIRECTORY TEST_NAME
+  OVERALL_NUM_MPI_PROCS 1
+  EXCLUDE_IF_NOT_TRUE TFW_Python_Testing
+  TEST_0 CMND ${PYTHON_EXECUTABLE} 
+    ARGS ${CMAKE_CURRENT_SOURCE_DIR}/clean_workspace/unittests/test_clean_all_jobs.py -v
+    PASS_REGULAR_EXPRESSION "OK"
+  )
+
+
+TRIBITS_ADD_ADVANCED_TEST( clean_sentinel_UnitTests
+  OVERALL_WORKING_DIRECTORY TEST_NAME
+  OVERALL_NUM_MPI_PROCS 1
+  EXCLUDE_IF_NOT_TRUE TFW_Python_Testing
+  TEST_0 CMND ${PYTHON_EXECUTABLE} 
+    ARGS ${CMAKE_CURRENT_SOURCE_DIR}//clean_workspace/unittests/test_clean_sentinel.py -v
+    PASS_REGULAR_EXPRESSION "OK"
+  )
+
+
+TRIBITS_ADD_ADVANCED_TEST( clean_workspace_UnitTests
+  OVERALL_WORKING_DIRECTORY TEST_NAME
+  OVERALL_NUM_MPI_PROCS 1
+  EXCLUDE_IF_NOT_TRUE TFW_Python_Testing
+  TEST_0 CMND ${PYTHON_EXECUTABLE} 
+    ARGS ${CMAKE_CURRENT_SOURCE_DIR}//clean_workspace/unittests/test_clean_workspace.py -v
+    PASS_REGULAR_EXPRESSION "OK"
+  )
+
+
+TRIBITS_ADD_ADVANCED_TEST( Modules_UnitTests
+  OVERALL_WORKING_DIRECTORY TEST_NAME
+  OVERALL_NUM_MPI_PROCS 1
+  EXCLUDE_IF_NOT_TRUE TFW_Python2_Testing
+  TEST_0 CMND ${PYTHON_EXECUTABLE} 
+    ARGS ${CMAKE_CURRENT_SOURCE_DIR}//clean_workspace/unittests/test_Modules.py -v
+    PASS_REGULAR_EXPRESSION "OK"
+  )
+
+
 FUNCTION(create_get_changed_trilinos_packages_test  TEST_POSTFIX
   FILES_CHANGED
   CHANGED_PACKAGES_FULL_LIST

commonTools/framework/clean_workspace/Modules.py

@@ -20,7 +20,8 @@ def __init__(self, name=None):
         self.command = self._setup_command()
         self.init_file = self._module_setup(name)
         if name is None and self.init_file:
-            execfile(self.init_file)
+            exec(compile(open(self.init_file, 'r').read(),
+                         self.init_file, 'exec'))
 
     @staticmethod
     def _setup_paths():
@@ -144,7 +145,7 @@ def module(self, command, *arguments):
         # The modules return a string of python commands to be executed on
         # stdout.  Execute those commands now.
         if output:
-            exec (output)  # pylint: disable=exec-used
+            exec(output)  # pylint: disable=exec-used
 
         # Check stderr for anything that looks like an error.
         if ":ERROR:" in stderr:

commonTools/framework/clean_workspace/clean_all_jobs.py

@@ -23,7 +23,7 @@ def run(self):
         """Run the actual routine in clean_sentinel"""
         try:
             set_reference_date()
-        except StandardError as e:
+        except BaseException as e:
             print(e.args, file=sys.stderr)
             sys.exit()
         except pickle.PicklingError:

commonTools/framework/clean_workspace/unittests/test_Modules.py

@@ -9,8 +9,16 @@
 
 import os
 import unittest
-import mock
-from cStringIO import StringIO
+try:
+    import mock
+except ImportError:
+    import unittest.mock as mock
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from io import StringIO
+
 
 sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir))
 import Modules
@@ -23,15 +31,29 @@ def setUp(self):
         modules_home = {'MODULESHOME': '/dummy/path/1'}
         which_side_effects = ['/path/to/modulecmd', None, None]
         find_side_effects = [None, '/fake/path/modules/init/python.py']
-        with mock.patch.dict(os.environ, modules_home), \
-                mock.patch('Modules.which',
-                           side_effect=which_side_effects), \
-                mock.patch('Modules.find_first_binary',
-                           return_value='/fake/path/modulecmd'), \
-                mock.patch('Modules.find_file_in_list',
-                           side_effect=find_side_effects), \
-                mock.patch('__builtin__.execfile'):
-            self.module_obj = Modules.Module()
+        major_version = sys.version_info[0]
+        if major_version is 3:
+            with mock.patch.dict(os.environ, modules_home), \
+                    mock.patch('Modules.which',
+                               side_effect=which_side_effects), \
+                    mock.patch('Modules.find_first_binary',
+                               return_value='/fake/path/modulecmd'), \
+                    mock.patch('Modules.find_file_in_list',
+                               side_effect=find_side_effects), \
+                    mock.patch('Modules.open'), \
+                    mock.patch('Modules.compile', return_value=''):
+                self.module_obj = Modules.Module()
+        elif major_version is 2:
+            with mock.patch.dict(os.environ, modules_home), \
+                    mock.patch('Modules.which',
+                               side_effect=which_side_effects), \
+                    mock.patch('Modules.find_first_binary',
+                               return_value='/fake/path/modulecmd'), \
+                    mock.patch('Modules.find_file_in_list',
+                               side_effect=find_side_effects), \
+                    mock.patch('Modules.open'), \
+                    mock.patch('Modules.compile', return_value=''):
+                self.module_obj = Modules.Module()
 
     def test_module_setup(self):
         """Test ability to instantiate the class"""
@@ -45,9 +67,10 @@ def test_module_setup(self):
                            return_value='/fake/path/modulecmd'), \
                 mock.patch('Modules.find_file_in_list',
                            side_effect=find_side_effects), \
-                mock.patch('__builtin__.execfile') as mock_exec:
+                mock.patch('Modules.open') as mock_open, \
+                mock.patch('Modules.compile', return_value=""):
             result = Modules.Module()
-        mock_exec.assert_called_once_with(find_side_effects[-1])
+        mock_open.assert_called_once_with(find_side_effects[-1], 'r')
         self.assertEqual('/fake/path/modulecmd', result.command)
         self.assertEqual('modulecmd', result.command_name)
         self.assertEqual('/fake/path/modules/init/python.py', result.init_file)
@@ -64,9 +87,10 @@ def test_module_setup_lmod(self):
                            return_value='/fake/path/lmodcmd'), \
                 mock.patch('Modules.find_file_in_list',
                            side_effect=find_side_effects), \
-                mock.patch('__builtin__.execfile') as mock_exec:
+                mock.patch('Modules.open') as mock_open, \
+                mock.patch('Modules.compile', return_value=""):
             result = Modules.Module()
-        mock_exec.assert_not_called()
+        mock_open.assert_not_called()
         self.assertEqual('lmodcmd', result.command)
         self.assertEqual('lmodcmd', result.command_name)
         self.assertEqual(None, result.init_file)

commonTools/framework/clean_workspace/unittests/test_clean_all_jobs.py

@@ -8,10 +8,16 @@
 import os
 sys.path.insert(1, os.path.dirname(os.path.dirname(__file__)))
 
-from cStringIO import StringIO
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from io import StringIO
 
 import unittest
-import mock
+try:
+    import mock
+except ImportError:
+    import unittest.mock as mock
 
 import pickle
 
@@ -21,11 +27,11 @@
 class TestRun(unittest.TestCase):
 
     def test_clean_run(self):
-        """If there is no dir attribute in the args raise SystemExit"""
+        """A clean run should call set_reference_date"""
         cleanRefInst = CleanReference()
         with mock.patch('clean_all_jobs.set_reference_date') as refDate:
             cleanRefInst.run()
-        refDate.assert_called_once()
+        refDate.assert_called_once_with()
 
     def test_pickling_exception(self):
         """A Pickling exception should give an error and exit"""