diff --git a/cmake/std/PullRequestLinuxDriver-Test.sh b/cmake/std/PullRequestLinuxDriver-Test.sh index 38c7ef033be7..5e5f4511d50f 100755 --- a/cmake/std/PullRequestLinuxDriver-Test.sh +++ b/cmake/std/PullRequestLinuxDriver-Test.sh @@ -62,6 +62,22 @@ function test_pr_constraints_master() } +function framework_tests_only() +{ + pkg_file=${1:?} + cat << EOF > ${pkg_file} + +MACRO(PR_ENABLE_BOOL VAR_NAME VAR_VAL) + MESSAGE("-- Setting ${VAR_NAME} = ${VAR_VAL}") + SET(\${VAR_NAME} \${VAR_VAL} CACHE BOOL "Set in packageEnables.cmake") +ENDMACRO() + +PR_ENABLE_BOOL(Trilinos_ENABLE_TrilinosFrameworkTests ON) + +EOF + +} + # This script expects to start out in the root level of the Jenkins workspace. # Let's make sure we're there. @@ -266,6 +282,20 @@ elif [ "Trilinos_pullrequest_intel_17.0.1" == "${JOB_BASE_NAME:?}" ] ; then echo -e "There was an issue loading the intel environment. The error code was: $ierror" exit $ierror fi +elif [ "Trilinos_pullrequest_python_2" == "${JOB_BASE_NAME:?}" ]; then + source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestPython2TestingEnv.sh + ierror=$? + if [[ $ierror != 0 ]]; then + echo -e "There was an issue loading the python2 environment. The error code was: $ierror" + exit $ierror + fi +elif [ "Trilinos_pullrequest_python_3" == "${JOB_BASE_NAME:?}" ]; then + source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestPython3TestingEnv.sh + ierror=$? + if [[ $ierror != 0 ]]; then + echo -e "There was an issue loading the python3 environment. The error code was: $ierror" + exit $ierror + fi else ierror=42 echo -e "ERROR: Unable to find matching environment for job: ${JOB_BASE_NAME:?}" @@ -380,6 +410,12 @@ else CONFIG_SCRIPT=PullRequestLinuxGCC7.3.0TestingSettings.cmake elif [ "Trilinos_pullrequest_cuda_9.2" == "${JOB_BASE_NAME:?}" ]; then CONFIG_SCRIPT=PullRequestLinuxCuda9.2TestingSettings.cmake + elif [ "Trilinos_pullrequest_python_2" == "${JOB_BASE_NAME:?}" ]; then + CONFIG_SCRIPT=PullRequestLinuxPython2.cmake + framework_tests_only ../packageEnables.cmake + elif [ "Trilinos_pullrequest_python_3" == "${JOB_BASE_NAME:?}" ]; then + CONFIG_SCRIPT=PullRequestLinuxPython3.cmake + framework_tests_only ../packageEnables.cmake fi fi diff --git a/cmake/std/PullRequestLinuxPython2.cmake b/cmake/std/PullRequestLinuxPython2.cmake new file mode 100644 index 000000000000..559c25a22f7f --- /dev/null +++ b/cmake/std/PullRequestLinuxPython2.cmake @@ -0,0 +1,18 @@ +# This file contains the options needed to both run the pull request testing +# for Trilinos for the Linux GCC 4.8.4 pull request testing builds, and to reproduce +# the errors reported by those builds. Prior to using this this file, the +# appropriate set of SEMS modules must be loaded and accessible through the +# SEMS NFS mount. (See the sems/PullRequestGCC*TestingEnv.sh files.) + +# Usage: cmake -C PullRequestLinuxGCC4.8.4TestingSettings.cmake + +# Misc options typically added by CI testing mode in TriBITS + +# Use the below option only when submitting to the dashboard +#set (CTEST_USE_LAUNCHERS ON CACHE BOOL "Set by default for PR testing") + +set (TFW_Python2_Testing ON CACHE BOOL "Set by default for PR testing") +set (TFW_Python_Testing ON CACHE BOOL "Set by default for PR testing") + +include("${CMAKE_CURRENT_LIST_DIR}/PullRequestLinuxCommonTestingSettings.cmake") + diff --git a/cmake/std/PullRequestLinuxPython3.cmake b/cmake/std/PullRequestLinuxPython3.cmake new file mode 100644 index 000000000000..2d9efda09d5f --- /dev/null +++ b/cmake/std/PullRequestLinuxPython3.cmake @@ -0,0 +1,18 @@ +# This file contains the options needed to both run the pull request testing +# for Trilinos for the Linux GCC 4.8.4 pull request testing builds, and to reproduce +# the errors reported by those builds. Prior to using this this file, the +# appropriate set of SEMS modules must be loaded and accessible through the +# SEMS NFS mount. (See the sems/PullRequestGCC*TestingEnv.sh files.) + +# Usage: cmake -C PullRequestLinuxGCC4.8.4TestingSettings.cmake + +# Misc options typically added by CI testing mode in TriBITS + +# Use the below option only when submitting to the dashboard +#set (CTEST_USE_LAUNCHERS ON CACHE BOOL "Set by default for PR testing") + +set (TFW_Python3_Testing ON CACHE BOOL "Set by default for PR testing") +set (TFW_Python_Testing ON CACHE BOOL "Set by default for PR testing") + +include("${CMAKE_CURRENT_LIST_DIR}/PullRequestLinuxCommonTestingSettings.cmake") + diff --git a/cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh b/cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh index 4f292e5bd83c..a15db7e19646 100644 --- a/cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh +++ b/cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh @@ -31,7 +31,6 @@ module load sems-cmake/3.10.3 # available in main-line Ninja) to significantly speed up # compile and link times. module load atdm-env -module load atdm-cmake/3.11.1 module load atdm-ninja_fortran/1.7.2 # add the OpenMP environment variable we need diff --git a/cmake/std/sems/PullRequestPython2TestingEnv.sh b/cmake/std/sems/PullRequestPython2TestingEnv.sh new file mode 100644 index 000000000000..f8183706b313 --- /dev/null +++ b/cmake/std/sems/PullRequestPython2TestingEnv.sh @@ -0,0 +1,43 @@ +# This script can be used to load the appropriate environment for the +# GCC 4.8.4 Pull Request testing build on a Linux machine that has access to +# the SEMS NFS mount. + +# usage: $ source PullRequestGCC4.8.4TestingEnv.sh + +# After the environment is no longer needed, it can be purged using +# $ module purge +# or Trilinos/cmake/unload_sems_dev_env.sh + +source /projects/sems/modulefiles/utils/sems-modules-init.sh + +module load sems-gcc/4.8.4 +module load sems-openmpi/1.10.1 +module load sems-git/2.10.1 +module load sems-boost/1.63.0/base +module load sems-zlib/1.2.8/base +module load sems-hdf5/1.8.12/parallel +module load sems-netcdf/4.4.1/exo_parallel +module load sems-parmetis/4.0.3/parallel +module load sems-scotch/6.0.3/nopthread_64bit_parallel +module load sems-superlu/4.3/base + +# Load the SEMS CMake Module +# - One of the SEMS modules will load CMake 3.4.x also, +# so this will pull in the SEMS cmake 3.10.3 version +# for Trilinos compatibility. +module load sems-cmake/3.10.3 +module load sems-ninja_fortran/1.8.2 + +# we will have implicitly gotten the sems python from +# the boost module above for whaever reason - reset it +# to one that has the mock packages installed +module unload sems-python +# module load sierra-python/2.7.15 - permissions do not allow this, but the execs are ok +PATH=/projects/sierra/linux_rh7/install/Python/2.7.15/bin:${PATH} +PATH=/projects/sierra/linux_rh7/install/Python/extras/bin:${PATH} +PYTHONPATH=/projects/sierra/linux_rh7/install/Python/extras/lib/python2.7/site-packages:${PYTHONPATH} +MANPATH=/projects/sierra/linux_rh7/install/Python/2.7.15/share/man:${MANPATH} +unset PYTHONHOME + +# add the OpenMP environment variable we need +export OMP_NUM_THREADS=2 diff --git a/cmake/std/sems/PullRequestPython3TestingEnv.sh b/cmake/std/sems/PullRequestPython3TestingEnv.sh new file mode 100644 index 000000000000..81c2cdf7d59f --- /dev/null +++ b/cmake/std/sems/PullRequestPython3TestingEnv.sh @@ -0,0 +1,48 @@ +# This script can be used to load the appropriate environment for the +# GCC 7.2.0 Pull Request testing build on a Linux machine that has access to +# the SEMS NFS mount. + +# usage: $ source PullRequestGCC7.2.0TestingEnv.sh + +source /projects/sems/modulefiles/utils/sems-modules-init.sh + +module load sems-gcc/7.2.0 +module load sems-openmpi/1.10.1 +module load sems-git/2.10.1 +module load sems-boost/1.63.0/base +module load sems-zlib/1.2.8/base +module load sems-hdf5/1.8.12/parallel +module load sems-netcdf/4.4.1/exo_parallel +module load sems-parmetis/4.0.3/parallel +module load sems-scotch/6.0.3/nopthread_64bit_parallel +module load sems-superlu/4.3/base + +# Load the SEMS CMake Module +# - One of the SEMS modules will load CMake 3.4.x also, +# so this will pull in the SEMS cmake 3.10.3 version +# for Trilinos compatibility. +module load sems-cmake/3.10.3 + +# Using CMake and Ninja modules from the ATDM project space. +# SEMS does not yet supply a recent enough version of CMake +# for the single configure/build/test capability. We are also +# using a custom version of Ninja (with Fortran support not +# available in main-line Ninja) to significantly speed up +# compile and link times. +module load atdm-env +module load atdm-ninja_fortran/1.7.2 + +# we will have implicitly gotten the sems python from +# the boost module above for whaever reason - reset it +# to one that has the proper sym-links from python -> python3 +module unload sems-python +# module load sierra-python/3.6.3 - permissions do not allow this, but the execs are ok +PATH=/projects/sierra/linux_rh7/install/Python/3.6..3/bin:${PATH} +PATH=/projects/sierra/linux_rh7/install/Python/extras/bin:${PATH} +PYTHONPATH=/projects/sierra/linux_rh7/install/Python/extras/lib/python3.6/site-packages:${PYTHONPATH} +MANPATH=/projects/sierra/linux_rh7/install/Python/3.6.3/share/man:${MANPATH} +unset PYTHONHOME + +# add the OpenMP environment variable we need +export OMP_NUM_THREADS=2 + diff --git a/commonTools/framework/CMakeLists.txt b/commonTools/framework/CMakeLists.txt index 44f2b24adc6d..ab899431f22d 100644 --- a/commonTools/framework/CMakeLists.txt +++ b/commonTools/framework/CMakeLists.txt @@ -10,6 +10,46 @@ TRIBITS_ADD_ADVANCED_TEST( ProjectCiFileChangeLogic_UnitTests ) +TRIBITS_ADD_ADVANCED_TEST( clean_all_jobs_UnitTests + OVERALL_WORKING_DIRECTORY TEST_NAME + OVERALL_NUM_MPI_PROCS 1 + EXCLUDE_IF_NOT_TRUE TFW_Python_Testing + TEST_0 CMND ${PYTHON_EXECUTABLE} + ARGS ${CMAKE_CURRENT_SOURCE_DIR}/clean_workspace/unittests/test_clean_all_jobs.py -v + PASS_REGULAR_EXPRESSION "OK" + ) + + +TRIBITS_ADD_ADVANCED_TEST( clean_sentinel_UnitTests + OVERALL_WORKING_DIRECTORY TEST_NAME + OVERALL_NUM_MPI_PROCS 1 + EXCLUDE_IF_NOT_TRUE TFW_Python_Testing + TEST_0 CMND ${PYTHON_EXECUTABLE} + ARGS ${CMAKE_CURRENT_SOURCE_DIR}//clean_workspace/unittests/test_clean_sentinel.py -v + PASS_REGULAR_EXPRESSION "OK" + ) + + +TRIBITS_ADD_ADVANCED_TEST( clean_workspace_UnitTests + OVERALL_WORKING_DIRECTORY TEST_NAME + OVERALL_NUM_MPI_PROCS 1 + EXCLUDE_IF_NOT_TRUE TFW_Python_Testing + TEST_0 CMND ${PYTHON_EXECUTABLE} + ARGS ${CMAKE_CURRENT_SOURCE_DIR}//clean_workspace/unittests/test_clean_workspace.py -v + PASS_REGULAR_EXPRESSION "OK" + ) + + +TRIBITS_ADD_ADVANCED_TEST( Modules_UnitTests + OVERALL_WORKING_DIRECTORY TEST_NAME + OVERALL_NUM_MPI_PROCS 1 + EXCLUDE_IF_NOT_TRUE TFW_Python2_Testing + TEST_0 CMND ${PYTHON_EXECUTABLE} + ARGS ${CMAKE_CURRENT_SOURCE_DIR}//clean_workspace/unittests/test_Modules.py -v + PASS_REGULAR_EXPRESSION "OK" + ) + + FUNCTION(create_get_changed_trilinos_packages_test TEST_POSTFIX FILES_CHANGED CHANGED_PACKAGES_FULL_LIST diff --git a/commonTools/framework/clean_workspace/Modules.py b/commonTools/framework/clean_workspace/Modules.py index ee223dd5af46..2addd0135546 100644 --- a/commonTools/framework/clean_workspace/Modules.py +++ b/commonTools/framework/clean_workspace/Modules.py @@ -20,7 +20,8 @@ def __init__(self, name=None): self.command = self._setup_command() self.init_file = self._module_setup(name) if name is None and self.init_file: - execfile(self.init_file) + exec(compile(open(self.init_file, 'r').read(), + self.init_file, 'exec')) @staticmethod def _setup_paths(): @@ -144,7 +145,7 @@ def module(self, command, *arguments): # The modules return a string of python commands to be executed on # stdout. Execute those commands now. if output: - exec (output) # pylint: disable=exec-used + exec(output) # pylint: disable=exec-used # Check stderr for anything that looks like an error. if ":ERROR:" in stderr: diff --git a/commonTools/framework/clean_workspace/clean_all_jobs.py b/commonTools/framework/clean_workspace/clean_all_jobs.py index c85a541084ae..1fa670916d2b 100755 --- a/commonTools/framework/clean_workspace/clean_all_jobs.py +++ b/commonTools/framework/clean_workspace/clean_all_jobs.py @@ -23,7 +23,7 @@ def run(self): """Run the actual routine in clean_sentinel""" try: set_reference_date() - except StandardError as e: + except BaseException as e: print(e.args, file=sys.stderr) sys.exit() except pickle.PicklingError: diff --git a/commonTools/framework/clean_workspace/unittests/test_Modules.py b/commonTools/framework/clean_workspace/unittests/test_Modules.py index df6261952d33..c3c1733520c7 100755 --- a/commonTools/framework/clean_workspace/unittests/test_Modules.py +++ b/commonTools/framework/clean_workspace/unittests/test_Modules.py @@ -9,8 +9,16 @@ import os import unittest -import mock -from cStringIO import StringIO +try: + import mock +except ImportError: + import unittest.mock as mock + +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir)) import Modules @@ -23,15 +31,29 @@ def setUp(self): modules_home = {'MODULESHOME': '/dummy/path/1'} which_side_effects = ['/path/to/modulecmd', None, None] find_side_effects = [None, '/fake/path/modules/init/python.py'] - with mock.patch.dict(os.environ, modules_home), \ - mock.patch('Modules.which', - side_effect=which_side_effects), \ - mock.patch('Modules.find_first_binary', - return_value='/fake/path/modulecmd'), \ - mock.patch('Modules.find_file_in_list', - side_effect=find_side_effects), \ - mock.patch('__builtin__.execfile'): - self.module_obj = Modules.Module() + major_version = sys.version_info[0] + if major_version is 3: + with mock.patch.dict(os.environ, modules_home), \ + mock.patch('Modules.which', + side_effect=which_side_effects), \ + mock.patch('Modules.find_first_binary', + return_value='/fake/path/modulecmd'), \ + mock.patch('Modules.find_file_in_list', + side_effect=find_side_effects), \ + mock.patch('Modules.open'), \ + mock.patch('Modules.compile', return_value=''): + self.module_obj = Modules.Module() + elif major_version is 2: + with mock.patch.dict(os.environ, modules_home), \ + mock.patch('Modules.which', + side_effect=which_side_effects), \ + mock.patch('Modules.find_first_binary', + return_value='/fake/path/modulecmd'), \ + mock.patch('Modules.find_file_in_list', + side_effect=find_side_effects), \ + mock.patch('Modules.open'), \ + mock.patch('Modules.compile', return_value=''): + self.module_obj = Modules.Module() def test_module_setup(self): """Test ability to instantiate the class""" @@ -45,9 +67,10 @@ def test_module_setup(self): return_value='/fake/path/modulecmd'), \ mock.patch('Modules.find_file_in_list', side_effect=find_side_effects), \ - mock.patch('__builtin__.execfile') as mock_exec: + mock.patch('Modules.open') as mock_open, \ + mock.patch('Modules.compile', return_value=""): result = Modules.Module() - mock_exec.assert_called_once_with(find_side_effects[-1]) + mock_open.assert_called_once_with(find_side_effects[-1], 'r') self.assertEqual('/fake/path/modulecmd', result.command) self.assertEqual('modulecmd', result.command_name) self.assertEqual('/fake/path/modules/init/python.py', result.init_file) @@ -64,9 +87,10 @@ def test_module_setup_lmod(self): return_value='/fake/path/lmodcmd'), \ mock.patch('Modules.find_file_in_list', side_effect=find_side_effects), \ - mock.patch('__builtin__.execfile') as mock_exec: + mock.patch('Modules.open') as mock_open, \ + mock.patch('Modules.compile', return_value=""): result = Modules.Module() - mock_exec.assert_not_called() + mock_open.assert_not_called() self.assertEqual('lmodcmd', result.command) self.assertEqual('lmodcmd', result.command_name) self.assertEqual(None, result.init_file) diff --git a/commonTools/framework/clean_workspace/unittests/test_clean_all_jobs.py b/commonTools/framework/clean_workspace/unittests/test_clean_all_jobs.py index 3c1a0930f2c7..4577551e39aa 100755 --- a/commonTools/framework/clean_workspace/unittests/test_clean_all_jobs.py +++ b/commonTools/framework/clean_workspace/unittests/test_clean_all_jobs.py @@ -8,10 +8,16 @@ import os sys.path.insert(1, os.path.dirname(os.path.dirname(__file__))) -from cStringIO import StringIO +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO import unittest -import mock +try: + import mock +except ImportError: + import unittest.mock as mock import pickle @@ -21,11 +27,11 @@ class TestRun(unittest.TestCase): def test_clean_run(self): - """If there is no dir attribute in the args raise SystemExit""" + """A clean run should call set_reference_date""" cleanRefInst = CleanReference() with mock.patch('clean_all_jobs.set_reference_date') as refDate: cleanRefInst.run() - refDate.assert_called_once() + refDate.assert_called_once_with() def test_pickling_exception(self): """A Pickling exception should give an error and exit""" diff --git a/commonTools/framework/clean_workspace/unittests/test_clean_sentinel.py b/commonTools/framework/clean_workspace/unittests/test_clean_sentinel.py index 28a76b84693d..6987ab12df03 100755 --- a/commonTools/framework/clean_workspace/unittests/test_clean_sentinel.py +++ b/commonTools/framework/clean_workspace/unittests/test_clean_sentinel.py @@ -11,7 +11,11 @@ import unittest -import mock +try: + import mock +except ImportError: + import unittest.mock as mock + from datetime import datetime import pickle @@ -113,12 +117,14 @@ class TestUpdateLastCleanDate(unittest.TestCase): def test_update_last_clean_date_writes(self): """Test that the current date is stored""" with mock.patch('clean_sentinel.pickle.dump') as p_save, \ + mock.patch('clean_sentinel.datetime') as dt, \ mock.patch('clean_sentinel.open') as pFile, \ mock.patch.dict('os.environ', {'WORKSPACE': '/scratch/trilinos/foo/bar'}): self.assertEqual(None, update_last_clean_date()) - p_save.assert_called_once() - pFile.assert_called_once() + p_save.assert_called_once_with(dt.now(), + pFile.return_value.__enter__()) + pFile.assert_called_once_with('/scratch/trilinos/foo/bar/lastCleanDate', 'w') def test_update_last_clean_date_raises_with_no_file(self): """Test that the current date is stored""" diff --git a/commonTools/framework/clean_workspace/unittests/test_clean_workspace.py b/commonTools/framework/clean_workspace/unittests/test_clean_workspace.py index 9172b5115469..3a50fa5a2d3d 100755 --- a/commonTools/framework/clean_workspace/unittests/test_clean_workspace.py +++ b/commonTools/framework/clean_workspace/unittests/test_clean_workspace.py @@ -8,10 +8,16 @@ import os sys.path.insert(1, os.path.dirname(os.path.dirname(__file__))) -from cStringIO import StringIO +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO import unittest -import mock +try: + import mock +except ImportError: + import unittest.mock as mock from argparse import Namespace from datetime import datetime @@ -28,8 +34,12 @@ def test_no_directory_raises(self): setattr(test_args, 'force_clean', False) with mock.patch.object(Cleaner, 'parse_args', return_value=test_args): cleanerInst = Cleaner() - with self.assertRaisesRegexp(SystemExit, "No directory passed - exiting!"): - cleanerInst.run() + if sys.version_info.major is not 3: + with self.assertRaisesRegexp(SystemExit, "No directory passed - exiting!"): + cleanerInst.run() + else: + with self.assertRaisesRegex(SystemExit, "No directory passed - exiting!"): + cleanerInst.run() def test_force_calls_clean(self): @@ -42,7 +52,7 @@ def test_force_calls_clean(self): with mock.patch.object(Cleaner, 'force_clean_space') as force_clean, \ mock.patch('clean_workspace.print') as m_print: cleanerInst.run() - force_clean.assert_called_once() + force_clean.assert_called_once_with() m_print.assert_called_once_with("Cleaning directory /dev/null/force_cleaned " "due to command line option") @@ -55,15 +65,19 @@ def test_dir_calls_clean_by_date(self): cleanerInst = Cleaner() with mock.patch.object(Cleaner, 'clean_space_by_date') as force_clean: cleanerInst.run() - force_clean.assert_called_once() + force_clean.assert_called_once_with() class TestParseArgs(unittest.TestCase): def test_no_args_gives_help_and_exits(self): """Test that the function does the right thing when given no arguments""" - usage_message = ('usage: programName [-h] [--force-clean] dir\n' - 'programName: error: too few arguments\n') + if sys.version_info.major is not 3: + usage_message = ('usage: programName [-h] [--force-clean] dir\n' + 'programName: error: too few arguments\n') + else: + usage_message = ('usage: programName [-h] [--force-clean] dir\n' + 'programName: error: the following arguments are required: dir\n') with self.assertRaises(SystemExit), \ mock.patch.object(sys, 'argv', ['programName']), \ mock.patch('sys.stderr', new_callable=StringIO) as cleanOut: @@ -144,8 +158,8 @@ def test_newer_reference_does_clean(self): mock.patch('clean_workspace.update_last_clean_date') as update, \ mock.patch('clean_workspace.print') as m_print: cleanerInst.clean_space_by_date() - force_clean.assert_called_once() - update.assert_called_once() + force_clean.assert_called_once_with() + update.assert_called_once_with() m_print.assert_called_once_with("Cleaning directory /dev/null/fake_directory " "due to newer reference date") @@ -162,8 +176,8 @@ def test_older_date_does_clean(self): mock.patch('clean_workspace.update_last_clean_date') as update, \ mock.patch('clean_workspace.print') as m_print: cleanerInst.clean_space_by_date() - force_clean.assert_called_once() - update.assert_called_once() + force_clean.assert_called_once_with() + update.assert_called_once_with() m_print.assert_called_once_with("Cleaning directory /dev/null/will_clean " "due to newer reference date")