From 2fdf9b2c6572fbad2df8c4b412ec9d3d919cc2a6 Mon Sep 17 00:00:00 2001 From: Paul Wolfenbarger Date: Thu, 18 Apr 2019 12:00:07 -0600 Subject: [PATCH 1/9] Adding the unit tests for the clean_workspace tool As we discussed getting these to always run is vital. I will be working on python 2/3 coexistence next. --- commonTools/framework/CMakeLists.txt | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/commonTools/framework/CMakeLists.txt b/commonTools/framework/CMakeLists.txt index 44f2b24adc6d..20c132d613bf 100644 --- a/commonTools/framework/CMakeLists.txt +++ b/commonTools/framework/CMakeLists.txt @@ -10,6 +10,42 @@ TRIBITS_ADD_ADVANCED_TEST( ProjectCiFileChangeLogic_UnitTests ) +TRIBITS_ADD_ADVANCED_TEST( clean_all_jobs_UnitTests + OVERALL_WORKING_DIRECTORY TEST_NAME + OVERALL_NUM_MPI_PROCS 1 + TEST_0 CMND ${PYTHON_EXECUTABLE} + ARGS ${CMAKE_CURRENT_SOURCE_DIR}/clean_workspace/unittests/test_clean_all_jobs.py -v + PASS_REGULAR_EXPRESSION "OK" + ) + + +TRIBITS_ADD_ADVANCED_TEST( clean_sentinel_UnitTests + OVERALL_WORKING_DIRECTORY TEST_NAME + OVERALL_NUM_MPI_PROCS 1 + TEST_0 CMND ${PYTHON_EXECUTABLE} + ARGS ${CMAKE_CURRENT_SOURCE_DIR}//clean_workspace/unittests/test_clean_sentinel.py -v + PASS_REGULAR_EXPRESSION "OK" + ) + + +TRIBITS_ADD_ADVANCED_TEST( clean_workspace_UnitTests + OVERALL_WORKING_DIRECTORY TEST_NAME + OVERALL_NUM_MPI_PROCS 1 + TEST_0 CMND ${PYTHON_EXECUTABLE} + ARGS ${CMAKE_CURRENT_SOURCE_DIR}//clean_workspace/unittests/test_clean_workspace.py -v + PASS_REGULAR_EXPRESSION "OK" + ) + + +TRIBITS_ADD_ADVANCED_TEST( Modules_UnitTests + OVERALL_WORKING_DIRECTORY TEST_NAME + OVERALL_NUM_MPI_PROCS 1 + TEST_0 CMND ${PYTHON_EXECUTABLE} + ARGS ${CMAKE_CURRENT_SOURCE_DIR}//clean_workspace/unittests/test_Modules.py -v + PASS_REGULAR_EXPRESSION "OK" + ) + + FUNCTION(create_get_changed_trilinos_packages_test TEST_POSTFIX FILES_CHANGED CHANGED_PACKAGES_FULL_LIST From 4eebda5faa42a9615f5f0b244a0af8b12772ff8f Mon Sep 17 00:00:00 2001 From: Paul Wolfenbarger Date: Wed, 24 Apr 2019 13:00:31 -0600 Subject: [PATCH 2/9] Change gcc 7.2 PR testing to use python 3 and disable Modules tests This assures us that we have unit testing for python 3 as well as 2 and disables a test that is known to fail in that condition. In fact it will fail until the Modules package is repaired. --- cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh b/cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh index 4f292e5bd83c..3964e1a7706e 100644 --- a/cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh +++ b/cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh @@ -8,7 +8,7 @@ source /projects/sems/modulefiles/utils/sems-modules-init.sh module load sems-gcc/7.2.0 module load sems-openmpi/1.10.1 -module load sems-python/2.7.9 +module load sems-python/3.5.2 module load sems-git/2.10.1 module load sems-boost/1.63.0/base module load sems-zlib/1.2.8/base @@ -31,7 +31,6 @@ module load sems-cmake/3.10.3 # available in main-line Ninja) to significantly speed up # compile and link times. module load atdm-env -module load atdm-cmake/3.11.1 module load atdm-ninja_fortran/1.7.2 # add the OpenMP environment variable we need From 976d6bc929b9b0ee5d33366a5ea10c728bf12d8b Mon Sep 17 00:00:00 2001 From: Paul Wolfenbarger Date: Wed, 24 Apr 2019 13:02:48 -0600 Subject: [PATCH 3/9] test_Modules.py: re-work to run under both python 2 and 3 Note that this still fails in python 3 not due to anything we can correct, but due to the underlying python interface to the Modules package. --- .../framework/clean_workspace/Modules.py | 5 +- .../clean_workspace/unittests/test_Modules.py | 54 +++++++++++++------ 2 files changed, 42 insertions(+), 17 deletions(-) diff --git a/commonTools/framework/clean_workspace/Modules.py b/commonTools/framework/clean_workspace/Modules.py index ee223dd5af46..2addd0135546 100644 --- a/commonTools/framework/clean_workspace/Modules.py +++ b/commonTools/framework/clean_workspace/Modules.py @@ -20,7 +20,8 @@ def __init__(self, name=None): self.command = self._setup_command() self.init_file = self._module_setup(name) if name is None and self.init_file: - execfile(self.init_file) + exec(compile(open(self.init_file, 'r').read(), + self.init_file, 'exec')) @staticmethod def _setup_paths(): @@ -144,7 +145,7 @@ def module(self, command, *arguments): # The modules return a string of python commands to be executed on # stdout. Execute those commands now. if output: - exec (output) # pylint: disable=exec-used + exec(output) # pylint: disable=exec-used # Check stderr for anything that looks like an error. if ":ERROR:" in stderr: diff --git a/commonTools/framework/clean_workspace/unittests/test_Modules.py b/commonTools/framework/clean_workspace/unittests/test_Modules.py index df6261952d33..c3c1733520c7 100755 --- a/commonTools/framework/clean_workspace/unittests/test_Modules.py +++ b/commonTools/framework/clean_workspace/unittests/test_Modules.py @@ -9,8 +9,16 @@ import os import unittest -import mock -from cStringIO import StringIO +try: + import mock +except ImportError: + import unittest.mock as mock + +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + sys.path.insert(1, os.path.join(os.path.dirname(__file__), os.pardir)) import Modules @@ -23,15 +31,29 @@ def setUp(self): modules_home = {'MODULESHOME': '/dummy/path/1'} which_side_effects = ['/path/to/modulecmd', None, None] find_side_effects = [None, '/fake/path/modules/init/python.py'] - with mock.patch.dict(os.environ, modules_home), \ - mock.patch('Modules.which', - side_effect=which_side_effects), \ - mock.patch('Modules.find_first_binary', - return_value='/fake/path/modulecmd'), \ - mock.patch('Modules.find_file_in_list', - side_effect=find_side_effects), \ - mock.patch('__builtin__.execfile'): - self.module_obj = Modules.Module() + major_version = sys.version_info[0] + if major_version is 3: + with mock.patch.dict(os.environ, modules_home), \ + mock.patch('Modules.which', + side_effect=which_side_effects), \ + mock.patch('Modules.find_first_binary', + return_value='/fake/path/modulecmd'), \ + mock.patch('Modules.find_file_in_list', + side_effect=find_side_effects), \ + mock.patch('Modules.open'), \ + mock.patch('Modules.compile', return_value=''): + self.module_obj = Modules.Module() + elif major_version is 2: + with mock.patch.dict(os.environ, modules_home), \ + mock.patch('Modules.which', + side_effect=which_side_effects), \ + mock.patch('Modules.find_first_binary', + return_value='/fake/path/modulecmd'), \ + mock.patch('Modules.find_file_in_list', + side_effect=find_side_effects), \ + mock.patch('Modules.open'), \ + mock.patch('Modules.compile', return_value=''): + self.module_obj = Modules.Module() def test_module_setup(self): """Test ability to instantiate the class""" @@ -45,9 +67,10 @@ def test_module_setup(self): return_value='/fake/path/modulecmd'), \ mock.patch('Modules.find_file_in_list', side_effect=find_side_effects), \ - mock.patch('__builtin__.execfile') as mock_exec: + mock.patch('Modules.open') as mock_open, \ + mock.patch('Modules.compile', return_value=""): result = Modules.Module() - mock_exec.assert_called_once_with(find_side_effects[-1]) + mock_open.assert_called_once_with(find_side_effects[-1], 'r') self.assertEqual('/fake/path/modulecmd', result.command) self.assertEqual('modulecmd', result.command_name) self.assertEqual('/fake/path/modules/init/python.py', result.init_file) @@ -64,9 +87,10 @@ def test_module_setup_lmod(self): return_value='/fake/path/lmodcmd'), \ mock.patch('Modules.find_file_in_list', side_effect=find_side_effects), \ - mock.patch('__builtin__.execfile') as mock_exec: + mock.patch('Modules.open') as mock_open, \ + mock.patch('Modules.compile', return_value=""): result = Modules.Module() - mock_exec.assert_not_called() + mock_open.assert_not_called() self.assertEqual('lmodcmd', result.command) self.assertEqual('lmodcmd', result.command_name) self.assertEqual(None, result.init_file) From 83c8f9396777c2904bc73fb29b2e506d60357b90 Mon Sep 17 00:00:00 2001 From: Paul Wolfenbarger Date: Mon, 29 Apr 2019 09:56:14 -0600 Subject: [PATCH 4/9] This commit allows all of the clean_workspace scripts and test to run under both pyython 2 and 3 --- .../clean_workspace/clean_all_jobs.py | 2 +- .../unittests/test_clean_all_jobs.py | 14 +++++-- .../unittests/test_clean_sentinel.py | 12 ++++-- .../unittests/test_clean_workspace.py | 38 +++++++++++++------ 4 files changed, 46 insertions(+), 20 deletions(-) diff --git a/commonTools/framework/clean_workspace/clean_all_jobs.py b/commonTools/framework/clean_workspace/clean_all_jobs.py index c85a541084ae..1fa670916d2b 100755 --- a/commonTools/framework/clean_workspace/clean_all_jobs.py +++ b/commonTools/framework/clean_workspace/clean_all_jobs.py @@ -23,7 +23,7 @@ def run(self): """Run the actual routine in clean_sentinel""" try: set_reference_date() - except StandardError as e: + except BaseException as e: print(e.args, file=sys.stderr) sys.exit() except pickle.PicklingError: diff --git a/commonTools/framework/clean_workspace/unittests/test_clean_all_jobs.py b/commonTools/framework/clean_workspace/unittests/test_clean_all_jobs.py index 3c1a0930f2c7..4577551e39aa 100755 --- a/commonTools/framework/clean_workspace/unittests/test_clean_all_jobs.py +++ b/commonTools/framework/clean_workspace/unittests/test_clean_all_jobs.py @@ -8,10 +8,16 @@ import os sys.path.insert(1, os.path.dirname(os.path.dirname(__file__))) -from cStringIO import StringIO +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO import unittest -import mock +try: + import mock +except ImportError: + import unittest.mock as mock import pickle @@ -21,11 +27,11 @@ class TestRun(unittest.TestCase): def test_clean_run(self): - """If there is no dir attribute in the args raise SystemExit""" + """A clean run should call set_reference_date""" cleanRefInst = CleanReference() with mock.patch('clean_all_jobs.set_reference_date') as refDate: cleanRefInst.run() - refDate.assert_called_once() + refDate.assert_called_once_with() def test_pickling_exception(self): """A Pickling exception should give an error and exit""" diff --git a/commonTools/framework/clean_workspace/unittests/test_clean_sentinel.py b/commonTools/framework/clean_workspace/unittests/test_clean_sentinel.py index 28a76b84693d..6987ab12df03 100755 --- a/commonTools/framework/clean_workspace/unittests/test_clean_sentinel.py +++ b/commonTools/framework/clean_workspace/unittests/test_clean_sentinel.py @@ -11,7 +11,11 @@ import unittest -import mock +try: + import mock +except ImportError: + import unittest.mock as mock + from datetime import datetime import pickle @@ -113,12 +117,14 @@ class TestUpdateLastCleanDate(unittest.TestCase): def test_update_last_clean_date_writes(self): """Test that the current date is stored""" with mock.patch('clean_sentinel.pickle.dump') as p_save, \ + mock.patch('clean_sentinel.datetime') as dt, \ mock.patch('clean_sentinel.open') as pFile, \ mock.patch.dict('os.environ', {'WORKSPACE': '/scratch/trilinos/foo/bar'}): self.assertEqual(None, update_last_clean_date()) - p_save.assert_called_once() - pFile.assert_called_once() + p_save.assert_called_once_with(dt.now(), + pFile.return_value.__enter__()) + pFile.assert_called_once_with('/scratch/trilinos/foo/bar/lastCleanDate', 'w') def test_update_last_clean_date_raises_with_no_file(self): """Test that the current date is stored""" diff --git a/commonTools/framework/clean_workspace/unittests/test_clean_workspace.py b/commonTools/framework/clean_workspace/unittests/test_clean_workspace.py index 9172b5115469..3a50fa5a2d3d 100755 --- a/commonTools/framework/clean_workspace/unittests/test_clean_workspace.py +++ b/commonTools/framework/clean_workspace/unittests/test_clean_workspace.py @@ -8,10 +8,16 @@ import os sys.path.insert(1, os.path.dirname(os.path.dirname(__file__))) -from cStringIO import StringIO +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO import unittest -import mock +try: + import mock +except ImportError: + import unittest.mock as mock from argparse import Namespace from datetime import datetime @@ -28,8 +34,12 @@ def test_no_directory_raises(self): setattr(test_args, 'force_clean', False) with mock.patch.object(Cleaner, 'parse_args', return_value=test_args): cleanerInst = Cleaner() - with self.assertRaisesRegexp(SystemExit, "No directory passed - exiting!"): - cleanerInst.run() + if sys.version_info.major is not 3: + with self.assertRaisesRegexp(SystemExit, "No directory passed - exiting!"): + cleanerInst.run() + else: + with self.assertRaisesRegex(SystemExit, "No directory passed - exiting!"): + cleanerInst.run() def test_force_calls_clean(self): @@ -42,7 +52,7 @@ def test_force_calls_clean(self): with mock.patch.object(Cleaner, 'force_clean_space') as force_clean, \ mock.patch('clean_workspace.print') as m_print: cleanerInst.run() - force_clean.assert_called_once() + force_clean.assert_called_once_with() m_print.assert_called_once_with("Cleaning directory /dev/null/force_cleaned " "due to command line option") @@ -55,15 +65,19 @@ def test_dir_calls_clean_by_date(self): cleanerInst = Cleaner() with mock.patch.object(Cleaner, 'clean_space_by_date') as force_clean: cleanerInst.run() - force_clean.assert_called_once() + force_clean.assert_called_once_with() class TestParseArgs(unittest.TestCase): def test_no_args_gives_help_and_exits(self): """Test that the function does the right thing when given no arguments""" - usage_message = ('usage: programName [-h] [--force-clean] dir\n' - 'programName: error: too few arguments\n') + if sys.version_info.major is not 3: + usage_message = ('usage: programName [-h] [--force-clean] dir\n' + 'programName: error: too few arguments\n') + else: + usage_message = ('usage: programName [-h] [--force-clean] dir\n' + 'programName: error: the following arguments are required: dir\n') with self.assertRaises(SystemExit), \ mock.patch.object(sys, 'argv', ['programName']), \ mock.patch('sys.stderr', new_callable=StringIO) as cleanOut: @@ -144,8 +158,8 @@ def test_newer_reference_does_clean(self): mock.patch('clean_workspace.update_last_clean_date') as update, \ mock.patch('clean_workspace.print') as m_print: cleanerInst.clean_space_by_date() - force_clean.assert_called_once() - update.assert_called_once() + force_clean.assert_called_once_with() + update.assert_called_once_with() m_print.assert_called_once_with("Cleaning directory /dev/null/fake_directory " "due to newer reference date") @@ -162,8 +176,8 @@ def test_older_date_does_clean(self): mock.patch('clean_workspace.update_last_clean_date') as update, \ mock.patch('clean_workspace.print') as m_print: cleanerInst.clean_space_by_date() - force_clean.assert_called_once() - update.assert_called_once() + force_clean.assert_called_once_with() + update.assert_called_once_with() m_print.assert_called_once_with("Cleaning directory /dev/null/will_clean " "due to newer reference date") From 454a9ae69eb39fcc8b66ab3a00ec20717d6b6071 Mon Sep 17 00:00:00 2001 From: Paul Wolfenbarger Date: Mon, 3 Jun 2019 09:48:02 -0600 Subject: [PATCH 5/9] Add stand-alone tests for framework python unittests In order to make these work I need to use a non-standard environment as the sems ones are off python2 is missing the mock package which is need for smooth transition to python3 python3 does not have a sym-link for pthon->python3 which breaks stand-alone scripts --- cmake/std/PullRequestLinuxDriver-Test.sh | 20 +++++++++ .../std/sems/PullRequestPython2TestingEnv.sh | 38 ++++++++++++++++ .../std/sems/PullRequestPython3TestingEnv.sh | 43 +++++++++++++++++++ commonTools/framework/CMakeLists.txt | 4 ++ 4 files changed, 105 insertions(+) create mode 100644 cmake/std/sems/PullRequestPython2TestingEnv.sh create mode 100644 cmake/std/sems/PullRequestPython3TestingEnv.sh diff --git a/cmake/std/PullRequestLinuxDriver-Test.sh b/cmake/std/PullRequestLinuxDriver-Test.sh index 38c7ef033be7..9a0e6c91c612 100755 --- a/cmake/std/PullRequestLinuxDriver-Test.sh +++ b/cmake/std/PullRequestLinuxDriver-Test.sh @@ -62,6 +62,20 @@ function test_pr_constraints_master() } +function framework_tests_only() +{ + pkg_file=${1:?} + framework_enables=" +MACRO(PR_ENABLE_BOOL VAR_NAME VAR_VAL) + MESSAGE("-- Setting ${VAR_NAME} = ${VAR_VAL}") + SET(${VAR_NAME} ${VAR_VAL} CACHE BOOL "Set in packageEnables.cmake") +ENDMACRO() + +PR_ENABLE_BOOL(Trilinos_ENABLE_TrilinosFrameworkTests ON) +" + cat ${framework_enables} > ${pkg_file} +} + # This script expects to start out in the root level of the Jenkins workspace. # Let's make sure we're there. @@ -380,6 +394,12 @@ else CONFIG_SCRIPT=PullRequestLinuxGCC7.3.0TestingSettings.cmake elif [ "Trilinos_pullrequest_cuda_9.2" == "${JOB_BASE_NAME:?}" ]; then CONFIG_SCRIPT=PullRequestLinuxCuda9.2TestingSettings.cmake + elif [ "Trilinos_pullrequest_python_2" == "${JOB_BASE_NAME:?}" ]; then + CONFIG_SCRIPT=PullRequestLinuxPython2.cmake + framework_tests_only ../packageEnables.cmake + elif [ "Trilinos_pullrequest_python_3" == "${JOB_BASE_NAME:?}" ]; then + CONFIG_SCRIPT=PullRequestLinuxPython3.cmake + framework_tests_only ../packageEnables.cmake fi fi diff --git a/cmake/std/sems/PullRequestPython2TestingEnv.sh b/cmake/std/sems/PullRequestPython2TestingEnv.sh new file mode 100644 index 000000000000..e23e34e8c906 --- /dev/null +++ b/cmake/std/sems/PullRequestPython2TestingEnv.sh @@ -0,0 +1,38 @@ +# This script can be used to load the appropriate environment for the +# GCC 4.8.4 Pull Request testing build on a Linux machine that has access to +# the SEMS NFS mount. + +# usage: $ source PullRequestGCC4.8.4TestingEnv.sh + +# After the environment is no longer needed, it can be purged using +# $ module purge +# or Trilinos/cmake/unload_sems_dev_env.sh + +source /projects/sems/modulefiles/utils/sems-modules-init.sh + +module load sems-gcc/4.8.4 +module load sems-openmpi/1.10.1 +module load sems-git/2.10.1 +module load sems-boost/1.63.0/base +module load sems-zlib/1.2.8/base +module load sems-hdf5/1.8.12/parallel +module load sems-netcdf/4.4.1/exo_parallel +module load sems-parmetis/4.0.3/parallel +module load sems-scotch/6.0.3/nopthread_64bit_parallel +module load sems-superlu/4.3/base + +# Load the SEMS CMake Module +# - One of the SEMS modules will load CMake 3.4.x also, +# so this will pull in the SEMS cmake 3.10.3 version +# for Trilinos compatibility. +module load sems-cmake/3.10.3 +module load sems-ninja_fortran/1.8.2 + +# we will have implicitly gotten the sems python from +# the boost module above for whaever reason - reset it +# to one that has the mock packages installed +module unload sems-python +module load sierra-python/2.7.15 + +# add the OpenMP environment variable we need +export OMP_NUM_THREADS=2 diff --git a/cmake/std/sems/PullRequestPython3TestingEnv.sh b/cmake/std/sems/PullRequestPython3TestingEnv.sh new file mode 100644 index 000000000000..68b8b79e19a6 --- /dev/null +++ b/cmake/std/sems/PullRequestPython3TestingEnv.sh @@ -0,0 +1,43 @@ +# This script can be used to load the appropriate environment for the +# GCC 7.2.0 Pull Request testing build on a Linux machine that has access to +# the SEMS NFS mount. + +# usage: $ source PullRequestGCC7.2.0TestingEnv.sh + +source /projects/sems/modulefiles/utils/sems-modules-init.sh + +module load sems-gcc/7.2.0 +module load sems-openmpi/1.10.1 +module load sems-git/2.10.1 +module load sems-boost/1.63.0/base +module load sems-zlib/1.2.8/base +module load sems-hdf5/1.8.12/parallel +module load sems-netcdf/4.4.1/exo_parallel +module load sems-parmetis/4.0.3/parallel +module load sems-scotch/6.0.3/nopthread_64bit_parallel +module load sems-superlu/4.3/base + +# Load the SEMS CMake Module +# - One of the SEMS modules will load CMake 3.4.x also, +# so this will pull in the SEMS cmake 3.10.3 version +# for Trilinos compatibility. +module load sems-cmake/3.10.3 + +# Using CMake and Ninja modules from the ATDM project space. +# SEMS does not yet supply a recent enough version of CMake +# for the single configure/build/test capability. We are also +# using a custom version of Ninja (with Fortran support not +# available in main-line Ninja) to significantly speed up +# compile and link times. +module load atdm-env +module load atdm-ninja_fortran/1.7.2 + +# we will have implicitly gotten the sems python from +# the boost module above for whaever reason - reset it +# to one that has the proper sym-links from python -> python3 +module unload sems-python +module load sierra-python/3.6.3 + +# add the OpenMP environment variable we need +export OMP_NUM_THREADS=2 + diff --git a/commonTools/framework/CMakeLists.txt b/commonTools/framework/CMakeLists.txt index 20c132d613bf..ab899431f22d 100644 --- a/commonTools/framework/CMakeLists.txt +++ b/commonTools/framework/CMakeLists.txt @@ -13,6 +13,7 @@ TRIBITS_ADD_ADVANCED_TEST( ProjectCiFileChangeLogic_UnitTests TRIBITS_ADD_ADVANCED_TEST( clean_all_jobs_UnitTests OVERALL_WORKING_DIRECTORY TEST_NAME OVERALL_NUM_MPI_PROCS 1 + EXCLUDE_IF_NOT_TRUE TFW_Python_Testing TEST_0 CMND ${PYTHON_EXECUTABLE} ARGS ${CMAKE_CURRENT_SOURCE_DIR}/clean_workspace/unittests/test_clean_all_jobs.py -v PASS_REGULAR_EXPRESSION "OK" @@ -22,6 +23,7 @@ TRIBITS_ADD_ADVANCED_TEST( clean_all_jobs_UnitTests TRIBITS_ADD_ADVANCED_TEST( clean_sentinel_UnitTests OVERALL_WORKING_DIRECTORY TEST_NAME OVERALL_NUM_MPI_PROCS 1 + EXCLUDE_IF_NOT_TRUE TFW_Python_Testing TEST_0 CMND ${PYTHON_EXECUTABLE} ARGS ${CMAKE_CURRENT_SOURCE_DIR}//clean_workspace/unittests/test_clean_sentinel.py -v PASS_REGULAR_EXPRESSION "OK" @@ -31,6 +33,7 @@ TRIBITS_ADD_ADVANCED_TEST( clean_sentinel_UnitTests TRIBITS_ADD_ADVANCED_TEST( clean_workspace_UnitTests OVERALL_WORKING_DIRECTORY TEST_NAME OVERALL_NUM_MPI_PROCS 1 + EXCLUDE_IF_NOT_TRUE TFW_Python_Testing TEST_0 CMND ${PYTHON_EXECUTABLE} ARGS ${CMAKE_CURRENT_SOURCE_DIR}//clean_workspace/unittests/test_clean_workspace.py -v PASS_REGULAR_EXPRESSION "OK" @@ -40,6 +43,7 @@ TRIBITS_ADD_ADVANCED_TEST( clean_workspace_UnitTests TRIBITS_ADD_ADVANCED_TEST( Modules_UnitTests OVERALL_WORKING_DIRECTORY TEST_NAME OVERALL_NUM_MPI_PROCS 1 + EXCLUDE_IF_NOT_TRUE TFW_Python2_Testing TEST_0 CMND ${PYTHON_EXECUTABLE} ARGS ${CMAKE_CURRENT_SOURCE_DIR}//clean_workspace/unittests/test_Modules.py -v PASS_REGULAR_EXPRESSION "OK" From 557cdd845cf5e28ea5feeea97c845e5acef9265c Mon Sep 17 00:00:00 2001 From: Paul Wolfenbarger Date: Mon, 3 Jun 2019 10:29:43 -0600 Subject: [PATCH 6/9] move gcc 7.2.0 environment back to python2 This fails due to the lack of proper sym-links in the sems python3 install --- cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh b/cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh index 3964e1a7706e..a15db7e19646 100644 --- a/cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh +++ b/cmake/std/sems/PullRequestGCC7.2.0TestingEnv.sh @@ -8,7 +8,7 @@ source /projects/sems/modulefiles/utils/sems-modules-init.sh module load sems-gcc/7.2.0 module load sems-openmpi/1.10.1 -module load sems-python/3.5.2 +module load sems-python/2.7.9 module load sems-git/2.10.1 module load sems-boost/1.63.0/base module load sems-zlib/1.2.8/base From 17d8ee341ff073db9ce26538956a7607e1313da7 Mon Sep 17 00:00:00 2001 From: Paul Wolfenbarger Date: Mon, 3 Jun 2019 12:55:50 -0600 Subject: [PATCH 7/9] Python testing - resolve issues found during testing * the new cmake files were not added * I was not sourceing the correct environment * sierra modules are not open for the trilinos entity to use --- cmake/std/PullRequestLinuxDriver-Test.sh | 14 ++++++++++++++ cmake/std/PullRequestLinuxPython2.cmake | 18 ++++++++++++++++++ cmake/std/PullRequestLinuxPython3.cmake | 18 ++++++++++++++++++ cmake/std/sems/PullRequestPython2TestingEnv.sh | 7 ++++++- cmake/std/sems/PullRequestPython3TestingEnv.sh | 7 ++++++- 5 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 cmake/std/PullRequestLinuxPython2.cmake create mode 100644 cmake/std/PullRequestLinuxPython3.cmake diff --git a/cmake/std/PullRequestLinuxDriver-Test.sh b/cmake/std/PullRequestLinuxDriver-Test.sh index 9a0e6c91c612..da375b0bb900 100755 --- a/cmake/std/PullRequestLinuxDriver-Test.sh +++ b/cmake/std/PullRequestLinuxDriver-Test.sh @@ -280,6 +280,20 @@ elif [ "Trilinos_pullrequest_intel_17.0.1" == "${JOB_BASE_NAME:?}" ] ; then echo -e "There was an issue loading the intel environment. The error code was: $ierror" exit $ierror fi +elif [ "Trilinos_pullrequest_python_2" == "${JOB_BASE_NAME:?}" ]; then + source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestPython2TestingEnv.sh + ierror=$? + if [[ $ierror != 0 ]]; then + echo -e "There was an issue loading the python2 environment. The error code was: $ierror" + exit $ierror + fi +elif [ "Trilinos_pullrequest_python_3" == "${JOB_BASE_NAME:?}" ]; then + source ${TRILINOS_DRIVER_SRC_DIR}/cmake/std/sems/PullRequestPython3TestingEnv.sh + ierror=$? + if [[ $ierror != 0 ]]; then + echo -e "There was an issue loading the python3 environment. The error code was: $ierror" + exit $ierror + fi else ierror=42 echo -e "ERROR: Unable to find matching environment for job: ${JOB_BASE_NAME:?}" diff --git a/cmake/std/PullRequestLinuxPython2.cmake b/cmake/std/PullRequestLinuxPython2.cmake new file mode 100644 index 000000000000..559c25a22f7f --- /dev/null +++ b/cmake/std/PullRequestLinuxPython2.cmake @@ -0,0 +1,18 @@ +# This file contains the options needed to both run the pull request testing +# for Trilinos for the Linux GCC 4.8.4 pull request testing builds, and to reproduce +# the errors reported by those builds. Prior to using this this file, the +# appropriate set of SEMS modules must be loaded and accessible through the +# SEMS NFS mount. (See the sems/PullRequestGCC*TestingEnv.sh files.) + +# Usage: cmake -C PullRequestLinuxGCC4.8.4TestingSettings.cmake + +# Misc options typically added by CI testing mode in TriBITS + +# Use the below option only when submitting to the dashboard +#set (CTEST_USE_LAUNCHERS ON CACHE BOOL "Set by default for PR testing") + +set (TFW_Python2_Testing ON CACHE BOOL "Set by default for PR testing") +set (TFW_Python_Testing ON CACHE BOOL "Set by default for PR testing") + +include("${CMAKE_CURRENT_LIST_DIR}/PullRequestLinuxCommonTestingSettings.cmake") + diff --git a/cmake/std/PullRequestLinuxPython3.cmake b/cmake/std/PullRequestLinuxPython3.cmake new file mode 100644 index 000000000000..2d9efda09d5f --- /dev/null +++ b/cmake/std/PullRequestLinuxPython3.cmake @@ -0,0 +1,18 @@ +# This file contains the options needed to both run the pull request testing +# for Trilinos for the Linux GCC 4.8.4 pull request testing builds, and to reproduce +# the errors reported by those builds. Prior to using this this file, the +# appropriate set of SEMS modules must be loaded and accessible through the +# SEMS NFS mount. (See the sems/PullRequestGCC*TestingEnv.sh files.) + +# Usage: cmake -C PullRequestLinuxGCC4.8.4TestingSettings.cmake + +# Misc options typically added by CI testing mode in TriBITS + +# Use the below option only when submitting to the dashboard +#set (CTEST_USE_LAUNCHERS ON CACHE BOOL "Set by default for PR testing") + +set (TFW_Python3_Testing ON CACHE BOOL "Set by default for PR testing") +set (TFW_Python_Testing ON CACHE BOOL "Set by default for PR testing") + +include("${CMAKE_CURRENT_LIST_DIR}/PullRequestLinuxCommonTestingSettings.cmake") + diff --git a/cmake/std/sems/PullRequestPython2TestingEnv.sh b/cmake/std/sems/PullRequestPython2TestingEnv.sh index e23e34e8c906..f8183706b313 100644 --- a/cmake/std/sems/PullRequestPython2TestingEnv.sh +++ b/cmake/std/sems/PullRequestPython2TestingEnv.sh @@ -32,7 +32,12 @@ module load sems-ninja_fortran/1.8.2 # the boost module above for whaever reason - reset it # to one that has the mock packages installed module unload sems-python -module load sierra-python/2.7.15 +# module load sierra-python/2.7.15 - permissions do not allow this, but the execs are ok +PATH=/projects/sierra/linux_rh7/install/Python/2.7.15/bin:${PATH} +PATH=/projects/sierra/linux_rh7/install/Python/extras/bin:${PATH} +PYTHONPATH=/projects/sierra/linux_rh7/install/Python/extras/lib/python2.7/site-packages:${PYTHONPATH} +MANPATH=/projects/sierra/linux_rh7/install/Python/2.7.15/share/man:${MANPATH} +unset PYTHONHOME # add the OpenMP environment variable we need export OMP_NUM_THREADS=2 diff --git a/cmake/std/sems/PullRequestPython3TestingEnv.sh b/cmake/std/sems/PullRequestPython3TestingEnv.sh index 68b8b79e19a6..81c2cdf7d59f 100644 --- a/cmake/std/sems/PullRequestPython3TestingEnv.sh +++ b/cmake/std/sems/PullRequestPython3TestingEnv.sh @@ -36,7 +36,12 @@ module load atdm-ninja_fortran/1.7.2 # the boost module above for whaever reason - reset it # to one that has the proper sym-links from python -> python3 module unload sems-python -module load sierra-python/3.6.3 +# module load sierra-python/3.6.3 - permissions do not allow this, but the execs are ok +PATH=/projects/sierra/linux_rh7/install/Python/3.6..3/bin:${PATH} +PATH=/projects/sierra/linux_rh7/install/Python/extras/bin:${PATH} +PYTHONPATH=/projects/sierra/linux_rh7/install/Python/extras/lib/python3.6/site-packages:${PYTHONPATH} +MANPATH=/projects/sierra/linux_rh7/install/Python/3.6.3/share/man:${MANPATH} +unset PYTHONHOME # add the OpenMP environment variable we need export OMP_NUM_THREADS=2 From 89646d0d3c08bcee0c9fb36ed176d187bc3cf605 Mon Sep 17 00:00:00 2001 From: Paul Wolfenbarger Date: Mon, 3 Jun 2019 14:15:29 -0600 Subject: [PATCH 8/9] Correct shell quoting in the package file replacement for python ... --- cmake/std/PullRequestLinuxDriver-Test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/std/PullRequestLinuxDriver-Test.sh b/cmake/std/PullRequestLinuxDriver-Test.sh index da375b0bb900..90f0b2a39902 100755 --- a/cmake/std/PullRequestLinuxDriver-Test.sh +++ b/cmake/std/PullRequestLinuxDriver-Test.sh @@ -67,13 +67,13 @@ function framework_tests_only() pkg_file=${1:?} framework_enables=" MACRO(PR_ENABLE_BOOL VAR_NAME VAR_VAL) - MESSAGE("-- Setting ${VAR_NAME} = ${VAR_VAL}") - SET(${VAR_NAME} ${VAR_VAL} CACHE BOOL "Set in packageEnables.cmake") + MESSAGE(\"-- Setting ${VAR_NAME} = ${VAR_VAL}\") + SET(${VAR_NAME} ${VAR_VAL} CACHE BOOL \"Set in packageEnables.cmake\") ENDMACRO() PR_ENABLE_BOOL(Trilinos_ENABLE_TrilinosFrameworkTests ON) " - cat ${framework_enables} > ${pkg_file} + echo ${framework_enables} > ${pkg_file} } From 9a754507c3a931199de518e6f88ebf908b810074 Mon Sep 17 00:00:00 2001 From: Paul Wolfenbarger Date: Tue, 4 Jun 2019 06:56:55 -0600 Subject: [PATCH 9/9] fix another type of quoting error The python only tests came back with an empty packageEnables file due to this one. --- cmake/std/PullRequestLinuxDriver-Test.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cmake/std/PullRequestLinuxDriver-Test.sh b/cmake/std/PullRequestLinuxDriver-Test.sh index 90f0b2a39902..5e5f4511d50f 100755 --- a/cmake/std/PullRequestLinuxDriver-Test.sh +++ b/cmake/std/PullRequestLinuxDriver-Test.sh @@ -65,15 +65,17 @@ function test_pr_constraints_master() function framework_tests_only() { pkg_file=${1:?} - framework_enables=" + cat << EOF > ${pkg_file} + MACRO(PR_ENABLE_BOOL VAR_NAME VAR_VAL) - MESSAGE(\"-- Setting ${VAR_NAME} = ${VAR_VAL}\") - SET(${VAR_NAME} ${VAR_VAL} CACHE BOOL \"Set in packageEnables.cmake\") + MESSAGE("-- Setting ${VAR_NAME} = ${VAR_VAL}") + SET(\${VAR_NAME} \${VAR_VAL} CACHE BOOL "Set in packageEnables.cmake") ENDMACRO() PR_ENABLE_BOOL(Trilinos_ENABLE_TrilinosFrameworkTests ON) -" - echo ${framework_enables} > ${pkg_file} + +EOF + }