Update develop-ref after #1697 (#1701)

Co-authored-by: George McCabe <[email protected]> Co-authored-by: Mrinal Biswas <[email protected]> Co-authored-by: Hank Fisher <[email protected]> Co-authored-by: Minna Win <[email protected]> Co-authored-by: bikegeek <[email protected]> Co-authored-by: Christina Kalb <[email protected]> Co-authored-by: Julie Prestopnik <[email protected]> Co-authored-by: johnhg <[email protected]> Co-authored-by: Hank Fisher <[email protected]> Co-authored-by: Mrinal Biswas <[email protected]> Co-authored-by: jprestop <[email protected]> Co-authored-by: j-opatz <[email protected]> Co-authored-by: Kathryn Newman <[email protected]> Co-authored-by: j-opatz <[email protected]> Co-authored-by: Lisa Goodrich <[email protected]> Co-authored-by: Julie Prestopnik <[email protected]> Co-authored-by: Christina Kalb <[email protected]> Co-authored-by: lisagoodrich <[email protected]> Co-authored-by: bikegeek <[email protected]> Co-authored-by: Hank Fisher <[email protected]> Co-authored-by: Christina Kalb <[email protected]> Co-authored-by: Howard Soh <[email protected]> Co-authored-by: Molly Smith <[email protected]> Co-authored-by: hsoh-u <[email protected]>
dtcenter · Jul 18, 2022 · df824e2 · df824e2
1 parent dde9411
commit df824e2
Show file tree

Hide file tree

Showing 244 changed files with 5,516 additions and 4,437 deletions.
diff --git a/.github/actions/run_tests/entrypoint.sh b/.github/actions/run_tests/entrypoint.sh
@@ -8,6 +8,8 @@ WS_PATH=$RUNNER_WORKSPACE/$REPO_NAME
 # set CI jobs directory variable to easily move it
 CI_JOBS_DIR=.github/jobs
 
+PYTESTS_GROUPS_FILEPATH=.github/parm/pytest_groups.txt
+
 source ${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/bash_functions.sh
 
 # get branch name for push or pull request events
@@ -30,10 +32,8 @@ if [ $? != 0 ]; then
    ${GITHUB_WORKSPACE}/${CI_JOBS_DIR}/docker_setup.sh
 fi
 
-#
 # running unit tests (pytests)
-#
-if [ "$INPUT_CATEGORIES" == "pytests" ]; then
+if [[ "$INPUT_CATEGORIES" == pytests* ]]; then
   export METPLUS_ENV_TAG="pytest"
   export METPLUS_IMG_TAG=${branch_name}
   echo METPLUS_ENV_TAG=${METPLUS_ENV_TAG}
@@ -56,14 +56,20 @@ if [ "$INPUT_CATEGORIES" == "pytests" ]; then
 	 .
 
   echo Running Pytests
-  command="export METPLUS_PYTEST_HOST=docker; cd internal_tests/pytests; /usr/local/envs/pytest/bin/pytest -vv --cov=../../metplus"
+  command="export METPLUS_PYTEST_HOST=docker; cd internal_tests/pytests;"
+  command+="status=0;"
+  for x in `cat $PYTESTS_GROUPS_FILEPATH`; do
+    marker="${x//_or_/ or }"
+    marker="${marker//not_/not }"
+    command+="/usr/local/envs/pytest/bin/pytest -vv --cov=../../metplus -m \"$marker\""
+    command+=";if [ \$? != 0 ]; then status=1; fi;"
+  done
+  command+="if [ \$status != 0 ]; then echo ERROR: Some pytests failed. Search for FAILED to review; false; fi"
   time_command docker run -v $WS_PATH:$GITHUB_WORKSPACE --workdir $GITHUB_WORKSPACE $RUN_TAG bash -c "$command"
   exit $?
 fi
 
-#
 # running use case tests
-#
 
 # split apart use case category and subset list from input
 CATEGORIES=`echo $INPUT_CATEGORIES | awk -F: '{print $1}'`

diff --git a/.github/jobs/free_disk_space.sh b/.github/jobs/free_disk_space.sh
@@ -0,0 +1,15 @@
+#! /bin/bash
+
+echo Checking disk usage before cleanup
+df -h
+
+printf "\nRemoving files as suggested by https://github.com/actions/virtual-environments/issues/2840"
+
+sudo rm -rf /usr/share/dotnet
+sudo rm -rf /opt/ghc
+sudo rm -rf "/usr/local/share/boost"
+sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+printf "\nChecking disk usage after cleanup"
+
+df -h
diff --git a/.github/jobs/get_metviewer.sh b/.github/jobs/get_metviewer.sh
@@ -21,10 +21,10 @@ sleep 20
 # print list of currently running containers to
 # verify mysql and metviewer are running
 docker ps -a
-
+# TODO: when METviewer Dockerfile changes METdatadb to METdataio, update path to sql file accordingly
 # commands to run inside METviewer container
 cmd="mysql -hmysql_mv -uroot -pmvuser -e\"create database mv_metplus_test;\";"
-cmd+="mysql -hmysql_mv -uroot -pmvuser mv_metplus_test < /METviewer/sql/mv_mysql.sql"
+cmd+="mysql -hmysql_mv -uroot -pmvuser mv_metplus_test < /METviewer-python/METdataio/METdbLoad/sql/mv_mysql.sql"
 cmd+=";mysql -hmysql_mv -uroot -pmvuser -e\"show databases;\""
 
 # execute commands inside metviewer container to create database

diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py
@@ -72,13 +72,13 @@ def handle_automation_env(host_name, reqs, work_dir):
             'cd -;'
         )
 
-    # if metdatadb is in requirements list, add command to obtain METdatadb
-    if 'metdatadb' in str(reqs).lower():
+    # if metdataio is in requirements list, add command to obtain METdataio
+    if 'metdataio' in str(reqs).lower():
         setup_env += (
             f'cd {METPLUS_DOCKER_LOC};'
             f'{work_dir}/manage_externals/checkout_externals'
-            f' -e {work_dir}/.github/parm/Externals_metdatadb.cfg;'
-            f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METdatadb;'
+            f' -e {work_dir}/.github/parm/Externals_metdataio.cfg;'
+            f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METdataio;'
             'cd -;'
         )
 

diff --git a/.github/jobs/get_use_cases_to_run.sh b/.github/jobs/get_use_cases_to_run.sh
@@ -1,6 +1,7 @@
 #! /bin/bash
 
 use_case_groups_filepath=.github/parm/use_case_groups.json
+
 # set matrix to string of an empty array in case no use cases will be run
 matrix="[]"
 
@@ -31,12 +32,14 @@ fi
 if [ "$run_unit_tests" == "true" ]; then
   echo Adding unit tests to list to run
 
+  pytests="\"pytests\","
+
   # if matrix is empty, set to an array that only includes pytests
   if [ "$matrix" == "[]" ]; then
-    matrix="[\"pytests\"]"
+    matrix="[${pytests:0: -1}]"
   # otherwise prepend item to list
   else
-    matrix="[\"pytests\", ${matrix:1}"
+    matrix="[${pytests}${matrix:1}"
   fi
 fi
 

diff --git a/.github/jobs/run_difference_tests.sh b/.github/jobs/run_difference_tests.sh
@@ -17,7 +17,7 @@ if [ "$( ls -A ${RUNNER_WORKSPACE}/diff)" ]; then
   echo ::set-output name=upload_diff::true
   mkdir -p artifact/diff-${artifact_name}
   cp -r ${RUNNER_WORKSPACE}/diff/* artifact/diff-${artifact_name}
-  exit 1
+  exit 2
 fi
 
 echo ::set-output name=upload_diff::false
diff --git a/.github/parm/Externals_metdatadb.cfg → .github/parm/Externals_metdataio.cfg b/.github/parm/Externals_metdatadb.cfg → .github/parm/Externals_metdataio.cfg
@@ -1,8 +1,8 @@
 [METdatadb]
-local_path = ../METdatadb
+local_path = ../METdataio
 protocol = git
 required = True
-repo_url = https://github.com/dtcenter/METdatadb
+repo_url = https://github.com/dtcenter/METdataio
 branch = develop
 
 [externals_description]

diff --git a/.github/parm/pytest_groups.txt b/.github/parm/pytest_groups.txt
@@ -0,0 +1,6 @@
+util
+wrapper
+wrapper_a
+wrapper_b
+wrapper_c
+plotting_or_long
diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json
@@ -25,32 +25,32 @@
     "run": false
   },
   {
-    "category": "convection_allowing_models",
+    "category": "short_range",
     "index_list": "0",
     "run": false
   },
   {
-    "category": "convection_allowing_models",
+    "category": "short_range",
     "index_list": "1",
     "run": false
   },
   {
-    "category": "convection_allowing_models",
+    "category": "short_range",
     "index_list": "2-6",
     "run": false
   },
   {
-    "category": "convection_allowing_models",
+    "category": "short_range",
     "index_list": "7",
     "run": false
   },
   {
-    "category": "convection_allowing_models",
+    "category": "short_range",
     "index_list": "8",
     "run": false
   },
   {
-    "category": "convection_allowing_models",
+    "category": "short_range",
     "index_list": "9",
     "run": false
   },
@@ -136,47 +136,52 @@
   },
   {
     "category": "s2s",
-    "index_list": "1-3",
+    "index_list": "1",
     "run": false
   },
   {
     "category": "s2s",
-    "index_list": "4",
+    "index_list": "2",
     "run": false
   },
   {
     "category": "s2s",
-    "index_list": "5",
+    "index_list": "3",
     "run": false
   },
   {
     "category": "s2s",
-    "index_list": "6-7",
+    "index_list": "4",
     "run": false
   },
   {
     "category": "s2s",
-    "index_list": "7-9",
+    "index_list": "5",
     "run": false
   },
   {
     "category": "s2s",
-    "index_list": "10",
+    "index_list": "6",
     "run": false
   },
   {
-    "category": "s2s",
-    "index_list": "11",
+    "category": "s2s_mid_lat",
+    "index_list": "0-2",
     "run": false
   },
   {
-    "category": "s2s",
-    "index_list": "12",
-    "run": false 
+    "category": "s2s_mid_lat",
+    "index_list": "3",
+    "run": false
   },
   {
-    "category": "s2s",
-    "index_list": "13",
+    "category": "s2s_mjo",
+    "index_list": "0-2",
+    "run": false
+  },
+  {
+    "category": "s2s_mjo",
+    "index_list": "3",
     "run": false
   },
   {

diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
@@ -117,6 +117,9 @@ jobs:
 
       - uses: actions/checkout@v2
 
+      - name: Free disk space
+        run: .github/jobs/free_disk_space.sh
+
       - name: Create directories for database
         run: .github/jobs/create_dirs_for_database.sh
 
@@ -139,24 +142,24 @@ jobs:
       # copy logs with errors to error_logs directory to save as artifact
       - name: Save error logs
         id: save-errors
-        if: ${{ always() && steps.run_tests.conclusion == 'failure' && matrix.categories != 'pytests' }}
+        if: ${{ always() && steps.run_tests.conclusion == 'failure' && !startsWith(matrix.categories,'pytests') }}
         run: .github/jobs/save_error_logs.sh
 
       # run difference testing
       - name: Run difference tests
         id: run-diff
-        if: ${{ needs.job_control.outputs.run_diff == 'true' && steps.run_tests.conclusion == 'success'  && matrix.categories != 'pytests' }}
+        if: ${{ needs.job_control.outputs.run_diff == 'true' && steps.run_tests.conclusion == 'success'  && !startsWith(matrix.categories,'pytests') }}
         run: .github/jobs/run_difference_tests.sh ${{ matrix.categories }} ${{ steps.get-artifact-name.outputs.artifact_name }}
 
       # copy output data to save as artifact
       - name: Save output data
         id: save-output
-        if: ${{ always() && steps.run_tests.conclusion != 'skipped'  && matrix.categories != 'pytests' }}
+        if: ${{ always() && steps.run_tests.conclusion != 'skipped'  && !startsWith(matrix.categories,'pytests') }}
         run: .github/jobs/copy_output_to_artifact.sh ${{ steps.get-artifact-name.outputs.artifact_name }}
 
       - name: Upload output data artifact
         uses: actions/upload-artifact@v2
-        if: ${{ always() && steps.run_tests.conclusion != 'skipped'  && matrix.categories != 'pytests' }}
+        if: ${{ always() && steps.run_tests.conclusion != 'skipped'  && !startsWith(matrix.categories,'pytests') }}
         with:
           name: ${{ steps.get-artifact-name.outputs.artifact_name }}
           path: artifact/${{ steps.get-artifact-name.outputs.artifact_name }}