Update develop-ref after #868 (#873)

Co-authored-by: George McCabe <[email protected]> Co-authored-by: George McCabe <[email protected]> Co-authored-by: j-opatz <[email protected]> Co-authored-by: jprestop <[email protected]> Co-authored-by: John Halley Gotway <[email protected]>
dtcenter · Apr 12, 2021 · 1b3776c · 1b3776c
1 parent f3bbc32
commit 1b3776c
Show file tree

Hide file tree

Showing 92 changed files with 1,678 additions and 1,000 deletions.
diff --git a/.github/jobs/build_documentation.sh b/.github/jobs/build_documentation.sh
@@ -0,0 +1,24 @@
+#! /bin/bash
+
+# path to docs directory relative to top level of repository
+# $GITHUB_WORKSPACE is set if the actions/checkout@v2 action is run first
+
+DOCS_DIR=${GITHUB_WORKSPACE}/docs
+
+# run Make to build the documentation and return to previous directory
+cd ${DOCS_DIR}
+make clean html
+cd -
+
+# copy HTML output into directory to create an artifact
+mkdir -p artifact/documentation
+cp -r ${DOCS_DIR}/_build/html/* artifact/documentation
+
+# check if the warnings.log file is empty
+# Copy it into the artifact and documeentation directories
+# so it will be available in the artifacts
+warning_file=${DOCS_DIR}/_build/warnings.log
+if [ -s $warning_file ]; then
+  cp -r ${DOCS_DIR}/_build/warnings.log artifact/doc_warnings.log
+  cp artifact/doc_warnings.log artifact/documentation
+fi
diff --git a/.github/labels/common_labels.txt b/.github/labels/common_labels.txt
@@ -35,11 +35,13 @@
 {"name": "requestor: NCAR/RAL","color": "3101c1","description": "NCAR Research Applications Laboratory"}
 {"name": "requestor: NCAR/MMM","color": "3101c1","description": "NCAR Meso and Microscale Meteorology Laboratory"}
 {"name": "requestor: NCAR/SIMA","color": "3101c1","description": "NCAR System for Integrated Modeling of the Atmosphere"}
+{"name": "requestor: NOAA/CPC","color": "3101c1","description": "NOAA Climate Prediction Center"}
 {"name": "requestor: NOAA/EMC","color": "3101c1","description": "NOAA Environmental Modeling Center"}
 {"name": "requestor: NOAA/GSL","color": "3101c1","description": "NOAA Global Systems Laboratory"}
 {"name": "requestor: NOAA/HFIP","color": "3101c1","description": "NOAA Hurricane Forecast Improvement Project"}
 {"name": "requestor: NOAA/MDL","color": "3101c1","description": "NOAA Meteorological Development Laboratory"}
 {"name": "requestor: NOAA/NSSL","color": "3101c1","description": "NOAA National Severe Storms Laboratory"}
+{"name": "requestor: NOAA/OPC","color": "3101c1","description": "NOAA Ocean Prediction Center"}
 {"name": "requestor: NOAA/other","color": "3101c1","description": "NOAA Laboratory, not otherwise specified"}
 {"name": "requestor: NOAA/PSD","color": "3101c1","description": "NOAA Physical Sciences Laboratory"}
 {"name": "requestor: NOAA/SWPC","color": "3101c1","description": "NOAA Space Weather Prediction Center"}

diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -0,0 +1,39 @@
+name: Documentation Workflow
+on:
+  push:
+    branches:
+      - develop
+      - develop-ref
+      - feature_*
+      - main_*
+      - bugfix_*
+    paths:
+      - docs/**
+  pull_request:
+    types: [opened, reopened, synchronize]
+
+jobs:
+  documentation:
+    name: Documentation
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.6'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade sphinx sphinx-gallery sphinx_rtd_theme
+          python -m pip install python-dateutil requests Pillow
+      - name: Build Documentation
+        continue-on-error: true
+        run: ./.github/jobs/build_documentation.sh
+      - uses: actions/upload-artifact@v2
+        with:
+          name: METplus_documentation
+          path: artifact/documentation
+      - uses: actions/upload-artifact@v2
+        with:
+          name: documentation_warnings.log
+          path: artifact/doc_warnings.log
+          if-no-files-found: ignore
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -1,12 +1,14 @@
-name: METplus CI/CD Workflow
+name: Testing Workflow
 on:
   push:
     branches:
       - develop
       - develop-ref
-      - 'feature_*'
-      - 'main_*'
-      - 'bugfix_*'
+      - feature_*
+      - main_*
+      - bugfix_*
+    paths-ignore:
+      - docs/**
   pull_request:
     types: [opened, reopened, synchronize]
 
@@ -29,47 +31,6 @@ jobs:
         with:
           name: job_control_status
           path: job_control_status
-
-  documentation:
-    name: Build Documentation
-    runs-on: ubuntu-latest
-    needs: job_control
-    steps:
-      - uses: actions/download-artifact@v2
-        with:
-          name: job_control_status
-      - run: cat job_control_status >> $GITHUB_ENV
-      - uses: actions/checkout@v2
-        if: ${{ env.run_docs == 'true' }}
-      - uses: actions/setup-python@v2
-        if: ${{ env.run_docs == 'true' }}
-        with:
-          python-version: '3.6'
-      - name: Install dependencies
-        if: ${{ env.run_docs == 'true' }}
-        run: |
-          python -m pip install --upgrade python-dateutil requests sphinx sphinx-gallery Pillow
-          pip install git+https://github.com/ESMCI/sphinx_rtd_theme@version-dropdown-with-fixes
-      - name: Run build_docs.py
-        if: ${{ env.run_docs == 'true' }}
-        continue-on-error: true
-        run: |
-          ${GITHUB_WORKSPACE}/docs/build_docs.py skip-doxygen
-          mkdir -p artifact/documentation
-          cp -r ${GITHUB_WORKSPACE}/docs/_build/html/* artifact/documentation
-          if [ "$( ls -A ${GITHUB_WORKSPACE}/docs/_build/warnings.log)" ]; then
-            cp -r ${GITHUB_WORKSPACE}/docs/_build/warnings.log artifact/doc_warnings.log
-            cp artifact/doc_warnings.log artifact/documentation
-          fi
-      - uses: actions/upload-artifact@v2
-        with:
-          name: METplus_documentation
-          path: artifact/documentation
-      - uses: actions/upload-artifact@v2
-        with:
-          name: documentation_warnings.log
-          path: artifact/doc_warnings.log
-          if-no-files-found: ignore
   get_image:
     name: Docker Setup - Get METplus Image
     runs-on: ubuntu-latest

diff --git a/ci/util/diff_util.py b/ci/util/diff_util.py
@@ -5,7 +5,6 @@
 import numpy
 
 IMAGE_EXTENSIONS = [
-    '.png',
     '.jpg',
     '.jpeg',
 ]
@@ -17,6 +16,14 @@
 
 SKIP_EXTENSIONS = [
     '.zip',
+    '.png',
+]
+
+PDF_EXTENSIONS = [
+    '.pdf',
+]
+
+UNSUPPORTED_EXTENSIONS = [
 ]
 
 PDF_EXTENSIONS = [
@@ -308,12 +315,43 @@ def compare_txt_files(filepath_a, filepath_b, dir_a=None, dir_b=None):
         print("Comparing stat file")
         header_a = lines_a.pop(0).split()[1:]
         header_b = lines_b.pop(0).split()[1:]
+    else:
+        header_a = header_b = None
 
     if len(lines_a) != len(lines_b):
         print(f"ERROR: Different number of lines in {filepath_b}")
         print(f" File_A: {len(lines_a)}\n File_B: {len(lines_b)}")
         return False
 
+    all_good = diff_text_lines(lines_a,
+                               lines_b,
+                               dir_a=dir_a,
+                               dir_b=dir_b,
+                               print_error=False,
+                               is_file_list=is_file_list,
+                               is_stat_file=is_stat_file,
+                               header_a=header_a)
+
+    # if differences found in text file, sort and try again
+    if not all_good:
+        lines_a.sort()
+        lines_b.sort()
+        all_good = diff_text_lines(lines_a,
+                                   lines_b,
+                                   dir_a=dir_a,
+                                   dir_b=dir_b,
+                                   print_error=True,
+                                   is_file_list=is_file_list,
+                                   is_stat_file=is_stat_file,
+                                   header_a=header_a)
+
+    return all_good
+
+def diff_text_lines(lines_a, lines_b,
+                    dir_a=None, dir_b=None,
+                    print_error=False,
+                    is_file_list=False, is_stat_file=False,
+                    header_a=None):
     all_good = True
     for line_a, line_b in zip(lines_a, lines_b):
         compare_a = line_a
@@ -331,12 +369,14 @@ def compare_txt_files(filepath_a, filepath_b, dir_a=None, dir_b=None):
                 cols_b = compare_b.split()[1:]
                 for col_a, col_b, label in zip(cols_a, cols_b, header_a):
                     if col_a != col_b:
-                        print(f"ERROR: {label} differs:\n"
-                              f" A: {col_a}\n B: {col_b}")
+                        if print_error:
+                            print(f"ERROR: {label} differs:\n"
+                                  f" A: {col_a}\n B: {col_b}")
                         all_good = False
             else:
-                print(f"ERROR: Line in {filepath_b} differs\n"
-                      f" A: {compare_a}\n B: {compare_b}")
+                if print_error:
+                    print(f"ERROR: Line differs\n"
+                          f" A: {compare_a}\n B: {compare_b}")
                 all_good = False
 
     return all_good

diff --git a/docs/Contributors_Guide/add_use_case.rst b/docs/Contributors_Guide/add_use_case.rst
@@ -16,32 +16,45 @@ instructions to fill out the template.
 This branch will be the source of the pull request to merge the changes into
 the develop branch.
 
+Types of Use Cases
+------------------
+
+* Use cases that involve a single MET tool/METplus wrapper will reside
+  in the *METplus/parm/use_cases/met_tool_wrapper* directory.
+
+* Use cases that involve multiple MET tools/METplus wrappers will reside
+  in the *METplus/parm/use_cases/model_applications* directory, under a
+  subdirectory that corresponds to a specific category.  
 
 .. _use_case_categories:
 
 Use Case Categories
 -------------------
 
-New use cases will be put in the repository under
+New MET tool wrapper use cases will be put in the repository under
+parm/use_cases/met_tool_wrapper/<MET TOOL NAME> where <MET TOOL NAME>
+is the name of the MET tool being wrapped.
+
+New model applications use cases will be put in the repository under
 parm/use_cases/model_applications/<CATEGORY> where <CATEGORY> is
 one of the following:
 
-* medium_range
-* s2s (Subseasonal to Seasonal)
+* air_quality_and_comp
+* climate
+* coastal
 * convection_allowing_models
-* data_assimilation
-* space_weather
-* marine
 * cryosphere
-* coastal
-* air_quality
-* pbl
-* land_surface
+* data_assimilation
 * extremes
-* climate
+* land_surface
+* marine_and_coastal
+* medium_range
+* miscellaneous
+* pbl
 * precipitation
+* s2s (Subseasonal to Seasonal)
+* space_weather
 * tc_and_extra_tc (Tropcial Cyclone and Extra Tropical Cyclone)
-* miscellaneous
 
 If you feel that the new use case does not fall into any of these categories
 or are unsure which category is the most appropriate, please contact MET Help
@@ -53,8 +66,19 @@ Use Case Content
 Configure New Use Case
 ^^^^^^^^^^^^^^^^^^^^^^
 
-In the category sub-directory (parm/use_cases/model_applications/<CATEGORY>),
-each use case should have the following:
+If creating a new MET tool wrapper use case, in the MET tool name
+sub-directory (parm/use_cases/met_tool_wrapper/<MET TOOL NAME>), each
+use case should have the following:
+
+* A METplus configuration file where the MET tool name follows PascalCase,
+  e.g. GridStat.conf or ASCII2NC.conf.
+  If the use case uses a Python embedding script, it should be
+  indicasted in the by adding "_python_embedding" to the MET tool name.
+  e.g. GridStat_python_embedding.conf
+
+If creating a new model applications use case, in the category sub-directory
+(parm/use_cases/model_applications/<CATEGORY>), each use case should have the
+following:
 
 * A METplus configuration file named
   \<MET-TOOL\>_fcst\<FCST\>_obs\<OBS\>_cilmo\<CLIMO\>\<DESCRIPTOR\>.conf where
@@ -150,13 +174,15 @@ would look something like this::
 
 The content of this file is rendered above the icons for the use cases in this
 category in the User's Guide > METplus Use Cases >
-`Model Applications <https://dtcenter.github.io/METplus/latest/generated/model_applications/index.html>`_
+`Model Applications <https://metplus.readthedocs.io/en/latest/generated/model_applications/index.html>`_
 page.
 
 Add Sphinx Documentation File
 """""""""""""""""""""""""""""
 
-In the corresponding documentation category directory
+In the corresponding documentation MET tool name directory
+(**docs**/use_cases/met_tool_wrapper/<MET TOOL NAME>) for a met_tool_wrappers
+use case OR category directory for a model_applications use case
 (**docs**/use_cases/model_applications/<CATEGORY>), add:
 
 * A Python Sphinx Documentation (.py) file with the same name as the METplus
@@ -179,7 +205,7 @@ In the corresponding documentation category directory
       a list of possible keywords to use (Note: The link text for the
       keywords must match the actual keyword exactly or it will not
       show up in the search, i.e. **ASCII2NCToolUseCase** must match
-      https://dtcenter.github.io/METplus/search.html?q=**ASCII2NCToolUseCase**
+      https://metplus.readthedocs.io/en/latest/search.html?q=**ASCII2NCToolUseCase**
 
     * Add an image to use as the thumbnail (if desired). Images can be added
       to the docs/_static directory and should be named <category>-<conf>.png
@@ -197,9 +223,23 @@ In the corresponding documentation category directory
     avoid ending a line with this character to avoid generating warnings in the
     documentation.
 
+Accessing the Documentation
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Build the Documentation
-^^^^^^^^^^^^^^^^^^^^^^^
+It is important to ensure that the new use case files is displayed and the
+formatting looks correct. Prior to the release of METplus v4.0.0 contributors
+were required to build the documentation manually.  However, the METplus
+components now use Read the Docs to build and display the documentation. For
+more information on how to view the newly added use case, see the 
+:ref:`Read the Docs METplus Documenation <read-the-docs>`.  Contributors can
+still build the documentation manually if desired. See the
+:ref:`Build the Documentation Manually <manual-build>` section below for more
+information.
+
+.. _manual-build:
+
+Build the Documentation Manually
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Build the documentation and ensure that the new use case file is
 displayed and the formatting looks correct. The Python packages sphinx,
@@ -1113,4 +1153,3 @@ Remove the tarfile and environment file from the staging directory::
 
     ls ${METPLUS_USER_ENV_FILE}
     rm ${METPLUS_USER_ENV_FILE}
-