diff --git a/.github/jobs/copy_diff_files.py b/.github/jobs/copy_diff_files.py index fe12f330e8..baf741b408 100755 --- a/.github/jobs/copy_diff_files.py +++ b/.github/jobs/copy_diff_files.py @@ -71,12 +71,9 @@ def copy_files_to_diff_dir(files_to_copy): shutil.copyfile(filename, output_path) def main(): - all_files_to_copy = set() - log_files = glob.glob(os.path.join(LOG_DIR, 'comp_dir*.log')) - for log_file in log_files: - print(f"Parsing {log_file}") - new_files = get_files_with_diffs(log_file) - all_files_to_copy.update(new_files) + log_file = glob.glob(os.path.join(LOG_DIR, 'comp_dir.log')) + print(f"Parsing {log_file}") + all_files_to_copy = get_files_with_diffs(log_file) copy_files_to_diff_dir(all_files_to_copy) diff --git a/.github/jobs/run_diff_docker.sh b/.github/jobs/run_diff_docker.sh index 2ed59c5804..6334769d23 100755 --- a/.github/jobs/run_diff_docker.sh +++ b/.github/jobs/run_diff_docker.sh @@ -19,11 +19,14 @@ mount_args="-v $LOCAL_OUTPUT_DIR:$DOCKER_OUTPUT_DIR -v $LOCAL_DIFF_DIR:$DOCKER_D # Set up data volumes volumes_from="--volumes-from met_truth" -export TESTS_TO_RUN=$TESTS - # run unit test script inside Docker, mount MET input and truth data cmd="\${MET_REPO_DIR}/.github/jobs/run_diff_tests.sh" -time_command docker run -e TESTS_TO_RUN ${volumes_from} ${mount_args} ${DOCKERHUB_TAG} bash -c \"${cmd}\" +time_command docker run ${volumes_from} ${mount_args} ${DOCKERHUB_TAG} bash -c \"${cmd}\" if [ $? != 0 ]; then exit 1 fi + +if [ "$(ls -A ${LOCAL_DIFF_DIR})" ]; then + echo "ERROR: Differences exist in the output" + exit 1 +fi diff --git a/.github/jobs/run_diff_tests.sh b/.github/jobs/run_diff_tests.sh index 911037c794..938eff4349 100755 --- a/.github/jobs/run_diff_tests.sh +++ b/.github/jobs/run_diff_tests.sh @@ -13,20 +13,15 @@ source ${MET_REPO_DIR}/.github/jobs/test_env_vars.sh ### echo "Running comparison on test output" -for testname in $TESTS_TO_RUN; do - CMD_LOGFILE=/met/logs/comp_dir_${testname}.log - time_command ${MET_TEST_BASE}/bin/comp_dir.sh ${MET_TEST_TRUTH}/${testname} ${MET_TEST_OUTPUT}/${testname} - if [ $? != 0 ]; then - echo "ERROR: Test ${testname} output comparison failed" - exit 1 - fi -done - -test_list=($TESTS_TO_RUN) -first_test=${test_list[0]} +CMD_LOGFILE=/met/logs/comp_dir.log +time_command ${MET_TEST_BASE}/bin/comp_dir.sh ${MET_TEST_TRUTH} ${MET_TEST_OUTPUT} +if [ $? != 0 ]; then + echo "ERROR: Test output comparison failed" + exit 1 +fi echo "Running copy_diff_files.py" -CMD_LOGFILE=/met/logs/copy_diff_files_${first_test}_group.log +CMD_LOGFILE=/met/logs/copy_diff_files.log time_command ${MET_REPO_DIR}/.github/jobs/copy_diff_files.py if [ $? != 0 ]; then echo "ERROR: Copy diff files script failed" diff --git a/.github/workflows/pr_unit_tests.yml b/.github/workflows/pr_unit_tests.yml index e9abf025e0..544a0417da 100644 --- a/.github/workflows/pr_unit_tests.yml +++ b/.github/workflows/pr_unit_tests.yml @@ -73,7 +73,7 @@ jobs: path: ${{ runner.workspace }}/logs if-no-files-found: ignore - unit_tests_1: + unit_tests_1a: name: Run MET Unit Tests runs-on: ubuntu-latest needs: [job_control, compile] @@ -81,9 +81,10 @@ jobs: strategy: matrix: tests: - - 'ref_config' - - 'tc_dland tc_pairs tc_stat plot_tc tc_rmw rmw_analysis tc_gen' - - 'met_test_scripts mode_graphics mtd regrid airnow gsi_tools netcdf modis series_analysis gen_ens_prod wwmca_regrid gen_vx_mask grid_weight interp_shape grid_diag test_grib_tables lidar2nc shift_data_plane trmm2nc aeronet wwmca_plot ioda2nc gaussian' + - 'pcp_combine' + - 'pb2nc' + - 'ascii2nc' + - 'madis2nc' fail-fast: false steps: - uses: actions/checkout@v2 @@ -96,21 +97,11 @@ jobs: TESTS: ${{ matrix.tests }} INPUT_DATA_VERSION: develop - - name: Run Diff Tests in Docker - run: .github/jobs/run_diff_docker.sh - env: - DOCKERHUB_REPO: ${{ needs.job_control.outputs.dockerhub_repo }} - SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} - TESTS: ${{ matrix.tests }} - TRUTH_DATA_VERSION: develop - - - name: Upload diff files as artifact - if: always() + - name: Upload output as artifact uses: actions/upload-artifact@v2 with: - name: diff - path: ${{ runner.workspace }}/diff - if-no-files-found: ignore + name: unit_tests_1a + path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() @@ -120,7 +111,7 @@ jobs: path: ${{ runner.workspace }}/logs if-no-files-found: ignore - unit_tests_2a: + unit_tests_1b: name: Run MET Unit Tests runs-on: ubuntu-latest needs: [job_control, compile] @@ -128,10 +119,9 @@ jobs: strategy: matrix: tests: - - 'pcp_combine' - - 'pb2nc' - - 'ascii2nc' - - 'madis2nc' + - 'ref_config' + - 'tc_dland tc_pairs tc_stat plot_tc tc_rmw rmw_analysis tc_gen' + - 'met_test_scripts mode_graphics mtd regrid airnow gsi_tools netcdf modis series_analysis gen_ens_prod wwmca_regrid gen_vx_mask grid_weight interp_shape grid_diag test_grib_tables lidar2nc shift_data_plane trmm2nc aeronet wwmca_plot ioda2nc gaussian' fail-fast: false steps: - uses: actions/checkout@v2 @@ -144,21 +134,55 @@ jobs: TESTS: ${{ matrix.tests }} INPUT_DATA_VERSION: develop - - name: Run Diff Tests in Docker - run: .github/jobs/run_diff_docker.sh + - name: Upload output as artifact + uses: actions/upload-artifact@v2 + with: + name: unit_tests_1b + path: ${{ runner.workspace }}/output + + - name: Upload logs as artifact + if: always() + uses: actions/upload-artifact@v2 + with: + name: logs + path: ${{ runner.workspace }}/logs + if-no-files-found: ignore + + unit_tests_2a: + name: Run MET Unit Tests + runs-on: ubuntu-latest + needs: unit_tests_1a + if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} + strategy: + matrix: + tests: + - 'point_stat' + - 'grid_stat' + - 'wavelet_stat' + - 'ensemble_stat' + fail-fast: false + steps: + - uses: actions/checkout@v2 + + - name: Download 1a output from artifact + uses: actions/download-artifact@v2 + with: + name: unit_tests_1a + path: ${{ runner.workspace }}/output + + - name: Run Unit Tests in Docker + run: .github/jobs/run_unit_docker.sh env: DOCKERHUB_REPO: ${{ needs.job_control.outputs.dockerhub_repo }} SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} TESTS: ${{ matrix.tests }} - TRUTH_DATA_VERSION: develop + INPUT_DATA_VERSION: develop - - name: Upload diff files as artifact - if: always() + - name: Upload output as artifact uses: actions/upload-artifact@v2 with: - name: diff - path: ${{ runner.workspace }}/diff - if-no-files-found: ignore + name: unit_tests_2a + path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() @@ -171,18 +195,23 @@ jobs: unit_tests_2b: name: Run MET Unit Tests runs-on: ubuntu-latest - needs: [job_control, compile, unit_tests_2a] + needs: unit_tests_1a if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} strategy: matrix: tests: - 'climatology' - - 'point_stat grid_stat wavelet_stat ensemble_stat' - 'python point2grid plot_data_plane mode mode_analysis perc_thresh hira plot_point_obs quality_filter obs_summary duplicate_flag' fail-fast: false steps: - uses: actions/checkout@v2 + - name: Download 1a output from artifact + uses: actions/download-artifact@v2 + with: + name: unit_tests_1a + path: ${{ runner.workspace }}/output + - name: Run Unit Tests in Docker run: .github/jobs/run_unit_docker.sh env: @@ -191,21 +220,11 @@ jobs: TESTS: ${{ matrix.tests }} INPUT_DATA_VERSION: develop - - name: Run Diff Tests in Docker - run: .github/jobs/run_diff_docker.sh - env: - DOCKERHUB_REPO: ${{ needs.job_control.outputs.dockerhub_repo }} - SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} - TESTS: ${{ matrix.tests }} - TRUTH_DATA_VERSION: develop - - - name: Upload diff files as artifact - if: always() + - name: Upload output as artifact uses: actions/upload-artifact@v2 with: - name: diff - path: ${{ runner.workspace }}/diff - if-no-files-found: ignore + name: unit_tests_2b + path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() @@ -215,10 +234,11 @@ jobs: path: ${{ runner.workspace }}/logs if-no-files-found: ignore - unit_tests_2c: + + unit_tests_3a: name: Run MET Unit Tests runs-on: ubuntu-latest - needs: [job_control, compile, unit_tests_2b] + needs: unit_tests_2a if: ${{ needs.job_control.outputs.run_unit_tests == 'true' }} strategy: matrix: @@ -228,6 +248,12 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Download 2a output from artifact + uses: actions/download-artifact@v2 + with: + name: unit_tests_2a + path: ${{ runner.workspace }}/output + - name: Run Unit Tests in Docker run: .github/jobs/run_unit_docker.sh env: @@ -236,21 +262,11 @@ jobs: TESTS: ${{ matrix.tests }} INPUT_DATA_VERSION: develop - - name: Run Diff Tests in Docker - run: .github/jobs/run_diff_docker.sh - env: - DOCKERHUB_REPO: ${{ needs.job_control.outputs.dockerhub_repo }} - SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} - TESTS: ${{ matrix.tests }} - TRUTH_DATA_VERSION: develop - - - name: Upload diff files as artifact - if: always() + - name: Upload output as artifact uses: actions/upload-artifact@v2 with: - name: diff - path: ${{ runner.workspace }}/diff - if-no-files-found: ignore + name: unit_tests_3a + path: ${{ runner.workspace }}/output - name: Upload logs as artifact if: always() @@ -260,17 +276,24 @@ jobs: path: ${{ runner.workspace }}/logs if-no-files-found: ignore - check_for_diffs: + run_diffs: name: Check if there were any differences runs-on: ubuntu-latest - needs: [unit_tests_1, unit_tests_2c] + needs: [unit_tests_1b, unit_tests_2b, unit_tests_3a] steps: - uses: actions/download-artifact@v2 + + - name: Run Diff Tests in Docker + run: .github/jobs/run_diff_docker.sh + env: + DOCKERHUB_REPO: ${{ needs.job_control.outputs.dockerhub_repo }} + SOURCE_BRANCH: ${{ needs.job_control.outputs.branch_name }} + TRUTH_DATA_VERSION: develop + + - name: Upload diff files as artifact + if: always() + uses: actions/upload-artifact@v2 with: name: diff - path: diff - - run: | - if [ "$(ls -A diff)" ]; then - echo "ERROR: Differences exist in the output" - exit 1 - fi + path: ${{ runner.workspace }}/diff + if-no-files-found: ignore