Initialize data version control for managing test images

.dvc/.gitignore

@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache

.dvc/config

@@ -0,0 +1,5 @@
+[core]
+    analytics = false
+    remote = origin
+['remote "origin"']
+    url = https://dagshub.com/GenericMappingTools/gmt.dvc

.dvcignore

@@ -0,0 +1,6 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore
+
+# macOS
+.DS_Store

.github/workflows/tests.yml

@@ -118,6 +118,10 @@ jobs:
           name: gmt-cache
           path: .gmt
 
+      # Pull baseline image data from dvc remote (DAGsHub)
+      - name: Pull baseline image data from dvc remote
+        run: dvc pull
+
       # Move downloaded files to ~/.gmt directory and list them
       - name: Move and list downloaded remote files
         run: |

BUILDING.md

@@ -55,6 +55,7 @@ Install for building GMT documentation and running tests (not required for gener
 
 - [Sphinx](http://www.sphinx-doc.org) (>=1.8, for building the documentation)
 - [GraphicsMagick](http://www.graphicsmagick.org/) (for running the tests)
+- [dvc](https://dvc.org/) (for running the tests)
 - [Ninja](https://ninja-build.org/) (optional, build system focused on speed)
 - [pngquant](https://pngquant.org/) (optional, for optimizing PNG images in the documentation)
 

ci/install-dependencies-linux.sh

@@ -47,4 +47,9 @@ if [ "$BUILD_DOCS" = "true" ]; then
     echo "$(python3 -m site --user-base)/bin" >> $GITHUB_PATH
 fi
 
+# Install dvc for running GMT tests
+if [ "$RUN_TESTS" = "true" ]; then
+    pip3 install --user dvc
+fi
+
 set +x +e

ci/install-dependencies-macos.sh

@@ -26,6 +26,7 @@ fi
 # packages for running GMT tests
 if [ "$RUN_TESTS" = "true" ]; then
     packages+=" graphicsmagick"
+    pip3 install --user dvc
 fi
 
 if [ "$PACKAGE" = "true" ]; then

ci/install-dependencies-windows.sh

@@ -49,6 +49,7 @@ fi
 
 if [ "$RUN_TESTS" = "true" ]; then
     choco install graphicsmagick
+    pip install --user dvc
     # Add GraphicsMagick to PATH
     echo 'C:\Program Files\GraphicsMagick-1.3.32-Q8' >> $GITHUB_PATH
 fi

doc/rst/source/devdocs/contributing.rst

@@ -275,7 +275,7 @@ The animations are built from the scripts in ``doc/examples/anim*/``. To add a n
 
     .. youtube:: Pvvc4vb8G4Y
       :width: 100%
-  
+
       :doc:`/animations/anim??`
 
 - :ref:`Submit a pull request <devdocs/contributing:Pull Request Workflow>` with your new animation. Please be sure
@@ -303,7 +303,6 @@ Here are some specific guidelines:
 - Use tabs, rather than spaces, for indentation.
 - Try to split lines at ~120 characters.
 
-
 Testing GMT
 ~~~~~~~~~~~
 
@@ -320,6 +319,10 @@ Tests that are known to fail are excluded by adding ``# GMT_KNOWN_FAILURE`` anyw
 Running tests
 ^^^^^^^^^^^^^
 
+First, pull any baseline images stored in the DAGsHub repository using dvc::
+
+  dvc pull
+
 After configuring CMake and building GMT, you can run all the tests by running this command in the build directory::
 
   cmake --build . --target check
@@ -408,6 +411,63 @@ To add a non-PostScript based test (e.g., `gmean.sh <https://github.com/GenericM
 - Check that your new test works using the instructions in the :ref:`running tests <devdocs/contributing:Running tests>`
   section.
 
+Managing Test Images Using Data Version Control (dvc)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+As the baseline images are large blob files that can change often, it is not ideal to store them in ``git``. Instead,
+we use `data version control (dvc) <https://dvc.org/>`_ to track the test images, which is like ``git`` but for data.
+``dvc`` stores the hash (md5sum) of a file or an md5sum that describes the contents of a directory. For each test
+``test/<module>/*.sh`` that generates a .PS file, there is a baseline image file in ``test/baseline/<module>/``
+that is compared to the test result using `GraphicsMagick <http://www.graphicsmagick.org/>`_. Each of the
+directories ``test/baseline/<module>`` are tracked by ``dvc`` using the file ``test/baseline/<module>.dvc``. This file
+contains the hash of a JSON .dir file stored in the .dvc cache. The .dir file contains information about each tracked
+file in the directory, which is used to push/pull the files to/from remote storage. The ``test/baseline/<module>.dvc``
+files are stored as usual on GitHub, while the .PS files are stored separately on the ``dvc`` remote at
+https://dagshub.com/GenericMappingTools/gmt.
+
+Setting up your local environment for dvc
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+#. `Install dvc <https://dvc.org/doc/install>`_
+#. If you will need to push baseline images to the remote, ask a GMT maintainer to add you as a collaborator on
+   `DAGsHub <https://dagshub.com/GenericMappingTools/gmt>`_.
+#. If you will need to push baseline imaged to the remote, set up
+   `authentication for the DVC remote <https://dagshub.com/docs/reference/dagshub_storage/#pushing-files-or-using-a-private-repo>`_.
+
+Pulling files from the remote for testing
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+To pull or sync files from the ``dvc`` remote to your local repository, the commands are similar to ``git``:
+
+::
+
+    dvc status  # should report any files 'not_in_cache'
+    dvc pull    # pull down files from DVC remote cache (fetch + checkout)
+
+
+Once the sync is complete, you should notice that there are images stored in the ``test/baseline/<module>``
+directories (e.g., ``test/baseline/api/api_matrix_as_grid.ps``). These images are technically reflinks/symlinks/copies
+of the files under the ``.dvc/cache`` directory. You can now run the test suite as usual.
+
+Migrating existing test images to dvc
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+#. Sync with git and dvc remotes (``git checkout master; git pull; dvc pull``).
+#. Create a branch for the module that you are working on (``git switch -c migrate-<module>-tests``).
+#. Navigate to the test directory (``cd test``).
+#. Remove the test image from git tracking (``git rm --cached <module>/<test-image>.ps``).
+#. Create a new directory for the image in ``baseline``, if one does not already exist (``mkdir baseline/<module>``).
+#. Move the test image to the new directory (``mv <module>/<test-image>.ps baseline/<module>/``).
+#. Add a comment ``# DVC_TEST`` to the test script that references the image. This is used to specify that the testing
+   mechanism should search in ``baseline/<module>`` for the baseline image.
+#. Repeat steps 4, 6, and 7 for other .PS based tests for the module.
+#. Run the GMT test suite to check that the tests work properly with the new structure.
+#. Add the directory to dvc (``dvc add baseline/<module>``).
+#. Commit the .dvc and modified .gitignore files to git (``git add baseline/<module>.dvc baseline/.gitignore; git commit``).
+#. Push the changes to git (``git push -u origin migrate-<module>-tests``).
+#. Push the changes to dvc (``dvc push``).
+#. Open a pull request for the changes.
+
 Debugging GMT
 ~~~~~~~~~~~~~
 

test/api/apicubeplot.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 #
 # Test the C API for reading via a matrix and making a plot
+# DVC_TEST
 ps=apicubeplot.ps
 if [ ! -f cube.nc ]; then
 	cp "${src}"/../grdinterpolate/cube.nc .

test/api/apigrdpix.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 #
 # Test the C API for passing matrices to grdimage as grids
+# DVC_TEST
 ps=api_matrix_as_grid.ps
 testapi_matrix_as_grid

test/api/apimat2grd.sh

@@ -2,6 +2,7 @@
 #
 # Test the C API for passing a global grid as a matrix to grdimage
 # and have the central meridian work correctly.
+# DVC_TEST
 
 ps=apimat2grd.ps
 testapi_grid2matrix

test/api/apimat_360.sh

@@ -2,6 +2,7 @@
 #
 # Test the C API for passing a global grid as a matrix to grdimage
 # and have the central meridian work correctly.
+# DVC_TEST
 
 ps=apimat_360.ps
 testapi_matrix_360 > $ps

test/api/apimat_360_ref.sh

@@ -2,6 +2,7 @@
 #
 # Test the C API for passing a global grid as a matrix to grdimage via reference
 # and have the central meridian work correctly.
+# DVC_TEST
 
 ps=apimat_360_ref.ps
 testapi_matrix_360_ref > $ps

test/api/apimat_io.sh

@@ -2,6 +2,7 @@
 #
 # Test the C API for reading and writing native matrices.
 # We read @hotspots.txt and write it out again, then plot it.
+# DVC_TEST
 
 ps=apimat_io.ps
 testapi_matrix_io

test/api/apimatgrd.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 #
 # Test the C API for reading grd plotting matrix
+# DVC_TEST
 ps=apimatgrd.ps
 testapi_matrix_plot > $ps

test/api/apimatimageplain.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 #
 # Test the C API for reading via a matrix and making a plot
+# DVC_TEST
 ps=apimatimageplain.ps
 testapi_imageshading > $ps

test/api/apimatimageshading.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 #
 # Test the C API for reading via a matrix and making a plot with shading
+# DVC_TEST
 ps=apimatimageshading.ps
 testapi_imageshading -I > $ps

test/api/apiuserstrings.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 #
 # Test the C API for passing vectors and strings to pstext
+# DVC_TEST
 ps=apiuserstrings.ps
 testapi_vector_strings > $ps

test/api/apivec_io.sh

@@ -2,6 +2,7 @@
 #
 # Test the C API for reading and writing native vectors.
 # We read @hotspots.txt and write it out again, then plot it.
+# DVC_TEST
 
 ps=apivec_io.ps
 testapi_vector_io

test/api/apivectorplot.sh

@@ -2,5 +2,6 @@
 #
 # Test the C API for plotting lines with arrows
 # See https://github.com/GenericMappingTools/gmt/pull/3528
+# DVC_TEST
 ps=apivectorplot.ps
 testapi_vector_plot > $ps

test/baseline/.gitignore

@@ -0,0 +1,2 @@
+/api
+/blockmean

test/baseline/api.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: f60c23e8437c61d49cd871550c311f8e.dir
+  size: 2988604
+  nfiles: 14
+  path: api

test/baseline/blockmean.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: 42d6184ac62a6af99f89ffcc0b888ae0.dir
+  size: 26174
+  nfiles: 1
+  path: blockmean

test/blockmean/xy2ij.sh

@@ -5,6 +5,7 @@
 # Points are prepared that will land in various tiles, some will
 # fall exactly on tile boundaries.  One point will fall outside -R
 # but should be considered for the gridline registrated case.
+# DVC_TEST
 
 ps=xy2ij.ps
 

test/gmtest.in

@@ -16,6 +16,7 @@ script_dir=$(dirname "${script_name}")
 local_script=$(basename "${script_name}")
 script="@GMT_SOURCE_DIR@/${script_name}"
 src="@GMT_SOURCE_DIR@/${script_dir}"
+baseline="@GMT_SOURCE_DIR@/test/baseline/${script_dir#test}"
 
 # Skip tests with label GMT_DISABLE_TEST
 skip_this_test=$(grep GMT_DISABLE_TEST "$script" -c)
@@ -24,6 +25,7 @@ if [ $skip_this_test -gt 0 ]; then	# Do not run this script, probably PS is too
 		exit 0
 fi
 
+dvc_test=$(grep DVC_TEST "$script" -c)
 # Is it a modern mode or one-liner script
 modern=$(grep "gmt begin\| -ps " "$script" -c)
 
@@ -156,7 +158,11 @@ for ps in *.ps ; do
       echo "$now ${script_dir}/${ps}: RMS Error = N/A Limit = $GRAPHICSMAGICK_RMS Failure = known" >> "@CMAKE_CURRENT_BINARY_DIR@/fail_count.txt"
   else
     # syntax: gm compare [ options ... ] reference-image [ options ... ] compare-image [ options ... ]
-    rms=$("${GRAPHICSMAGICK}" compare -density 200 -maximum-error $GRAPHICSMAGICK_RMS -highlight-color magenta -highlight-style assign -metric rmse -file "${ps%.ps}.png" "$ps" "$src/${psref:-$ps}") || pscmpfailed="yes"
+    if [ $dvc_test -gt 0 ]; then
+      rms=$("${GRAPHICSMAGICK}" compare -density 200 -maximum-error $GRAPHICSMAGICK_RMS -highlight-color magenta -highlight-style assign -metric rmse -file "${ps%.ps}_diff.png" "$ps" "$baseline/${psref:-$ps}") || pscmpfailed="yes"
+    else
+      rms=$("${GRAPHICSMAGICK}" compare -density 200 -maximum-error $GRAPHICSMAGICK_RMS -highlight-color magenta -highlight-style assign -metric rmse -file "${ps%.ps}_diff.png" "$ps" "$src/${psref:-$ps}") || pscmpfailed="yes"
+    fi
     rms=$(perl -ne 'print $1 if /Total: ([0-9.]+)/' <<< "$rms")
     if [ -z "$rms" ]; then
       rms="NA"