Proposed new future minimum of blosc 1.17.0

Signed-off-by: Nick Avramoussis <[email protected]>

.github/workflows/weekly.yml

@@ -12,7 +12,7 @@ on:
   workflow_dispatch:
     inputs:
       type:
-        description: 'The type of CI to run (all, houdini, sanitize, mac, extra, ax)'
+        description: 'The type of CI to run (all, houdini, sanitize, mac, extra, ax, blosc)'
         required: true
         default: 'all'
 
@@ -43,12 +43,12 @@ jobs:
   # succeeds, put it into the GitHub Actions cache
   linux_houdini18_5:
     needs: [checksecret]
-    if: >
-      ${{ (needs.checksecret.outputs.HOUDINI_SECRETS == 'true' ||
-           github.repository_owner == 'AcademySoftwareFoundation') &&
-          (github.event_name != 'workflow_dispatch' ||
-           github.event.inputs.type == 'all' ||
-           github.event.inputs.type == 'houdini') }}
+    if: |
+      (needs.checksecret.outputs.HOUDINI_SECRETS == 'true' ||
+      github.repository_owner == 'AcademySoftwareFoundation') &&
+      (github.event_name != 'workflow_dispatch' ||
+      github.event.inputs.type == 'all' ||
+      github.event.inputs.type == 'houdini')
     runs-on: ubuntu-latest
     env:
       CXX: clang++
@@ -315,3 +315,38 @@ jobs:
         run: cd build && ctest -V
       - name: test_doxygen_examples
         run: ./ci/extract_test_examples.sh
+
+  #############################################################################
+  ################################## Blosc ####################################
+  #############################################################################
+
+  linux-blosc:
+    if: |
+      github.event_name != 'workflow_dispatch' ||
+      github.event.inputs.type == 'all' ||
+      github.event.inputs.type == 'blosc'
+    runs-on: ubuntu-latest
+    name: linux-blosc:${{ matrix.blosc }}
+    strategy:
+      matrix:
+        blosc: ['1.17.0','1.18.0'] # Need to fix unit tests for '1.19.0','1.20.0','1.21.0'
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v2
+    - name: install_boost
+      run: sudo apt-get -q install -y libboost-dev libboost-system-dev libboost-iostreams-dev libboost-python-dev
+    - name: install_tbb
+      run: sudo apt-get -q install -y libtbb-dev
+    - name: install_gtest
+      run: sudo apt-get -q install -y libgtest-dev
+    - name: install_blosc
+      shell: bash
+      run: sudo ./ci/install_blosc.sh ${{ matrix.blosc }}
+    - name: build
+      run: >
+        sudo ./ci/build.sh -v
+        --build-type=Release
+        --components=\"core,test\"
+    - name: test
+      run: cd build && sudo ctest -V
+

CMakeLists.txt

@@ -217,7 +217,6 @@ set(MINIMUM_CXX_STANDARD 14)
 
 # @note  Blosc version is currently treated as exception which must be adhered
 # to. The minimum version must be at least 1.5.0. Previous versions are incompatible.
-# Later versions (including 1.5.4), can be buggy on certain platforms.
 set(MINIMUM_BLOSC_VERSION 1.5.0)
 # @note  ABI always enforced so the correct deprecation messages are available.
 # OPENVDB_USE_DEPRECATED_ABI_<VERSION> should be used to circumvent this
@@ -262,6 +261,7 @@ set(FUTURE_MINIMUM_ICC_VERSION 19)
 # set(FUTURE_MINIMUM_ILMBASE_VERSION 2.4)
 # set(FUTURE_MINIMUM_OPENEXR_VERSION 2.4)
 set(FUTURE_MINIMUM_BOOST_VERSION 1.73)
+set(FUTURE_MINIMUM_BLOSC_VERSION 1.17.0)
 set(FUTURE_MINIMUM_TBB_VERSION 2020.2)
 set(FUTURE_MINIMUM_PYTHON_VERSION 3.7)
 set(FUTURE_MINIMUM_NUMPY_VERSION 1.17.0)

README.md

@@ -43,17 +43,7 @@ The following provides basic installation examples for the core OpenVDB library.
 apt-get install -y libboost-iostreams-dev
 apt-get install -y libboost-system-dev
 apt-get install -y libtbb-dev
-```
-```
-git clone [email protected]:Blosc/c-blosc.git
-cd c-blosc
-git checkout tags/v1.5.0 -b v1.5.0
-mkdir build
-cd build
-cmake ..
-make -j4
-make install
-cd ../..
+apt-get install -y libblosc-dev
 ```
 
 ##### Building OpenVDB
@@ -71,17 +61,7 @@ make install
 ```
 brew install boost
 brew install tbb
-```
-```
-git clone [email protected]:Blosc/c-blosc.git
-cd c-blosc
-git checkout tags/v1.5.0 -b v1.5.0
-mkdir build
-cd build
-cmake ..
-make -j4
-make install
-cd ../..
+brew install c-blosc
 ```
 ##### Building OpenVDB
 ```

ci/install_blosc.sh

@@ -3,6 +3,11 @@
 set -ex
 
 BLOSC_VERSION="$1"
+INSTALL_ROOT="$2"
+CMAKE_EXTRA=()
+if [ ! -z "${INSTALL_ROOT}" ]; then
+    CMAKE_EXTRA+=("-DCMAKE_INSTALL_PREFIX=${INSTALL_ROOT}")
+fi
 
 git clone https://github.com/Blosc/c-blosc.git
 cd c-blosc
@@ -13,6 +18,19 @@ fi
 
 mkdir build
 cd build
-cmake ../.
+
+# On MacOS there's a bug between blosc 1.14-1.20 where unistd isn't included
+# in zlib-1.2.8/gzlib.c. Provide -DPREFER_EXTERNAL_ZLIB to use the installed
+# version of zlib.
+# https://github.com/Blosc/python-blosc/issues/229
+cmake \
+    -DPREFER_EXTERNAL_ZLIB=ON \
+    -DBUILD_STATIC=OFF \
+    -DBUILD_TESTS=OFF \
+    -DBUILD_FUZZERS=OFF \
+    -DBUILD_BENCHMARKS=OFF \
+    "${CMAKE_EXTRA[@]}" \
+    ..
+
 make -j4
 make install

doc/build.txt

@@ -154,8 +154,8 @@ for this:
    that were created using blosc compression. This includes OpenVDB files from
    Houdini.
 
-Note however that OpenVDB currently uses a very old version of Blosc (v1.5.0).
-See the note in the [known issues list.](@ref depKnownIssues).
+See the note in the [known issues list](@ref depKnownIssues) regarding supported
+versions of blosc.
 
 @subsection buildZLibSupport ZLIB Support
 

doc/dependencies.txt

@@ -73,16 +73,15 @@ Python         | 2.7     | 2.7         | The python interpreter and libraries fo
 NumPy          | 1.14.0  | 1.14.0      | Scientific computing with Python                                  | Y       | Y        | http://www.numpy.org
 GoogleTest     | 1.8     | Latest      | A unit testing framework module for C++                           | Y       | Y        | https://github.com/google/googletest
 CppUnit        | 1.10    | Latest      | A unit testing framework module for C++                           | N       | Y        | https://freedesktop.org/wiki/Software/cppunit
-Blosc          | 1.5.0   | 1.5.0       | Recommended dependency for improved disk compression              | N       | N*       | https://github.com/Blosc/c-blosc/releases
+Blosc          | 1.5.0*  | 1.17.0      | Recommended dependency for improved disk compression              | Y       | Y        | https://github.com/Blosc/c-blosc/releases
 Log4cplus      | 1.1.2   | Latest      | An optional dependency for improved OpenVDB Logging               | Y       | Y        | https://github.com/log4cplus/log4cplus
 libpng         | -       | Latest      | Library for manipulating PNG images                               | Y       | Y        | http://www.libpng.org/pub/png/libpng.html
 GLFW           | 3.1     | Latest      | Simple API for OpenGL development                                 | Y       | Y        | https://www.glfw.org
 OpenGL         | 3.2     | Latest      | Environment for developing portable graphics applications         | Y       | Y        | https://www.opengl.org
 GLEW           | 1.0.0   | Latest      | A cross-platform OpenGL extension loading library.                | Y       | Y        | http://glew.sourceforge.net
 Doxygen        | 1.8.8   | 1.8.11      | Documentation generation from C++                                 | Y       | Y        | http://www.doxygen.nl
 
- - @b * Blosc 1.16 is only available through Homebrew and currently requires
- [manual](@ref depManInstall) installation for 1.5.
+ - @b * See the note in [known issues](@ref depKnownIssues) regarding supported blosc versions.
 
  - @b ** IlmBase/OpenEXR 2.2 is supported as Houdini 18.0 is shipped with
  VFX Reference Platform 2019, except for using older versions of these
@@ -101,11 +100,22 @@ Doxygen        | 1.8.8   | 1.8.11      | Documentation generation from C++
 RuntimeError: FATAL: module compiled as little endian, but detected different endianness at runtime.
 @endcode
 
- - @b Blosc: OpenVDB currently uses a very old version of blosc (v1.5.0). This
- is due to issues with compatibility between certain blosc versions. You can
- [read more about this problem here.](https://www.blosc.org/posts/new-forward-compat-policy/)
- Using blosc versions equal to or later than 1.14.0 should be forwards
- compatible but this remains untested.
+ - @b Blosc: OpenVDB has historically used an old version of blosc (1.5.0) to
+ serialize .vdb files. OpenVDB is moving to a new blosc version minimum of 1.17.0.
+ We have tested a range of blosc versions and found that the following versions
+ are NOT compatible with OpenVDB caches written using blosc 1.5.0. Additionally
+ these versions of blosc may generally not be compatible with OpenVDB itself.
+ Avoid using these versions and upgrade to the new minimum of 1.17.0 where
+ possible.
+
+ - All versions from 1.11.0 through to, but excluding, 1.14.0.
+ - Versions 1.16.0 and 1.16.1.
+
+ For more information see:
+
+ - https://www.blosc.org/posts/new-forward-compat-policy/
+ - https://github.com/Blosc/c-blosc/issues/260
+ - https://github.com/Blosc/c-blosc/issues/261
 
 @section depInstallingDependencies Installing Dependencies
 
@@ -114,9 +124,8 @@ required OpenVDB dependencies.
 
 @subsection depUsingAptGet Using UNIX apt-get
 
-@b Note: Blosc and CppUnit are unavailable using apt-get. See the other
-package manager methods or the [manual installation](@ref depManInstall)
-options.
+@b Note:CppUnit is unavailable using apt-get. See the other package manager
+methods or the [manual installation](@ref depManInstall) options.
 
 @code{.sh}
 #!/bin/bash
@@ -126,6 +135,7 @@ apt-get install libtbb-dev              # TBB
 apt-get install zlibc                   # zlib
 apt-get install libboost-system-dev     # Boost::system
 apt-get install libboost-iostreams-dev  # Boost::iostream
+apt-get install libblosc-dev            # Blosc
 # AX
 apt-get install llvm-8-dev              # LLVM
 # Python
@@ -150,18 +160,14 @@ apt-get install doxygen                 # doxygen
 [Homebrew](https://brew.sh/) is a package manager for UNIX systems and is
 an alternative to apt-get or manual installation from source.
 
-@b Note: Homebrew installs blosc 1.16 by default with no option for previous
-versions. OpenVDB recommends Blosc version 1.5 as there are known issues with
-using later versions of blosc. See the
-[manual installation](@ref depManInstall) options.
-
 @code{.sh}
 #!/bin/bash
 # Core library
 brew install cmake                     # CMake
 brew install tbb                       # TBB
 brew install zlib                      # zlib
 brew install boost                     # Boost
+brew install c-blosc                   # blosc
 # AX
 brew install llvm                      # LLVM
 # Python
@@ -188,21 +194,6 @@ Below are manual installation methods for these packages. Note that you will
 have to provide your chosen installation locations to the OpenVDB CMake build
 system when building OpenVDB.
 
-@b Blosc
-
-@code{.sh}
-#!/bin/bash
-wget https://github.com/Blosc/c-blosc/archive/v1.5.0.zip
-unzip v1.5.0.zip
-cd c-blosc-1.5.0
-# Install to users home directory - you may customise this location
-mkdir -p $HOME/blosc
-# Requires CMake
-cmake -DCMAKE_INSTALL_PREFIX=$HOME/blosc
-make
-make install
-@endcode
-
 @b CppUnit (for AX Unit Tests)
 
 @code{.sh}

openvdb/openvdb/CMakeLists.txt

@@ -158,12 +158,24 @@ endif()
 if(USE_BLOSC)
   # Find Blosc libraries
   find_package(Blosc ${MINIMUM_BLOSC_VERSION} REQUIRED)
-  if(Blosc_FOUND AND Blosc_VERSION VERSION_GREATER MINIMUM_BLOSC_VERSION)
-    message(WARNING "The version of Blosc located is greater than ${MINIMUM_BLOSC_VERSION}. "
-      "There have been reported issues with using later versions of Blosc with OpenVDB. "
-      "OpenVDB has been tested fully against Blosc ${MINIMUM_BLOSC_VERSION}, it is "
-      "recommended that you use this version where possible."
-    )
+  if(Blosc_VERSION)
+    if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_BLOSC_VERSION)
+      if(${Blosc_VERSION} VERSION_LESS FUTURE_MINIMUM_BLOSC_VERSION)
+        message(DEPRECATION "Support for Blosc versions < ${FUTURE_MINIMUM_BLOSC_VERSION} "
+          "is deprecated and will be removed.")
+      endif()
+    endif()
+    # Print a warning if using of the blosc versions with regression issues
+    # with Blosc 1.5.0 caches.
+    if((Blosc_VERSION VERSION_GREATER_EQUAL 1.11.0 AND
+        Blosc_VERSION VERSION_LESS 1.14.0) OR
+       (Blosc_VERSION VERSION_GREATER_EQUAL 1.16.0 AND
+        Blosc_VERSION VERSION_LESS 1.16.2))
+      message(WARNING "The following Blosc versions are incompatible with the "
+        "recommended builds of OpenVDB: [1.11.0 -> 1.14.0) [1.16.0, 1.16.1]. "
+        "Found Blosc version '${Blosc_VERSION}' which falls in this range. We "
+        "strongly recommend using the new future minimum version '${FUTURE_MINIMUM_BLOSC_VERSION}'")
+    endif()
   endif()
 else()
   message(WARNING "Blosc support is disabled. It is strongly recommended to "