diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml index abf9fea5cb..128de5d90a 100644 --- a/.github/workflows/weekly.yml +++ b/.github/workflows/weekly.yml @@ -12,7 +12,7 @@ on: workflow_dispatch: inputs: type: - description: 'The type of CI to run (all, houdini, sanitize, mac, extra, ax)' + description: 'The type of CI to run (all, houdini, sanitize, mac, extra, ax, blosc)' required: true default: 'all' @@ -43,12 +43,12 @@ jobs: # succeeds, put it into the GitHub Actions cache linux_houdini18_5: needs: [checksecret] - if: > - ${{ (needs.checksecret.outputs.HOUDINI_SECRETS == 'true' || - github.repository_owner == 'AcademySoftwareFoundation') && - (github.event_name != 'workflow_dispatch' || - github.event.inputs.type == 'all' || - github.event.inputs.type == 'houdini') }} + if: | + (needs.checksecret.outputs.HOUDINI_SECRETS == 'true' || + github.repository_owner == 'AcademySoftwareFoundation') && + (github.event_name != 'workflow_dispatch' || + github.event.inputs.type == 'all' || + github.event.inputs.type == 'houdini') runs-on: ubuntu-latest env: CXX: clang++ @@ -315,3 +315,38 @@ jobs: run: cd build && ctest -V - name: test_doxygen_examples run: ./ci/extract_test_examples.sh + + ############################################################################# + ################################## Blosc #################################### + ############################################################################# + + linux-blosc: + if: | + github.event_name != 'workflow_dispatch' || + github.event.inputs.type == 'all' || + github.event.inputs.type == 'blosc' + runs-on: ubuntu-latest + name: linux-blosc:${{ matrix.blosc }} + strategy: + matrix: + blosc: ['1.17.0','1.18.0'] # Need to fix unit tests for '1.19.0','1.20.0','1.21.0' + fail-fast: false + steps: + - uses: actions/checkout@v2 + - name: install_boost + run: sudo apt-get -q install -y libboost-dev libboost-system-dev libboost-iostreams-dev libboost-python-dev + - name: install_tbb + run: sudo apt-get -q install -y libtbb-dev + - name: install_gtest + run: sudo apt-get -q install -y libgtest-dev + - name: install_blosc + shell: bash + run: sudo ./ci/install_blosc.sh ${{ matrix.blosc }} + - name: build + run: > + sudo ./ci/build.sh -v + --build-type=Release + --components=\"core,test\" + - name: test + run: cd build && sudo ctest -V + diff --git a/CMakeLists.txt b/CMakeLists.txt index dd38750fec..19e9c2a1ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -217,7 +217,6 @@ set(MINIMUM_CXX_STANDARD 14) # @note Blosc version is currently treated as exception which must be adhered # to. The minimum version must be at least 1.5.0. Previous versions are incompatible. -# Later versions (including 1.5.4), can be buggy on certain platforms. set(MINIMUM_BLOSC_VERSION 1.5.0) # @note ABI always enforced so the correct deprecation messages are available. # OPENVDB_USE_DEPRECATED_ABI_ should be used to circumvent this @@ -262,6 +261,7 @@ set(FUTURE_MINIMUM_ICC_VERSION 19) # set(FUTURE_MINIMUM_ILMBASE_VERSION 2.4) # set(FUTURE_MINIMUM_OPENEXR_VERSION 2.4) set(FUTURE_MINIMUM_BOOST_VERSION 1.73) +set(FUTURE_MINIMUM_BLOSC_VERSION 1.17.0) set(FUTURE_MINIMUM_TBB_VERSION 2020.2) set(FUTURE_MINIMUM_PYTHON_VERSION 3.7) set(FUTURE_MINIMUM_NUMPY_VERSION 1.17.0) diff --git a/README.md b/README.md index c820dcf607..5eae2b9729 100644 --- a/README.md +++ b/README.md @@ -43,17 +43,7 @@ The following provides basic installation examples for the core OpenVDB library. apt-get install -y libboost-iostreams-dev apt-get install -y libboost-system-dev apt-get install -y libtbb-dev -``` -``` -git clone git@github.com:Blosc/c-blosc.git -cd c-blosc -git checkout tags/v1.5.0 -b v1.5.0 -mkdir build -cd build -cmake .. -make -j4 -make install -cd ../.. +apt-get install -y libblosc-dev ``` ##### Building OpenVDB @@ -71,17 +61,7 @@ make install ``` brew install boost brew install tbb -``` -``` -git clone git@github.com:Blosc/c-blosc.git -cd c-blosc -git checkout tags/v1.5.0 -b v1.5.0 -mkdir build -cd build -cmake .. -make -j4 -make install -cd ../.. +brew install c-blosc ``` ##### Building OpenVDB ``` diff --git a/ci/install_blosc.sh b/ci/install_blosc.sh index 8b13229ecf..bf4ab27907 100755 --- a/ci/install_blosc.sh +++ b/ci/install_blosc.sh @@ -3,6 +3,11 @@ set -ex BLOSC_VERSION="$1" +INSTALL_ROOT="$2" +CMAKE_EXTRA=() +if [ ! -z "${INSTALL_ROOT}" ]; then + CMAKE_EXTRA+=("-DCMAKE_INSTALL_PREFIX=${INSTALL_ROOT}") +fi git clone https://github.com/Blosc/c-blosc.git cd c-blosc @@ -13,6 +18,19 @@ fi mkdir build cd build -cmake ../. + +# On MacOS there's a bug between blosc 1.14-1.20 where unistd isn't included +# in zlib-1.2.8/gzlib.c. Provide -DPREFER_EXTERNAL_ZLIB to use the installed +# version of zlib. +# https://github.com/Blosc/python-blosc/issues/229 +cmake \ + -DPREFER_EXTERNAL_ZLIB=ON \ + -DBUILD_STATIC=OFF \ + -DBUILD_TESTS=OFF \ + -DBUILD_FUZZERS=OFF \ + -DBUILD_BENCHMARKS=OFF \ + "${CMAKE_EXTRA[@]}" \ + .. + make -j4 make install diff --git a/doc/build.txt b/doc/build.txt index e6face01c3..48ad902cf4 100644 --- a/doc/build.txt +++ b/doc/build.txt @@ -154,8 +154,8 @@ for this: that were created using blosc compression. This includes OpenVDB files from Houdini. -Note however that OpenVDB currently uses a very old version of Blosc (v1.5.0). -See the note in the [known issues list.](@ref depKnownIssues). +See the note in the [known issues list](@ref depKnownIssues) regarding supported +versions of blosc. @subsection buildZLibSupport ZLIB Support diff --git a/doc/dependencies.txt b/doc/dependencies.txt index ca55856453..d88b117082 100644 --- a/doc/dependencies.txt +++ b/doc/dependencies.txt @@ -73,7 +73,7 @@ Python | 2.7 | 2.7 | The python interpreter and libraries fo NumPy | 1.14.0 | 1.14.0 | Scientific computing with Python | Y | Y | http://www.numpy.org GoogleTest | 1.8 | Latest | A unit testing framework module for C++ | Y | Y | https://github.com/google/googletest CppUnit | 1.10 | Latest | A unit testing framework module for C++ | N | Y | https://freedesktop.org/wiki/Software/cppunit -Blosc | 1.5.0 | 1.5.0 | Recommended dependency for improved disk compression | N | N* | https://github.com/Blosc/c-blosc/releases +Blosc | 1.5.0* | 1.17.0 | Recommended dependency for improved disk compression | Y | Y | https://github.com/Blosc/c-blosc/releases Log4cplus | 1.1.2 | Latest | An optional dependency for improved OpenVDB Logging | Y | Y | https://github.com/log4cplus/log4cplus libpng | - | Latest | Library for manipulating PNG images | Y | Y | http://www.libpng.org/pub/png/libpng.html GLFW | 3.1 | Latest | Simple API for OpenGL development | Y | Y | https://www.glfw.org @@ -81,8 +81,7 @@ OpenGL | 3.2 | Latest | Environment for developing portable gra GLEW | 1.0.0 | Latest | A cross-platform OpenGL extension loading library. | Y | Y | http://glew.sourceforge.net Doxygen | 1.8.8 | 1.8.11 | Documentation generation from C++ | Y | Y | http://www.doxygen.nl - - @b * Blosc 1.16 is only available through Homebrew and currently requires - [manual](@ref depManInstall) installation for 1.5. + - @b * See the note in [known issues](@ref depKnownIssues) regarding supported blosc versions. - @b ** IlmBase/OpenEXR 2.2 is supported as Houdini 18.0 is shipped with VFX Reference Platform 2019, except for using older versions of these @@ -101,11 +100,22 @@ Doxygen | 1.8.8 | 1.8.11 | Documentation generation from C++ RuntimeError: FATAL: module compiled as little endian, but detected different endianness at runtime. @endcode - - @b Blosc: OpenVDB currently uses a very old version of blosc (v1.5.0). This - is due to issues with compatibility between certain blosc versions. You can - [read more about this problem here.](https://www.blosc.org/posts/new-forward-compat-policy/) - Using blosc versions equal to or later than 1.14.0 should be forwards - compatible but this remains untested. + - @b Blosc: OpenVDB has historically used an old version of blosc (1.5.0) to + serialize .vdb files. OpenVDB is moving to a new blosc version minimum of 1.17.0. + We have tested a range of blosc versions and found that the following versions + are NOT compatible with OpenVDB caches written using blosc 1.5.0. Additionally + these versions of blosc may generally not be compatible with OpenVDB itself. + Avoid using these versions and upgrade to the new minimum of 1.17.0 where + possible. + + - All versions from 1.11.0 through to, but excluding, 1.14.0. + - Versions 1.16.0 and 1.16.1. + + For more information see: + + - https://www.blosc.org/posts/new-forward-compat-policy/ + - https://github.com/Blosc/c-blosc/issues/260 + - https://github.com/Blosc/c-blosc/issues/261 @section depInstallingDependencies Installing Dependencies @@ -114,9 +124,8 @@ required OpenVDB dependencies. @subsection depUsingAptGet Using UNIX apt-get -@b Note: Blosc and CppUnit are unavailable using apt-get. See the other -package manager methods or the [manual installation](@ref depManInstall) -options. +@b Note:CppUnit is unavailable using apt-get. See the other package manager +methods or the [manual installation](@ref depManInstall) options. @code{.sh} #!/bin/bash @@ -126,6 +135,7 @@ apt-get install libtbb-dev # TBB apt-get install zlibc # zlib apt-get install libboost-system-dev # Boost::system apt-get install libboost-iostreams-dev # Boost::iostream +apt-get install libblosc-dev # Blosc # AX apt-get install llvm-8-dev # LLVM # Python @@ -150,11 +160,6 @@ apt-get install doxygen # doxygen [Homebrew](https://brew.sh/) is a package manager for UNIX systems and is an alternative to apt-get or manual installation from source. -@b Note: Homebrew installs blosc 1.16 by default with no option for previous -versions. OpenVDB recommends Blosc version 1.5 as there are known issues with -using later versions of blosc. See the -[manual installation](@ref depManInstall) options. - @code{.sh} #!/bin/bash # Core library @@ -162,6 +167,7 @@ brew install cmake # CMake brew install tbb # TBB brew install zlib # zlib brew install boost # Boost +brew install c-blosc # blosc # AX brew install llvm # LLVM # Python @@ -188,21 +194,6 @@ Below are manual installation methods for these packages. Note that you will have to provide your chosen installation locations to the OpenVDB CMake build system when building OpenVDB. -@b Blosc - -@code{.sh} -#!/bin/bash -wget https://github.com/Blosc/c-blosc/archive/v1.5.0.zip -unzip v1.5.0.zip -cd c-blosc-1.5.0 -# Install to users home directory - you may customise this location -mkdir -p $HOME/blosc -# Requires CMake -cmake -DCMAKE_INSTALL_PREFIX=$HOME/blosc -make -make install -@endcode - @b CppUnit (for AX Unit Tests) @code{.sh} diff --git a/openvdb/openvdb/CMakeLists.txt b/openvdb/openvdb/CMakeLists.txt index fc0fe45b6f..4750c29f36 100644 --- a/openvdb/openvdb/CMakeLists.txt +++ b/openvdb/openvdb/CMakeLists.txt @@ -158,12 +158,24 @@ endif() if(USE_BLOSC) # Find Blosc libraries find_package(Blosc ${MINIMUM_BLOSC_VERSION} REQUIRED) - if(Blosc_FOUND AND Blosc_VERSION VERSION_GREATER MINIMUM_BLOSC_VERSION) - message(WARNING "The version of Blosc located is greater than ${MINIMUM_BLOSC_VERSION}. " - "There have been reported issues with using later versions of Blosc with OpenVDB. " - "OpenVDB has been tested fully against Blosc ${MINIMUM_BLOSC_VERSION}, it is " - "recommended that you use this version where possible." - ) + if(Blosc_VERSION) + if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_BLOSC_VERSION) + if(${Blosc_VERSION} VERSION_LESS FUTURE_MINIMUM_BLOSC_VERSION) + message(DEPRECATION "Support for Blosc versions < ${FUTURE_MINIMUM_BLOSC_VERSION} " + "is deprecated and will be removed.") + endif() + endif() + # Print a warning if using of the blosc versions with regression issues + # with Blosc 1.5.0 caches. + if((Blosc_VERSION VERSION_GREATER_EQUAL 1.11.0 AND + Blosc_VERSION VERSION_LESS 1.14.0) OR + (Blosc_VERSION VERSION_GREATER_EQUAL 1.16.0 AND + Blosc_VERSION VERSION_LESS 1.16.2)) + message(WARNING "The following Blosc versions are incompatible with the " + "recommended builds of OpenVDB: [1.11.0 -> 1.14.0) [1.16.0, 1.16.1]. " + "Found Blosc version '${Blosc_VERSION}' which falls in this range. We " + "strongly recommend using the new future minimum version '${FUTURE_MINIMUM_BLOSC_VERSION}'") + endif() endif() else() message(WARNING "Blosc support is disabled. It is strongly recommended to "