content_encoding: add zstd decoding support

include zstd curl patch for Makefile.m32 from vszakats
and include Add CMake support for zstd from Peter Wu

Helped-by: Viktor Szakats
Helped-by: Peter Wu
Closes #5453

.azure-pipelines.yml

@@ -76,7 +76,7 @@ stages:
           configure: --enable-debug --disable-shared --disable-threaded-resolver --enable-alt-svc
           tflags: -n -t --shallow=40 !FTP
     steps:
-    - script: sudo apt-get update && sudo apt-get install -y stunnel4 python-impacket $(install)
+    - script: sudo apt-get update && sudo apt-get install -y stunnel4 python-impacket libzstd-dev libbrotli-dev $(install)
       displayName: 'apt install'
 
     - script: ./buildconf && ./configure $(configure)

.travis.yml

@@ -94,6 +94,7 @@ jobs:
         - *common_packages
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
     - T=novalgrind BORINGSSL=yes C="--with-ssl=$HOME/boringssl" LD_LIBRARY_PATH=/home/travis/boringssl/lib:/usr/local/lib
     - OVERRIDE_CC="CC=gcc-8" OVERRIDE_CXX="CXX=g++-8"
@@ -111,6 +112,7 @@ jobs:
         - *common_packages
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
     - T=novalgrind NGTCP2=yes C="--with-ssl=$HOME/ngbuild --with-ngtcp2=$HOME/ngbuild --with-nghttp3=$HOME/ngbuild --enable-alt-svc" NOTESTS=
     - OVERRIDE_CC="CC=gcc-8" OVERRIDE_CXX="CXX=g++-8"
@@ -121,6 +123,7 @@ jobs:
         - *common_packages
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
     - T=novalgrind NGTCP2=yes GNUTLS=yes C="PKG_CONFIG_PATH=$HOME/ngbuild --without-ssl --with-gnutls=$HOME/ngbuild --with-ngtcp2=$HOME/ngbuild --with-nghttp3=$HOME/ngbuild --enable-alt-svc" NOTESTS=
     - OVERRIDE_CC="CC=gcc-8" OVERRIDE_CXX="CXX=g++-8"
@@ -131,6 +134,7 @@ jobs:
         - *common_packages
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
         - autogen
         - automake
         - autopoint
@@ -151,6 +155,7 @@ jobs:
         - *common_packages
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
     - T=debug-mesalink C="--with-mesalink --without-ssl"
     - OVERRIDE_CC="CC=gcc-8" OVERRIDE_CXX="CXX=g++-8"
@@ -161,6 +166,7 @@ jobs:
         - *common_packages
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
     - T=debug
     - &clang OVERRIDE_CC="CC=clang-9" OVERRIDE_CXX="CXX=clang++-9"
@@ -172,6 +178,7 @@ jobs:
         - &clang_packages [*common_packages, clang-9]
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
     - T=debug C="--enable-alt-svc"
     - *clang
@@ -183,6 +190,7 @@ jobs:
         - *clang_packages
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
     - T=debug C="--with-mbedtls --without-ssl"
     - *clang
@@ -194,6 +202,7 @@ jobs:
         - *clang_packages
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
         - libmbedtls-dev
   - env:
     - T=debug C="--with-gnutls --without-ssl"
@@ -207,6 +216,7 @@ jobs:
         - libgnutls28-dev
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
     - T=debug C="--with-nss --without-ssl" NOTESTS=1 CPPFLAGS="-isystem /usr/include/nss"
     - *clang
@@ -219,11 +229,12 @@ jobs:
         - libnss3-dev
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
     - T=iconv
     - OVERRIDE_CC="CC=gcc-8" OVERRIDE_CXX="CXX=g++-8"
   - env:
-    - T=cmake BORINGSSL=yes QUICHE=yes C="-DUSE_QUICHE=1 -DOPENSSL_ROOT_DIR=$HOME/boringssl"
+    - T=cmake BORINGSSL=yes QUICHE=yes C="-DUSE_QUICHE=1 -DOPENSSL_ROOT_DIR=$HOME/boringssl -DCURL_BROTLI=1 -DCURL_ZSTD=1"
     - OVERRIDE_CC="CC=gcc-8" OVERRIDE_CXX="CXX=g++-8"
     - PKG_CONFIG_PATH="$HOME/quiche/target/release"
     before_install:
@@ -235,8 +246,9 @@ jobs:
         - *common_packages
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
-    - T=cmake NGTCP2=yes C="-DUSE_NGTCP2=ON"
+    - T=cmake NGTCP2=yes C="-DUSE_NGTCP2=ON -DCURL_BROTLI=1 -DCURL_ZSTD=1"
     - *clang
     - PKG_CONFIG_PATH="$HOME/ngbuild/lib/pkgconfig"
     compiler: clang
@@ -247,6 +259,7 @@ jobs:
         - *clang_packages
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
     - T=torture
     - OVERRIDE_CC="CC=gcc-8" OVERRIDE_CXX="CXX=g++-8"
@@ -258,6 +271,7 @@ jobs:
         - lcov
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
         - libssh2-1-dev
   - env:
     - T=distcheck
@@ -269,6 +283,7 @@ jobs:
         - *common_packages
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
     - T=fuzzer
     - *clang
@@ -280,6 +295,7 @@ jobs:
         - *clang_packages
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
     - T=tidy
     - *clang
@@ -292,6 +308,7 @@ jobs:
         - clang-tidy-9
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
     - T=scan-build
     - *clang
@@ -303,6 +320,7 @@ jobs:
         - *clang_packages
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
     - T=debug CFLAGS="-fsanitize=address,undefined,signed-integer-overflow -fno-sanitize-recover=undefined,integer -Wformat -Werror=format-security -Werror=array-bounds -g" LDFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=undefined,integer" LIBS="-ldl -lubsan"
     - *clang
@@ -314,6 +332,7 @@ jobs:
         - *clang_packages
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
   - env:
     - T=debug C="--enable-alt-svc"
     - OVERRIDE_CC="CC=gcc-8" OVERRIDE_CXX="CXX=g++-8"
@@ -325,6 +344,7 @@ jobs:
         - *common_packages
         - libpsl-dev
         - libbrotli-dev
+        - libzstd-dev
         - libev-dev
         - libssl-dev
         - libtool

CMake/FindZstd.cmake

@@ -0,0 +1,69 @@
+#***************************************************************************
+#                                  _   _ ____  _
+#  Project                     ___| | | |  _ \| |
+#                             / __| | | | |_) | |
+#                            | (__| |_| |  _ <| |___
+#                             \___|\___/|_| \_\_____|
+#
+# Copyright (C) 1998 - 2020, Daniel Stenberg, <[email protected]>, et al.
+#
+# This software is licensed as described in the file COPYING, which
+# you should have received as part of this distribution. The terms
+# are also available at https://curl.haxx.se/docs/copyright.html.
+#
+# You may opt to use, copy, modify, merge, publish, distribute and/or sell
+# copies of the Software, and permit persons to whom the Software is
+# furnished to do so, under the terms of the COPYING file.
+#
+# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+# KIND, either express or implied.
+#
+###########################################################################
+
+#[=======================================================================[.rst:
+FindZstd
+----------
+
+Find the zstd library
+
+Result Variables
+^^^^^^^^^^^^^^^^
+
+``Zstd_FOUND``
+  System has zstd
+``Zstd_INCLUDE_DIRS``
+  The zstd include directories.
+``Zstd_LIBRARIES``
+  The libraries needed to use zstd
+#]=======================================================================]
+
+if(UNIX)
+  find_package(PkgConfig QUIET)
+  pkg_search_module(PC_Zstd libzstd)
+endif()
+
+find_path(Zstd_INCLUDE_DIR zstd.h
+  HINTS
+    ${PC_Zstd_INCLUDEDIR}
+    ${PC_Zstd_INCLUDE_DIRS}
+)
+
+find_library(Zstd_LIBRARY NAMES zstd
+  HINTS
+    ${PC_Zstd_LIBDIR}
+    ${PC_Zstd_LIBRARY_DIRS}
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Zstd
+  REQUIRED_VARS
+    Zstd_LIBRARY
+    Zstd_INCLUDE_DIR
+)
+
+if(Zstd_FOUND)
+  set(Zstd_LIBRARIES    ${Zstd_LIBRARY})
+  set(Zstd_INCLUDE_DIRS ${Zstd_INCLUDE_DIR})
+endif()
+
+mark_as_advanced(Zstd_INCLUDE_DIRS Zstd_LIBRARIES)

CMakeLists.txt

@@ -641,6 +641,22 @@ if(CURL_BROTLI)
   endif()
 endif()
 
+option(CURL_ZSTD "Set to ON to enable building curl with zstd support." OFF)
+set(HAVE_ZSTD OFF)
+if(CURL_ZSTD)
+  find_package(Zstd REQUIRED)
+  cmake_push_check_state()
+  set(CMAKE_REQUIRED_INCLUDES ${Zstd_INCLUDE_DIRS})
+  set(CMAKE_REQUIRED_LIBRARIES ${Zstd_LIBRARIES})
+  check_symbol_exists(ZSTD_createDStream "zstd.h" HAVE_ZSTD_CREATEDSTREAM)
+  cmake_pop_check_state()
+  if(Zstd_FOUND AND HAVE_ZSTD_CREATEDSTREAM)
+    set(HAVE_ZSTD ON)
+    list(APPEND CURL_LIBS ${Zstd_LIBRARIES})
+    include_directories(${Zstd_INCLUDE_DIRS})
+  endif()
+endif()
+
 #libSSH2
 option(CMAKE_USE_LIBSSH2 "Use libSSH2" ON)
 mark_as_advanced(CMAKE_USE_LIBSSH2)
@@ -1322,6 +1338,8 @@ _add_if("SSL"           SSL_ENABLED)
 _add_if("IPv6"          ENABLE_IPV6)
 _add_if("unix-sockets"  USE_UNIX_SOCKETS)
 _add_if("libz"          HAVE_LIBZ)
+_add_if("brotli"        HAVE_BROTLI)
+_add_if("zstd"          HAVE_ZSTD)
 _add_if("AsynchDNS"     USE_ARES OR USE_THREADS_POSIX OR USE_THREADS_WIN32)
 _add_if("IDN"           HAVE_LIBIDN2)
 _add_if("Largefile"     (CURL_SIZEOF_CURL_OFF_T GREATER 4) AND

configure.ac

@@ -1115,6 +1115,93 @@ if test X"$OPT_BROTLI" != Xno; then
   fi
 fi
 
+dnl **********************************************************************
+dnl Check for libzstd
+dnl **********************************************************************
+
+dnl Default to compiler & linker defaults for libzstd
+OPT_ZSTD=off
+AC_ARG_WITH(zstd,dnl
+AC_HELP_STRING([--with-zstd=PATH],[Where to look for libzstd, PATH points to the libzstd installation; when possible, set the PKG_CONFIG_PATH environment variable instead of using this option])
+AC_HELP_STRING([--without-zstd], [disable libzstd]),
+  OPT_ZSTD=$withval)
+
+if test X"$OPT_ZSTD" != Xno; then
+  dnl backup the pre-zstd variables
+  CLEANLDFLAGS="$LDFLAGS"
+  CLEANCPPFLAGS="$CPPFLAGS"
+  CLEANLIBS="$LIBS"
+
+  case "$OPT_ZSTD" in
+  yes)
+    dnl --with-zstd (without path) used
+    CURL_CHECK_PKGCONFIG(libzstd)
+
+    if test "$PKGCONFIG" != "no" ; then
+      LIB_ZSTD=`$PKGCONFIG --libs-only-l libzstd`
+      LD_ZSTD=`$PKGCONFIG --libs-only-L libzstd`
+      CPP_ZSTD=`$PKGCONFIG --cflags-only-I libzstd`
+      version=`$PKGCONFIG --modversion libzstd`
+      DIR_ZSTD=`echo $LD_ZSTD | $SED -e 's/-L//'`
+    fi
+
+    ;;
+  off)
+    dnl no --with-zstd option given, just check default places
+    ;;
+  *)
+    dnl use the given --with-zstd spot
+    PREFIX_ZSTD=$OPT_ZSTD
+    ;;
+  esac
+
+  dnl if given with a prefix, we set -L and -I based on that
+  if test -n "$PREFIX_ZSTD"; then
+    LIB_ZSTD="-lzstd"
+    LD_ZSTD=-L${PREFIX_ZSTD}/lib$libsuff
+    CPP_ZSTD=-I${PREFIX_ZSTD}/include
+    DIR_ZSTD=${PREFIX_ZSTD}/lib$libsuff
+  fi
+
+  LDFLAGS="$LDFLAGS $LD_ZSTD"
+  CPPFLAGS="$CPPFLAGS $CPP_ZSTD"
+  LIBS="$LIB_ZSTD $LIBS"
+
+  AC_CHECK_LIB(zstd, ZSTD_createDStream)
+
+  AC_CHECK_HEADERS(zstd.h,
+    curl_zstd_msg="enabled (libzstd)"
+    HAVE_ZSTD=1
+    AC_DEFINE(HAVE_ZSTD, 1, [if libzstd is in use])
+    AC_SUBST(HAVE_ZSTD, [1])
+  )
+
+  if test X"$OPT_ZSTD" != Xoff &&
+     test "$HAVE_ZSTD" != "1"; then
+    AC_MSG_ERROR([libzstd was not found where specified!])
+  fi
+
+  if test "$HAVE_ZSTD" = "1"; then
+    if test -n "$DIR_ZSTD"; then
+       dnl when the zstd shared lib were found in a path that the run-time
+       dnl linker doesn't search through, we need to add it to
+       dnl CURL_LIBRARY_PATH to prevent further configure tests to fail due to
+       dnl this
+
+       if test "x$cross_compiling" != "xyes"; then
+         CURL_LIBRARY_PATH="$CURL_LIBRARY_PATH:$DIR_ZSTD"
+         export CURL_LIBRARY_PATH
+         AC_MSG_NOTICE([Added $DIR_ZSTD to CURL_LIBRARY_PATH])
+       fi
+    fi
+  else
+    dnl no zstd, revert back to clean variables
+    LDFLAGS=$CLEANLDFLAGS
+    CPPFLAGS=$CLEANCPPFLAGS
+    LIBS=$CLEANLIBS
+  fi
+fi
+
 dnl **********************************************************************
 dnl Check for LDAP
 dnl **********************************************************************
@@ -4825,6 +4912,9 @@ fi
 if test "x$HAVE_BROTLI" = "x1"; then
   SUPPORT_FEATURES="$SUPPORT_FEATURES brotli"
 fi
+if test "x$HAVE_ZSTD" = "x1"; then
+  SUPPORT_FEATURES="$SUPPORT_FEATURES zstd"
+fi
 if test "x$USE_ARES" = "x1" -o "x$USE_THREADS_POSIX" = "x1" \
                             -o "x$USE_THREADS_WIN32" = "x1"; then
   SUPPORT_FEATURES="$SUPPORT_FEATURES AsynchDNS"
@@ -5067,6 +5157,7 @@ AC_MSG_NOTICE([Configured to build curl/libcurl:
   SSH:              ${curl_ssh_msg}
   zlib:             ${curl_zlib_msg}
   brotli:           ${curl_brotli_msg}
+  zstd:             ${curl_zstd_msg}
   GSS-API:          ${curl_gss_msg}
   TLS-SRP:          ${curl_tls_srp_msg}
   resolver:         ${curl_res_msg}