add pcre2 support

changes/next/pcre2-support

@@ -0,0 +1,30 @@
+Description:
+
+Adds pcre2 support.  Prefers pcre2 over pcre if both are available.
+
+
+Config changes:
+
+None
+
+
+Upgrade instructions:
+
+Cyrus will prefer pcre2 over pcre if both are installed.  If you have both
+installed and wish to use pcre rather than pcre2, run configure with
+--disable-pcre2.
+
+If you haven't specifically installed libpcre2-dev (or whatever your system's
+equivalent is), you might still have parts of pcre2 installed due to other
+packages on your system depending on it.  This can confuse configure into
+thinking you have a usable pcre2 when you don't.  Either properly install
+libpcre2-dev so Cyrus can use it, or configure Cyrus with --disable-pcre2
+so that it ignores the partial installation.
+
+Please note that on Debian-based systems, pcre (the old one, no longer
+maintained) is called "pcre3".  Yes, this is confusing.
+
+
+GitHub issue:
+
+#3861 #4711

configure.ac

@@ -675,65 +675,111 @@ AM_CONDITIONAL([SQUATTER],
     [test "${enable_squat}" != "no" -o "${enable_xapian}" != "no" ])
 
 AC_ARG_ENABLE(sieve,
-        [AS_HELP_STRING([--disable-sieve], [disable Sieve support])],,[enable_sieve="yes";])
-AC_ARG_ENABLE(pcre,
-        [AS_HELP_STRING([--disable-pcre], [disable PCRE library])],[cyrus_cv_pcre_utf8="$enableval"])
+    [AS_HELP_STRING([--disable-sieve], [disable Sieve support])],,[enable_sieve="yes";])
 
 if test "$enable_sieve" != "no"; then
-        AC_DEFINE(USE_SIEVE,[],[Build in Sieve support?])
+    AC_DEFINE(USE_SIEVE,[],[Build in Sieve support?])
 
-        if test "x$HAVE_SQLITE" != x1; then
-            AC_MSG_ERROR([Need sqlite3 for sieve])
-        else
-            use_sqlite="yes"
-        fi
+    if test "x$HAVE_SQLITE" != x1; then
+        AC_MSG_ERROR([Need sqlite3 for sieve])
+    else
+        use_sqlite="yes"
+    fi
 
-        dnl Sieve configure stuff
-        AC_PROG_YACC
-        AM_PROG_LEX
+    dnl Sieve configure stuff
+    AC_PROG_YACC
+    AM_PROG_LEX
 
-        if test -z "$ac_cv_prog_YACC"; then
-            AC_MSG_ERROR([Sieve requires bison/byacc/yacc, but none is installed])
-        fi
+    if test -z "$ac_cv_prog_YACC"; then
+        AC_MSG_ERROR([Sieve requires bison/byacc/yacc, but none is installed])
+    fi
 
-        if test -z "$ac_cv_prog_LEX"; then
-            AC_MSG_ERROR([Sieve requires flex/lex, but none is installed])
-        fi
+    if test -z "$ac_cv_prog_LEX"; then
+        AC_MSG_ERROR([Sieve requires flex/lex, but none is installed])
+    fi
 fi
 
 AM_CONDITIONAL([SIEVE], [test "${enable_sieve}" != "no"])
 
+AC_ARG_ENABLE(pcre,
+    [AS_HELP_STRING([--disable-pcre], [disable PCRE library])],[cyrus_cv_pcre_utf8="$enableval"])
+AC_ARG_ENABLE(pcre2,
+    [AS_HELP_STRING([--disable-pcre2], [disable PCRE2 library])],[cyrus_cv_pcre2_utf8="$enableval"])
+
 if test "$enable_pcre" != "no"; then
-        AC_CHECK_HEADER(pcreposix.h)
-        if test "$ac_cv_header_pcreposix_h" = "yes"; then
-            AC_MSG_CHECKING(for utf8 enabled pcre)
-            AC_CACHE_VAL(cyrus_cv_pcre_utf8, AC_TRY_CPP([#include <pcreposix.h>
-#ifndef REG_UTF8
-#include </nonexistent>
-#endif],cyrus_cv_pcre_utf8=yes,cyrus_cv_pcre_utf8=no))
-            AC_MSG_RESULT($cyrus_cv_pcre_utf8)
-        else
-            cyrus_cv_pcre_utf8="no"
-        fi
+    PKG_CHECK_MODULES([PCRE],
+                      [libpcreposix libpcre],
+                      [ AC_MSG_CHECKING(for utf8 enabled pcre)
+                        saved_CFLAGS="$CFLAGS"
+                        saved_LIBS="$LIBS"
+                        CFLAGS="$CFLAGS PCRE_CFLAGS"
+                        LIBS="$LIBS PCRE_LIBS"
+                        AC_CACHE_VAL(cyrus_cv_pcre_utf8,
+                                     AC_TRY_CPP([   #include <pcreposix.h>
+                                                    #ifndef REG_UTF8
+                                                    #include </nonexistent>
+                                                    #endif
+                                                ],
+                                                [cyrus_cv_pcre_utf8=yes],
+                                                [cyrus_cv_pcre_utf8=no]))
+                        AC_MSG_RESULT($cyrus_cv_pcre_utf8)
+                        CFLAGS="$saved_CFLAGS"
+                        LIBS="$saved_LIBS"
+                      ],
+                      [cyrus_cv_pcre_utf8="no"])
+fi
+
+if test "$enable_pcre2" != "no"; then
+    PKG_CHECK_MODULES([PCRE2],
+                      [libpcre2-posix libpcre2-8],
+                      [ AC_MSG_CHECKING(for utf8 enabled pcre2)
+                        saved_CFLAGS="$CFLAGS"
+                        saved_LIBS="$LIBS"
+                        CFLAGS="$CFLAGS PCRE2_CFLAGS"
+                        LIBS="$LIBS PCRE2_LIBS"
+                        AC_CACHE_VAL(cyrus_cv_pcre2_utf8,
+                                     AC_TRY_CPP([   #include <pcre2posix.h>
+                                                    #ifndef REG_UTF
+                                                    #include </nonexistent>
+                                                    #endif
+                                                ],
+                                                [cyrus_cv_pcre2_utf8=yes],
+                                                [cyrus_cv_pcre2_utf8=no]))
+                        AC_MSG_RESULT($cyrus_cv_pcre2_utf8)
+                        CFLAGS="$saved_CFLAGS"
+                        LIBS="$saved_LIBS"
+                      ],
+                      [cyrus_cv_pcre2_utf8="no"])
 fi
 
 LIB_REGEX=
-if test "$cyrus_cv_pcre_utf8" = "yes"; then
-        LIB_REGEX="-lpcre -lpcreposix";
-        AC_DEFINE(ENABLE_REGEX, [], [Do we have a regex library?])
-        AC_DEFINE(HAVE_PCREPOSIX_H, [], [Do we have usable pcre library?])
+CFLAGS_REGEX=
+if test "$cyrus_cv_pcre2_utf8" = "yes"; then
+    CFLAGS_REGEX="$PCRE2_CFLAGS"
+    LIB_REGEX="$PCRE2_LIBS"
+    AC_DEFINE(ENABLE_REGEX, [], [Do we have a regex library?])
+    AC_DEFINE(HAVE_PCRE2POSIX_H, [], [Do we have usable pcre2 library?])
+    cyrus_cv_pcre_utf8="no"
+elif test "$cyrus_cv_pcre_utf8" = "yes"; then
+    CFLAGS_REGEX="$PCRE_CFLAGS"
+    LIB_REGEX="$PCRE_LIBS";
+    AC_DEFINE(ENABLE_REGEX, [], [Do we have a regex library?])
+    AC_DEFINE(HAVE_PCREPOSIX_H, [], [Do we have usable pcre library?])
+    cyrus_cv_pcre2_utf8="no"
 else
-        AC_CHECK_HEADERS(rxposix.h)
-        if test "$ac_cv_header_rxposix_h" = "yes"; then
-            LIB_REGEX="-lrx"
-            AC_DEFINE(ENABLE_REGEX, [], [Do we have a regex library?])
-        else
-            AC_SEARCH_LIBS(regcomp, regex,
-                           AC_DEFINE(ENABLE_REGEX, [], [Do we have a regex library?]), [])
-        fi
+    AC_CHECK_HEADERS(rxposix.h)
+    if test "$ac_cv_header_rxposix_h" = "yes"; then
+        LIB_REGEX="-lrx"
+        AC_DEFINE(ENABLE_REGEX, [], [Do we have a regex library?])
+    else
+        AC_SEARCH_LIBS(regcomp, regex,
+                        AC_DEFINE(ENABLE_REGEX, [], [Do we have a regex library?]), [])
+    fi
 fi
 AC_SUBST(LIB_REGEX)
+AC_SUBST(CFLAGS_REGEX)
 LIBS="$LIBS $LIB_REGEX"
+CFLAGS="$CFLAGS $CFLAGS_REGEX"
 
 dnl
 dnl see if we're compiling with SRS
@@ -2569,6 +2615,7 @@ External dependencies:
    zlib:               $with_zlib
    jansson:            $with_jansson
    pcre:               $cyrus_cv_pcre_utf8
+   pcre2:              $cyrus_cv_pcre2_utf8
    clamav:             $with_clamav
    -----------------------
    caringo:            $with_caringo

docsrc/assets/cyrus-build-devpkg.rst

@@ -8,7 +8,7 @@ automated test facility.
         debhelper flex g++ git gperf groff heimdal-dev libbsd-resource-perl libclone-perl libconfig-inifiles-perl \
         libcunit1-dev libdatetime-perl libbsd-dev libdigest-sha-perl libencode-imaputf7-perl \ libfile-chdir-perl libglib2.0-dev libical-dev libio-socket-inet6-perl \
         libio-stringy-perl libldap2-dev libmysqlclient-dev \
-        libnet-server-perl libnews-nntpclient-perl libpam0g-dev libpcre3-dev libsasl2-dev \
+        libnet-server-perl libnews-nntpclient-perl libpam0g-dev libpcre2-dev libsasl2-dev \
         libsqlite3-dev libssl-dev libtest-unit-perl libtool libunix-syslog-perl liburi-perl \
         libxapian-dev libxml-generator-perl libxml-xpath-perl libxml2-dev libwrap0-dev libzephyr-dev lsb-base \
         net-tools perl php-cli php-curl pkg-config po-debconf tcl-dev \

docsrc/assets/cyrus-build-reqpkg.rst

@@ -2,6 +2,6 @@
 
     sudo apt-get install git build-essential autoconf automake libtool \
         pkg-config bison flex libssl-dev libjansson-dev libxml2-dev \
-        libsqlite3-dev libical-dev libsasl2-dev libpcre3-dev uuid-dev \
+        libsqlite3-dev libical-dev libsasl2-dev libpcre2-dev uuid-dev \
         libicu-dev
     sudo apt-get -t jessie-backports install libxapian-dev

docsrc/imap/developer/compiling.rst

@@ -194,7 +194,7 @@ Other
     to enable **ptloader** to interface with LDAP directly, for canonification
     of login usernames to mailbox names, and verification of login usernames,
     ACL subjects and group membership. Configure option: ``--with-ldap``."
-    `pcre`_, libpcre3-dev, pcre-devel, "yes", "PCRE 1 (8.x) - for utf-8/unicode
+    `pcre2`_, libpcre2-dev, pcre2-devel, "yes", "PCRE 2 (10.x) - for utf-8/unicode
     regular expression matching. Could be replaced by something else in the
     future. See `issues/1731`_ for more information."
     `perl(Term::ReadLine)`_,,, "no", "Perl library needed by **cyradm**."
@@ -219,6 +219,7 @@ Other
 .. _nghttp2: https://nghttp2.org/
 .. _openldap: http://www.openldap.org/
 .. _pcre: http://www.pcre.org/
+.. _pcre2: http://www.pcre.org/
 .. _perl(Term::ReadLine): https://metacpan.org/pod/Term::ReadLine
 .. _perl(ExtUtils::MakeMaker): http://search.cpan.org/dist/ExtUtils-MakeMaker/
 .. _perl(Pod::POM::View::Restructured): https://metacpan.org/pod/Pod::POM::View::Restructured

docsrc/imap/download/upgrade.rst

@@ -223,7 +223,7 @@ packages) for Debian is::
     libcunit1-dev libdatetime-perl libdigest-sha-perl libencode-imaputf7-perl \
     libfile-chdir-perl libglib2.0-dev libical-dev libio-socket-inet6-perl \
     libio-stringy-perl libjansson-dev libldap2-dev libmysqlclient-dev \
-    libnet-server-perl libnews-nntpclient-perl libpam0g-dev libpcre3-dev \
+    libnet-server-perl libnews-nntpclient-perl libpam0g-dev libpcre2-dev \
     libsasl2-dev libsqlite3-dev libssl-dev libtest-unit-perl libtool \
     libunix-syslog-perl liburi-perl libxapian-dev libxml-generator-perl \
     libxml-xpath-perl libxml2-dev libwrap0-dev libzephyr-dev lsb-base \

imap/cyr_buildinfo.c

@@ -202,6 +202,11 @@ static json_t *buildinfo()
 #else
     json_object_set_new(dependency, "pcre", json_false());
 #endif
+#if defined(ENABLE_REGEX) && defined(HAVE_PCRE2POSIX_H)
+    json_object_set_new(dependency, "pcre2", json_true());
+#else
+    json_object_set_new(dependency, "pcre2", json_false());
+#endif
 #ifdef HAVE_CLAMAV
     json_object_set_new(dependency, "clamav", json_true());
 #else

lib/util.h

@@ -71,17 +71,20 @@
 extern const char CYRUS_VERSION[];
 
 #ifdef ENABLE_REGEX
-# ifdef HAVE_PCREPOSIX_H
+# if defined HAVE_PCREPOSIX_H
 #  include <pcre.h>
 #  include <pcreposix.h>
-# else /* !HAVE_PCREPOSIX_H */
-#  ifdef HAVE_RXPOSIX_H
-#   include <rxposix.h>
-#  else /* !HAVE_RXPOSIX_H */
-#   include <regex.h>
-#  endif /* HAVE_RXPOSIX_H */
-# endif /* HAVE_PCREPOSIX_H */
-#endif /* ENABLE_REGEX */
+# elif defined HAVE_PCRE2POSIX_H
+#  ifndef PCRE2POSIX_H_INCLUDED
+#   include <pcre2posix.h>
+#   define PCRE2POSIX_H_INCLUDED
+#  endif
+# elif defined HAVE_RXPOSIX_H
+#  include <rxposix.h>
+# else
+#  include <regex.h>
+# endif
+#endif
 
 #ifdef HAVE_LIBUUID
 #include <uuid/uuid.h>

ptclient/test3.c

@@ -1,6 +1,6 @@
 /*
  * test ptsmodule_standard_root_dn
- * gcc -lz -lpcreposix -I.. -I ../lib ../lib/.libs/util.o ../lib/.libs/xmalloc.o ../lib/.libs/map_shared.o ../lib/.libs/retry.o ../lib/.libs/assert.o test3.c
+ * gcc -I.. -I ../lib ../lib/.libs/util.o ../lib/.libs/xmalloc.o ../lib/.libs/map_shared.o ../lib/.libs/retry.o ../lib/.libs/assert.o test3.c -lz -lpcre2-posix 
  */
 #include <assert.h>
 #include <stdio.h>

sieve/bc_eval.c

@@ -321,9 +321,11 @@ static int regcomp_flags(int comparator, int requires)
 {
     int cflags = REG_EXTENDED;
 
-#ifdef HAVE_PCREPOSIX_H
     /* support UTF8 comparisons */
+#if defined HAVE_PCREPOSIX_H
     cflags |= REG_UTF8;
+#elif defined HAVE_PCRE2POSIX_H
+    cflags |= REG_UTF;
 #endif
 
     if (comparator == B_ASCIICASEMAP) {

sieve/comparator.h

@@ -47,17 +47,20 @@
 #include <sys/types.h>
 
 #ifdef ENABLE_REGEX
-# ifdef HAVE_PCREPOSIX_H
+# if defined HAVE_PCREPOSIX_H
 #  include <pcre.h>
 #  include <pcreposix.h>
-# else /* !HAVE_PCREPOSIX_H */
-#  ifdef HAVE_RXPOSIX_H
-#   include <rxposix.h>
-#  else /* !HAVE_RXPOSIX_H */
-#   include <regex.h>
-#  endif /* HAVE_RXPOSIX_H */
-# endif /* HAVE_PCREPOSIX_H */
-#endif /* ENABLE_REGEX */
+# elif defined HAVE_PCRE2POSIX_H
+#  ifndef PCRE2POSIX_H_INCLUDED
+#   include <pcre2posix.h>
+#   define PCRE2POSIX_H_INCLUDED
+#  endif
+# elif defined HAVE_RXPOSIX_H
+#  include <rxposix.h>
+# else
+#  include <regex.h>
+# endif
+#endif
 
 #include "sieve_interface.h"
 #include "strarray.h"

sieve/sieve.y

@@ -2143,9 +2143,11 @@ static int verify_regexlist(sieve_script_t *sscript,
     regex_t reg;
     int cflags = REG_EXTENDED | REG_NOSUB;
 
-#ifdef HAVE_PCREPOSIX_H
     /* support UTF8 comparisons */
+#if defined HAVE_PCREPOSIX_H
     cflags |= REG_UTF8;
+#elif defined HAVE_PCRE2POSIX_H
+    cflags |= REG_UTF;
 #endif
 
     if (collation == B_ASCIICASEMAP) {