X86: Fast CRC32 computation using PCLMULQDQ instruction

Based on:
"Fast CRC Computation for Generic Polynomials Using PCLMULQDQ Instruction"
V. Gopal, E. Ozturk, et al., 2009, http://intel.ly/2ySEwL0

Signed-off-by: Frank Du <[email protected]>

Closes GH-6018

Zend/zend_cpuinfo.h

@@ -159,6 +159,17 @@ static zend_always_inline int zend_cpu_supports_sse42() {
 	return __builtin_cpu_supports("sse4.2");
 }
 
+/* __builtin_cpu_supports has pclmul from gcc9 */
+#if (!defined(__GNUC__) || (ZEND_GCC_VERSION >= 9000))
+ZEND_NO_SANITIZE_ADDRESS
+static zend_always_inline int zend_cpu_supports_pclmul() {
+#if PHP_HAVE_BUILTIN_CPU_INIT
+        __builtin_cpu_init();
+#endif
+        return __builtin_cpu_supports("pclmul");
+}
+#endif
+
 ZEND_NO_SANITIZE_ADDRESS
 static zend_always_inline int zend_cpu_supports_avx() {
 #if PHP_HAVE_BUILTIN_CPU_INIT
@@ -196,6 +207,10 @@ static zend_always_inline int zend_cpu_supports_sse42() {
 	return zend_cpu_supports(ZEND_CPU_FEATURE_SSE42);
 }
 
+static zend_always_inline int zend_cpu_supports_pclmul() {
+        return zend_cpu_supports(ZEND_CPU_FEATURE_PCLMULQDQ);
+}
+
 static zend_always_inline int zend_cpu_supports_avx() {
 	return zend_cpu_supports(ZEND_CPU_FEATURE_AVX);
 }

Zend/zend_portability.h

@@ -487,6 +487,10 @@ extern "C++" {
 #  define PHP_HAVE_SSE4_2
 # endif
 
+# if defined(HAVE_WMMINTRIN_H)
+#  define PHP_HAVE_PCLMUL
+# endif
+
 /*
  * AVX2 support was added in gcc 4.7, but AVX2 intrinsics don't work in
  * __attribute__((target("avx2"))) functions until gcc 4.9.
@@ -547,6 +551,58 @@ extern "C++" {
 # define ZEND_INTRIN_SSE4_2_FUNC_DECL(func)
 #endif
 
+#ifdef __PCLMUL__
+/* Instructions compiled directly. */
+# define ZEND_INTRIN_PCLMUL_NATIVE 1
+#elif (defined(HAVE_FUNC_ATTRIBUTE_TARGET) && defined(PHP_HAVE_PCLMUL)) || defined(ZEND_WIN32)
+/* Function resolved by ifunc or MINIT. */
+# define ZEND_INTRIN_PCLMUL_RESOLVER 1
+#endif
+
+/* Do not use for conditional declaration of API functions! */
+#if defined(ZEND_INTRIN_PCLMUL_RESOLVER) && defined(ZEND_INTRIN_HAVE_IFUNC_TARGET) && (!defined(__GNUC__) || (ZEND_GCC_VERSION >= 9000))
+/* __builtin_cpu_supports has pclmul from gcc9 */
+# define ZEND_INTRIN_PCLMUL_FUNC_PROTO 1
+#elif defined(ZEND_INTRIN_PCLMUL_RESOLVER)
+# define ZEND_INTRIN_PCLMUL_FUNC_PTR 1
+#endif
+
+#ifdef ZEND_INTRIN_PCLMUL_RESOLVER
+# ifdef HAVE_FUNC_ATTRIBUTE_TARGET
+#  define ZEND_INTRIN_PCLMUL_FUNC_DECL(func) ZEND_API func __attribute__((target("pclmul")))
+# else
+#  define ZEND_INTRIN_PCLMUL_FUNC_DECL(func) func
+# endif
+#else
+# define ZEND_INTRIN_PCLMUL_FUNC_DECL(func)
+#endif
+
+#if defined(ZEND_INTRIN_SSE4_2_NATIVE) && defined(ZEND_INTRIN_PCLMUL_NATIVE)
+/* Instructions compiled directly. */
+# define ZEND_INTRIN_SSE4_2_PCLMUL_NATIVE 1
+#elif (defined(HAVE_FUNC_ATTRIBUTE_TARGET) && defined(PHP_HAVE_SSE4_2) && defined(PHP_HAVE_PCLMUL)) || defined(ZEND_WIN32)
+/* Function resolved by ifunc or MINIT. */
+# define ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER 1
+#endif
+
+/* Do not use for conditional declaration of API functions! */
+#if defined(ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER) && defined(ZEND_INTRIN_HAVE_IFUNC_TARGET) && (!defined(__GNUC__) || (ZEND_GCC_VERSION >= 9000))
+/* __builtin_cpu_supports has pclmul from gcc9 */
+# define ZEND_INTRIN_SSE4_2_PCLMUL_FUNC_PROTO 1
+#elif defined(ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER)
+# define ZEND_INTRIN_SSE4_2_PCLMUL_FUNC_PTR 1
+#endif
+
+#ifdef ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER
+# ifdef HAVE_FUNC_ATTRIBUTE_TARGET
+#  define ZEND_INTRIN_SSE4_2_PCLMUL_FUNC_DECL(func) ZEND_API func __attribute__((target("sse4.2,pclmul")))
+# else
+#  define ZEND_INTRIN_SSE4_2_PCLMUL_FUNC_DECL(func) func
+# endif
+#else
+# define ZEND_INTRIN_SSE4_2_PCLMUL_FUNC_DECL(func)
+#endif
+
 #ifdef __AVX2__
 # define ZEND_INTRIN_AVX2_NATIVE 1
 #elif (defined(HAVE_FUNC_ATTRIBUTE_TARGET) && defined(PHP_HAVE_AVX2)) || defined(ZEND_WIN32)

configure.ac

@@ -409,6 +409,7 @@ sys/ipc.h \
 dlfcn.h \
 tmmintrin.h \
 nmmintrin.h \
+wmmintrin.h \
 immintrin.h
 ],[],[],[
 #ifdef HAVE_SYS_PARAM_H

ext/hash/hash_crc32.c

@@ -18,6 +18,7 @@
 #include "php_hash.h"
 #include "php_hash_crc32.h"
 #include "php_hash_crc32_tables.h"
+#include "ext/standard/crc32_x86.h"
 
 PHP_HASH_API void PHP_CRC32Init(PHP_CRC32_CTX *context)
 {
@@ -26,27 +27,39 @@ PHP_HASH_API void PHP_CRC32Init(PHP_CRC32_CTX *context)
 
 PHP_HASH_API void PHP_CRC32Update(PHP_CRC32_CTX *context, const unsigned char *input, size_t len)
 {
-	size_t i;
+	size_t i = 0;
 
-	for (i = 0; i < len; ++i) {
+#if ZEND_INTRIN_SSE4_2_PCLMUL_NATIVE || ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER
+	i += crc32_x86_simd_update(X86_CRC32, &context->state, input, len);
+#endif
+
+	for (; i < len; ++i) {
 		context->state = (context->state << 8) ^ crc32_table[(context->state >> 24) ^ (input[i] & 0xff)];
 	}
 }
 
 PHP_HASH_API void PHP_CRC32BUpdate(PHP_CRC32_CTX *context, const unsigned char *input, size_t len)
 {
-	size_t i;
+	size_t i = 0;
+
+#if ZEND_INTRIN_SSE4_2_PCLMUL_NATIVE || ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER
+	i += crc32_x86_simd_update(X86_CRC32B, &context->state, input, len);
+#endif
 
-	for (i = 0; i < len; ++i) {
+	for (; i < len; ++i) {
 		context->state = (context->state >> 8) ^ crc32b_table[(context->state ^ input[i]) & 0xff];
 	}
 }
 
 PHP_HASH_API void PHP_CRC32CUpdate(PHP_CRC32_CTX *context, const unsigned char *input, size_t len)
 {
-	size_t i;
+	size_t i = 0;
+
+#if ZEND_INTRIN_SSE4_2_PCLMUL_NATIVE || ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER
+	i += crc32_x86_simd_update(X86_CRC32C, &context->state, input, len);
+#endif
 
-	for (i = 0; i < len; ++i) {
+	for (; i < len; ++i) {
 		context->state = (context->state >> 8) ^ crc32c_table[(context->state ^ input[i]) & 0xff];
 	}
 }

ext/hash/tests/crc32.phpt

@@ -10,6 +10,20 @@ echo hash('crc32', 'message digest'), "\n";
 echo hash('crc32', 'abcdefghijklmnopqrstuvwxyz'), "\n";
 echo hash('crc32', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'), "\n";
 echo hash('crc32', '12345678901234567890123456789012345678901234567890123456789012345678901234567890'), "\n";
+echo hash('crc32', '1234567890123456'), "\n";
+echo hash('crc32', '1234567890123456abc'), "\n";
+echo hash('crc32', '12345678901234561234567890123456'), "\n";
+echo hash('crc32', '12345678901234561234567890123456abc'), "\n";
+echo hash('crc32', '123456789012345612345678901234561234567890123456'), "\n";
+echo hash('crc32', '123456789012345612345678901234561234567890123456abc'), "\n";
+echo hash('crc32', '1234567890123456123456789012345612345678901234561234567890123456'), "\n";
+echo hash('crc32', '1234567890123456123456789012345612345678901234561234567890123456abc'), "\n";
+echo hash('crc32', '12345678901234561234567890123456123456789012345612345678901234561234567890123456'), "\n";
+echo hash('crc32', '12345678901234561234567890123456123456789012345612345678901234561234567890123456abc'), "\n";
+echo hash('crc32', '12345678901234561234567890123456123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456'), "\n";
+echo hash('crc32', '12345678901234561234567890123456123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456abc'), "\n";
+echo hash('crc32', '123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456'), "\n";
+echo hash('crc32', '123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456abc'), "\n";
 
 echo "crc32b\n";
 echo hash('crc32b', ''), "\n";
@@ -19,6 +33,20 @@ echo hash('crc32b', 'message digest'), "\n";
 echo hash('crc32b', 'abcdefghijklmnopqrstuvwxyz'), "\n";
 echo hash('crc32b', 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'), "\n";
 echo hash('crc32b', '12345678901234567890123456789012345678901234567890123456789012345678901234567890'), "\n";
+echo hash('crc32b', '1234567890123456'), "\n";
+echo hash('crc32b', '1234567890123456abc'), "\n";
+echo hash('crc32b', '12345678901234561234567890123456'), "\n";
+echo hash('crc32b', '12345678901234561234567890123456abc'), "\n";
+echo hash('crc32b', '123456789012345612345678901234561234567890123456'), "\n";
+echo hash('crc32b', '123456789012345612345678901234561234567890123456abc'), "\n";
+echo hash('crc32b', '1234567890123456123456789012345612345678901234561234567890123456'), "\n";
+echo hash('crc32b', '1234567890123456123456789012345612345678901234561234567890123456abc'), "\n";
+echo hash('crc32b', '12345678901234561234567890123456123456789012345612345678901234561234567890123456'), "\n";
+echo hash('crc32b', '12345678901234561234567890123456123456789012345612345678901234561234567890123456abc'), "\n";
+echo hash('crc32b', '12345678901234561234567890123456123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456'), "\n";
+echo hash('crc32b', '12345678901234561234567890123456123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456abc'), "\n";
+echo hash('crc32b', '123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456'), "\n";
+echo hash('crc32b', '123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456abc'), "\n";
 
 echo "crc32c\n";
 echo hash('crc32c', ''), "\n";
@@ -59,6 +87,20 @@ echo hash('crc32c', "Even if I could be Shakespeare, I think I should still choo
 echo hash('crc32c', "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"), "\n";
 echo hash('crc32c', "How can you write a big system without C++?  -Paul Glick"), "\n";
 echo hash('crc32c', "\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\v\f\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\"#\$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff"), "\n";
+echo hash('crc32c', '1234567890123456'), "\n";
+echo hash('crc32c', '1234567890123456abc'), "\n";
+echo hash('crc32c', '12345678901234561234567890123456'), "\n";
+echo hash('crc32c', '12345678901234561234567890123456abc'), "\n";
+echo hash('crc32c', '123456789012345612345678901234561234567890123456'), "\n";
+echo hash('crc32c', '123456789012345612345678901234561234567890123456abc'), "\n";
+echo hash('crc32c', '1234567890123456123456789012345612345678901234561234567890123456'), "\n";
+echo hash('crc32c', '1234567890123456123456789012345612345678901234561234567890123456abc'), "\n";
+echo hash('crc32c', '12345678901234561234567890123456123456789012345612345678901234561234567890123456'), "\n";
+echo hash('crc32c', '12345678901234561234567890123456123456789012345612345678901234561234567890123456abc'), "\n";
+echo hash('crc32c', '12345678901234561234567890123456123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456'), "\n";
+echo hash('crc32c', '12345678901234561234567890123456123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456abc'), "\n";
+echo hash('crc32c', '123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456'), "\n";
+echo hash('crc32c', '123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456123456789012345612345678901234561234567890123456abc'), "\n";
 
 ?>
 --EXPECT--
@@ -70,6 +112,20 @@ crc32
 9693bf77
 882174a0
 96790816
+98b0e78d
+a6f33d71
+900a1d38
+396978fe
+adfc6afe
+d3ef9388
+c53911dc
+37006f1b
+4a54af3a
+98d05c71
+5a26f5b4
+b9108715
+cc684112
+b2ac45af
 crc32b
 00000000
 e8b7be43
@@ -78,6 +134,20 @@ e8b7be43
 4c2750bd
 1fc2e6d2
 7ca94a72
+1e5fcdb7
+70b54c2f
+094fb11e
+38210c49
+7399c6ef
+83e98d04
+1f26a94e
+e2e8634a
+0642542d
+43b42c9b
+262e1ded
+b7a463c4
+dfa1bbae
+4022d57a
 crc32c
 00000000
 c1d04330
@@ -116,4 +186,18 @@ de2e65c5
 297a88ed
 66ed1d8b
 dcded527
-9c44184b
+9c44184b
+9aa4287f
+ab2761c5
+cd486b4b
+c19c4a41
+1ea5b441
+36d20512
+31d11ffa
+65d5bb9e
+a0e3e317
+8dc10a7c
+7ab04135
+c292a38d
+e3e558ec
+b6c5e13e

ext/standard/basic_functions.c

@@ -33,6 +33,7 @@
 #include "ext/standard/php_dns.h"
 #include "ext/standard/php_uuencode.h"
 #include "ext/standard/php_mt_rand.h"
+#include "ext/standard/crc32_x86.h"
 
 #ifdef PHP_WIN32
 #include "win32/php_win32_globals.h"
@@ -363,6 +364,10 @@ PHP_MINIT_FUNCTION(basic) /* {{{ */
 	BASIC_MINIT_SUBMODULE(string_intrin)
 #endif
 
+#if ZEND_INTRIN_SSE4_2_PCLMUL_FUNC_PTR
+	BASIC_MINIT_SUBMODULE(crc32_x86_intrin)
+#endif
+
 #if ZEND_INTRIN_AVX2_FUNC_PTR || ZEND_INTRIN_SSSE3_FUNC_PTR
 	BASIC_MINIT_SUBMODULE(base64_intrin)
 #endif

ext/standard/config.m4

@@ -449,7 +449,7 @@ PHP_NEW_EXTENSION(standard, array.c base64.c basic_functions.c browscap.c crc32.
                             http_fopen_wrapper.c php_fopen_wrapper.c credits.c css.c \
                             var_unserializer.c ftok.c sha1.c user_filters.c uuencode.c \
                             filters.c proc_open.c streamsfuncs.c http.c password.c \
-                            random.c net.c hrtime.c,,,
+                            random.c net.c hrtime.c crc32_x86.c,,,
 			    -DZEND_ENABLE_STATIC_TSRMLS_CACHE=1)
 
 PHP_ADD_MAKEFILE_FRAGMENT

ext/standard/config.w32

@@ -25,7 +25,7 @@ ADD_FLAG("LIBS_STANDARD", "iphlpapi.lib");
 
 EXTENSION("standard", "array.c base64.c basic_functions.c browscap.c \
 	crc32.c crypt.c crypt_freesec.c crypt_blowfish.c crypt_sha256.c \
-	crypt_sha512.c  php_crypt_r.c \
+	crypt_sha512.c  php_crypt_r.c crc32_x86.c \
 	datetime.c dir.c dl.c dns.c dns_win32.c exec.c \
 	file.c filestat.c formatted_print.c fsock.c head.c html.c image.c \
 	info.c iptc.c lcg.c link.c mail.c math.c md5.c metaphone.c microtime.c \

ext/standard/crc32.c

@@ -17,6 +17,7 @@
 #include "php.h"
 #include "basic_functions.h"
 #include "crc32.h"
+#include "crc32_x86.h"
 
 #if HAVE_AARCH64_CRC32
 # include <arm_acle.h>
@@ -74,7 +75,7 @@ PHP_FUNCTION(crc32)
 	char *p;
 	size_t nr;
 	uint32_t crcinit = 0;
-	register uint32_t crc;
+	uint32_t crc;
 
 	ZEND_PARSE_PARAMETERS_START(1, 1)
 		Z_PARAM_STRING(p, nr)
@@ -89,6 +90,12 @@ PHP_FUNCTION(crc32)
 	}
 #endif
 
+#if ZEND_INTRIN_SSE4_2_PCLMUL_NATIVE || ZEND_INTRIN_SSE4_2_PCLMUL_RESOLVER
+	size_t nr_simd = crc32_x86_simd_update(X86_CRC32B, &crc, (const unsigned char *)p, nr);
+	nr -= nr_simd;
+	p += nr_simd;
+#endif
+
 	for (; nr--; ++p) {
 		crc = ((crc >> 8) & 0x00FFFFFF) ^ crc32tab[(crc ^ (*p)) & 0xFF ];
 	}