Implemented vectorscan match engine

docker/build/entrypoint.sh

@@ -95,6 +95,8 @@ if [ "$PACKAGE_TYPE" = 'standalone' ]; then
     --hidden-import wordfence.cli.remediate.definition \
     --hidden-import wordfence.cli.countsites.countsites \
     --hidden-import wordfence.cli.countsites.definition \
+    --hidden-import wordfence.scanning.matching.pcre \
+    --hidden-import wordfence.scanning.matching.vectorscan \
     main.py
 
   # compress and copy to output volume

wordfence/api/licensing.py

@@ -1,4 +1,4 @@
-from typing import Union
+from typing import Union, Optional
 
 from .exceptions import ApiException
 
@@ -36,8 +36,14 @@ def __init__(self):
 
 class LicenseSpecific:
 
-    def __init__(self, license: License):
+    def __init__(self, license: Optional[License]):
         self.license = license
 
     def is_compatible_with_license(self, license: License):
-        return self.license == license
+        return self.license is None or self.license == license
+
+    def assign_license(self, license: Optional[License]):
+        self.license = license
+
+    def clear_license(self):
+        self.assign_license(None)

wordfence/api/noc1.py

@@ -1,13 +1,17 @@
 import json
 import re
+import base64
 from typing import Callable, Optional
 
 from .noc_client import NocClient
 from .exceptions import ApiException
 from .licensing import License
 
-from ..intel.signatures import CommonString, Signature, SignatureSet
-from ..util.validation import DictionaryValidator, ListValidator, Validator
+from ..intel.signatures import CommonString, Signature, SignatureSet, \
+    PrecompiledSignatureSet, deserialize_precompiled_signature_set
+from ..util.validation import DictionaryValidator, ListValidator, Validator, \
+    OptionalValueValidator
+from ..util.platform import Platform
 
 NOC1_BASE_URL = 'https://noc1.wordfence.com/v2.27/'
 
@@ -137,6 +141,52 @@ def get_malware_signatures(self) -> SignatureSet:
                         ) from index_error
         return SignatureSet(common_strings, signatures, self.license)
 
+    def get_precompiled_patterns(
+                self,
+                platform: str,
+                library_version: str,
+                library_type: Optional[str] = None,
+                database_version: int = PrecompiledSignatureSet.VERSION
+            ) -> dict:
+        parameters = {
+                'platform': platform,
+                'library_version': library_version,
+                'database_version': database_version
+            }
+        if library_type is not None:
+            parameters['library_type'] = library_type
+        response = self.request('get_precompiled_patterns', parameters)
+        validator = DictionaryValidator({
+                'data': OptionalValueValidator(str)
+            })
+        self.validate_response(response, validator)
+        return response
+
+    def get_precompiled_malware_signatures(
+                self,
+                platform: Platform,
+                library_version: str,
+                library_type: Optional[str] = None,
+                database_version: int = PrecompiledSignatureSet.VERSION
+            ) -> Optional[PrecompiledSignatureSet]:
+        response = self.get_precompiled_patterns(
+                platform.key,
+                library_version,
+                library_type,
+                database_version
+            )
+        data = response['data']
+        if data is None:
+            return None
+        data = base64.b64decode(data)
+        signature_set = deserialize_precompiled_signature_set(data)
+        signature_set.assign_license(self.license)
+        if isinstance(signature_set, PrecompiledSignatureSet):
+            return signature_set
+        raise ApiException(
+                'Malformed signature set data received from Wordfence API'
+            )
+
     def ping_api_key(self) -> bool:
         return self.process_simple_request('ping_api_key')
 

wordfence/cli/context.py

@@ -2,7 +2,8 @@
 from typing import Optional, Any, Callable, Set, Union
 
 from ..version import __version__, __version_name__
-from ..util import pcre
+from ..util import pcre, vectorscan
+from ..util.text import yes_no
 from ..api import noc1, intelligence
 from ..util.caching import Cache, CacheDirectory, RuntimeCache, \
         InvalidCachedValueException, CacheException
@@ -156,17 +157,35 @@ def get_mailer(self) -> Mailer:
             self._mailer = Mailer(self.config)
         return self._mailer
 
+    def has_pcre(self) -> bool:
+        return pcre.AVAILABLE
+
+    def has_vectorscan(self) -> bool:
+        return vectorscan.AVAILABLE
+
     def display_version(self) -> None:
         if __version_name__ is None:
             name_suffix = ''
         else:
             name_suffix = f' "{__version_name__}"'
         print(f"Wordfence CLI {__version__}{name_suffix}")
-        jit_support_text = 'Yes' if pcre.HAS_JIT_SUPPORT else 'No'
-        print(
-                f"PCRE Version: {pcre.VERSION} - "
-                f"JIT Supported: {jit_support_text}"
-            )
+        has_pcre = self.has_pcre()
+        pcre_support_text = yes_no(has_pcre)
+        if has_pcre:
+            jit_support_text = yes_no(pcre.HAS_JIT_SUPPORT)
+            pcre_support_text += (
+                    f" - PCRE Version: {pcre.VERSION}"
+                    f" (JIT Supported: {jit_support_text})"
+                )
+        print(f'PCRE Supported: {pcre_support_text}')
+        has_vectorscan = self.has_vectorscan()
+        vectorscan_support_text = yes_no(has_vectorscan)
+        if has_vectorscan:
+            vectorscan_support_text += (
+                    f' - Version: {vectorscan.VERSION} (API Version: '
+                    f'{vectorscan.API_VERSION})'
+                )
+        print(f'Vectorscan Supported: {vectorscan_support_text}')
 
     def has_terminal_output(self) -> bool:
         return has_terminal_output()

wordfence/cli/malwarescan/definition.py

@@ -2,6 +2,7 @@
         PCRE_DEFAULT_MATCH_LIMIT_RECURSION
 from wordfence.util.units import byte_length
 
+from ...scanning.matching import MatchEngine
 from ..subcommands import SubcommandDefinition, UsageExample
 from ..config.typing import ConfigDefinitions
 from .reporting import SCAN_REPORT_CONFIG_OPTIONS
@@ -146,6 +147,15 @@
             "value_type": byte_length
         }
     },
+    "match-engine": {
+        "description": "The regex engine to use for malware scanning.",
+        "context": "ALL",
+        "argument_type": "OPTION",
+        "default": MatchEngine.get_default_option(),
+        "meta": {
+            "valid_options": MatchEngine.get_options()
+        }
+    },
     "match-all": {
         "description": "If set, all possible signatures will be checked "
                        "against each scanned file. Otherwise, only the "
@@ -187,13 +197,79 @@
         "argument_type": "FLAG",
         "default": False
     },
+    "pre-compile": {
+        "description": "Pre-compile and cache the signature set without "
+                       "actually running a scan",
+        "context": "CLI",
+        "argument_type": "FLAG",
+        "default": False,
+        "category": "Signature Compilation"
+    },
+    "pre-compile-generic": {
+        "description": "Pre-compile and cache the signature set without "
+                       "any CPU-specific optimizations and without running "
+                       "a scan",
+        "context": "CLI",
+        "argument_type": "FLAG",
+        "default": False,
+        "category": "Signature Compilation"
+    },
+    "pattern-database-path": {
+        "description": "Use an alternate path for storage of the pattern "
+                       "database",
+        "context": "ALL",
+        "argument_type": "OPTION",
+        "meta": {
+            "accepts_file": True
+        },
+        "default": None,
+        "category": "Signature Compilation"
+    },
+    "compile-local": {
+        "description": "Always compile the signature set locally rather than "
+                       "attempting to download a pre-compiled set",
+        "context": "ALL",
+        "argument_type": "FLAG",
+        "default": False,
+        "category": "Signature Compilation"
+    },
+    "re-compile": {
+        "description": "Always re-compile the signature set rather than using "
+                       "a cached version (when compiling locally)",
+        "context": "CLI",
+        "argument_type": "FLAG",
+        "default": False,
+        "category": "Signature Compilation"
+    },
+    "profile": {
+        "description": "Profile scan performance",
+        "context": "CLI",
+        "argument_type": "FLAG",
+        "default": False,
+        "hidden": True
+    },
+    "profile-path": {
+        "description": "Path at which to save profiling results",
+        "context": "CLI",
+        "argument_type": "OPTION",
+        "default": None,
+        "hidden": True
+    },
+    "direct-io": {
+        "short_name": "D",
+        "description": "Use direct IO when opening files to avoid caching",
+        "context": "ALL",
+        "argument_type": "FLAG",
+        "default": False
+    }
 }
 
 
 cacheable_types = {
     'wordfence.intel.signatures.SignatureSet',
     'wordfence.intel.signatures.CommonString',
     'wordfence.intel.signatures.Signature',
+    'wordfence.intel.signatures.PrecompiledSignatureSet',
     'wordfence.api.licensing.License'
 }