Further adjusted handling of binary/string data

wordfence/cli/remediate/reporting.py

@@ -15,19 +15,25 @@
         REPORT_FORMAT_CSV, REPORT_FORMAT_TSV, REPORT_FORMAT_NULL_DELIMITED, \
         REPORT_FORMAT_LINE_DELIMITED
 from ..context import CliContext
+from ...util.encoding import bytes_to_str
 
 
 class RemediationReportColumn(ReportColumnEnum):
     PATH = 'path', lambda record: os.fsdecode(record.result.path),
     STATUS = 'status', lambda record: record.get_status(),
     TYPE = 'type', lambda record: record.result.identity.type,
     SITE = 'site', \
-        lambda record: record.result.identity.site.core_path \
+        lambda record: os.fsdecode(
+                record.result.identity.site.core_path
+            ) \
         if record.result.identity.site is not None \
         else None
-    TARGET_PATH = 'target_path', lambda record: record.result.target_path,
+    TARGET_PATH = 'target_path', \
+        lambda record: os.fsdecode(record.result.target_path),
     WORDPRESS_VERSION = 'wordpress_version', \
-        lambda record: record.result.identity.site.get_version() \
+        lambda record: bytes_to_str(
+                record.result.identity.site.get_version()
+            ) \
         if record.result.identity.site is not None \
         else None
     EXTENSION_SLUG = 'extension_slug', \
@@ -37,7 +43,9 @@ class RemediationReportColumn(ReportColumnEnum):
         lambda record: record.result.identity.extension.get_name() \
         if record.result.identity.extension is not None else None
     EXTENSION_VERSION = 'extension_version', \
-        lambda record: record.result.identity.extension.version \
+        lambda record: bytes_to_str(
+                record.result.identity.extension.version
+            ) \
         if record.result.identity.extension is not None else None
 
 

wordfence/cli/vulnscan/reporting.py

@@ -9,6 +9,7 @@
 from ...api.intelligence import VulnerabilityFeedVariant
 from ...util.terminal import Color, escape, RESET
 from ...util.html import Tag
+from ...util.versioning import version_to_str
 from ..reporting import Report, ReportColumnEnum, ReportFormatEnum, \
         ReportRecord, ReportManager, ReportFormat, ReportColumn, \
         BaseHumanReadableWriter, ReportEmail, get_config_options, \
@@ -22,7 +23,7 @@
 class VulnScanReportColumn(ReportColumnEnum):
     SOFTWARE_TYPE = 'software_type', lambda record: record.software.type.value
     SLUG = 'slug', lambda record: record.software.slug
-    VERSION = 'version', lambda record: record.software.version.decode('ascii')
+    VERSION = 'version', lambda record: version_to_str(record.software.version)
     ID = 'id', \
         lambda record: record.vulnerability.identifier
     TITLE = 'title', lambda record: record.vulnerability.title
@@ -92,7 +93,7 @@ def get_severity_color(self, severity: str) -> str:
     def format_record(self, record) -> str:
         vuln = record.vulnerability
         sw = record.software
-        sw_version = sw.version.decode('ascii')
+        sw_version = version_to_str(sw.version)
         yellow = escape(color=Color.YELLOW)
         link = vuln.get_wordfence_link()
         blue = escape(color=Color.BLUE)

wordfence/cli/vulnscan/vulnscan.py

@@ -8,6 +8,7 @@
         is_cve_id
 from ...api.intelligence import VulnerabilityFeedVariant
 from ...util.caching import Cacheable, DURATION_ONE_DAY
+from ...util.versioning import version_to_str
 from ...wordpress.site import WordpressSite, WordpressStructureOptions, \
         WordpressLocator, WordpressException
 from ...wordpress.plugin import PluginLoader, Plugin
@@ -47,7 +48,7 @@ def _scan_plugins(
         for plugin in plugins:
             log.debug(
                     f'Plugin {plugin.slug}, version: ' +
-                    plugin.version.decode('ascii')
+                    version_to_str(plugin.version)
                 )
             scanner.scan_plugin(plugin, path)
 
@@ -75,7 +76,7 @@ def _scan_themes(
         for theme in themes:
             log.debug(
                     f'Theme {theme.slug}, version: ' +
-                    theme.version.decode('ascii')
+                    version_to_str(theme.version)
                 )
             scanner.scan_theme(theme, path)
 
@@ -119,7 +120,7 @@ def _scan(
         version = site.get_version()
         log.debug(
                 'WordPress Core Version: ' +
-                version.decode('ascii', 'replace')
+                version_to_str(version)
             )
         if scan_path is None:
             scan_path = path

wordfence/php/parsing.py

@@ -3,6 +3,8 @@
 from enum import Enum
 from collections import deque
 
+from ..util.encoding import str_to_bytes
+
 from .lexing import Lexer, Token, TokenType, CharacterType, STRING_ESCAPE
 
 
@@ -45,7 +47,7 @@ class PhpStateType:
 
 def make_strings_binary(value: Any) -> Any:
     if isinstance(value, str):
-        return value.encode('ascii', 'ignore')
+        return str_to_bytes(value)
     return value
 
 

wordfence/util/encoding.py

@@ -0,0 +1,13 @@
+from typing import Optional
+
+
+def bytes_to_str(value: Optional[bytes]) -> Optional[str]:
+    if value is None:
+        return None
+    return value.decode('latin1', 'replace')
+
+
+def str_to_bytes(value: Optional[str]) -> Optional[bytes]:
+    if value is None:
+        return None
+    return value.encode('latin1', 'replace')

wordfence/util/pcre/bindings.py

@@ -4,6 +4,7 @@
 from typing import Optional
 
 from ..library import load_library, LibraryNotAvailableException
+from ..encoding import bytes_to_str
 
 from .pcre import PcreException, PcreLibraryNotAvailableException, \
     PcreOptions, \
@@ -18,7 +19,7 @@
 _pcre_version = pcre.pcre_version
 _pcre_version.argtypes = []
 _pcre_version.restype = c_char_p
-VERSION = _pcre_version().decode('ascii')
+VERSION = bytes_to_str(_pcre_version())
 
 
 class PcreError(IntEnum):

wordfence/util/vectorscan/bindings.py

@@ -4,6 +4,7 @@
 from typing import Dict, Optional, Callable, Union, Any
 
 from ..library import load_library, LibraryNotAvailableException
+from ..encoding import bytes_to_str
 from .. import signals
 
 from .vectorscan import VectorscanException, \
@@ -19,7 +20,7 @@
 _hs_version = hs.hs_version
 _hs_version.argtypes = []
 _hs_version.restype = c_char_p
-VERSION = _hs_version().decode('ascii')
+VERSION = bytes_to_str(_hs_version())
 API_VERSION = ''.join(VERSION.split()[:1])
 
 

wordfence/util/versioning.py

@@ -1,6 +1,8 @@
 import re
 from typing import List, Dict, Union, Optional
 
+from .encoding import str_to_bytes, bytes_to_str
+
 
 PHP_VERSION_DELIMITER = b'.'
 PHP_VERSION_ALTERNATE_DELIMITERS = [b'_', b'-', b'+']
@@ -101,7 +103,7 @@ class PhpVersion:
 
     def __init__(self, version: Union[str, bytes]):
         if isinstance(version, str):
-            version = version.encode('ascii')
+            version = str_to_bytes(version)
         self.version = version
         self._components = self.extract_components(version)
 
@@ -153,3 +155,9 @@ def compare_php_versions(
         if comparison != 0:
             return comparison
     return 0
+
+
+def version_to_str(version: Optional[bytes]) -> str:
+    if version is None:
+        return 'unknown'
+    return bytes_to_str(version)

wordfence/wordpress/extension.py

@@ -4,6 +4,7 @@
 
 from ..logging import log
 from ..util.io import SYMLINK_IO_ERRORS
+from ..util.encoding import str_to_bytes
 from .exceptions import ExtensionException
 
 
@@ -93,7 +94,7 @@ def load(
         try:
             version = header['Version']
             if isinstance(version, str):
-                version = version.encode('ascii')
+                version = str_to_bytes(version)
         except KeyError:
             version = None
         if base_path is None:

wordfence/wordpress/identifier.py

@@ -4,6 +4,7 @@
 from typing import Optional
 
 from ..util.io import resolve_path, get_path_components
+from ..util.encoding import bytes_to_str
 from .site import WordpressSite
 from .exceptions import WordpressException
 from .extension import Extension
@@ -73,7 +74,7 @@ def __str__(self) -> str:
             software = self.extension.get_name()
             version = self.extension.version
         if isinstance(version, bytes):
-            version = version.decode('ascii')
+            version = bytes_to_str(version)
         return (
                 os.fsdecode(self.local_path) +
                 f' of {self.type.value} {software} ({version})'