From 54606d267e57e09d16e774a199e390ad508d99ba Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Thu, 1 Aug 2024 15:50:32 -0400 Subject: [PATCH] Further adjusted handling of binary/string data --- wordfence/cli/remediate/reporting.py | 16 ++++++++++++---- wordfence/cli/vulnscan/reporting.py | 5 +++-- wordfence/cli/vulnscan/vulnscan.py | 7 ++++--- wordfence/php/parsing.py | 4 +++- wordfence/util/encoding.py | 13 +++++++++++++ wordfence/util/pcre/bindings.py | 3 ++- wordfence/util/vectorscan/bindings.py | 3 ++- wordfence/util/versioning.py | 10 +++++++++- wordfence/wordpress/extension.py | 3 ++- wordfence/wordpress/identifier.py | 3 ++- 10 files changed, 52 insertions(+), 15 deletions(-) create mode 100644 wordfence/util/encoding.py diff --git a/wordfence/cli/remediate/reporting.py b/wordfence/cli/remediate/reporting.py index 482039c..1f7cf20 100644 --- a/wordfence/cli/remediate/reporting.py +++ b/wordfence/cli/remediate/reporting.py @@ -15,6 +15,7 @@ REPORT_FORMAT_CSV, REPORT_FORMAT_TSV, REPORT_FORMAT_NULL_DELIMITED, \ REPORT_FORMAT_LINE_DELIMITED from ..context import CliContext +from ...util.encoding import bytes_to_str class RemediationReportColumn(ReportColumnEnum): @@ -22,12 +23,17 @@ class RemediationReportColumn(ReportColumnEnum): STATUS = 'status', lambda record: record.get_status(), TYPE = 'type', lambda record: record.result.identity.type, SITE = 'site', \ - lambda record: record.result.identity.site.core_path \ + lambda record: os.fsdecode( + record.result.identity.site.core_path + ) \ if record.result.identity.site is not None \ else None - TARGET_PATH = 'target_path', lambda record: record.result.target_path, + TARGET_PATH = 'target_path', \ + lambda record: os.fsdecode(record.result.target_path), WORDPRESS_VERSION = 'wordpress_version', \ - lambda record: record.result.identity.site.get_version() \ + lambda record: bytes_to_str( + record.result.identity.site.get_version() + ) \ if record.result.identity.site is not None \ else None EXTENSION_SLUG = 'extension_slug', \ @@ -37,7 +43,9 @@ class RemediationReportColumn(ReportColumnEnum): lambda record: record.result.identity.extension.get_name() \ if record.result.identity.extension is not None else None EXTENSION_VERSION = 'extension_version', \ - lambda record: record.result.identity.extension.version \ + lambda record: bytes_to_str( + record.result.identity.extension.version + ) \ if record.result.identity.extension is not None else None diff --git a/wordfence/cli/vulnscan/reporting.py b/wordfence/cli/vulnscan/reporting.py index 962cc51..f910a25 100644 --- a/wordfence/cli/vulnscan/reporting.py +++ b/wordfence/cli/vulnscan/reporting.py @@ -9,6 +9,7 @@ from ...api.intelligence import VulnerabilityFeedVariant from ...util.terminal import Color, escape, RESET from ...util.html import Tag +from ...util.versioning import version_to_str from ..reporting import Report, ReportColumnEnum, ReportFormatEnum, \ ReportRecord, ReportManager, ReportFormat, ReportColumn, \ BaseHumanReadableWriter, ReportEmail, get_config_options, \ @@ -22,7 +23,7 @@ class VulnScanReportColumn(ReportColumnEnum): SOFTWARE_TYPE = 'software_type', lambda record: record.software.type.value SLUG = 'slug', lambda record: record.software.slug - VERSION = 'version', lambda record: record.software.version.decode('ascii') + VERSION = 'version', lambda record: version_to_str(record.software.version) ID = 'id', \ lambda record: record.vulnerability.identifier TITLE = 'title', lambda record: record.vulnerability.title @@ -92,7 +93,7 @@ def get_severity_color(self, severity: str) -> str: def format_record(self, record) -> str: vuln = record.vulnerability sw = record.software - sw_version = sw.version.decode('ascii') + sw_version = version_to_str(sw.version) yellow = escape(color=Color.YELLOW) link = vuln.get_wordfence_link() blue = escape(color=Color.BLUE) diff --git a/wordfence/cli/vulnscan/vulnscan.py b/wordfence/cli/vulnscan/vulnscan.py index e32388f..e473391 100644 --- a/wordfence/cli/vulnscan/vulnscan.py +++ b/wordfence/cli/vulnscan/vulnscan.py @@ -8,6 +8,7 @@ is_cve_id from ...api.intelligence import VulnerabilityFeedVariant from ...util.caching import Cacheable, DURATION_ONE_DAY +from ...util.versioning import version_to_str from ...wordpress.site import WordpressSite, WordpressStructureOptions, \ WordpressLocator, WordpressException from ...wordpress.plugin import PluginLoader, Plugin @@ -47,7 +48,7 @@ def _scan_plugins( for plugin in plugins: log.debug( f'Plugin {plugin.slug}, version: ' + - plugin.version.decode('ascii') + version_to_str(plugin.version) ) scanner.scan_plugin(plugin, path) @@ -75,7 +76,7 @@ def _scan_themes( for theme in themes: log.debug( f'Theme {theme.slug}, version: ' + - theme.version.decode('ascii') + version_to_str(theme.version) ) scanner.scan_theme(theme, path) @@ -119,7 +120,7 @@ def _scan( version = site.get_version() log.debug( 'WordPress Core Version: ' + - version.decode('ascii', 'replace') + version_to_str(version) ) if scan_path is None: scan_path = path diff --git a/wordfence/php/parsing.py b/wordfence/php/parsing.py index 332f19e..a9fdddf 100644 --- a/wordfence/php/parsing.py +++ b/wordfence/php/parsing.py @@ -3,6 +3,8 @@ from enum import Enum from collections import deque +from ..util.encoding import str_to_bytes + from .lexing import Lexer, Token, TokenType, CharacterType, STRING_ESCAPE @@ -45,7 +47,7 @@ class PhpStateType: def make_strings_binary(value: Any) -> Any: if isinstance(value, str): - return value.encode('ascii', 'ignore') + return str_to_bytes(value) return value diff --git a/wordfence/util/encoding.py b/wordfence/util/encoding.py new file mode 100644 index 0000000..c4ac27a --- /dev/null +++ b/wordfence/util/encoding.py @@ -0,0 +1,13 @@ +from typing import Optional + + +def bytes_to_str(value: Optional[bytes]) -> Optional[str]: + if value is None: + return None + return value.decode('latin1', 'replace') + + +def str_to_bytes(value: Optional[str]) -> Optional[bytes]: + if value is None: + return None + return value.encode('latin1', 'replace') diff --git a/wordfence/util/pcre/bindings.py b/wordfence/util/pcre/bindings.py index f08c27d..0df7d52 100644 --- a/wordfence/util/pcre/bindings.py +++ b/wordfence/util/pcre/bindings.py @@ -4,6 +4,7 @@ from typing import Optional from ..library import load_library, LibraryNotAvailableException +from ..encoding import bytes_to_str from .pcre import PcreException, PcreLibraryNotAvailableException, \ PcreOptions, \ @@ -18,7 +19,7 @@ _pcre_version = pcre.pcre_version _pcre_version.argtypes = [] _pcre_version.restype = c_char_p -VERSION = _pcre_version().decode('ascii') +VERSION = bytes_to_str(_pcre_version()) class PcreError(IntEnum): diff --git a/wordfence/util/vectorscan/bindings.py b/wordfence/util/vectorscan/bindings.py index 7056a80..0a5e953 100644 --- a/wordfence/util/vectorscan/bindings.py +++ b/wordfence/util/vectorscan/bindings.py @@ -4,6 +4,7 @@ from typing import Dict, Optional, Callable, Union, Any from ..library import load_library, LibraryNotAvailableException +from ..encoding import bytes_to_str from .. import signals from .vectorscan import VectorscanException, \ @@ -19,7 +20,7 @@ _hs_version = hs.hs_version _hs_version.argtypes = [] _hs_version.restype = c_char_p -VERSION = _hs_version().decode('ascii') +VERSION = bytes_to_str(_hs_version()) API_VERSION = ''.join(VERSION.split()[:1]) diff --git a/wordfence/util/versioning.py b/wordfence/util/versioning.py index 59287f2..856475c 100644 --- a/wordfence/util/versioning.py +++ b/wordfence/util/versioning.py @@ -1,6 +1,8 @@ import re from typing import List, Dict, Union, Optional +from .encoding import str_to_bytes, bytes_to_str + PHP_VERSION_DELIMITER = b'.' PHP_VERSION_ALTERNATE_DELIMITERS = [b'_', b'-', b'+'] @@ -101,7 +103,7 @@ class PhpVersion: def __init__(self, version: Union[str, bytes]): if isinstance(version, str): - version = version.encode('ascii') + version = str_to_bytes(version) self.version = version self._components = self.extract_components(version) @@ -153,3 +155,9 @@ def compare_php_versions( if comparison != 0: return comparison return 0 + + +def version_to_str(version: Optional[bytes]) -> str: + if version is None: + return 'unknown' + return bytes_to_str(version) diff --git a/wordfence/wordpress/extension.py b/wordfence/wordpress/extension.py index 60f26fc..81e7a81 100644 --- a/wordfence/wordpress/extension.py +++ b/wordfence/wordpress/extension.py @@ -4,6 +4,7 @@ from ..logging import log from ..util.io import SYMLINK_IO_ERRORS +from ..util.encoding import str_to_bytes from .exceptions import ExtensionException @@ -93,7 +94,7 @@ def load( try: version = header['Version'] if isinstance(version, str): - version = version.encode('ascii') + version = str_to_bytes(version) except KeyError: version = None if base_path is None: diff --git a/wordfence/wordpress/identifier.py b/wordfence/wordpress/identifier.py index 135a525..54a64cd 100644 --- a/wordfence/wordpress/identifier.py +++ b/wordfence/wordpress/identifier.py @@ -4,6 +4,7 @@ from typing import Optional from ..util.io import resolve_path, get_path_components +from ..util.encoding import bytes_to_str from .site import WordpressSite from .exceptions import WordpressException from .extension import Extension @@ -73,7 +74,7 @@ def __str__(self) -> str: software = self.extension.get_name() version = self.extension.version if isinstance(version, bytes): - version = version.decode('ascii') + version = bytes_to_str(version) return ( os.fsdecode(self.local_path) + f' of {self.type.value} {software} ({version})'