From 6a0841ef4e9273bd2c19ad58c713e44a4b529615 Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Fri, 6 Sep 2024 16:10:05 -0400 Subject: [PATCH 01/18] Set up initial structure and subcommand for DB scanning --- README.md | 1 + pyproject.toml | 3 +- requirements.txt | 1 + wordfence/cli/dbscan/dbscan.py | 44 ++++++++++++ wordfence/cli/dbscan/definition.py | 82 ++++++++++++++++++++++ wordfence/cli/subcommands.py | 1 + wordfence/databasescanning/__init__.py | 3 + wordfence/databasescanning/scanner.py | 37 ++++++++++ wordfence/intel/database_rules.py | 6 ++ wordfence/wordpress/database.py | 94 ++++++++++++++++++++++++++ 10 files changed, 271 insertions(+), 1 deletion(-) create mode 100644 wordfence/cli/dbscan/dbscan.py create mode 100644 wordfence/cli/dbscan/definition.py create mode 100644 wordfence/databasescanning/__init__.py create mode 100644 wordfence/databasescanning/scanner.py create mode 100644 wordfence/intel/database_rules.py create mode 100644 wordfence/wordpress/database.py diff --git a/README.md b/README.md index 857bdabe..4ce0e53e 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ If you'd like to install Wordfence CLI manually or use CLI for development, you - Python packages: - `packaging` >= 21.0 - `requests` >= 2.3 + - `mysql-connector-python` >= 8.0 ### Obtaining a license diff --git a/pyproject.toml b/pyproject.toml index b67f6bc4..61f1406c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,8 @@ classifiers = [ ] dependencies = [ "packaging>=21.0", - "requests>=2.3" + "requests>=2.3", + "mysql-connector-python>=8.0" ] dynamic = [ "version" ] diff --git a/requirements.txt b/requirements.txt index a2c08548..86309c26 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ # Runtime dependencies packaging >= 21.0 requests >= 2.3 +mysql-connector-python >= 8.0 # Build requirements build ~= 0.10 setuptools ~= 68.0 diff --git a/wordfence/cli/dbscan/dbscan.py b/wordfence/cli/dbscan/dbscan.py new file mode 100644 index 00000000..9b1cd53f --- /dev/null +++ b/wordfence/cli/dbscan/dbscan.py @@ -0,0 +1,44 @@ +from wordfence.wordpress.database import WordpressDatabase, \ + WordpressDatabaseServer +from wordfence.intel.database_rules import DatabaseRuleSet +from wordfence.databasescanning.scanner import DatabaseScanner +from getpass import getpass +from typing import Optional +import os + +from ...logging import log +from ..subcommands import Subcommand + + +class DbScanSubcommand(Subcommand): + + def resolve_password(self) -> Optional[str]: + if self.config.password is not None: + log.warning( + 'Providing passwords via command line parameters is ' + 'insecure as they can be exposed to other users' + ) + return self.config.password + elif self.config.prompt_for_password: + return getpass() + return os.environ.get(self.config.password_env) + + def invoke(self) -> int: + ruleSet = DatabaseRuleSet() + scanner = DatabaseScanner(ruleSet) + server = WordpressDatabaseServer( + host=self.config.host, + port=self.config.port, + user=self.config.user, + password=self.resolve_password() + ) + for database_name in self.config.trailing_arguments: + database = WordpressDatabase( + name=database_name, + server=server + ) + scanner.scan(database) + return 0 + + +factory = DbScanSubcommand diff --git a/wordfence/cli/dbscan/definition.py b/wordfence/cli/dbscan/definition.py new file mode 100644 index 00000000..db5dd2ea --- /dev/null +++ b/wordfence/cli/dbscan/definition.py @@ -0,0 +1,82 @@ +from wordfence.wordpress.database import DEFAULT_HOST, DEFAULT_PORT, \ + DEFAULT_USER, DEFAULT_PREFIX + +from ..subcommands import SubcommandDefinition, UsageExample +from ..config.typing import ConfigDefinitions + +config_definitions: ConfigDefinitions = { + "host": { + "short_name": "H", + "description": "The database hostname", + "context": "CLI", + "argument_type": "OPTION", + "default": DEFAULT_HOST, + "category": "Database Connectivity" + }, + "port": { + "short_name": "P", + "description": "The database port", + "context": "CLI", + "argument_type": "OPTION", + "default": DEFAULT_PORT, + "meta": { + "value_type": int + }, + "category": "Database Connectivity" + }, + "user": { + "short_name": "u", + "description": "The database user", + "context": "CLI", + "argument_type": "OPTION", + "default": DEFAULT_USER, + "category": "Database Connectivity" + }, + "password": { + "description": "The database password (this option is insecure)", + "context": "CLI", + "argument_type": "OPTION", + "default": None, + "category": "Database Connectivity" + }, + "prompt-for-password": { + "short_name": "p", + "description": "Prompt for a database password", + "context": "CLI", + "argument_type": "FLAG", + "default": False, + "category": "Database Connectivity" + }, + "password-env": { + "description": "The environment variable name to check for a password", + "context": "ALL", + "argument_type": "OPTION", + "default": "WFCLI_DB_PASSWORD", + "category": "Database Connectivity" + }, + "prefix": { + "short_name": "x", + "description": "The WordPress database prefix", + "context": "CLI", + "argument_type": "OPTION", + "default": DEFAULT_PREFIX, + "category": "Database Connectivity" + } +} + +examples = [ + UsageExample( + 'Scan the WordPress database at db.example.com', + 'wordfence db-scan -h db.example.com -p wordpress' + ) +] + +definition = SubcommandDefinition( + name='db-scan', + usage='[OPTIONS] [DATABASE_NAME]...', + description='Scan for malicious content in a WordPress databases', + config_definitions=config_definitions, + config_section='DB_SCAN', + cacheable_types=set(), + examples=examples +) diff --git a/wordfence/cli/subcommands.py b/wordfence/cli/subcommands.py index f127547d..89baa459 100644 --- a/wordfence/cli/subcommands.py +++ b/wordfence/cli/subcommands.py @@ -14,6 +14,7 @@ 'vuln-scan', 'remediate', 'count-sites', + 'db-scan', 'help', 'version', 'terms' diff --git a/wordfence/databasescanning/__init__.py b/wordfence/databasescanning/__init__.py new file mode 100644 index 00000000..f7f624bc --- /dev/null +++ b/wordfence/databasescanning/__init__.py @@ -0,0 +1,3 @@ +from . import scanner + +__all__ = ['scanner'] diff --git a/wordfence/databasescanning/scanner.py b/wordfence/databasescanning/scanner.py new file mode 100644 index 00000000..d34aea10 --- /dev/null +++ b/wordfence/databasescanning/scanner.py @@ -0,0 +1,37 @@ +from wordfence.intel.database_rules import DatabaseRuleSet +from wordfence.wordpress.database import WordpressDatabase, \ + WordpressDatabaseConnection +from ..logging import log +from typing import Union + + +class DatabaseScanner: + + def __init__( + self, + ruleSet: DatabaseRuleSet + ): + self.ruleSet = ruleSet + + def _scan_connection( + self, + connection: WordpressDatabaseConnection + ) -> None: + log.debug(f'Scanning database: {connection.database.debug_string}...') + pass + log.debug(f'Scan completed for: {connection.database.debug_string}...') + + def scan( + self, + database: Union[WordpressDatabase, WordpressDatabaseConnection] + ) -> None: + if isinstance(database, WordpressDatabaseConnection): + return self._scan_connection(database) + else: + log.debug(f'Connecting to database: {database.debug_string}...') + with database.connect() as connection: + log.debug( + 'Successfully connected to database: ' + f'{database.debug_string}' + ) + return self._scan_connection(connection) diff --git a/wordfence/intel/database_rules.py b/wordfence/intel/database_rules.py new file mode 100644 index 00000000..8cdcba5e --- /dev/null +++ b/wordfence/intel/database_rules.py @@ -0,0 +1,6 @@ +class DatabaseRule: + pass + + +class DatabaseRuleSet: + pass diff --git a/wordfence/wordpress/database.py b/wordfence/wordpress/database.py new file mode 100644 index 00000000..28c9b51b --- /dev/null +++ b/wordfence/wordpress/database.py @@ -0,0 +1,94 @@ +import mysql.connector +from typing import Optional, Generator + + +DEFAULT_HOST = 'localhost' +DEFAULT_PORT = 3306 +DEFAULT_USER = 'root' +DEFAULT_PREFIX = 'wp_' + + +class WordpressDatabaseException(Exception): + + def __init__(self, database, message): + self.database = database + + +class WordpressDatabaseConnection: + + def __init__(self, database): + self.database = database + try: + self.connection = mysql.connector.connect( + host=database.server.host, + port=database.server.port, + user=database.server.user, + password=database.server.password, + database=database.name + ) + except mysql.connector.Error: + raise WordpressDatabaseException( + database, + f'Failed to connect to database: {self.debug_string}' + ) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + return + + def query( + self, + query: str, + parameters: tuple = () + ) -> Generator[tuple, None, None]: + try: + cursor = self.connection.cursor() + cursor.execute(query, parameters) + for result in cursor: + yield result + cursor.close() + except mysql.connector.Error: + raise WordpressDatabaseException( + self.database, + 'Failed to execute query' + ) + + +class WordpressDatabaseServer: + + def __init__( + self, + host: str = DEFAULT_HOST, + port: int = DEFAULT_PORT, + user: str = DEFAULT_USER, + password: Optional[str] = None, + ): + self.host = host + self.port = port + self.user = user + self.password = password + + +class WordpressDatabase: + + def __init__( + self, + name: str, + server: WordpressDatabaseServer, + prefix: str = DEFAULT_PREFIX + ): + self.name = name + self.server = server + self.prefix = prefix + self.debug_string = self._build_debug_string() + + def connect(self) -> WordpressDatabaseConnection: + return WordpressDatabaseConnection(self) + + def _build_debug_string(self) -> str: + return ( + f'{self.server.user}@{self.server.host}:' + f'{self.server.port}/{self.name}' + ) From 0e385010e1069601ed41817e4bf818e3a2f0c2ff Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Tue, 10 Sep 2024 16:19:58 -0400 Subject: [PATCH 02/18] Expanded input options for db-scan --- wordfence/cli/countsites/definition.py | 2 +- wordfence/cli/dbscan/dbscan.py | 129 ++++++++++++++++++++++--- wordfence/cli/dbscan/definition.py | 66 ++++++++++++- wordfence/databasescanning/scanner.py | 6 +- wordfence/intel/database_rules.py | 11 ++- wordfence/wordpress/database.py | 10 +- wordfence/wordpress/exceptions.py | 6 ++ wordfence/wordpress/site.py | 109 ++++++++++++++++++--- 8 files changed, 303 insertions(+), 36 deletions(-) diff --git a/wordfence/cli/countsites/definition.py b/wordfence/cli/countsites/definition.py index 5dde9db0..7afa1793 100644 --- a/wordfence/cli/countsites/definition.py +++ b/wordfence/cli/countsites/definition.py @@ -20,7 +20,7 @@ "default_type": "base64" }, "require-path": { - "description": "When enabled, invoking the remediate command without " + "description": "When enabled, invoking the count command without " "specifying at least one path will trigger an error. " "This is the default behavior when running in a " "terminal.", diff --git a/wordfence/cli/dbscan/dbscan.py b/wordfence/cli/dbscan/dbscan.py index 9b1cd53f..cc157282 100644 --- a/wordfence/cli/dbscan/dbscan.py +++ b/wordfence/cli/dbscan/dbscan.py @@ -1,18 +1,26 @@ from wordfence.wordpress.database import WordpressDatabase, \ - WordpressDatabaseServer + WordpressDatabaseServer, DEFAULT_PORT +from wordfence.wordpress.site import WordpressLocator, \ + WordpressSite +from wordfence.wordpress.exceptions import WordpressException from wordfence.intel.database_rules import DatabaseRuleSet from wordfence.databasescanning.scanner import DatabaseScanner +from wordfence.util.validation import ListValidator, DictionaryValidator, \ + OptionalValueValidator from getpass import getpass -from typing import Optional +from typing import Optional, List, Generator import os +import json from ...logging import log from ..subcommands import Subcommand +from ..io import IoManager +from ..exceptions import ConfigurationException class DbScanSubcommand(Subcommand): - def resolve_password(self) -> Optional[str]: + def _resolve_password(self) -> Optional[str]: if self.config.password is not None: log.warning( 'Providing passwords via command line parameters is ' @@ -23,21 +31,120 @@ def resolve_password(self) -> Optional[str]: return getpass() return os.environ.get(self.config.password_env) - def invoke(self) -> int: - ruleSet = DatabaseRuleSet() - scanner = DatabaseScanner(ruleSet) + def _get_base_database(self) -> Optional[WordpressDatabase]: + name = self.config.database_name + if name is None: + return None server = WordpressDatabaseServer( host=self.config.host, port=self.config.port, user=self.config.user, - password=self.resolve_password() + password=self._resolve_password() + ) + return WordpressDatabase( + name=name, + server=server + ) + + def _get_search_paths( + self, + include_current: bool = False + ) -> Generator[bytes, None, None]: + io_manager = IoManager( + self.config.read_stdin, + self.config.path_separator, + binary=True ) - for database_name in self.config.trailing_arguments: - database = WordpressDatabase( - name=database_name, - server=server + if len(self.config.trailing_arguments): + yield from self.config.trailing_arguments + elif include_current and not io_manager.should_read_stdin(): + yield os.fsencode(os.getcwd()) + if io_manager.should_read_stdin(): + for path in io_manager.get_input_reader().read_all_entries(): + yield path + + def _locate_site_databases( + self + ) -> Generator[WordpressDatabase, None, None]: + for path in self._get_search_paths(include_current=True): + locator = WordpressLocator( + path=path, + allow_nested=self.config.allow_nested, + allow_io_errors=self.config.allow_io_errors ) + for core_path in locator.locate_core_paths(): + site = WordpressSite(core_path) + log.debug( + 'Located WordPress site at ' + os.fsdecode(core_path) + ) + try: + database = site.get_database() + yield database + except WordpressException: + if self.config.allow_io_errors: + log.warning( + 'Failed to extract database credentials ' + 'for site at ' + os.fsdecode(core_path) + ) + else: + raise + + def _get_json_validator(self) -> ListValidator: + return ListValidator( + DictionaryValidator({ + 'name': str, + 'user': str, + 'password': str, + 'host': str, + 'port': OptionalValueValidator(int) + }, optional_keys={'port'}) + ) + + def _parse_configured_databases( + self + ) -> Generator[WordpressDatabase, None, None]: + validator = self._get_json_validator() + for path in self._get_search_paths(): + with open(path, 'rb') as file: + configList = json.load(file) + validator.validate(configList) + for config in configList: + try: + port = config['port'] + except KeyError: + port = DEFAULT_PORT + yield WordpressDatabase( + name=config['name'], + server=WordpressDatabaseServer( + host=config['host'], + port=port, + user=config['user'], + password=config['password'] + ) + ) + + def _get_databases(self) -> List[WordpressDatabase]: + databases = [] + base = self._get_base_database() + if base is not None: + databases.append(base) + generator = self._locate_site_databases() if \ + self.config.locate_sites else \ + self._parse_configured_databases() + for database in generator: + databases.append(database) + return databases + + def invoke(self) -> int: + rule_set = DatabaseRuleSet() + scanner = DatabaseScanner(rule_set) + for database in self._get_databases(): scanner.scan(database) + if self.context.requires_input(self.config.require_database) \ + and scanner.scan_count == 0: + raise ConfigurationException( + 'At least one database to scan must be specified' + ) return 0 diff --git a/wordfence/cli/dbscan/definition.py b/wordfence/cli/dbscan/definition.py index db5dd2ea..f4d367dc 100644 --- a/wordfence/cli/dbscan/definition.py +++ b/wordfence/cli/dbscan/definition.py @@ -61,6 +61,69 @@ "argument_type": "OPTION", "default": DEFAULT_PREFIX, "category": "Database Connectivity" + }, + "database-name": { + "short_name": "D", + "description": "The MySQL database name", + "context": "CLI", + "argument_type": "OPTION", + "default": None + }, + "read-stdin": { + "description": "Read paths from stdin. If not specified, paths will " + "automatically be read from stdin when input is not " + "from a TTY.", + "context": "ALL", + "argument_type": "OPTIONAL_FLAG", + "default": None + }, + "path-separator": { + "short_name": "s", + "description": "Separator used to delimit paths when reading from " + "stdin. Defaults to the null byte.", + "context": "ALL", + "argument_type": "OPTION", + "default": "AA==", + "default_type": "base64" + }, + "require-database": { + "description": "When enabled, invoking the db-scan command without " + "specifying at least one database will trigger an " + "error. This is the default behavior when running in " + "a terminal.", + "context": "CLI", + "argument_type": "OPTIONAL_FLAG", + "default": None + }, + "locate-sites": { + "short_name": "S", + "description": ( + "Automatically locate WordPress config files to extract " + "database connection details" + ), + "context": "CLI", + "argument_type": "FLAG", + "default": None, + "category": "Site Location" + }, + "allow-nested": { + "description": "Allow WordPress installations nested below other " + "installations to be identified as targets for " + "database scanning", + "context": "ALL", + "argument_type": "FLAG", + "default": True, + "category": "Site Location" + }, + "allow-io-errors": { + "description": "Allow scanning to continue even if an IO error occurs" + "while locating WordPress sites. Sites that cannot " + "be identified due to IO errors will be excluded from " + "scanning. This is the default behavior.", + "context": "ALL", + "argument_type": "FLAG", + "default": True, + "category": "Site Location" } } @@ -78,5 +141,6 @@ config_definitions=config_definitions, config_section='DB_SCAN', cacheable_types=set(), - examples=examples + examples=examples, + accepts_directories=True ) diff --git a/wordfence/databasescanning/scanner.py b/wordfence/databasescanning/scanner.py index d34aea10..9d7aaae1 100644 --- a/wordfence/databasescanning/scanner.py +++ b/wordfence/databasescanning/scanner.py @@ -9,9 +9,10 @@ class DatabaseScanner: def __init__( self, - ruleSet: DatabaseRuleSet + rule_set: DatabaseRuleSet ): - self.ruleSet = ruleSet + self.rule_set = rule_set + self.scan_count = 0 def _scan_connection( self, @@ -25,6 +26,7 @@ def scan( self, database: Union[WordpressDatabase, WordpressDatabaseConnection] ) -> None: + self.scan_count += 1 if isinstance(database, WordpressDatabaseConnection): return self._scan_connection(database) else: diff --git a/wordfence/intel/database_rules.py b/wordfence/intel/database_rules.py index 8cdcba5e..62b895bc 100644 --- a/wordfence/intel/database_rules.py +++ b/wordfence/intel/database_rules.py @@ -1,5 +1,14 @@ +from typing import Optional, Set + + class DatabaseRule: - pass + + def __init__( + self, + tables: Optional[Set[str]] = None, + condition: Optional[str] = None + ): + self.tables = tables class DatabaseRuleSet: diff --git a/wordfence/wordpress/database.py b/wordfence/wordpress/database.py index 28c9b51b..ff848760 100644 --- a/wordfence/wordpress/database.py +++ b/wordfence/wordpress/database.py @@ -1,6 +1,8 @@ import mysql.connector from typing import Optional, Generator +from .exceptions import WordpressDatabaseException + DEFAULT_HOST = 'localhost' DEFAULT_PORT = 3306 @@ -8,12 +10,6 @@ DEFAULT_PREFIX = 'wp_' -class WordpressDatabaseException(Exception): - - def __init__(self, database, message): - self.database = database - - class WordpressDatabaseConnection: def __init__(self, database): @@ -29,7 +25,7 @@ def __init__(self, database): except mysql.connector.Error: raise WordpressDatabaseException( database, - f'Failed to connect to database: {self.debug_string}' + f'Failed to connect to database: {database.debug_string}' ) def __enter__(self): diff --git a/wordfence/wordpress/exceptions.py b/wordfence/wordpress/exceptions.py index d957c64d..e1d680c3 100644 --- a/wordfence/wordpress/exceptions.py +++ b/wordfence/wordpress/exceptions.py @@ -4,3 +4,9 @@ class WordpressException(Exception): class ExtensionException(WordpressException): pass + + +class WordpressDatabaseException(Exception): + + def __init__(self, database, message): + self.database = database diff --git a/wordfence/wordpress/site.py b/wordfence/wordpress/site.py index e2e82909..1688f10c 100644 --- a/wordfence/wordpress/site.py +++ b/wordfence/wordpress/site.py @@ -1,7 +1,7 @@ import os import os.path from dataclasses import dataclass, field -from typing import Optional, List, Generator +from typing import Optional, List, Generator, Dict, Callable, Any from ..php.parsing import parse_php_file, PhpException, PhpState, \ PhpEvaluationOptions @@ -11,6 +11,7 @@ from .exceptions import WordpressException, ExtensionException from .plugin import Plugin, PluginLoader from .theme import Theme, ThemeLoader +from .database import WordpressDatabase, WordpressDatabaseServer, DEFAULT_PORT WP_BLOG_HEADER_NAME = b'wp-blog-header.php' WP_CONFIG_NAME = b'wp-config.php' @@ -32,11 +33,18 @@ b'../app' ] +DATABASE_CONFIG_CONSTANTS = { + b'DB_NAME': 'name', + b'DB_USER': 'user', + b'DB_PASSWORD': 'password', + b'DB_HOST': 'host' + } + @dataclass class WordpressStructureOptions: relative_content_paths: List[str] = field(default_factory=list) - relative_plugins_paths: List[str] = field(default_factory=list) + relaGtive_plugins_paths: List[str] = field(default_factory=list) relative_mu_plugins_paths: List[str] = field(default_factory=list) @@ -307,18 +315,16 @@ def _get_parsed_config_state(self) -> PhpState: def _extract_string_from_config( self, - constant: str, - default: Optional[str] = None - ) -> str: + constant: bytes, + default: Optional[bytes], + extractor: Callable[[PhpState], Any] + ) -> bytes: try: state = self._get_parsed_config_state() if state is not None: - path = state.get_constant_value( - name=constant, - default_to_name=False - ) - if isinstance(path, str): - return path + value = extractor(state) + if isinstance(value, bytes): + return value except PhpException as exception: # Just use the default if parsing errors occur log.warning( @@ -327,8 +333,43 @@ def _extract_string_from_config( ) return default + def _extract_string_from_config_constant( + self, + constant: bytes, + default: Optional[bytes] = None + ): + def get_constant_value(state: PhpState): + return state.get_constant_value( + name=constant, + default_to_name=False + ) + return self._extract_string_from_config( + constant, + default, + get_constant_value + ) + + def _extract_string_from_config_variable( + self, + variable: bytes, + default: Optional[bytes] = None + ): + def get_variable_value(state: PhpState): + return state.get_variable_value(variable) + return self._extract_string_from_config( + variable, + default, + get_variable_value + ) + + def get_config_constant(self, constant: bytes) -> bytes: + return self._extract_string_from_config_constant(constant) + + def get_config_variable(self, variable: bytes) -> bytes: + return self._extract_string_from_config_variable(variable) + def _generate_possible_content_paths(self) -> Generator[str, None, None]: - configured = self._extract_string_from_config( + configured = self._extract_string_from_config_constant( 'WP_CONTENT_DIR' ) if configured is not None: @@ -357,7 +398,7 @@ def get_content_directory(self) -> str: return self.content_path def get_configured_plugins_directory(self, mu: bool = False) -> str: - return self._extract_string_from_config( + return self._extract_string_from_config_constant( 'WPMU_PLUGIN_DIR' if mu else 'WP_PLUGIN_DIR', ) @@ -434,3 +475,45 @@ def get_themes(self, allow_io_errors: bool = False) -> List[Theme]: raise loader = ThemeLoader(directory, allow_io_errors) return loader.load_all() + + def _extract_database_config(self) -> Dict[str, str]: + config = {} + + def add_config(key: str, value: Any): + if value is None: + raise WordpressException( + 'Unable to extract database connection details from ' + f'WordPress config (Key: {key}, Value: ' + + repr(value) + ')' + ) + config[key] = value.decode('latin1') + + for constant, attribute in DATABASE_CONFIG_CONSTANTS.items(): + add_config( + key=attribute, + value=self.get_config_constant(constant) + ) + add_config( + key='prefix', + value=self.get_config_variable(b'table_prefix') + ) + return config + + def get_database(self) -> WordpressDatabase: + config = self._extract_database_config() + host_components = config['host'].split(':', 1) + host = host_components[0] + try: + port = host_components[1] + except IndexError: + port = DEFAULT_PORT + server = WordpressDatabaseServer( + host=host, + port=port, + user=config['user'], + password=config['password'] + ) + return WordpressDatabase( + name=config['name'], + server=server + ) From 5549c71fea1453dba8a4c2a029c671ce39e4c527 Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Tue, 17 Sep 2024 14:07:49 -0400 Subject: [PATCH 03/18] Added support for configuring database collation --- wordfence/cli/dbscan/dbscan.py | 15 +++++++++++---- wordfence/cli/dbscan/definition.py | 9 ++++++++- wordfence/intel/database_rules.py | 4 ++++ wordfence/wordpress/database.py | 10 +++++++--- wordfence/wordpress/site.py | 13 ++++++++++--- 5 files changed, 40 insertions(+), 11 deletions(-) diff --git a/wordfence/cli/dbscan/dbscan.py b/wordfence/cli/dbscan/dbscan.py index cc157282..5c73ed0e 100644 --- a/wordfence/cli/dbscan/dbscan.py +++ b/wordfence/cli/dbscan/dbscan.py @@ -1,5 +1,5 @@ from wordfence.wordpress.database import WordpressDatabase, \ - WordpressDatabaseServer, DEFAULT_PORT + WordpressDatabaseServer, DEFAULT_PORT, DEFAULT_COLLATION from wordfence.wordpress.site import WordpressLocator, \ WordpressSite from wordfence.wordpress.exceptions import WordpressException @@ -43,7 +43,8 @@ def _get_base_database(self) -> Optional[WordpressDatabase]: ) return WordpressDatabase( name=name, - server=server + server=server, + collation=self.config.collation ) def _get_search_paths( @@ -96,7 +97,8 @@ def _get_json_validator(self) -> ListValidator: 'user': str, 'password': str, 'host': str, - 'port': OptionalValueValidator(int) + 'port': OptionalValueValidator(int), + 'collation': OptionalValueValidator(str) }, optional_keys={'port'}) ) @@ -113,6 +115,10 @@ def _parse_configured_databases( port = config['port'] except KeyError: port = DEFAULT_PORT + try: + collation = config['collation'] + except KeyError: + collation = DEFAULT_COLLATION yield WordpressDatabase( name=config['name'], server=WordpressDatabaseServer( @@ -120,7 +126,8 @@ def _parse_configured_databases( port=port, user=config['user'], password=config['password'] - ) + ), + collation=collation ) def _get_databases(self) -> List[WordpressDatabase]: diff --git a/wordfence/cli/dbscan/definition.py b/wordfence/cli/dbscan/definition.py index f4d367dc..61f7b0f2 100644 --- a/wordfence/cli/dbscan/definition.py +++ b/wordfence/cli/dbscan/definition.py @@ -1,5 +1,5 @@ from wordfence.wordpress.database import DEFAULT_HOST, DEFAULT_PORT, \ - DEFAULT_USER, DEFAULT_PREFIX + DEFAULT_USER, DEFAULT_PREFIX, DEFAULT_COLLATION from ..subcommands import SubcommandDefinition, UsageExample from ..config.typing import ConfigDefinitions @@ -69,6 +69,13 @@ "argument_type": "OPTION", "default": None }, + "collation": { + "short_name": "C", + "description": "The collation to use when connecting to MySQL", + "context": "CLI", + "argument_type": "OPTION", + "default": DEFAULT_COLLATION + }, "read-stdin": { "description": "Read paths from stdin. If not specified, paths will " "automatically be read from stdin when input is not " diff --git a/wordfence/intel/database_rules.py b/wordfence/intel/database_rules.py index 62b895bc..19c19950 100644 --- a/wordfence/intel/database_rules.py +++ b/wordfence/intel/database_rules.py @@ -13,3 +13,7 @@ def __init__( class DatabaseRuleSet: pass + + +def load_rules_file(path: str) -> None: + pass diff --git a/wordfence/wordpress/database.py b/wordfence/wordpress/database.py index ff848760..3cc3139c 100644 --- a/wordfence/wordpress/database.py +++ b/wordfence/wordpress/database.py @@ -8,6 +8,7 @@ DEFAULT_PORT = 3306 DEFAULT_USER = 'root' DEFAULT_PREFIX = 'wp_' +DEFAULT_COLLATION = 'utf8mb4_unicode_ci' class WordpressDatabaseConnection: @@ -20,7 +21,8 @@ def __init__(self, database): port=database.server.port, user=database.server.user, password=database.server.password, - database=database.name + database=database.name, + collation=database.collation ) except mysql.connector.Error: raise WordpressDatabaseException( @@ -59,7 +61,7 @@ def __init__( host: str = DEFAULT_HOST, port: int = DEFAULT_PORT, user: str = DEFAULT_USER, - password: Optional[str] = None, + password: Optional[str] = None ): self.host = host self.port = port @@ -73,11 +75,13 @@ def __init__( self, name: str, server: WordpressDatabaseServer, - prefix: str = DEFAULT_PREFIX + prefix: str = DEFAULT_PREFIX, + collation: str = DEFAULT_COLLATION ): self.name = name self.server = server self.prefix = prefix + self.collation = collation self.debug_string = self._build_debug_string() def connect(self) -> WordpressDatabaseConnection: diff --git a/wordfence/wordpress/site.py b/wordfence/wordpress/site.py index 1688f10c..9b572526 100644 --- a/wordfence/wordpress/site.py +++ b/wordfence/wordpress/site.py @@ -11,7 +11,8 @@ from .exceptions import WordpressException, ExtensionException from .plugin import Plugin, PluginLoader from .theme import Theme, ThemeLoader -from .database import WordpressDatabase, WordpressDatabaseServer, DEFAULT_PORT +from .database import WordpressDatabase, WordpressDatabaseServer, \ + DEFAULT_PORT, DEFAULT_COLLATION WP_BLOG_HEADER_NAME = b'wp-blog-header.php' WP_CONFIG_NAME = b'wp-config.php' @@ -37,7 +38,8 @@ b'DB_NAME': 'name', b'DB_USER': 'user', b'DB_PASSWORD': 'password', - b'DB_HOST': 'host' + b'DB_HOST': 'host', + b'DB_COLLATE': 'collation' } @@ -507,6 +509,10 @@ def get_database(self) -> WordpressDatabase: port = host_components[1] except IndexError: port = DEFAULT_PORT + try: + collation = config['collation;'] + except KeyError: + collation = DEFAULT_COLLATION server = WordpressDatabaseServer( host=host, port=port, @@ -515,5 +521,6 @@ def get_database(self) -> WordpressDatabase: ) return WordpressDatabase( name=config['name'], - server=server + server=server, + collation=collation ) From 0cb153c4131c52e324cb18af765335fbcdeea174 Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Mon, 23 Sep 2024 10:21:26 -0400 Subject: [PATCH 04/18] Added initial support for loading database rules --- wordfence/databasescanning/scanner.py | 7 +++ wordfence/intel/database_rules.py | 67 +++++++++++++++++++++++++-- 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/wordfence/databasescanning/scanner.py b/wordfence/databasescanning/scanner.py index 9d7aaae1..6b146283 100644 --- a/wordfence/databasescanning/scanner.py +++ b/wordfence/databasescanning/scanner.py @@ -14,6 +14,13 @@ def __init__( self.rule_set = rule_set self.scan_count = 0 + def _scan_table( + self, + connection: WordpressDatabaseConnection, + table: str + ): + pass + def _scan_connection( self, connection: WordpressDatabaseConnection diff --git a/wordfence/intel/database_rules.py b/wordfence/intel/database_rules.py index 62b895bc..88044f34 100644 --- a/wordfence/intel/database_rules.py +++ b/wordfence/intel/database_rules.py @@ -1,15 +1,76 @@ -from typing import Optional, Set +from wordfence.util.validation import ListValidator, DictionaryValidator, \ + OptionalValueValidator +from typing import Optional, Set, List +import json class DatabaseRule: def __init__( self, + identifier: int, tables: Optional[Set[str]] = None, - condition: Optional[str] = None + condition: Optional[str] = None, + description: Optional[str] = None ): + self.identifier = identifier self.tables = tables + self.condition = condition + self.description = description class DatabaseRuleSet: - pass + + def __init__(self): + self.rules = {} + self.table_rules = {} + self.global_rules = set() + + def add_rule(self, rule: DatabaseRule) -> None: + if rule.identifier in self.rules: + raise Exception('Duplicate rule ID: {rule.identifier}') + self.rules[rule.identifier] = rule + if rule.tables is None: + self.global_rules.append(rule) + else: + for table in rule.tables: + if table not in self.table_rules: + self.table_rules[table] = [] + self.table_rules[table].append(rule) + + def get_rules(self, table: str) -> List[DatabaseRule]: + rules = [] + try: + rules.extend(self.table_rules[table]) + except KeyError: + pass # There are no table rules + rules.extend(self.global_rules) + return rules + + def get_targeted_tables(self) -> List[str]: + return self.table_rules.keys() + + +JSON_VALIDATOR = ListValidator( + DictionaryValidator({ + 'id': int, + 'tables': ListValidator(str), + 'condition': str, + 'description': OptionalValueValidator(str) + }, optional_keys={'description'}) + ) + + +def load_database_rules(path: bytes) -> DatabaseRuleSet: + with open(path, 'rb') as file: + data = json.load(file) + JSON_VALIDATOR.validate(data) + rule_set = DatabaseRuleSet() + for rule_data in data: + rule = DatabaseRule( + identifier=rule_data['id'], + tables=rule_data['tables'], + condition=rule_data['condition'] + ) + rule_set.add_rule(rule) + return rule_set From 00663c4331f1ee20580b9b8afb6bb2d9e856816a Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Mon, 23 Sep 2024 10:32:00 -0400 Subject: [PATCH 05/18] Merged concurrent work on database scanning --- wordfence/cli/dbscan/dbscan.py | 11 +++++++++-- wordfence/cli/dbscan/definition.py | 11 ++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/wordfence/cli/dbscan/dbscan.py b/wordfence/cli/dbscan/dbscan.py index 5c73ed0e..595b6bb8 100644 --- a/wordfence/cli/dbscan/dbscan.py +++ b/wordfence/cli/dbscan/dbscan.py @@ -3,7 +3,7 @@ from wordfence.wordpress.site import WordpressLocator, \ WordpressSite from wordfence.wordpress.exceptions import WordpressException -from wordfence.intel.database_rules import DatabaseRuleSet +from wordfence.intel.database_rules import DatabaseRuleSet, load_database_rules from wordfence.databasescanning.scanner import DatabaseScanner from wordfence.util.validation import ListValidator, DictionaryValidator, \ OptionalValueValidator @@ -142,8 +142,15 @@ def _get_databases(self) -> List[WordpressDatabase]: databases.append(database) return databases - def invoke(self) -> int: + def _load_rules(self) -> DatabaseRuleSet: rule_set = DatabaseRuleSet() + for rules_file in self.config.rules_file: + load_database_rules(rules_file, rule_set) + return rule_set + + def invoke(self) -> int: + rule_set = self._load_rules() + print(repr(vars((rule_set.rules[1])))) scanner = DatabaseScanner(rule_set) for database in self._get_databases(): scanner.scan(database) diff --git a/wordfence/cli/dbscan/definition.py b/wordfence/cli/dbscan/definition.py index 61f7b0f2..bdf1ea73 100644 --- a/wordfence/cli/dbscan/definition.py +++ b/wordfence/cli/dbscan/definition.py @@ -131,7 +131,16 @@ "argument_type": "FLAG", "default": True, "category": "Site Location" - } + }, + "rules-file": { + "short_name": "R", + "description": "Path to a JSON file containing scanning rules", + "context": "ALL", + "argument_type": "OPTION_REPEATABLE", + "meta": { + "accepts_file": True + } + } } examples = [ From a5050c3762feece3ab342e7080229a96c69f1774 Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Mon, 23 Sep 2024 10:33:04 -0400 Subject: [PATCH 06/18] Fixed mixed indentation --- wordfence/cli/dbscan/definition.py | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/wordfence/cli/dbscan/definition.py b/wordfence/cli/dbscan/definition.py index bdf1ea73..e68a1ecb 100644 --- a/wordfence/cli/dbscan/definition.py +++ b/wordfence/cli/dbscan/definition.py @@ -69,13 +69,13 @@ "argument_type": "OPTION", "default": None }, - "collation": { - "short_name": "C", - "description": "The collation to use when connecting to MySQL", - "context": "CLI", - "argument_type": "OPTION", - "default": DEFAULT_COLLATION - }, + "collation": { + "short_name": "C", + "description": "The collation to use when connecting to MySQL", + "context": "CLI", + "argument_type": "OPTION", + "default": DEFAULT_COLLATION + }, "read-stdin": { "description": "Read paths from stdin. If not specified, paths will " "automatically be read from stdin when input is not " @@ -132,15 +132,15 @@ "default": True, "category": "Site Location" }, - "rules-file": { - "short_name": "R", - "description": "Path to a JSON file containing scanning rules", - "context": "ALL", - "argument_type": "OPTION_REPEATABLE", - "meta": { - "accepts_file": True - } - } + "rules-file": { + "short_name": "R", + "description": "Path to a JSON file containing scanning rules", + "context": "ALL", + "argument_type": "OPTION_REPEATABLE", + "meta": { + "accepts_file": True + } + } } examples = [ From c7d12d714f2f78be19399861d936c91826ff6b25 Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Thu, 26 Sep 2024 12:34:15 -0400 Subject: [PATCH 07/18] Implemented database rule fetching from NOC1 and added basic reporting for db-scan --- wordfence/api/noc1.py | 10 ++ wordfence/cli/dbscan/dbscan.py | 70 ++++++++++---- wordfence/cli/dbscan/definition.py | 15 ++- wordfence/cli/dbscan/reporting.py | 129 ++++++++++++++++++++++++++ wordfence/cli/reporting.py | 3 +- wordfence/databasescanning/scanner.py | 55 +++++++++-- wordfence/intel/database_rules.py | 19 +++- wordfence/wordpress/database.py | 8 +- 8 files changed, 272 insertions(+), 37 deletions(-) create mode 100644 wordfence/cli/dbscan/reporting.py diff --git a/wordfence/api/noc1.py b/wordfence/api/noc1.py index 7b3f7553..8b5fb947 100644 --- a/wordfence/api/noc1.py +++ b/wordfence/api/noc1.py @@ -9,6 +9,8 @@ from ..intel.signatures import CommonString, Signature, SignatureSet, \ PrecompiledSignatureSet, deserialize_precompiled_signature_set +from ..intel.database_rules import DatabaseRuleSet, JSON_VALIDATOR as \ + DATABASE_RULES_JSON_VALIDATOR, parse_database_rules from ..util.validation import DictionaryValidator, ListValidator, Validator, \ OptionalValueValidator from ..util.platform import Platform @@ -258,3 +260,11 @@ def get_wp_file_content( body=parameters ) return response + + def get_database_rules(self) -> DatabaseRuleSet: + response = self.request('get_database_rules') + validator = DictionaryValidator({ + 'rules': DATABASE_RULES_JSON_VALIDATOR + }) + self.validate_response(response, validator) + return parse_database_rules(response['rules'], pre_validated=True) diff --git a/wordfence/cli/dbscan/dbscan.py b/wordfence/cli/dbscan/dbscan.py index 595b6bb8..98d4bcf1 100644 --- a/wordfence/cli/dbscan/dbscan.py +++ b/wordfence/cli/dbscan/dbscan.py @@ -7,6 +7,7 @@ from wordfence.databasescanning.scanner import DatabaseScanner from wordfence.util.validation import ListValidator, DictionaryValidator, \ OptionalValueValidator +from wordfence.util import caching from getpass import getpass from typing import Optional, List, Generator import os @@ -16,6 +17,9 @@ from ..subcommands import Subcommand from ..io import IoManager from ..exceptions import ConfigurationException +from ..config import not_set_token + +from .reporting import DatabaseScanReportManager class DbScanSubcommand(Subcommand): @@ -49,13 +53,9 @@ def _get_base_database(self) -> Optional[WordpressDatabase]: def _get_search_paths( self, + io_manager: IoManager, include_current: bool = False ) -> Generator[bytes, None, None]: - io_manager = IoManager( - self.config.read_stdin, - self.config.path_separator, - binary=True - ) if len(self.config.trailing_arguments): yield from self.config.trailing_arguments elif include_current and not io_manager.should_read_stdin(): @@ -65,9 +65,10 @@ def _get_search_paths( yield path def _locate_site_databases( - self + self, + io_manager: IoManager ) -> Generator[WordpressDatabase, None, None]: - for path in self._get_search_paths(include_current=True): + for path in self._get_search_paths(io_manager, include_current=True): locator = WordpressLocator( path=path, allow_nested=self.config.allow_nested, @@ -99,14 +100,15 @@ def _get_json_validator(self) -> ListValidator: 'host': str, 'port': OptionalValueValidator(int), 'collation': OptionalValueValidator(str) - }, optional_keys={'port'}) + }, optional_keys={'port', 'collation'}) ) def _parse_configured_databases( - self + self, + io_manager: IoManager ) -> Generator[WordpressDatabase, None, None]: validator = self._get_json_validator() - for path in self._get_search_paths(): + for path in self._get_search_paths(io_manager): with open(path, 'rb') as file: configList = json.load(file) validator.validate(configList) @@ -130,35 +132,65 @@ def _parse_configured_databases( collation=collation ) - def _get_databases(self) -> List[WordpressDatabase]: + def _get_databases( + self, + io_manager: IoManager + ) -> List[WordpressDatabase]: databases = [] base = self._get_base_database() if base is not None: databases.append(base) - generator = self._locate_site_databases() if \ + generator = self._locate_site_databases(io_manager) if \ self.config.locate_sites else \ - self._parse_configured_databases() + self._parse_configured_databases(io_manager) for database in generator: databases.append(database) return databases + def _load_remote_rules(self) -> DatabaseRuleSet: + + def fetch_rules() -> DatabaseRuleSet: + client = self.context.get_noc1_client() + return client.get_database_rules() + + cacheable = caching.Cacheable( + 'database_rules', + fetch_rules, + caching.DURATION_ONE_DAY + ) + + return cacheable.get(self.cache) + def _load_rules(self) -> DatabaseRuleSet: - rule_set = DatabaseRuleSet() - for rules_file in self.config.rules_file: - load_database_rules(rules_file, rule_set) + rule_set = self._load_remote_rules() \ + if self.config.use_remote_rules \ + else DatabaseRuleSet() + if self.config.rules_file is not not_set_token: + for rules_file in self.config.rules_file: + load_database_rules(rules_file, rule_set) return rule_set def invoke(self) -> int: + report_manager = DatabaseScanReportManager(self.context) + io_manager = report_manager.get_io_manager() rule_set = self._load_rules() - print(repr(vars((rule_set.rules[1])))) scanner = DatabaseScanner(rule_set) - for database in self._get_databases(): - scanner.scan(database) + with report_manager.open_output_file() as output_file: + report = report_manager.initialize_report(output_file) + for database in self._get_databases(io_manager): + for result in scanner.scan(database): + report.add_result(result) + report.database_count = scanner.scan_count + report.complete() if self.context.requires_input(self.config.require_database) \ and scanner.scan_count == 0: raise ConfigurationException( 'At least one database to scan must be specified' ) + log.info( + f'Found {report.result_count} result(s) after scanning ' + f'{scanner.scan_count} database(s)' + ) return 0 diff --git a/wordfence/cli/dbscan/definition.py b/wordfence/cli/dbscan/definition.py index e68a1ecb..2edf9b01 100644 --- a/wordfence/cli/dbscan/definition.py +++ b/wordfence/cli/dbscan/definition.py @@ -4,6 +4,8 @@ from ..subcommands import SubcommandDefinition, UsageExample from ..config.typing import ConfigDefinitions +from .reporting import DATABASE_SCAN_REPORT_CONFIG_OPTIONS + config_definitions: ConfigDefinitions = { "host": { "short_name": "H", @@ -67,7 +69,8 @@ "description": "The MySQL database name", "context": "CLI", "argument_type": "OPTION", - "default": None + "default": None, + "category": "Database Connectivity" }, "collation": { "short_name": "C", @@ -93,6 +96,7 @@ "default": "AA==", "default_type": "base64" }, + **DATABASE_SCAN_REPORT_CONFIG_OPTIONS, "require-database": { "description": "When enabled, invoking the db-scan command without " "specifying at least one database will trigger an " @@ -132,6 +136,13 @@ "default": True, "category": "Site Location" }, + "use-remote-rules": { + "description": "If enabled, scanning rules will be pulled from " + "the Wordfence API", + "context": "ALL", + "argument_type": "FLAG", + "default": True + }, "rules-file": { "short_name": "R", "description": "Path to a JSON file containing scanning rules", @@ -152,7 +163,7 @@ definition = SubcommandDefinition( name='db-scan', - usage='[OPTIONS] [DATABASE_NAME]...', + usage='[OPTIONS] [DATABASE_CONFIG_PATH or WORDPRESS_INSTALLATION_PATH]...', description='Scan for malicious content in a WordPress databases', config_definitions=config_definitions, config_section='DB_SCAN', diff --git a/wordfence/cli/dbscan/reporting.py b/wordfence/cli/dbscan/reporting.py new file mode 100644 index 00000000..efb71c5a --- /dev/null +++ b/wordfence/cli/dbscan/reporting.py @@ -0,0 +1,129 @@ +import json +from typing import List, Optional, Dict + +from wordfence.databasescanning.scanner import DatabaseScanResult +from ..reporting import ReportManager, ReportColumnEnum, ReportFormatEnum, \ + ReportRecord, Report, ReportFormat, ReportColumn, ReportEmail, \ + get_config_options, generate_html_table, generate_report_email_html, \ + REPORT_FORMAT_CSV, REPORT_FORMAT_TSV, REPORT_FORMAT_NULL_DELIMITED, \ + REPORT_FORMAT_LINE_DELIMITED +from ..context import CliContext +from ..email import Mailer + + +class DatabaseScanReportColumn(ReportColumnEnum): + TABLE = 'table', lambda record: record.result.table + RULE_ID = 'rule_id', lambda record: record.result.rule.identifier + RULE_DESCRIPTION = 'rule_description', \ + lambda record: record.result.rule.description + # TODO: Ensure rows can be safely represented as JSON + ROW = 'row', lambda record: json.dumps(record.result.row) + + +class DatabaseScanReportFormat(ReportFormatEnum): + CSV = REPORT_FORMAT_CSV + TSV = REPORT_FORMAT_TSV + NULL_DELIMITED = REPORT_FORMAT_NULL_DELIMITED + LINE_DELIMITED = REPORT_FORMAT_LINE_DELIMITED + + +class DatabaseScanReportRecord(ReportRecord): + + def __init__(self, result: DatabaseScanResult): + self.result = result + + +class DatabaseScanReport(Report): + + def __init__( + self, + format: ReportFormat, + columns: List[ReportColumn], + email_addresses: List[str], + mailer: Optional[Mailer], + write_headers: bool = False, + only_unremediated: bool = False + ): + super().__init__( + format, + columns, + email_addresses, + mailer, + write_headers + ) + self.result_count = 0 + self.database_count = 0 + + def add_result(self, result: DatabaseScanResult): + self.result_count += 1 + self.write_record( + DatabaseScanReportRecord(result) + ) + + def generate_email( + self, + recipient: str, + attachments: Dict[str, str], + hostname: str + ) -> ReportEmail: + plain = ( + 'Database Scan Complete\n\n' + f'Scanned Databases: {self.database_count}\n\n' + f'Results Found: {self.result_count}\n\n' + ) + + results = { + 'Scanned Databases': self.database_count, + 'Results Found': self.result_count + } + + table = generate_html_table(results) + + document = generate_report_email_html( + table, + 'Database Scan Results', + hostname + ) + + return ReportEmail( + recipient=recipient, + subject=f'Database Scan Results for {hostname}', + plain_content=plain, + html_content=document.to_html() + ) + + +class DatabaseScanReportManager(ReportManager): + + def __init__(self, context: CliContext): + super().__init__( + formats=DatabaseScanReportFormat, + columns=DatabaseScanReportColumn, + context=context, + read_stdin=context.config.read_stdin, + input_delimiter=context.config.path_separator, + binary_input=True + ) + + def _instantiate_report( + self, + format: ReportFormat, + columns: List[ReportColumn], + email_addresses: List[str], + mailer: Optional[Mailer], + write_headers: bool + ) -> Report: + return DatabaseScanReport( + format, + columns, + email_addresses, + mailer, + write_headers + ) + + +DATABASE_SCAN_REPORT_CONFIG_OPTIONS = get_config_options( + DatabaseScanReportFormat, + DatabaseScanReportColumn, + default_format='csv' + ) diff --git a/wordfence/cli/reporting.py b/wordfence/cli/reporting.py index 29c94c21..8721a87c 100644 --- a/wordfence/cli/reporting.py +++ b/wordfence/cli/reporting.py @@ -455,7 +455,8 @@ def has_writers(self) -> bool: def generate_email( self, recipient: str, - attachments: Dict[str, str] + attachments: Dict[str, str], + hostname: str ) -> ReportEmail: raise NotImplementedError( 'This report does not support email generation' diff --git a/wordfence/databasescanning/scanner.py b/wordfence/databasescanning/scanner.py index 6b146283..2b564e2b 100644 --- a/wordfence/databasescanning/scanner.py +++ b/wordfence/databasescanning/scanner.py @@ -1,8 +1,21 @@ -from wordfence.intel.database_rules import DatabaseRuleSet +from wordfence.intel.database_rules import DatabaseRuleSet, DatabaseRule from wordfence.wordpress.database import WordpressDatabase, \ WordpressDatabaseConnection from ..logging import log -from typing import Union +from typing import Union, Generator + + +class DatabaseScanResult: + + def __init__( + self, + rule: DatabaseRule, + table: str, + row: dict + ): + self.rule = rule + self.table = table + self.row = row class DatabaseScanner: @@ -18,24 +31,46 @@ def _scan_table( self, connection: WordpressDatabaseConnection, table: str - ): - pass + ) -> Generator[DatabaseScanResult, None, None]: + prefixed_table = connection.prefix_table(table) + conditions = [] + rule_selects = [] + for rule in self.rule_set.get_rules(table): + conditions.append(f'({rule.condition})') + rule_selects.append( + f'WHEN {rule.condition} THEN {rule.identifier}' + ) + rule_case = 'CASE\n' + '\n'.join(rule_selects) + '\nEND' + query = ( + f'SELECT {rule_case} AS rule_id, {prefixed_table}.* FROM ' + f'{prefixed_table} WHERE ' + + ' OR '.join(conditions) + ) + for result in connection.query(query): + rule = self.rule_set.get_rule(result['rule_id']) + del result['rule_id'] + yield DatabaseScanResult( + rule=rule, + table=table, + row=result + ) def _scan_connection( self, connection: WordpressDatabaseConnection - ) -> None: + ) -> Generator[DatabaseScanResult, None, None]: log.debug(f'Scanning database: {connection.database.debug_string}...') - pass - log.debug(f'Scan completed for: {connection.database.debug_string}...') + for table in self.rule_set.get_targeted_tables(): + yield from self._scan_table(connection, table) + log.debug(f'Scan completed for: {connection.database.debug_string}') def scan( self, database: Union[WordpressDatabase, WordpressDatabaseConnection] - ) -> None: + ) -> Generator[DatabaseScanResult, None, None]: self.scan_count += 1 if isinstance(database, WordpressDatabaseConnection): - return self._scan_connection(database) + yield from self._scan_connection(database) else: log.debug(f'Connecting to database: {database.debug_string}...') with database.connect() as connection: @@ -43,4 +78,4 @@ def scan( 'Successfully connected to database: ' f'{database.debug_string}' ) - return self._scan_connection(connection) + yield from self._scan_connection(connection) diff --git a/wordfence/intel/database_rules.py b/wordfence/intel/database_rules.py index 88044f34..d909dc16 100644 --- a/wordfence/intel/database_rules.py +++ b/wordfence/intel/database_rules.py @@ -50,6 +50,9 @@ def get_rules(self, table: str) -> List[DatabaseRule]: def get_targeted_tables(self) -> List[str]: return self.table_rules.keys() + def get_rule(self, identifier: int) -> DatabaseRule: + return self.rules[identifier] + JSON_VALIDATOR = ListValidator( DictionaryValidator({ @@ -61,10 +64,12 @@ def get_targeted_tables(self) -> List[str]: ) -def load_database_rules(path: bytes) -> DatabaseRuleSet: - with open(path, 'rb') as file: - data = json.load(file) - JSON_VALIDATOR.validate(data) +def parse_database_rules( + data, + pre_validated: bool = False + ) -> DatabaseRuleSet: + if not pre_validated: + JSON_VALIDATOR.validate(data) rule_set = DatabaseRuleSet() for rule_data in data: rule = DatabaseRule( @@ -74,3 +79,9 @@ def load_database_rules(path: bytes) -> DatabaseRuleSet: ) rule_set.add_rule(rule) return rule_set + + +def load_database_rules(path: bytes) -> DatabaseRuleSet: + with open(path, 'rb') as file: + data = json.load(file) + return parse_database_rules(data) diff --git a/wordfence/wordpress/database.py b/wordfence/wordpress/database.py index 3cc3139c..469fd1e0 100644 --- a/wordfence/wordpress/database.py +++ b/wordfence/wordpress/database.py @@ -36,13 +36,16 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): return + def prefix_table(self, table: str) -> str: + return self.database.prefix_table(table) + def query( self, query: str, parameters: tuple = () ) -> Generator[tuple, None, None]: try: - cursor = self.connection.cursor() + cursor = self.connection.cursor(dictionary=True) cursor.execute(query, parameters) for result in cursor: yield result @@ -92,3 +95,6 @@ def _build_debug_string(self) -> str: f'{self.server.user}@{self.server.host}:' f'{self.server.port}/{self.name}' ) + + def prefix_table(self, table: str) -> str: + return self.prefix + table From b4c9ead2763ebc28f60fba0e31ce31cebe1e5ff8 Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Thu, 26 Sep 2024 16:09:43 -0400 Subject: [PATCH 08/18] Added human readable format, time output, column conflict checking, and safe JSON encoding to db-scan --- wordfence/cli/dbscan/dbscan.py | 4 ++- wordfence/cli/dbscan/definition.py | 5 +++- wordfence/cli/dbscan/reporting.py | 29 +++++++++++++++++-- wordfence/databasescanning/scanner.py | 30 +++++++++++++++++--- wordfence/intel/database_rules.py | 3 +- wordfence/util/json.py | 40 +++++++++++++++++++++++++++ wordfence/util/timing.py | 29 ++++++++++++++----- wordfence/wordpress/database.py | 14 +++++++++- 8 files changed, 136 insertions(+), 18 deletions(-) create mode 100644 wordfence/util/json.py diff --git a/wordfence/cli/dbscan/dbscan.py b/wordfence/cli/dbscan/dbscan.py index 98d4bcf1..b53b5c69 100644 --- a/wordfence/cli/dbscan/dbscan.py +++ b/wordfence/cli/dbscan/dbscan.py @@ -187,9 +187,11 @@ def invoke(self) -> int: raise ConfigurationException( 'At least one database to scan must be specified' ) + elapsed_time = round(scanner.get_elapsed_time()) log.info( f'Found {report.result_count} result(s) after scanning ' - f'{scanner.scan_count} database(s)' + f'{scanner.scan_count} database(s) over {elapsed_time} ' + 'second(s)' ) return 0 diff --git a/wordfence/cli/dbscan/definition.py b/wordfence/cli/dbscan/definition.py index 2edf9b01..f833ae94 100644 --- a/wordfence/cli/dbscan/definition.py +++ b/wordfence/cli/dbscan/definition.py @@ -167,7 +167,10 @@ description='Scan for malicious content in a WordPress databases', config_definitions=config_definitions, config_section='DB_SCAN', - cacheable_types=set(), + cacheable_types={ + 'wordfence.intel.database_rules.DatabaseRuleSet', + 'wordfence.intel.database_rules.DatabaseRule' + }, examples=examples, accepts_directories=True ) diff --git a/wordfence/cli/dbscan/reporting.py b/wordfence/cli/dbscan/reporting.py index efb71c5a..229d1c14 100644 --- a/wordfence/cli/dbscan/reporting.py +++ b/wordfence/cli/dbscan/reporting.py @@ -2,8 +2,11 @@ from typing import List, Optional, Dict from wordfence.databasescanning.scanner import DatabaseScanResult +from wordfence.util.terminal import Color, escape, RESET +from wordfence.util.json import safe_json_encode from ..reporting import ReportManager, ReportColumnEnum, ReportFormatEnum, \ ReportRecord, Report, ReportFormat, ReportColumn, ReportEmail, \ + BaseHumanReadableWriter, \ get_config_options, generate_html_table, generate_report_email_html, \ REPORT_FORMAT_CSV, REPORT_FORMAT_TSV, REPORT_FORMAT_NULL_DELIMITED, \ REPORT_FORMAT_LINE_DELIMITED @@ -16,8 +19,27 @@ class DatabaseScanReportColumn(ReportColumnEnum): RULE_ID = 'rule_id', lambda record: record.result.rule.identifier RULE_DESCRIPTION = 'rule_description', \ lambda record: record.result.rule.description - # TODO: Ensure rows can be safely represented as JSON - ROW = 'row', lambda record: json.dumps(record.result.row) + ROW = 'row', lambda record: safe_json_encode(record.result.row) + + +class HumanReadableWriter(BaseHumanReadableWriter): + + def format_record(self, record) -> str: + result = record.result + return ( + escape(Color.YELLOW) + + 'Suspicious database record found in table ' + f'"{result.table}" matching rule "{result.rule.description}"' + ': ' + safe_json_encode(record.result.row) + RESET + ) + + +REPORT_FORMAT_HUMAN = ReportFormat( + 'human', + lambda stream, columns: HumanReadableWriter(stream), + allows_headers=False, + allows_column_customization=False + ) class DatabaseScanReportFormat(ReportFormatEnum): @@ -25,6 +47,7 @@ class DatabaseScanReportFormat(ReportFormatEnum): TSV = REPORT_FORMAT_TSV NULL_DELIMITED = REPORT_FORMAT_NULL_DELIMITED LINE_DELIMITED = REPORT_FORMAT_LINE_DELIMITED + HUMAN = REPORT_FORMAT_HUMAN class DatabaseScanReportRecord(ReportRecord): @@ -125,5 +148,5 @@ def _instantiate_report( DATABASE_SCAN_REPORT_CONFIG_OPTIONS = get_config_options( DatabaseScanReportFormat, DatabaseScanReportColumn, - default_format='csv' + default_format='human' ) diff --git a/wordfence/databasescanning/scanner.py b/wordfence/databasescanning/scanner.py index 2b564e2b..fd820a10 100644 --- a/wordfence/databasescanning/scanner.py +++ b/wordfence/databasescanning/scanner.py @@ -1,8 +1,9 @@ +from typing import Union, Generator, List from wordfence.intel.database_rules import DatabaseRuleSet, DatabaseRule from wordfence.wordpress.database import WordpressDatabase, \ WordpressDatabaseConnection -from ..logging import log -from typing import Union, Generator +from wordfence.logging import log +from wordfence.util.timing import Timer class DatabaseScanResult: @@ -26,6 +27,19 @@ def __init__( ): self.rule_set = rule_set self.scan_count = 0 + self.timer = Timer(start=False) + + def _get_valid_columns( + self, + connection: WordpressDatabaseConnection, + prefixed_table: str + ) -> List: + columns = connection.get_column_types(prefixed_table) + try: + del columns['rule_id'] + except KeyError: + pass # If the column doesn't exist, that's fine + return list(columns.keys()) def _scan_table( self, @@ -41,8 +55,11 @@ def _scan_table( f'WHEN {rule.condition} THEN {rule.identifier}' ) rule_case = 'CASE\n' + '\n'.join(rule_selects) + '\nEND' + selected_columns = self._get_valid_columns(connection, prefixed_table) + selected_columns.append(f'{rule_case} as rule_id') + selected_columns = ', '.join(selected_columns) query = ( - f'SELECT {rule_case} AS rule_id, {prefixed_table}.* FROM ' + f'SELECT {selected_columns} FROM ' f'{prefixed_table} WHERE ' + ' OR '.join(conditions) ) @@ -51,7 +68,7 @@ def _scan_table( del result['rule_id'] yield DatabaseScanResult( rule=rule, - table=table, + table=prefixed_table, row=result ) @@ -59,10 +76,12 @@ def _scan_connection( self, connection: WordpressDatabaseConnection ) -> Generator[DatabaseScanResult, None, None]: + self.timer.resume() log.debug(f'Scanning database: {connection.database.debug_string}...') for table in self.rule_set.get_targeted_tables(): yield from self._scan_table(connection, table) log.debug(f'Scan completed for: {connection.database.debug_string}') + self.timer.stop() def scan( self, @@ -79,3 +98,6 @@ def scan( f'{database.debug_string}' ) yield from self._scan_connection(connection) + + def get_elapsed_time(self) -> int: + return self.timer.get_elapsed() diff --git a/wordfence/intel/database_rules.py b/wordfence/intel/database_rules.py index d909dc16..557aa5bd 100644 --- a/wordfence/intel/database_rules.py +++ b/wordfence/intel/database_rules.py @@ -75,7 +75,8 @@ def parse_database_rules( rule = DatabaseRule( identifier=rule_data['id'], tables=rule_data['tables'], - condition=rule_data['condition'] + condition=rule_data['condition'], + description=rule_data['description'] ) rule_set.add_rule(rule) return rule_set diff --git a/wordfence/util/json.py b/wordfence/util/json.py new file mode 100644 index 00000000..11b9037f --- /dev/null +++ b/wordfence/util/json.py @@ -0,0 +1,40 @@ +import json +from typing import Any +from base64 import b64encode + + +UNFILTERED_TYPES = { + bool, + int, + float, + str + } + + +def encode_invalid_data(data) -> Any: + for unfiltered_type in UNFILTERED_TYPES: + if isinstance(data, unfiltered_type): + return data + if isinstance(data, dict): + filtered = {} + for key, value in data.items(): + filtered[encode_invalid_data(key)] = encode_invalid_data(value) + return filtered + elif isinstance(data, list): + filtered = [] + for value in data: + filtered.append(encode_invalid_data(value)) + return filtered + elif isinstance(data, bytes): + return b64encode(data).decode('utf-8') + else: + try: + json.dumps(data) + except Exception: + return None + + +# Encode any data that cannot be represented as valid JSON +# prior to attempting to encode data as JSON +def safe_json_encode(data) -> str: + return json.dumps(encode_invalid_data(data)) diff --git a/wordfence/util/timing.py b/wordfence/util/timing.py index bcd8d3cc..993bf5e7 100644 --- a/wordfence/util/timing.py +++ b/wordfence/util/timing.py @@ -1,6 +1,10 @@ import time +def unit_nanoseconds(ns: int) -> int: + return ns + + def unit_seconds(ns: int) -> int: return ns / 1000000000 @@ -17,21 +21,32 @@ def __init__(self, start: bool = True): else: self.start_time = None self.end_time = None + self.previous_time = 0 def _capture_time(self) -> int: return time.monotonic_ns() - def start(self): + def start(self) -> None: self.start_time = self._capture_time() + self.end_time = None - def reset(self): + def reset(self) -> None: self.start() - def stop(self): + def stop(self) -> None: self.end_time = self._capture_time() - def get_elapsed(self, unit=unit_seconds): + def resume(self) -> None: + if self.start_time is not None: + self.previous_time += self.get_elapsed( + unit=unit_nanoseconds, + total=False + ) + self.start() + + def get_elapsed(self, unit=unit_seconds, total: bool = True) -> int: + previous_time = self.previous_time if total else 0 end_time = \ - self.end_time if self.end_time is not None \ - else self._capture_time() - return unit(end_time - self.start_time) + self.end_time if self.end_time is not None \ + else self._capture_time() + return unit(previous_time + end_time - self.start_time) diff --git a/wordfence/wordpress/database.py b/wordfence/wordpress/database.py index 469fd1e0..567648e6 100644 --- a/wordfence/wordpress/database.py +++ b/wordfence/wordpress/database.py @@ -1,5 +1,5 @@ import mysql.connector -from typing import Optional, Generator +from typing import Optional, Generator, Dict from .exceptions import WordpressDatabaseException @@ -56,6 +56,18 @@ def query( 'Failed to execute query' ) + def get_column_types( + self, + table: str, + prefix: bool = False + ) -> Dict[str, str]: + if prefix: + table = self.prefix_table(table) + columns = {} + for result in self.query(f'SHOW COLUMNS FROM {table}'): + columns[result['Field'].lower()] = result['Type'] + return columns + class WordpressDatabaseServer: From 75a168cdd9d289b79f048fa1c8d048c1287cdba2 Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Fri, 27 Sep 2024 11:45:49 -0400 Subject: [PATCH 09/18] Added support for filtering rules and fixed issue with %s placeholders in literal queries --- wordfence/cli/dbscan/dbscan.py | 10 +++++++ wordfence/cli/dbscan/definition.py | 24 +++++++++++++++ wordfence/databasescanning/scanner.py | 5 +++- wordfence/intel/database_rules.py | 43 +++++++++++++++++++++++++-- 4 files changed, 78 insertions(+), 4 deletions(-) diff --git a/wordfence/cli/dbscan/dbscan.py b/wordfence/cli/dbscan/dbscan.py index b53b5c69..c2910e1b 100644 --- a/wordfence/cli/dbscan/dbscan.py +++ b/wordfence/cli/dbscan/dbscan.py @@ -161,6 +161,15 @@ def fetch_rules() -> DatabaseRuleSet: return cacheable.get(self.cache) + def _filter_rules(self, rule_set: DatabaseRuleSet) -> None: + included = None + if self.config.include_rules: + included = set(self.config.include_rules) + excluded = None + if self.config.exclude_rules: + excluded = set(self.config.exclude_rules) + rule_set.filter_rules(included, excluded) + def _load_rules(self) -> DatabaseRuleSet: rule_set = self._load_remote_rules() \ if self.config.use_remote_rules \ @@ -168,6 +177,7 @@ def _load_rules(self) -> DatabaseRuleSet: if self.config.rules_file is not not_set_token: for rules_file in self.config.rules_file: load_database_rules(rules_file, rule_set) + self._filter_rules(rule_set) return rule_set def invoke(self) -> int: diff --git a/wordfence/cli/dbscan/definition.py b/wordfence/cli/dbscan/definition.py index f833ae94..683a8e01 100644 --- a/wordfence/cli/dbscan/definition.py +++ b/wordfence/cli/dbscan/definition.py @@ -151,6 +151,30 @@ "meta": { "accepts_file": True } + }, + "exclude-rules": { + "short_name": "e", + "description": "Specify rule IDs to ignore when scanning. May be " + "comma-delimited and/or repeated.", + "context": "ALL", + "argument_type": "OPTION_REPEATABLE", + "default": None, + "meta": { + "separator": ",", + "value_type": int + } + }, + "include-rules": { + "short_name": "i", + "description": "Specify rule IDs to include when scanning. May be " + "comma-delimited and/or repeated.", + "context": "ALL", + "argument_type": "OPTION_REPEATABLE", + "default": None, + "meta": { + "separator": ",", + "value_type": int + } } } diff --git a/wordfence/databasescanning/scanner.py b/wordfence/databasescanning/scanner.py index fd820a10..4d865995 100644 --- a/wordfence/databasescanning/scanner.py +++ b/wordfence/databasescanning/scanner.py @@ -63,7 +63,10 @@ def _scan_table( f'{prefixed_table} WHERE ' + ' OR '.join(conditions) ) - for result in connection.query(query): + # Using a dict as the query parameters avoids %s from being + # interpreted as a placeholder (there is apparently no way + # to escape "%s" ("%%s" doesn't work) + for result in connection.query(query, {}): rule = self.rule_set.get_rule(result['rule_id']) del result['rule_id'] yield DatabaseScanResult( diff --git a/wordfence/intel/database_rules.py b/wordfence/intel/database_rules.py index 557aa5bd..89fdcc96 100644 --- a/wordfence/intel/database_rules.py +++ b/wordfence/intel/database_rules.py @@ -18,6 +18,15 @@ def __init__( self.condition = condition self.description = description + def __hash__(self): + return hash(self.identifier) + + def __eq__(self, other) -> bool: + return ( + type(other) is type(self) + and other.identifier == self.identifier + ) + class DatabaseRuleSet: @@ -31,12 +40,27 @@ def add_rule(self, rule: DatabaseRule) -> None: raise Exception('Duplicate rule ID: {rule.identifier}') self.rules[rule.identifier] = rule if rule.tables is None: - self.global_rules.append(rule) + self.global_rules.add(rule) else: for table in rule.tables: if table not in self.table_rules: - self.table_rules[table] = [] - self.table_rules[table].append(rule) + self.table_rules[table] = set() + self.table_rules[table].add(rule) + + def remove_rule(self, rule_id: int) -> None: + try: + rule = self.rules.pop(rule_id) + if rule.tables is None: + self.global_rules.discard(rule) + else: + for table in rule.tables: + if table in list(self.table_rules.keys()): + table_rules = self.table_rules[table] + table_rules.discard(rule) + if len(table_rules) == 0: + del self.table_rules[table] + except KeyError: + pass # Rule doesn't exist, no need to remove def get_rules(self, table: str) -> List[DatabaseRule]: rules = [] @@ -53,6 +77,19 @@ def get_targeted_tables(self) -> List[str]: def get_rule(self, identifier: int) -> DatabaseRule: return self.rules[identifier] + def filter_rules( + self, + included: Optional[Set[int]] = None, + excluded: Optional[Set[int]] = None + ): + if included is not None: + for rule_id in list(self.rules.keys()): + if rule_id not in included: + self.remove_rule(rule_id) + if excluded is not None: + for rule_id in excluded: + self.remove_rule(rule_id) + JSON_VALIDATOR = ListValidator( DictionaryValidator({ From 3c1094875bf106c6674f46c057eb91328ab31d5e Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Fri, 27 Sep 2024 12:02:31 -0400 Subject: [PATCH 10/18] Added Python dependencies to unit testing workflow setup --- .github/workflows/unit-testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index b32268f9..13299bee 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -5,4 +5,5 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 + - run: apt-get update && apt-get install -y python3-mysql.connector python3-requests - run: python3 -m unittest From 364a4431ba931df57106fb2514539c1f642b7ede Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Fri, 27 Sep 2024 12:04:31 -0400 Subject: [PATCH 11/18] Added sudo to apt-get calls in unit-testing workflow --- .github/workflows/unit-testing.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-testing.yml b/.github/workflows/unit-testing.yml index 13299bee..5d73a758 100644 --- a/.github/workflows/unit-testing.yml +++ b/.github/workflows/unit-testing.yml @@ -5,5 +5,5 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 - - run: apt-get update && apt-get install -y python3-mysql.connector python3-requests + - run: sudo apt-get update && sudo apt-get install -y python3-mysql.connector python3-requests - run: python3 -m unittest From b017237c91c57d5ad5066f636ab6d6497cf6ca1e Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Fri, 27 Sep 2024 12:11:28 -0400 Subject: [PATCH 12/18] Attempted to fix issue with code style validation workflow --- .github/workflows/validate-code-styles.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-code-styles.yml b/.github/workflows/validate-code-styles.yml index 60ba8d9e..8781b332 100644 --- a/.github/workflows/validate-code-styles.yml +++ b/.github/workflows/validate-code-styles.yml @@ -6,5 +6,5 @@ jobs: steps: - uses: actions/checkout@v3 - run: sudo apt-get install -y flake8 python3-pip - - run: pip3 install flake8-bugbear + - run: pip3 install flake8-bugbear attrs - run: flake8 --require-plugins pycodestyle,flake8-bugbear From 83dda04a93396ba3bea8db66b62dc322691837c4 Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Fri, 27 Sep 2024 12:17:34 -0400 Subject: [PATCH 13/18] Made another attempt to fix flake8 workflow --- .github/workflows/validate-code-styles.yml | 4 ++-- wordfence/cli/dbscan/reporting.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate-code-styles.yml b/.github/workflows/validate-code-styles.yml index 8781b332..659cef41 100644 --- a/.github/workflows/validate-code-styles.yml +++ b/.github/workflows/validate-code-styles.yml @@ -5,6 +5,6 @@ jobs: runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v3 - - run: sudo apt-get install -y flake8 python3-pip - - run: pip3 install flake8-bugbear attrs + - run: sudo apt-get update && sudo apt-get install -y python3-pip + - run: pip3 install flake8 flake8-bugbear - run: flake8 --require-plugins pycodestyle,flake8-bugbear diff --git a/wordfence/cli/dbscan/reporting.py b/wordfence/cli/dbscan/reporting.py index 229d1c14..c3af162b 100644 --- a/wordfence/cli/dbscan/reporting.py +++ b/wordfence/cli/dbscan/reporting.py @@ -1,4 +1,3 @@ -import json from typing import List, Optional, Dict from wordfence.databasescanning.scanner import DatabaseScanResult From 615f58e3dba9d992a55ed766e727e4c1ffd34971 Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Fri, 27 Sep 2024 12:20:43 -0400 Subject: [PATCH 14/18] Updated Ubuntu version for flake8 workflow --- .github/workflows/validate-code-styles.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-code-styles.yml b/.github/workflows/validate-code-styles.yml index 659cef41..1a993054 100644 --- a/.github/workflows/validate-code-styles.yml +++ b/.github/workflows/validate-code-styles.yml @@ -2,7 +2,7 @@ name: "Validate Code Styles" on: [push] jobs: flake8: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 - run: sudo apt-get update && sudo apt-get install -y python3-pip From f3ec2a6819d517502d11a5670181741b21e4ef15 Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Fri, 27 Sep 2024 12:24:12 -0400 Subject: [PATCH 15/18] Updated flake8 workflow to use Python venv --- .github/workflows/validate-code-styles.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/validate-code-styles.yml b/.github/workflows/validate-code-styles.yml index 1a993054..74ae1677 100644 --- a/.github/workflows/validate-code-styles.yml +++ b/.github/workflows/validate-code-styles.yml @@ -5,6 +5,7 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v3 - - run: sudo apt-get update && sudo apt-get install -y python3-pip - - run: pip3 install flake8 flake8-bugbear - - run: flake8 --require-plugins pycodestyle,flake8-bugbear + - run: sudo apt-get update && sudo apt-get install -y python3-full + - run: python3 -m venv ./python-venv + - run: ./python-venv/bin/pip install flake8 flake8-bugbear + - run: ./python-venv/bin/python -m flake8 --require-plugins pycodestyle,flake8-bugbear From eed7479ededaf2ae07d5268fcf62ffabfe4e3b77 Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Fri, 27 Sep 2024 12:26:13 -0400 Subject: [PATCH 16/18] Excluded venv from flake8 validation in GitHub workflow --- .github/workflows/validate-code-styles.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-code-styles.yml b/.github/workflows/validate-code-styles.yml index 74ae1677..0b8d8730 100644 --- a/.github/workflows/validate-code-styles.yml +++ b/.github/workflows/validate-code-styles.yml @@ -8,4 +8,4 @@ jobs: - run: sudo apt-get update && sudo apt-get install -y python3-full - run: python3 -m venv ./python-venv - run: ./python-venv/bin/pip install flake8 flake8-bugbear - - run: ./python-venv/bin/python -m flake8 --require-plugins pycodestyle,flake8-bugbear + - run: ./python-venv/bin/python -m flake8 --exclude python-venv --require-plugins pycodestyle,flake8-bugbear From 481735a5d18f8ec1bd558c09e6839ebb1424bfec Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Fri, 27 Sep 2024 12:34:05 -0400 Subject: [PATCH 17/18] Adjusted venv directory name in workflow and fixed style violation --- .github/workflows/validate-code-styles.yml | 6 +++--- wordfence/cli/config/base_config_definitions.py | 4 +--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/validate-code-styles.yml b/.github/workflows/validate-code-styles.yml index 0b8d8730..549118b2 100644 --- a/.github/workflows/validate-code-styles.yml +++ b/.github/workflows/validate-code-styles.yml @@ -6,6 +6,6 @@ jobs: steps: - uses: actions/checkout@v3 - run: sudo apt-get update && sudo apt-get install -y python3-full - - run: python3 -m venv ./python-venv - - run: ./python-venv/bin/pip install flake8 flake8-bugbear - - run: ./python-venv/bin/python -m flake8 --exclude python-venv --require-plugins pycodestyle,flake8-bugbear + - run: python3 -m venv ./venv + - run: ./venv/bin/pip install flake8 flake8-bugbear + - run: ./venv/bin/python -m flake8 --exclude venv --require-plugins pycodestyle,flake8-bugbear diff --git a/wordfence/cli/config/base_config_definitions.py b/wordfence/cli/config/base_config_definitions.py index 7a34ef4c..3cbce529 100644 --- a/wordfence/cli/config/base_config_definitions.py +++ b/wordfence/cli/config/base_config_definitions.py @@ -86,9 +86,7 @@ "if supported using the STARTTLS SMTP command.", "context": "ALL", "argument_type": "OPTION", - "meta": { - "valid_options": [mode.value for mode in SmtpTlsMode] - }, + "meta": {"valid_options": [mode.value for mode in SmtpTlsMode]}, "default": SmtpTlsMode.STARTTLS.value, "category": "Email" }, From 67159f5e444f48bd9130040e54a031ae85a734b0 Mon Sep 17 00:00:00 2001 From: Alex Kenion Date: Fri, 27 Sep 2024 12:54:46 -0400 Subject: [PATCH 18/18] Restored support for parsing local rule files --- wordfence/intel/database_rules.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/wordfence/intel/database_rules.py b/wordfence/intel/database_rules.py index 89fdcc96..9205ac6a 100644 --- a/wordfence/intel/database_rules.py +++ b/wordfence/intel/database_rules.py @@ -103,11 +103,13 @@ def filter_rules( def parse_database_rules( data, - pre_validated: bool = False + pre_validated: bool = False, + rule_set: Optional[DatabaseRuleSet] = None ) -> DatabaseRuleSet: if not pre_validated: JSON_VALIDATOR.validate(data) - rule_set = DatabaseRuleSet() + if rule_set is None: + rule_set = DatabaseRuleSet() for rule_data in data: rule = DatabaseRule( identifier=rule_data['id'], @@ -119,7 +121,10 @@ def parse_database_rules( return rule_set -def load_database_rules(path: bytes) -> DatabaseRuleSet: +def load_database_rules( + path: bytes, + rule_set: Optional[DatabaseRuleSet] = None + ) -> DatabaseRuleSet: with open(path, 'rb') as file: data = json.load(file) - return parse_database_rules(data) + return parse_database_rules(data, rule_set=rule_set)