Merge branch 'vulnerability-scanner' into milestone/voodoo-child

wordfence/api/noc1.py

@@ -23,16 +23,19 @@ def build_query(self, action: str, base_query: dict = None) -> dict:
         query['s'] = self._generate_site_stats()
         return query
 
-    def register_terms_update_hook(self, callable: Callable[[], None]) -> None:
+    def register_terms_update_hook(
+                self,
+                callable: Callable[[bool], None]
+            ) -> None:
         if not hasattr(self, 'terms_update_hooks'):
             self.terms_update_hooks = []
         self.terms_update_hooks.append(callable)
 
-    def _trigger_terms_update_hooks(self) -> None:
+    def _trigger_terms_update_hooks(self, paid: bool = False) -> None:
         if not hasattr(self, 'terms_update_hooks'):
             return
         for hook in self.terms_update_hooks:
-            hook()
+            hook(paid)
 
     def validate_response(self, response, validator: Validator) -> None:
         if isinstance(response, dict):
@@ -42,7 +45,8 @@ def validate_response(self, response, validator: Validator) -> None:
                         response['errorMsg']
                     )
             if '_termsUpdated' in response:
-                self._trigger_terms_update_hooks()
+                paid = '_isPaidKey' in response and response['_isPaidKey']
+                self._trigger_terms_update_hooks(paid)
         return super().validate_response(response, validator)
 
     def process_simple_request(self, action: str) -> bool:

wordfence/cli/config/__init__.py

@@ -55,13 +55,20 @@ def create_config_object(
             # later values always replace previous values
             if new_value is not not_set_token:
                 setattr(target, item_definition.property_name, new_value)
+                try:
+                    target.defaulted_options.remove(
+                            item_definition.property_name
+                        )
+                except KeyError:
+                    pass  # Ignore options that weren't previously defaulted
             elif not hasattr(target, item_definition.property_name):
                 default = item_definition.default
                 if item_definition.has_separator() and \
                         isinstance(default, str):
                     default = default.split(item_definition.meta.separator)
                 setattr(target, item_definition.property_name,
                         default)
+                target.defaulted_options.add(item_definition.property_name)
     target.trailing_arguments = trailing_arguments
     return target
 

wordfence/cli/config/base_config_definitions.py

@@ -4,8 +4,7 @@
 config_definitions = {
     "configuration": {
         "short_name": "c",
-        "description": "Path to a configuration INI file to use (defaults to"
-                       f" \"{INI_DEFAULT_PATH}\").",
+        "description": "Path to a configuration INI file to use.",
         "context": "CLI",
         "argument_type": "OPTION",
         "default": INI_DEFAULT_PATH

wordfence/cli/config/config.py

@@ -20,6 +20,7 @@ def __init__(
         self.subcommand = subcommand
         self.ini_path = ini_path
         self.trailing_arguments = None
+        self.defaulted_options = set()
 
     def values(self) -> Dict[str, Any]:
         result: Dict[str, Any] = dict()
@@ -42,3 +43,6 @@ def has_ini_file(self) -> bool:
     def display_help(self) -> None:
         self._parser.print_help()
         print()
+
+    def is_specified(self, option: str) -> bool:
+        return option not in self.defaulted_options

wordfence/cli/context.py

@@ -1,3 +1,4 @@
+import sys
 from typing import Optional, Any
 
 from ..version import __version__, __version_name__
@@ -74,3 +75,11 @@ def display_version(self) -> None:
                 f"PCRE Version: {pcre.VERSION} - "
                 f"JIT Supported: {jit_support_text}"
             )
+
+    def has_terminal_output(self) -> bool:
+        return sys.stdout is not None \
+                and sys.stdout.isatty()
+
+    def has_terminal_input(self) -> bool:
+        return sys.stdin is not None \
+                and sys.stdin.isatty()

wordfence/cli/helper.py

@@ -17,13 +17,15 @@ def __init__(
                 short_name: Optional[str],
                 description: str,
                 category: str,
+                default: str,
                 valid_values: Optional[List[str]],
                 is_flag: bool = False
             ):
         self.long_name = long_name
         self.short_name = short_name
         self.description = description
         self.category = category
+        self.default = default
         self.valid_values = valid_values
         self.is_flag = is_flag
         self.label = self.generate_label()
@@ -45,12 +47,12 @@ def split_line(
                 self,
                 line: str,
                 max_length: int,
-                offset: int = 0
+                offset: int = 0,
+                first: bool = True
             ) -> List[str]:
         if offset > max_length:
             offset = 0
         lines = []
-        first = True
         while len(line) > 0:
             end = max_length - 1
             if len(line) > max_length:
@@ -59,7 +61,7 @@ def split_line(
                 except ValueError:
                     next_break = end
             else:
-                next_break = len(line) - 1
+                next_break = len(line)
             next = line[:next_break]
             line = line[next_break + 1:]
             if not first:
@@ -79,13 +81,20 @@ def join_lines(
         final_lines = []
         max_length = self.terminal_size.columns
         for input_line in lines:
+            initial = True
             for real_line in input_line.splitlines():
-                if len(real_line) > max_length:
+                if len(real_line) > max_length or not initial:
                     final_lines.extend(
-                            self.split_line(real_line, max_length, offset)
+                            self.split_line(
+                                    real_line,
+                                    max_length,
+                                    offset,
+                                    first=initial
+                                )
                         )
                 else:
                     final_lines.append(real_line)
+                initial = False
         return delimiter.join(final_lines)
 
     def join_chunks(
@@ -136,6 +145,7 @@ def _load_options(
                     item.short_name,
                     item.description,
                     item.category,
+                    item.default,
                     valid_values,
                     item.is_flag()
                 )
@@ -171,6 +181,11 @@ def format_category(
                         f'(use --no-{option.long_name} to disable)',
                         offset
                     ))
+            elif isinstance(option.default, str) and len(option.default):
+                lines.append(self._offset(
+                        f'(default: {option.default})',
+                        offset
+                    ))
         return self.line_formatter.join_lines(lines, offset=offset)
 
     def format_options(self) -> str:

wordfence/cli/malwarescan/definition.py

@@ -115,7 +115,7 @@
                        "be skipped and a warning will be logged.",
         "context": "ALL",
         "argument_type": "FLAG",
-        "default": False
+        "default": True
     },
     "chunk-size": {
         "short_name": "z",

wordfence/cli/malwarescan/malwarescan.py

@@ -204,8 +204,8 @@ def generate_exception_message(
             ) -> Optional[str]:
         if isinstance(exception, ProgressException):
             return (
-                    'The current terminal size is inadequate for '
-                    'displaying progress output for the current scan '
+                    'The current terminal is too small to '
+                    'display progress output with the current scan '
                     'options'
                 )
         return super().generate_exception_message(exception)

wordfence/cli/malwarescan/reporting.py

@@ -66,7 +66,8 @@ def allows_headers(self) -> bool:
 
 REPORT_FORMAT_HUMAN = ReportFormat(
         'human',
-        lambda stream, columns: HumanReadableWriter(stream, columns)
+        lambda stream, columns: HumanReadableWriter(stream, columns),
+        allows_headers=False
     )
 
 

wordfence/cli/reporting.py

@@ -5,6 +5,7 @@
 from enum import Enum
 from contextlib import nullcontext
 
+from wordfence.logging import log
 from .config import Config
 from .io import IoManager
 
@@ -67,6 +68,9 @@ def write_row(self, data: List[str]):
     def allows_headers(self) -> bool:
         return True
 
+    def allows_column_customization(self) -> bool:
+        return True
+
 
 class CsvReportWriter(ReportWriter):
 
@@ -106,6 +110,9 @@ class RowlessWriter(ReportWriter):
     def allows_headers(self) -> bool:
         return False
 
+    def allows_column_customization(self) -> bool:
+        return False
+
     def write_row(self, data: List[str]) -> None:
         pass
 
@@ -118,10 +125,14 @@ class ReportFormat:
     def __init__(
                 self,
                 option: str,
-                initializer: Callable[[IO, List[ReportColumn]], ReportWriter]
+                initializer: Callable[[IO, List[ReportColumn]], ReportWriter],
+                allows_headers: bool = True,
+                allows_column_customization: bool = True
             ):
         self.option = option
         self.initializer = initializer
+        self.allows_headers = allows_headers
+        self.allows_column_customization = allows_column_customization
 
     def initialize_writer(
                 self,
@@ -130,9 +141,6 @@ def initialize_writer(
             ) -> ReportWriter:
         return self.initializer(stream, columns)
 
-    def get_valid_options() -> List[str]:
-        return [format.value for format in ReportFormat]
-
 
 REPORT_FORMAT_CSV = ReportFormat(
         'csv',
@@ -183,9 +191,21 @@ def __init__(
         self.write_headers = write_headers
         self.headers_written = False
         self.writers = []
+        self.has_custom_columns = False
 
     def add_target(self, stream: IO) -> None:
         writer = self.format.initialize_writer(stream, self.columns)
+        if self.write_headers and not writer.allows_headers():
+            log.warning(
+                    'Headers are not supported when using the '
+                    f'{self.format.option} output format'
+                )
+        if self.has_custom_columns \
+                and not writer.allows_column_customization():
+            log.warning(
+                    'Columns cannot be specified when using the '
+                    f'{self.format.option} output format'
+                )
         self.writers.append(writer)
 
     def _write_row(self, data: List[str], record: ReportRecord):
@@ -198,9 +218,10 @@ def _write_row(self, data: List[str], record: ReportRecord):
     def _write_headers(self) -> None:
         if self.headers_written or not self.write_headers:
             return
+        headers = [column.header for column in self.columns]
         for writer in self.writers:
             if writer.allows_headers():
-                writer.write_row(self.columns)
+                writer.write_row(headers)
         self.headers_written = True
 
     def _format_record(self, record: ReportRecord) -> List[str]:
@@ -231,6 +252,15 @@ def get_config_options(
             default_columns: List[ReportColumnEnum],
             default_format: str = 'csv'
         ) -> Dict[str, Dict[str, Any]]:
+    header_formats = []
+    column_formats = []
+    for format in formats:
+        if format.value.allows_headers:
+            header_formats.append(format.value.option)
+        if format.value.allows_column_customization:
+            column_formats.append(format.value.option)
+    header_format_string = ', '.join(header_formats)
+    column_format_string = ', '.join(column_formats)
     return {
         "output": {
             "description": "Write results to stdout. This is the default "
@@ -251,7 +281,9 @@ def get_config_options(
         "output-columns": {
             "description": ("An ordered, comma-delimited list of columns to"
                             " include in the output. Available columns: "
-                            + columns.get_options_as_string()),
+                            + columns.get_options_as_string()
+                            + f"\nCompatible formats: {column_format_string}"
+                            ),
             "context": "ALL",
             "argument_type": "OPTION",
             "default": ','.join([column.header for column in default_columns]),
@@ -273,7 +305,8 @@ def get_config_options(
         },
         "output-headers": {
             "description": "Whether or not to include column headers in "
-                           "output",
+                           "output.\n"
+                           f"Compatible formats: {header_format_string}",
             "context": "ALL",
             "argument_type": "FLAG",
             "default": None,
@@ -356,6 +389,7 @@ def initialize_report(self, output_file: Optional[IO] = None) -> Report:
                 columns,
                 self.config.output_headers
             )
+        report.has_custom_columns = self.config.is_specified('output_columns')
         self._add_targets(report, output_file)
         if not report.has_writers():
             raise ReportingException(

wordfence/cli/terms.py

@@ -18,21 +18,25 @@ def prompt_acceptance_if_needed(self):
         if not accepted:
             self.prompt_acceptance()
 
-    def trigger_update(self):
+    def trigger_update(self, paid: bool = False):
         self.context.cache.put(TERMS_CACHE_KEY, False)
-        self.prompt_acceptance()
+        self.prompt_acceptance(paid)
 
     def record_acceptance(self, remote: bool = True):
         if remote:
             client = self.context.get_noc1_client()
             client.record_toupp()
         self.context.cache.put(TERMS_CACHE_KEY, True)
 
-    def prompt_acceptance(self):
+    def prompt_acceptance(self, paid: bool = False):
         if not (sys.stdout.isatty() and sys.stdin.isatty()):
             return
+        if paid:
+            edition = ''
+        else:
+            edition = ' Free edition'
         terms_accepted = prompt_yes_no(
-                'Your access to and use of Wordfence CLI Free edition is '
+                f'Your access to and use of Wordfence CLI{edition} is '
                 'subject to the updated Wordfence CLI License Terms and '
                 f'Conditions set forth at {TERMS_URL}. By entering "y" and '
                 'selecting Enter, you agree that you have read and accept the '

wordfence/cli/vulnscan/definition.py

@@ -135,6 +135,14 @@
                     variant.path for variant in VulnerabilityFeedVariant
                 ]
         }
+    },
+    "require-path": {
+        "description": "When enabled, an error will be issued if at least one "
+                       "path to scan is not specified. This is the default "
+                       "behavior when running in a terminal.",
+        "context": "CLI",
+        "argument_type": "OPTIONAL_FLAG",
+        "default": None
     }
 }
 

wordfence/cli/vulnscan/reporting.py

@@ -109,7 +109,9 @@ def write_record(self, record) -> None:
 
 REPORT_FORMAT_HUMAN = ReportFormat(
         'human',
-        lambda stream, columns: HumanReadableWriter(stream)
+        lambda stream, columns: HumanReadableWriter(stream),
+        allows_headers=False,
+        allows_column_customization=False
     )