From f252410e8f917372edd707475a05fbf5a6f8c6ad Mon Sep 17 00:00:00 2001 From: vringar Date: Tue, 29 Sep 2020 14:52:40 +0200 Subject: [PATCH 001/139] First steps in the rewrite --- automation/DataAggregator/S3Aggregator.py | 5 +- automation/data_aggregator/__init__.py | 2 + automation/data_aggregator/arrow_storage.py | 29 +++++++++ .../data_aggregator/in_memory_storage.py | 32 ++++++++++ .../parquet_schema.py | 0 .../data_aggregator/storage_providers.py | 60 +++++++++++++++++++ scripts/environment-unpinned-dev.yaml | 1 + 7 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 automation/data_aggregator/__init__.py create mode 100644 automation/data_aggregator/arrow_storage.py create mode 100644 automation/data_aggregator/in_memory_storage.py rename automation/{DataAggregator => data_aggregator}/parquet_schema.py (100%) create mode 100644 automation/data_aggregator/storage_providers.py diff --git a/automation/DataAggregator/S3Aggregator.py b/automation/DataAggregator/S3Aggregator.py index d81c3bf24..f27a391dd 100644 --- a/automation/DataAggregator/S3Aggregator.py +++ b/automation/DataAggregator/S3Aggregator.py @@ -18,6 +18,8 @@ from botocore.exceptions import ClientError, EndpointConnectionError from pyarrow.filesystem import S3FSWrapper # noqa +from automation.data_aggregator.parquet_schema import PQ_SCHEMAS + from .BaseAggregator import ( RECORD_TYPE_CONTENT, RECORD_TYPE_CREATE, @@ -26,7 +28,6 @@ BaseListener, BaseParams, ) -from .parquet_schema import PQ_SCHEMAS CACHE_SIZE = 500 SITE_VISITS_INDEX = "_site_visits_index" @@ -76,7 +77,7 @@ def __init__( def factory_function(): return defaultdict(list) - self._records: Dict[int, DefaultDict[str, List[Any]]] = defaultdict( + self._records: DefaultDict[int, DefaultDict[str, List[Any]]] = defaultdict( factory_function ) # maps visit_id and table to records self._batches: DefaultDict[str, List[pa.RecordBatch]] = defaultdict( diff --git a/automation/data_aggregator/__init__.py b/automation/data_aggregator/__init__.py new file mode 100644 index 000000000..fc91c51e2 --- /dev/null +++ b/automation/data_aggregator/__init__.py @@ -0,0 +1,2 @@ +class DataAggregator: + pass diff --git a/automation/data_aggregator/arrow_storage.py b/automation/data_aggregator/arrow_storage.py new file mode 100644 index 000000000..fb271db79 --- /dev/null +++ b/automation/data_aggregator/arrow_storage.py @@ -0,0 +1,29 @@ +from abc import abstractmethod + +from pyarrow import Table + +from .parquet_schema import PQ_SCHEMAS +from .storage_providers import StructuredStorageProvider + + +class ArrowAggregator(StructuredStorageProvider): + """This class implements a StructuredStorage provider that + serializes records into the arrow format + """ + + def flush_cache(self): + pass + + def store_record(self, table: str, record: Dict[str, Any], visit_id: int) -> None: + records = self._records[visit_id] + # Add nulls + for item in PQ_SCHEMAS[table].names: + if item not in record: + data[item] = None + # Add instance_id (for partitioning) + record["instance_id"] = self._instance_id + records[table].append(record) + + @abstractmethod + def write_table(self, table: Table) -> None: + """Write out the table to persistent storage""" diff --git a/automation/data_aggregator/in_memory_storage.py b/automation/data_aggregator/in_memory_storage.py new file mode 100644 index 000000000..b578ddd81 --- /dev/null +++ b/automation/data_aggregator/in_memory_storage.py @@ -0,0 +1,32 @@ +from collections import defaultdict +from typing import Any, DefaultDict, Dict, List, Tuple + +from .storage_providers import StructuredStorageProvider, UnstructuredStorageProvider + + +class MemoryStructuredProvider(StructuredStorageProvider): + def __init__(self): + super().__init__() + self._storage: DefaultDict[str, List[Any]] = defaultdict(list) + self._completed_visit_ids: List[Tuple[int, bool]] = list() + + def flush_cache(self) -> None: + pass + + def store_record(self, table: str, record: Dict[str, Any]) -> None: + self._storage[table].append(record) + pass + + def run_visit_completion_tasks( + self, visit_id: int, interrupted: bool = False + ) -> None: + self._completed_visit_ids.append((visit_id, interrupted)) + pass + + def saved_visit_ids(self) -> List[Tuple[int, bool]]: + temp = self._completed_visit_ids + self._completed_visit_ids = list() + return temp + + def shutdown(self) -> None: + pass diff --git a/automation/DataAggregator/parquet_schema.py b/automation/data_aggregator/parquet_schema.py similarity index 100% rename from automation/DataAggregator/parquet_schema.py rename to automation/data_aggregator/parquet_schema.py diff --git a/automation/data_aggregator/storage_providers.py b/automation/data_aggregator/storage_providers.py new file mode 100644 index 000000000..afde7ff75 --- /dev/null +++ b/automation/data_aggregator/storage_providers.py @@ -0,0 +1,60 @@ +from abc import ABC, abstractmethod +from typing import Any, Dict, List, Tuple + +""" +This module contains all base classes of the storage provider hierachy +Any subclass of these classes should be able to be used in OpenWPM +without any changes to the rest of the code base +""" + + +class StorageProvider(ABC): + """Base class that defines some general helper methods + Do not inherit from this class directly + Inherit from StructuredStorageProvider or UnstructuredStorageProvider instead + """ + + @abstractmethod + def flush_cache(self) -> None: + """ Blockingly write out any cached data to the respective storage """ + pass + + @abstractmethod + def shutdown(self) -> None: + """Close all open ressources + After this method has been called no further calls should be made to the object + """ + pass + + +class StructuredStorageProvider(StorageProvider): + @abstractmethod + def store_record(self, table: str, record: Dict[str, Any]) -> None: + """Submit a record to be stored + The storing might not happen immediately + """ + pass + + @abstractmethod + def run_visit_completion_tasks( + self, visit_id: int, interrupted: bool = False + ) -> None: + """This method is invoked to inform the StrucuturedStorageProvider that no more + records for this visit_id will be submitted + """ + pass + + @abstractmethod + def saved_visit_ids(self) -> List[Tuple[int, bool]]: + """Return the list of all visit_ids that have been saved to permanent storage + since the last time this method was called + + For each visit_id it's also noted whether they were interrupted + """ + pass + + +class UnstructuredStorageProvider(StorageProvider): + def store_blob(self, blob: str) -> None: + """Stores the data passed in as an base64 encoded string""" + pass diff --git a/scripts/environment-unpinned-dev.yaml b/scripts/environment-unpinned-dev.yaml index 45a0d4cc0..eb61a75de 100644 --- a/scripts/environment-unpinned-dev.yaml +++ b/scripts/environment-unpinned-dev.yaml @@ -11,6 +11,7 @@ dependencies: - pip - pre-commit - pytest + - mypy - pip: # Select depenedencies from localstack[full] that we need - amazon-kclpy From 0ba7de76e6f1aaa91b61277e187c3cd5d9e58630 Mon Sep 17 00:00:00 2001 From: vringar Date: Tue, 29 Sep 2020 15:16:17 +0200 Subject: [PATCH 002/139] Fixed import paths --- automation/DataAggregator/S3Aggregator.py | 3 +-- test/test_s3_aggregator.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/automation/DataAggregator/S3Aggregator.py b/automation/DataAggregator/S3Aggregator.py index f27a391dd..3940e6ee9 100644 --- a/automation/DataAggregator/S3Aggregator.py +++ b/automation/DataAggregator/S3Aggregator.py @@ -18,8 +18,7 @@ from botocore.exceptions import ClientError, EndpointConnectionError from pyarrow.filesystem import S3FSWrapper # noqa -from automation.data_aggregator.parquet_schema import PQ_SCHEMAS - +from ..data_aggregator.parquet_schema import PQ_SCHEMAS from .BaseAggregator import ( RECORD_TYPE_CONTENT, RECORD_TYPE_CREATE, diff --git a/test/test_s3_aggregator.py b/test/test_s3_aggregator.py index a598b21d5..0089b2edc 100644 --- a/test/test_s3_aggregator.py +++ b/test/test_s3_aggregator.py @@ -9,7 +9,7 @@ from ..automation import TaskManager from ..automation.CommandSequence import CommandSequence -from ..automation.DataAggregator.parquet_schema import PQ_SCHEMAS +from ..automation.data_aggregator.parquet_schema import PQ_SCHEMAS from .openwpmtest import OpenWPMTest from .utilities import BASE_TEST_URL, LocalS3Dataset, LocalS3Session, local_s3_bucket From d65af1e4a8a6f11412af70fdaf1080dc6e36b1dc Mon Sep 17 00:00:00 2001 From: vringar Date: Tue, 29 Sep 2020 17:16:56 +0200 Subject: [PATCH 003/139] One giant refactor --- .travis.yml | 6 +----- automation/BrowserManager.py | 4 ++-- automation/Commands/browser_commands.py | 8 ++++---- automation/DataAggregator/BaseAggregator.py | 6 +++--- automation/MPLogger.py | 4 ++-- automation/SocketInterface.py | 8 ++++---- automation/TaskManager.py | 4 ++-- automation/data_aggregator/__init__.py | 12 +++++++++++- automation/data_aggregator/in_memory_storage.py | 9 +++++++-- scripts/travis.sh | 3 +-- test/openwpm_jstest.py | 3 ++- test/openwpmtest.py | 3 ++- test/storage_providers/__init__.py | 0 test/storage_providers/test_data_aggregator.py | 0 .../test_memory_storage_provider.py | 17 +++++++++++++++++ test/test_callback.py | 5 +++-- test/test_callstack_instrument.py | 7 ++++--- test/test_crawl.py | 5 +++-- test/test_custom_function_command.py | 9 +++++---- test/test_dns_instrument.py | 3 ++- test/test_env.py | 3 ++- test/test_extension.py | 5 +++-- test/test_http_instrumentation.py | 5 +++-- test/test_js_instrument.py | 3 ++- test/test_js_instrument_py.py | 2 +- test/test_mp_logger.py | 5 +++-- test/test_profile.py | 9 +++++---- test/test_s3_aggregator.py | 7 ++++--- test/test_simple_commands.py | 5 +++-- test/test_storage_vectors.py | 5 +++-- test/test_webdriver_utils.py | 7 ++++--- 31 files changed, 108 insertions(+), 64 deletions(-) create mode 100644 test/storage_providers/__init__.py create mode 100644 test/storage_providers/test_data_aggregator.py create mode 100644 test/storage_providers/test_memory_storage_provider.py diff --git a/.travis.yml b/.travis.yml index acd3be359..5a067ec20 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,15 +36,11 @@ after_success: jobs: include: - - language: - python: + - language: "python" env: - TESTS="Docker" services: - docker - before_install: - before_script: - install: script: - docker build -f Dockerfile -t openwpm . - ./scripts/deploy-to-dockerhub.sh diff --git a/automation/BrowserManager.py b/automation/BrowserManager.py index 1bb96183b..a9b5f7599 100644 --- a/automation/BrowserManager.py +++ b/automation/BrowserManager.py @@ -20,7 +20,7 @@ from .Commands.Types import ShutdownCommand from .DeployBrowsers import deploy_browser from .Errors import BrowserConfigError, BrowserCrashError, ProfileLoadError -from .SocketInterface import clientsocket +from .SocketInterface import ClientSocket from .utilities.multiprocess_utils import ( Process, kill_process_and_children, @@ -478,7 +478,7 @@ def BrowserManager( "BROWSER %i: Connecting to extension on port %i" % (browser_params["browser_id"], port) ) - extension_socket = clientsocket(serialization="json") + extension_socket = ClientSocket(serialization="json") extension_socket.connect("127.0.0.1", int(port)) else: extension_socket = None diff --git a/automation/Commands/browser_commands.py b/automation/Commands/browser_commands.py index e9d2f4d79..06acb9e31 100644 --- a/automation/Commands/browser_commands.py +++ b/automation/Commands/browser_commands.py @@ -20,7 +20,7 @@ from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.ui import WebDriverWait -from ..SocketInterface import clientsocket +from ..SocketInterface import ClientSocket from .utils.webdriver_utils import ( execute_in_all_frames, execute_script_with_retry, @@ -112,7 +112,7 @@ def tab_restart_browser(webdriver): def get_website( - url, sleep, visit_id, webdriver, browser_params, extension_socket: clientsocket + url, sleep, visit_id, webdriver, browser_params, extension_socket: ClientSocket ): """ goes to using the given instance @@ -372,7 +372,7 @@ def collect_source(driver, frame_stack, rv={}): def finalize( - visit_id: int, webdriver: WebDriver, extension_socket: clientsocket, sleep: int + visit_id: int, webdriver: WebDriver, extension_socket: ClientSocket, sleep: int ) -> None: """ Informs the extension that a visit is done """ tab_restart_browser(webdriver) @@ -383,6 +383,6 @@ def finalize( extension_socket.send(msg) -def initialize(visit_id: int, extension_socket: clientsocket) -> None: +def initialize(visit_id: int, extension_socket: ClientSocket) -> None: msg = {"action": "Initialize", "visit_id": visit_id} extension_socket.send(msg) diff --git a/automation/DataAggregator/BaseAggregator.py b/automation/DataAggregator/BaseAggregator.py index 335a8956e..49b3cc31d 100644 --- a/automation/DataAggregator/BaseAggregator.py +++ b/automation/DataAggregator/BaseAggregator.py @@ -7,7 +7,7 @@ from multiprocess import Queue -from ..SocketInterface import serversocket +from ..SocketInterface import ServerSocket from ..utilities.multiprocess_utils import Process RECORD_TYPE_CONTENT = "page_content" @@ -61,7 +61,7 @@ def __init__( self.record_queue: Queue = None # Initialized on `startup` self.logger = logging.getLogger("openwpm") self.curent_visit_ids: List[int] = list() # All visit_ids in flight - self.sock: Optional[serversocket] = None + self.sock: Optional[ServerSocket] = None @abc.abstractmethod def process_record(self, record): @@ -98,7 +98,7 @@ def startup(self): """Run listener startup tasks Note: Child classes should call this method""" - self.sock = serversocket(name=type(self).__name__) + self.sock = ServerSocket(name=type(self).__name__) self.status_queue.put(self.sock.sock.getsockname()) self.sock.start_accepting() self.record_queue = self.sock.queue diff --git a/automation/MPLogger.py b/automation/MPLogger.py index 7c22c0519..3b7256c1d 100644 --- a/automation/MPLogger.py +++ b/automation/MPLogger.py @@ -17,7 +17,7 @@ from tblib import pickling_support from .Commands.utils.webdriver_utils import parse_neterror -from .SocketInterface import serversocket +from .SocketInterface import ServerSocket pickling_support.install() @@ -218,7 +218,7 @@ def _initialize_sentry(self): def _start_listener(self): """Start listening socket for remote logs from extension""" - socket = serversocket(name="loggingserver") + socket = ServerSocket(name="loggingserver") self._status_queue.put(socket.sock.getsockname()) socket.start_accepting() self._status_queue.join() # block to allow parent to retrieve address diff --git a/automation/SocketInterface.py b/automation/SocketInterface.py index 835965343..2b8628782 100644 --- a/automation/SocketInterface.py +++ b/automation/SocketInterface.py @@ -11,7 +11,7 @@ # see: https://stackoverflow.com/a/1148237 -class serversocket: +class ServerSocket: """ A server socket to receive and process string messages from client sockets to a central queue @@ -111,7 +111,7 @@ def close(self): self.sock.close() -class clientsocket: +class ClientSocket: """A client socket for sending messages""" def __init__(self, serialization="json", verbose=False): @@ -171,7 +171,7 @@ def main(): # Just for testing if sys.argv[1] == "s": - sock = serversocket(verbose=True) + sock = ServerSocket(verbose=True) sock.start_accepting() input("Press enter to exit...") sock.close() @@ -181,7 +181,7 @@ def main(): serialization = input("Enter the serialization type (default: 'json'):\n") if serialization == "": serialization = "json" - sock = clientsocket(serialization=serialization) + sock = ClientSocket(serialization=serialization) sock.connect(host, int(port)) msg = None diff --git a/automation/TaskManager.py b/automation/TaskManager.py index 838a1c221..bc632fa91 100644 --- a/automation/TaskManager.py +++ b/automation/TaskManager.py @@ -20,7 +20,7 @@ from .Errors import CommandExecutionError from .js_instrumentation import clean_js_instrumentation_settings from .MPLogger import MPLogger -from .SocketInterface import clientsocket +from .SocketInterface import ClientSocket from .utilities.multiprocess_utils import kill_process_and_children from .utilities.platform_utils import get_configuration_string, get_version @@ -299,7 +299,7 @@ def _launch_aggregators(self) -> None: ] = self.data_aggregator.listener_address # open connection to aggregator for saving crawl details - self.sock = clientsocket(serialization="dill") + self.sock = ClientSocket(serialization="dill") self.sock.connect(*self.manager_params["aggregator_address"]) def _shutdown_manager( diff --git a/automation/data_aggregator/__init__.py b/automation/data_aggregator/__init__.py index fc91c51e2..2eca004d5 100644 --- a/automation/data_aggregator/__init__.py +++ b/automation/data_aggregator/__init__.py @@ -1,2 +1,12 @@ +from ..SocketInterface import ServerSocket + + class DataAggregator: - pass + def startup(self): + """Run listener startup tasks + + Note: Child classes should call this method""" + self.sock = ServerSocket(name=type(self).__name__) + self.status_queue.put(self.sock.sock.getsockname()) + self.sock.start_accepting() + self.record_queue = self.sock.queue diff --git a/automation/data_aggregator/in_memory_storage.py b/automation/data_aggregator/in_memory_storage.py index b578ddd81..3ca05c90c 100644 --- a/automation/data_aggregator/in_memory_storage.py +++ b/automation/data_aggregator/in_memory_storage.py @@ -5,16 +5,21 @@ class MemoryStructuredProvider(StructuredStorageProvider): + """This storage provider stores all data in memory under + self.storage. + This makes it ideal for testing and for small crawls where no persitence is required + """ + def __init__(self): super().__init__() - self._storage: DefaultDict[str, List[Any]] = defaultdict(list) + self.storage: DefaultDict[str, List[Any]] = defaultdict(list) self._completed_visit_ids: List[Tuple[int, bool]] = list() def flush_cache(self) -> None: pass def store_record(self, table: str, record: Dict[str, Any]) -> None: - self._storage[table].append(record) + self.storage[table].append(record) pass def run_visit_completion_tasks( diff --git a/scripts/travis.sh b/scripts/travis.sh index f59c39ba2..4553b3829 100755 --- a/scripts/travis.sh +++ b/scripts/travis.sh @@ -3,8 +3,7 @@ if [[ "$TESTS" == "webextension" ]]; then cd automation/Extension/webext-instrumentation; npm test; else - cd test; - python -m pytest --cov=../automation --cov-report=xml $TESTS -s -v --durations=10; + python -m pytest --cov=automation --cov-report=xml $TESTS -s -v --durations=0; exit_code=$?; if [[ "$exit_code" -ne 0 ]]; then exit $exit_code; diff --git a/test/openwpm_jstest.py b/test/openwpm_jstest.py index b83c9e677..927dc2ae0 100644 --- a/test/openwpm_jstest.py +++ b/test/openwpm_jstest.py @@ -1,6 +1,7 @@ import re -from ..automation.utilities import db_utils +from automation.utilities import db_utils + from .openwpmtest import OpenWPMTest diff --git a/test/openwpmtest.py b/test/openwpmtest.py index 578557376..bc6f5119b 100644 --- a/test/openwpmtest.py +++ b/test/openwpmtest.py @@ -3,7 +3,8 @@ import pytest -from ..automation import TaskManager +from automation import TaskManager + from . import utilities diff --git a/test/storage_providers/__init__.py b/test/storage_providers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/storage_providers/test_data_aggregator.py b/test/storage_providers/test_data_aggregator.py new file mode 100644 index 000000000..e69de29bb diff --git a/test/storage_providers/test_memory_storage_provider.py b/test/storage_providers/test_memory_storage_provider.py new file mode 100644 index 000000000..ded3ad84d --- /dev/null +++ b/test/storage_providers/test_memory_storage_provider.py @@ -0,0 +1,17 @@ +import pytest + +from automation.data_aggregator.in_memory_storage import MemoryStructuredProvider + +pytestmark = pytest.mark.pyonly + + +def test_construction(): + assert MemoryStructuredProvider() + + +def test_basic_acess(): + prov = MemoryStructuredProvider() + prov.store_record("test", {"visit_id": 2, "data": "test"}) + prov.run_visit_completion_tasks(2) + assert prov.saved_visit_ids() == [(2, False)] + assert prov.storage == {"test": [{"visit_id": 2, "data": "test"}]} diff --git a/test/test_callback.py b/test/test_callback.py index 71ea98e0d..34188bc5f 100644 --- a/test/test_callback.py +++ b/test/test_callback.py @@ -1,8 +1,9 @@ from functools import partial from typing import List -from ..automation.CommandSequence import CommandSequence -from ..automation.TaskManager import TaskManager +from automation.CommandSequence import CommandSequence +from automation.TaskManager import TaskManager + from .openwpmtest import OpenWPMTest from .utilities import BASE_TEST_URL diff --git a/test/test_callstack_instrument.py b/test/test_callstack_instrument.py index b8c64967d..dee0cbd1a 100644 --- a/test/test_callstack_instrument.py +++ b/test/test_callstack_instrument.py @@ -1,6 +1,7 @@ -from ..automation import TaskManager -from ..automation.utilities import db_utils -from ..automation.utilities.platform_utils import parse_http_stack_trace_str +from automation import TaskManager +from automation.utilities import db_utils +from automation.utilities.platform_utils import parse_http_stack_trace_str + from . import utilities from .openwpmtest import OpenWPMTest diff --git a/test/test_crawl.py b/test/test_crawl.py index 8286ae3c3..9d45c9d4e 100644 --- a/test/test_crawl.py +++ b/test/test_crawl.py @@ -4,8 +4,9 @@ import domain_utils as du import pytest -from ..automation import TaskManager -from ..automation.utilities import db_utils +from automation import TaskManager +from automation.utilities import db_utils + from .openwpmtest import OpenWPMTest TEST_SITES = [ diff --git a/test/test_custom_function_command.py b/test/test_custom_function_command.py index 76dc4269e..aa5450c4a 100644 --- a/test/test_custom_function_command.py +++ b/test/test_custom_function_command.py @@ -1,5 +1,6 @@ -from ..automation import CommandSequence, TaskManager -from ..automation.utilities import db_utils +from automation import CommandSequence, TaskManager +from automation.utilities import db_utils + from . import utilities from .openwpmtest import OpenWPMTest @@ -30,7 +31,7 @@ def get_config(self, data_dir=""): def test_custom_function(self): """ Test `custom_function` with an inline func that collects links """ - from ..automation.SocketInterface import clientsocket + from automation.SocketInterface import ClientSocket def collect_links(table_name, scheme, **kwargs): """ Collect links with `scheme` and save in table `table_name` """ @@ -48,7 +49,7 @@ def collect_links(table_name, scheme, **kwargs): ] current_url = driver.current_url - sock = clientsocket() + sock = ClientSocket() sock.connect(*manager_params["aggregator_address"]) query = ( diff --git a/test/test_dns_instrument.py b/test/test_dns_instrument.py index 4b547d066..71af7bf7d 100644 --- a/test/test_dns_instrument.py +++ b/test/test_dns_instrument.py @@ -1,4 +1,5 @@ -from ..automation.utilities import db_utils +from automation.utilities import db_utils + from .openwpmtest import OpenWPMTest diff --git a/test/test_env.py b/test/test_env.py index a049ce2a7..9c5c83973 100644 --- a/test/test_env.py +++ b/test/test_env.py @@ -1,6 +1,7 @@ from os.path import isfile -from ..automation.utilities.platform_utils import get_firefox_binary_path +from automation.utilities.platform_utils import get_firefox_binary_path + from .openwpmtest import OpenWPMTest diff --git a/test/test_extension.py b/test/test_extension.py index 8c0a4c7da..774aaea69 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -3,8 +3,9 @@ import pytest -from ..automation import TaskManager -from ..automation.utilities import db_utils +from automation import TaskManager +from automation.utilities import db_utils + from . import utilities from .openwpmtest import OpenWPMTest diff --git a/test/test_http_instrumentation.py b/test/test_http_instrumentation.py index 7d67ad600..5a05516f5 100644 --- a/test/test_http_instrumentation.py +++ b/test/test_http_instrumentation.py @@ -10,8 +10,9 @@ import pytest -from ..automation import CommandSequence, TaskManager -from ..automation.utilities import db_utils +from automation import CommandSequence, TaskManager +from automation.utilities import db_utils + from . import utilities from .openwpmtest import OpenWPMTest diff --git a/test/test_js_instrument.py b/test/test_js_instrument.py index 3ca579d5f..56a6302d8 100644 --- a/test/test_js_instrument.py +++ b/test/test_js_instrument.py @@ -1,4 +1,5 @@ -from ..automation.utilities import db_utils +from automation.utilities import db_utils + from . import utilities as util from .openwpm_jstest import OpenWPMJSTest diff --git a/test/test_js_instrument_py.py b/test/test_js_instrument_py.py index f32c31722..08053674e 100644 --- a/test/test_js_instrument_py.py +++ b/test/test_js_instrument_py.py @@ -1,7 +1,7 @@ import pytest from jsonschema.exceptions import ValidationError -from ..automation import js_instrumentation as jsi +from automation import js_instrumentation as jsi pytestmark = pytest.mark.pyonly diff --git a/test/test_mp_logger.py b/test/test_mp_logger.py index 3d92c0ee3..85ae07276 100644 --- a/test/test_mp_logger.py +++ b/test/test_mp_logger.py @@ -4,8 +4,9 @@ import pytest -from ..automation import MPLogger -from ..automation.utilities.multiprocess_utils import Process +from automation import MPLogger +from automation.utilities.multiprocess_utils import Process + from .openwpmtest import OpenWPMTest CHILD_INFO_STR_1 = "Child %d - INFO1" diff --git a/test/test_profile.py b/test/test_profile.py index af136f967..12731728e 100644 --- a/test/test_profile.py +++ b/test/test_profile.py @@ -2,10 +2,11 @@ import pytest -from ..automation import TaskManager -from ..automation.CommandSequence import CommandSequence -from ..automation.Errors import CommandExecutionError, ProfileLoadError -from ..automation.utilities import db_utils +from automation import TaskManager +from automation.CommandSequence import CommandSequence +from automation.Errors import CommandExecutionError, ProfileLoadError +from automation.utilities import db_utils + from .openwpmtest import OpenWPMTest # TODO update these tests to make use of blocking commands diff --git a/test/test_s3_aggregator.py b/test/test_s3_aggregator.py index 0089b2edc..87aaa7dfc 100644 --- a/test/test_s3_aggregator.py +++ b/test/test_s3_aggregator.py @@ -7,9 +7,10 @@ from localstack.services import infra from multiprocess import Queue -from ..automation import TaskManager -from ..automation.CommandSequence import CommandSequence -from ..automation.data_aggregator.parquet_schema import PQ_SCHEMAS +from automation import TaskManager +from automation.CommandSequence import CommandSequence +from automation.data_aggregator.parquet_schema import PQ_SCHEMAS + from .openwpmtest import OpenWPMTest from .utilities import BASE_TEST_URL, LocalS3Dataset, LocalS3Session, local_s3_bucket diff --git a/test/test_simple_commands.py b/test/test_simple_commands.py index 1e43dc5f0..f0a41c8d8 100644 --- a/test/test_simple_commands.py +++ b/test/test_simple_commands.py @@ -7,8 +7,9 @@ from PIL import Image -from ..automation import CommandSequence, TaskManager -from ..automation.utilities import db_utils +from automation import CommandSequence, TaskManager +from automation.utilities import db_utils + from . import utilities from .openwpmtest import OpenWPMTest diff --git a/test/test_storage_vectors.py b/test/test_storage_vectors.py index 85b1991f9..b5ae5bbc6 100644 --- a/test/test_storage_vectors.py +++ b/test/test_storage_vectors.py @@ -1,5 +1,6 @@ -from ..automation import CommandSequence, TaskManager -from ..automation.utilities import db_utils +from automation import CommandSequence, TaskManager +from automation.utilities import db_utils + from . import utilities from .openwpmtest import OpenWPMTest diff --git a/test/test_webdriver_utils.py b/test/test_webdriver_utils.py index 14366fb00..8a23ab2ee 100644 --- a/test/test_webdriver_utils.py +++ b/test/test_webdriver_utils.py @@ -1,6 +1,7 @@ -from ..automation import TaskManager -from ..automation.Commands.utils.webdriver_utils import parse_neterror -from ..automation.utilities import db_utils +from automation import TaskManager +from automation.Commands.utils.webdriver_utils import parse_neterror +from automation.utilities import db_utils + from .openwpmtest import OpenWPMTest From 3fb70b88ca550ade4c421b81897915484e0f1cdf Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 30 Sep 2020 15:27:36 +0200 Subject: [PATCH 004/139] Fixing tests --- .travis.yml | 8 ++++---- test/pytest.ini => pytest.ini | 0 2 files changed, 4 insertions(+), 4 deletions(-) rename test/pytest.ini => pytest.ini (100%) diff --git a/.travis.yml b/.travis.yml index 5a067ec20..cd57240de 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,11 +6,11 @@ env: # Once we add and remove tests, this distribution may become unbalanced. # Feel free to move tests around to make the running time of the jobs # as close as possible. - - TESTS=test_[a-e]* - - TESTS=test_[f-h]* - - TESTS=test_[i-r,t-z]* + - TESTS=test/test_[a-e]* + - TESTS=test/test_[f-h]* + - TESTS=test/test_[i-r,t-z]* # test_simple_commands.py is slow due to parametrization. - - TESTS=test_[s]* + - TESTS=test/test_[s]* - TESTS=webextension git: depth: 3 diff --git a/test/pytest.ini b/pytest.ini similarity index 100% rename from test/pytest.ini rename to pytest.ini From 53b55cdeaf79121297e18cdb5de070bc2104f168 Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 30 Sep 2020 17:24:25 +0200 Subject: [PATCH 005/139] Adding mypy --- .pre-commit-config.yaml | 5 +++ automation/CommandSequence.py | 2 +- automation/DataAggregator/BaseAggregator.py | 14 +++++--- automation/DataAggregator/LocalAggregator.py | 8 +++-- automation/DataAggregator/S3Aggregator.py | 6 ++-- automation/SocketInterface.py | 8 ++--- automation/TaskManager.py | 12 ++++--- automation/data_aggregator/__init__.py | 34 +++++++++++++++++++ automation/data_aggregator/arrow_storage.py | 15 ++++++-- automation/data_aggregator/s3_storage.py | 2 ++ .../data_aggregator/storage_providers.py | 10 ++++-- automation/types.py | 6 ++++ environment.yaml | 17 +++++----- pytest.ini | 1 + setup.cfg | 7 ++++ test/openwpmtest.py | 9 +++-- test/test_callback.py | 6 ++-- test/test_http_instrumentation.py | 3 +- 18 files changed, 128 insertions(+), 37 deletions(-) create mode 100644 automation/data_aggregator/s3_storage.py create mode 100644 automation/types.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4810485eb..2b6de5182 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,4 +8,9 @@ repos: hooks: - id: black language_version: python3 # Should be a command that runs python3.6+ + - repo: https://github.com/pre-commit/mirrors-mypy + rev: 'v0.782' # Use the sha / tag you want to point at + hooks: + - id: mypy + diff --git a/automation/CommandSequence.py b/automation/CommandSequence.py index 70df815f6..3eaba19a5 100644 --- a/automation/CommandSequence.py +++ b/automation/CommandSequence.py @@ -190,7 +190,7 @@ def run_custom_function(self, function_handle, func_args=(), timeout=30): command = RunCustomFunctionCommand(function_handle, func_args) self._commands_with_timeout.append((command, timeout)) - def mark_done(self, success: bool): + def mark_done(self, success: bool) -> None: if self.callback is not None: self.callback(success) diff --git a/automation/DataAggregator/BaseAggregator.py b/automation/DataAggregator/BaseAggregator.py index 49b3cc31d..48d277e1f 100644 --- a/automation/DataAggregator/BaseAggregator.py +++ b/automation/DataAggregator/BaseAggregator.py @@ -84,7 +84,9 @@ def process_content(self, record): for (content, content_hash)""" @abc.abstractmethod - def run_visit_completion_tasks(self, visit_id: int, interrupted: bool = False): + def run_visit_completion_tasks( + self, visit_id: int, interrupted: bool = False + ) -> None: """Will be called once a visit_id will receive no new records Parameters @@ -162,7 +164,7 @@ def mark_visit_complete(self, visit_id: int) -> None: relating to a certain visit_id have been saved""" self.completion_queue.put((visit_id, False)) - def mark_visit_incomplete(self, visit_id: int): + def mark_visit_incomplete(self, visit_id: int) -> None: """This function should be called to indicate that a certain visit has been interrupted and will forever be incomplete """ @@ -224,6 +226,10 @@ def get_next_visit_id(self): def get_next_browser_id(self): """Return a unique crawl ID used as a key for a browser instance""" + @abc.abstractmethod + def launch(self): + """Launch the aggregator listener process""" + def get_most_recent_status(self): """Return the most recent queue size sent from the listener process""" @@ -273,7 +279,7 @@ def get_new_completed_visits(self) -> List[Tuple[int, bool]]: finished_visit_ids.append(self.completion_queue.get()) return finished_visit_ids - def launch(self, listener_process_runner, *args): + def _launch(self, listener_process_runner, *args): """Launch the aggregator listener process""" args = ((self.status_queue, self.completion_queue, self.shutdown_queue),) + args self.listener_process = Process(target=listener_process_runner, args=args) @@ -281,7 +287,7 @@ def launch(self, listener_process_runner, *args): self.listener_process.start() self.listener_address = self.status_queue.get() - def shutdown(self, relaxed: bool = True): + def shutdown(self, relaxed: bool = True) -> None: """ Terminate the aggregator listener process""" self.logger.debug( "Sending the shutdown signal to the %s listener process..." diff --git a/automation/DataAggregator/LocalAggregator.py b/automation/DataAggregator/LocalAggregator.py index 2e662c6ed..72e1ef2c5 100644 --- a/automation/DataAggregator/LocalAggregator.py +++ b/automation/DataAggregator/LocalAggregator.py @@ -87,7 +87,7 @@ def _generate_insert(self, table, data): statement = statement + ") " + value_str + ")" return statement, values - def process_record(self, record: Tuple[str, Union[str, Dict[str, Any]]]): + def process_record(self, record: Tuple[str, Union[str, Dict[str, Any]]]) -> None: """Add `record` to database""" if len(record) != 2: @@ -181,7 +181,9 @@ def maybe_commit_records(self): self._ldb_counter = 0 self._ldb_commit_time = time.time() - def run_visit_completion_tasks(self, visit_id: int, interrupted: bool = False): + def run_visit_completion_tasks( + self, visit_id: int, interrupted: bool = False + ) -> None: if interrupted: self.logger.warning("Visit with visit_id %d got interrupted", visit_id) self.cur.execute("INSERT INTO incomplete_visits VALUES (?)", (visit_id,)) @@ -283,7 +285,7 @@ def get_next_browser_id(self): def launch(self): """Launch the aggregator listener process""" - super(LocalAggregator, self).launch( + super(LocalAggregator, self)._launch( listener_process_runner, self.manager_params, self.ldb_enabled ) diff --git a/automation/DataAggregator/S3Aggregator.py b/automation/DataAggregator/S3Aggregator.py index 3940e6ee9..d750065e0 100644 --- a/automation/DataAggregator/S3Aggregator.py +++ b/automation/DataAggregator/S3Aggregator.py @@ -302,7 +302,9 @@ def drain_queue(self): super(S3Listener, self).drain_queue() self._send_to_s3(force=True) - def run_visit_completion_tasks(self, visit_id: int, interrupted: bool = False): + def run_visit_completion_tasks( + self, visit_id: int, interrupted: bool = False + ) -> None: if interrupted: self.logger.error("Visit with visit_id %d got interrupted", visit_id) self._write_record("incomplete_visits", {"visit_id": visit_id}, visit_id) @@ -406,6 +408,6 @@ def get_next_browser_id(self): def launch(self): """Launch the aggregator listener process""" - super(S3Aggregator, self).launch( + super(S3Aggregator, self)._launch( listener_process_runner, self.manager_params, self._instance_id ) diff --git a/automation/SocketInterface.py b/automation/SocketInterface.py index 2b8628782..f5ffbab4c 100644 --- a/automation/SocketInterface.py +++ b/automation/SocketInterface.py @@ -171,17 +171,17 @@ def main(): # Just for testing if sys.argv[1] == "s": - sock = ServerSocket(verbose=True) - sock.start_accepting() + ssock: ServerSocket = ServerSocket(verbose=True) + ssock.start_accepting() input("Press enter to exit...") - sock.close() + ssock.close() elif sys.argv[1] == "c": host = input("Enter the host name:\n") port = input("Enter the port:\n") serialization = input("Enter the serialization type (default: 'json'):\n") if serialization == "": serialization = "json" - sock = ClientSocket(serialization=serialization) + sock: ClientSocket = ClientSocket(serialization=serialization) sock.connect(host, int(port)) msg = None diff --git a/automation/TaskManager.py b/automation/TaskManager.py index bc632fa91..50a4d7898 100644 --- a/automation/TaskManager.py +++ b/automation/TaskManager.py @@ -21,6 +21,7 @@ from .js_instrumentation import clean_js_instrumentation_settings from .MPLogger import MPLogger from .SocketInterface import ClientSocket +from .types import BrowserParams, ManagerParams from .utilities.multiprocess_utils import kill_process_and_children from .utilities.platform_utils import get_configuration_string, get_version @@ -34,7 +35,7 @@ def load_default_params( num_browsers: int = 1, -) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]: +) -> Tuple[ManagerParams, List[BrowserParams]]: """ Loads num_browsers copies of the default browser_params dictionary. Also loads a single copy of the default TaskManager params dictionary. @@ -64,8 +65,8 @@ class TaskManager: def __init__( self, - manager_params: Dict[str, Any], - browser_params: List[Dict[str, Any]], + manager_params: ManagerParams, + browser_params: List[BrowserParams], process_watchdog: bool = False, logger_kwargs: Dict[Any, Any] = {}, ) -> None: @@ -182,7 +183,7 @@ def __init__( self.callback_thread.start() def _initialize_browsers( - self, browser_params: List[Dict[str, Any]] + self, browser_params: List[BrowserParams] ) -> List[Browser]: """ initialize the browser classes, each its unique set of params """ browsers = list() @@ -460,6 +461,9 @@ def _issue_command( browser.curr_visit_id, browser.browser_id, ) + assert browser.command_queue is not None + assert browser.status_queue is not None + for command_and_timeout in command_sequence.get_commands_with_timeout(): command, timeout = command_and_timeout command.set_visit_browser_id(browser.curr_visit_id, browser.browser_id) diff --git a/automation/data_aggregator/__init__.py b/automation/data_aggregator/__init__.py index 2eca004d5..2f0d7b898 100644 --- a/automation/data_aggregator/__init__.py +++ b/automation/data_aggregator/__init__.py @@ -1,7 +1,41 @@ +import logging +import time +from typing import Any, Dict, List, Optional, Tuple + +from multiprocess import Queue + from ..SocketInterface import ServerSocket class DataAggregator: + def __init__( + self, status_queue: Queue, completion_queue: Queue, shutdown_queue: Queue + ) -> None: + """ + Creates a BaseListener instance + + Parameters + ---------- + status_queue + queue that the current amount of records to be processed will + be sent to + also used for initialization + completion_queue + queue containing the visitIDs of saved records + shutdown_queue + queue that the main process can use to shut down the listener + """ + self.status_queue = status_queue + self.completion_queue = completion_queue + self.shutdown_queue = shutdown_queue + self._shutdown_flag = False + self._relaxed = False + self._last_update = time.time() # last status update time + self.record_queue: Queue = None # Initialized on `startup` + self.logger = logging.getLogger("openwpm") + self.curent_visit_ids: List[int] = list() # All visit_ids in flight + self.sock: Optional[ServerSocket] = None + def startup(self): """Run listener startup tasks diff --git a/automation/data_aggregator/arrow_storage.py b/automation/data_aggregator/arrow_storage.py index fb271db79..3c8a80ea0 100644 --- a/automation/data_aggregator/arrow_storage.py +++ b/automation/data_aggregator/arrow_storage.py @@ -1,7 +1,11 @@ +import random from abc import abstractmethod +from typing import Any, Dict, List from pyarrow import Table +from automation.types import VisitId + from .parquet_schema import PQ_SCHEMAS from .storage_providers import StructuredStorageProvider @@ -11,15 +15,22 @@ class ArrowAggregator(StructuredStorageProvider): serializes records into the arrow format """ + def __init__(self): + super().__init__() + self._records: Dict[VisitId, Dict[str, List[Dict[str, Any]]]] = {} + self._instance_id = random.getrandbits(32) + def flush_cache(self): pass - def store_record(self, table: str, record: Dict[str, Any], visit_id: int) -> None: + def store_record( + self, table: str, visit_id: VisitId, record: Dict[str, Any] + ) -> None: records = self._records[visit_id] # Add nulls for item in PQ_SCHEMAS[table].names: if item not in record: - data[item] = None + record[item] = None # Add instance_id (for partitioning) record["instance_id"] = self._instance_id records[table].append(record) diff --git a/automation/data_aggregator/s3_storage.py b/automation/data_aggregator/s3_storage.py new file mode 100644 index 000000000..f6e098783 --- /dev/null +++ b/automation/data_aggregator/s3_storage.py @@ -0,0 +1,2 @@ +from .arrow_storage import ArrowAggregator +from .storage_providers import UnstructuredStorageProvider diff --git a/automation/data_aggregator/storage_providers.py b/automation/data_aggregator/storage_providers.py index afde7ff75..c94bec97d 100644 --- a/automation/data_aggregator/storage_providers.py +++ b/automation/data_aggregator/storage_providers.py @@ -1,6 +1,8 @@ from abc import ABC, abstractmethod from typing import Any, Dict, List, Tuple +from automation.types import VisitId + """ This module contains all base classes of the storage provider hierachy Any subclass of these classes should be able to be used in OpenWPM @@ -29,7 +31,9 @@ def shutdown(self) -> None: class StructuredStorageProvider(StorageProvider): @abstractmethod - def store_record(self, table: str, record: Dict[str, Any]) -> None: + def store_record( + self, table: str, visit_id: VisitId, record: Dict[str, Any] + ) -> None: """Submit a record to be stored The storing might not happen immediately """ @@ -37,7 +41,7 @@ def store_record(self, table: str, record: Dict[str, Any]) -> None: @abstractmethod def run_visit_completion_tasks( - self, visit_id: int, interrupted: bool = False + self, visit_id: VisitId, interrupted: bool = False ) -> None: """This method is invoked to inform the StrucuturedStorageProvider that no more records for this visit_id will be submitted @@ -45,7 +49,7 @@ def run_visit_completion_tasks( pass @abstractmethod - def saved_visit_ids(self) -> List[Tuple[int, bool]]: + def saved_visit_ids(self) -> List[Tuple[VisitId, bool]]: """Return the list of all visit_ids that have been saved to permanent storage since the last time this method was called diff --git a/automation/types.py b/automation/types.py new file mode 100644 index 000000000..f62b0d80c --- /dev/null +++ b/automation/types.py @@ -0,0 +1,6 @@ +from typing import Any, Dict, NewType + +BrowserParams = NewType("BrowserParams", Dict[str, Any]) +ManagerParams = NewType("ManagerParams", Dict[str, Any]) +VisitId = NewType("VisitId", int) +BrowserId = NewType("BrowserId", int) diff --git a/environment.yaml b/environment.yaml index 7ba7a0d5f..4bc46a4c6 100644 --- a/environment.yaml +++ b/environment.yaml @@ -3,32 +3,33 @@ channels: - main dependencies: - autopep8=1.5.4 -- beautifulsoup4=4.9.1 +- beautifulsoup4=4.9.2 - click=7.1.2 - codecov=2.1.9 - dill=0.3.2 - flake8-isort=4.0.0 - flake8=3.8.3 - geckodriver=0.27.0 -- ipython=7.17.0 +- ipython=7.18.1 - leveldb=1.22 - localstack=0.11.1.1 - multiprocess=0.70.10 -- nodejs=14.8.0 -- pandas=1.1.1 +- mypy=0.782 +- nodejs=14.12.0 +- pandas=1.1.2 - pillow=7.2.0 -- pip=20.2.2 +- pip=20.2.3 - pre-commit=2.7.1 - psutil=5.7.2 - pyarrow=1.0.1 - pytest-cov=2.10.1 -- pytest=6.0.1 +- pytest=6.1.0 - python=3.8.5 - pyvirtualdisplay=0.2.5 - redis-py=3.5.3 - s3fs=0.4.0 - selenium=3.141.0 -- sentry-sdk=0.17.0 +- sentry-sdk=0.18.0 - tabulate=0.8.7 - tblib=1.6.0 - wget=1.20.1 @@ -36,7 +37,7 @@ dependencies: - amazon-kclpy==2.0.1 - crontab==0.22.9 - domain-utils==0.7.1 - - flask-cors==3.0.8 + - flask-cors==3.0.9 - jsonschema==3.2.0 - moto-ext==1.3.15.15 - plyvel==1.2.0 diff --git a/pytest.ini b/pytest.ini index 866c7203a..0ee2310c5 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,6 @@ [pytest] python_files=test_*.py +testpaths=test markers = pyonly: marks a test as being python only and so server and xpi not needed diff --git a/setup.cfg b/setup.cfg index cdc19bc16..fe8fdee8e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,3 +4,10 @@ known_future_library = future known_first_party = automation,openwpmtest,test default_section = THIRDPARTY skip = venv,automation/Extension,firefox-bin + +[mypy] +follow_imports = silent +python_version = 3.8 +warn_unused_configs = True +ignore_missing_imports = True +disallow_incomplete_defs = True \ No newline at end of file diff --git a/test/openwpmtest.py b/test/openwpmtest.py index bc6f5119b..987e91cc7 100644 --- a/test/openwpmtest.py +++ b/test/openwpmtest.py @@ -1,9 +1,11 @@ import os from os.path import isfile, join +from typing import List, Tuple import pytest from automation import TaskManager +from automation.types import BrowserParams, ManagerParams from . import utilities @@ -31,8 +33,11 @@ def visit(self, page_url, data_dir="", sleep_after=0): return manager_params["db"] def get_test_config( - self, data_dir="", num_browsers=NUM_BROWSERS, display_mode="headless" - ): + self, + data_dir: str = "", + num_browsers: int = NUM_BROWSERS, + display_mode: str = "headless", + ) -> Tuple[ManagerParams, List[BrowserParams]]: """Load and return the default test parameters.""" if not data_dir: data_dir = self.tmpdir diff --git a/test/test_callback.py b/test/test_callback.py index 34188bc5f..d338e6e0b 100644 --- a/test/test_callback.py +++ b/test/test_callback.py @@ -15,15 +15,15 @@ class TestCallbackCommand(OpenWPMTest): def get_config(self, data_dir=""): return self.get_test_config(data_dir) - def test_local_callbacks(self): + def test_local_callbacks(self) -> None: manager_params, browser_params = self.get_config() TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html" manager = TaskManager(manager_params, browser_params) - def callback(argument: List[int], success: bool): + def callback(argument: List[int], success: bool) -> None: argument.extend([1, 2, 3]) - my_list = [] + my_list: List[int] = [] sequence = CommandSequence( TEST_SITE, reset=True, blocking=True, callback=partial(callback, my_list) ) diff --git a/test/test_http_instrumentation.py b/test/test_http_instrumentation.py index 5a05516f5..bf25035f6 100644 --- a/test/test_http_instrumentation.py +++ b/test/test_http_instrumentation.py @@ -6,6 +6,7 @@ import os from hashlib import sha256 from time import sleep +from typing import Set from urllib.parse import urlparse import pytest @@ -623,7 +624,7 @@ def test_page_visit(self): # HTTP Responses rows = db_utils.query_db(db, "SELECT * FROM http_responses") - observed_records = set() + observed_records: Set[str, str] = set() for row in rows: observed_records.add( ( From ec4c7df8ae5e966ea0cd0b01bfeaf6d94ef50de2 Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 30 Sep 2020 19:57:57 +0200 Subject: [PATCH 006/139] Removed mypy from pre-commit workflow --- .pre-commit-config.yaml | 4 ---- automation/data_aggregator/in_memory_storage.py | 12 ++++++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2b6de5182..17fa3be38 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,9 +8,5 @@ repos: hooks: - id: black language_version: python3 # Should be a command that runs python3.6+ - - repo: https://github.com/pre-commit/mirrors-mypy - rev: 'v0.782' # Use the sha / tag you want to point at - hooks: - - id: mypy diff --git a/automation/data_aggregator/in_memory_storage.py b/automation/data_aggregator/in_memory_storage.py index 3ca05c90c..c7d153d5e 100644 --- a/automation/data_aggregator/in_memory_storage.py +++ b/automation/data_aggregator/in_memory_storage.py @@ -1,6 +1,8 @@ from collections import defaultdict from typing import Any, DefaultDict, Dict, List, Tuple +from automation.types import VisitId + from .storage_providers import StructuredStorageProvider, UnstructuredStorageProvider @@ -13,22 +15,24 @@ class MemoryStructuredProvider(StructuredStorageProvider): def __init__(self): super().__init__() self.storage: DefaultDict[str, List[Any]] = defaultdict(list) - self._completed_visit_ids: List[Tuple[int, bool]] = list() + self._completed_visit_ids: List[Tuple[VisitId, bool]] = list() def flush_cache(self) -> None: pass - def store_record(self, table: str, record: Dict[str, Any]) -> None: + def store_record( + self, table: str, visit_id: VisitId, record: Dict[str, Any] + ) -> None: self.storage[table].append(record) pass def run_visit_completion_tasks( - self, visit_id: int, interrupted: bool = False + self, visit_id: VisitId, interrupted: bool = False ) -> None: self._completed_visit_ids.append((visit_id, interrupted)) pass - def saved_visit_ids(self) -> List[Tuple[int, bool]]: + def saved_visit_ids(self) -> List[Tuple[VisitId, bool]]: temp = self._completed_visit_ids self._completed_visit_ids = list() return temp From 7288783082f5b71f557c9a78e266941979ad710d Mon Sep 17 00:00:00 2001 From: vringar Date: Thu, 1 Oct 2020 14:01:08 +0200 Subject: [PATCH 007/139] First draft on DataAggregator --- automation/data_aggregator/__init__.py | 182 +++++++++++++++++- .../data_aggregator/in_memory_storage.py | 39 +++- .../data_aggregator/storage_providers.py | 24 ++- .../test_memory_storage_provider.py | 25 ++- 4 files changed, 255 insertions(+), 15 deletions(-) diff --git a/automation/data_aggregator/__init__.py b/automation/data_aggregator/__init__.py index 2f0d7b898..ae4dc25bc 100644 --- a/automation/data_aggregator/__init__.py +++ b/automation/data_aggregator/__init__.py @@ -1,15 +1,62 @@ +import base64 import logging +import queue +import threading import time from typing import Any, Dict, List, Optional, Tuple from multiprocess import Queue from ..SocketInterface import ServerSocket +from ..types import ManagerParams, VisitId +from .storage_providers import StructuredStorageProvider, UnstructuredStorageProvider + +RECORD_TYPE_CONTENT = "page_content" +RECORD_TYPE_META = "meta_information" +ACTION_TYPE_FINALIZE = "Finalize" +ACTION_TYPE_INITIALIZE = "Initialize" +RECORD_TYPE_CREATE = "create_table" +STATUS_TIMEOUT = 120 # seconds +SHUTDOWN_SIGNAL = "SHUTDOWN" +BATCH_COMMIT_TIMEOUT = 30 # commit a batch if no new records for N seconds + + +STATUS_UPDATE_INTERVAL = 5 # seconds + + +def listener_process_runner( + status_queue: Queue, + completion_queue: Queue, + shutdown_queue: Queue, + structured_storage: StructuredStorageProvider, + unstructured_storage: UnstructuredStorageProvider, +) -> None: + aggregator = DataAggregator( + structured_storage, + unstructured_storage, + status_queue=status_queue, + completion_queue=completion_queue, + shutdown_queue=shutdown_queue, + ) + aggregator.startup() + + while not aggregator.should_shutdown(): + aggregator.update_status_queue() + aggregator.save_batch_if_past_timeout() + aggregator.poll_queue() + + aggregator.drain_queue() + aggregator.shutdown() class DataAggregator: def __init__( - self, status_queue: Queue, completion_queue: Queue, shutdown_queue: Queue + self, + structured_storage: StructuredStorageProvider, + unstructured_storage: UnstructuredStorageProvider, + status_queue: Queue, + completion_queue: Queue, + shutdown_queue: Queue, ) -> None: """ Creates a BaseListener instance @@ -33,14 +80,139 @@ def __init__( self._last_update = time.time() # last status update time self.record_queue: Queue = None # Initialized on `startup` self.logger = logging.getLogger("openwpm") - self.curent_visit_ids: List[int] = list() # All visit_ids in flight + self.curent_visit_ids: List[VisitId] = list() # All visit_ids in flight self.sock: Optional[ServerSocket] = None + self.structured_storage = structured_storage + self.unstructured_storage = unstructured_storage def startup(self): - """Run listener startup tasks - - Note: Child classes should call this method""" + """Puts the DataAggregator into a runable state + by starting up the ServerSocket""" self.sock = ServerSocket(name=type(self).__name__) self.status_queue.put(self.sock.sock.getsockname()) self.sock.start_accepting() self.record_queue = self.sock.queue + + def poll_queue(self) -> None: + assert self.record_queue is not None + try: + record: Tuple[str, Any] = self.record_queue.get(block=True, timeout=5) + except queue.Empty: + return + if len(record) != 2: + self.logger.error("Query is not the correct length %s", repr(record)) + return + self._last_record_received = time.time() + record_type, data = record + if record_type == RECORD_TYPE_CREATE: + raise RuntimeError( + f"""{RECORD_TYPE_CREATE} is no longer supported. + since the user now has access to the DB before it + goes into use, they should set up all schemas before + launching the DataAggregator + """ + ) + return + if record_type == RECORD_TYPE_CONTENT: + assert isinstance(data, tuple) + assert len(data) == 2 + content, content_hash = data + content = base64.b64decode(content) + self.unstructured_storage.store_blob(filename=content_hash, blob=content) + return + if record_type == RECORD_TYPE_META: + self.handle_meta(data) + return + + self.structured_storage.store_record + + def handle_meta(self, data: Dict[str, Any]) -> None: + """ + Messages for the table RECORD_TYPE_SPECIAL are metainformation + communicated to the aggregator + Supported message types: + - finalize: A message sent by the extension to + signal that a visit_id is complete. + """ + if data["action"] == ACTION_TYPE_INITIALIZE: + self.curent_visit_ids.append(data["visit_id"]) + elif data["action"] == ACTION_TYPE_FINALIZE: + try: + self.curent_visit_ids.remove(data["visit_id"]) + except ValueError: + self.logger.error( + "Trying to remove visit_id %i " "from current_visit_ids failed", + data["visit_id"], + ) + + self.structured_storage.run_visit_completion_tasks( + data["visit_id"], interrupted=not data["success"] + ) + else: + raise ValueError( + "Unexpected meta " "information type: %s" % data["meta_type"] + ) + + def update_status_queue(self): + """Send manager process a status update.""" + if (time.time() - self._last_update) < STATUS_UPDATE_INTERVAL: + return + qsize = self.record_queue.qsize() + self.status_queue.put(qsize) + self.logger.debug( + "Status update; current record queue size: %d. " + "current number of threads: %d." % (qsize, threading.active_count()) + ) + self._last_update = time.time() + + def drain_queue(self) -> None: + """ Ensures queue is empty before closing """ + time.sleep(3) # TODO: the socket needs a better way of closing + while not self.record_queue.empty(): + self.poll_queue() + self.logger.info("Queue was flushed completely") + + def shutdown(self) -> None: + self.structured_storage.flush_cache() + self.unstructured_storage.flush_cache() + self.structured_storage.shutdown() + self.unstructured_storage.shutdown() + + def should_shutdown(self): + """Return `True` if the listener has received a shutdown signal + Sets `self._relaxed` and `self.shutdown_flag` + `self._relaxed means this shutdown is + happening after all visits have completed and + all data can be seen as complete + """ + if not self.shutdown_queue.empty(): + _, relaxed = self.shutdown_queue.get() + self._relaxed = relaxed + self._shutdown_flag = True + self.logger.info("Received shutdown signal!") + return True + return False + + def save_batch_if_past_timeout(self): + """Save the current batch of records if no new data has been received. + + If we aren't receiving new data for this batch we commit early + regardless of the current batch size.""" + if self._last_record_received is None: + return + if time.time() - self._last_record_received < BATCH_COMMIT_TIMEOUT: + return + self.logger.debug( + "Saving current record batches to S3 since no new data has " + "been written for %d seconds." % (time.time() - self._last_record_received) + ) + self.drain_queue() + self._last_record_received = None + + +class DataAggregatorHandle: + """This class contains all methods relevant for the TaskManager + to interact with the DataAggregator + """ + + ... diff --git a/automation/data_aggregator/in_memory_storage.py b/automation/data_aggregator/in_memory_storage.py index c7d153d5e..e2d9af399 100644 --- a/automation/data_aggregator/in_memory_storage.py +++ b/automation/data_aggregator/in_memory_storage.py @@ -1,15 +1,18 @@ from collections import defaultdict from typing import Any, DefaultDict, Dict, List, Tuple +import pytest + from automation.types import VisitId from .storage_providers import StructuredStorageProvider, UnstructuredStorageProvider +pytestmark = pytest.mark.pyonly + class MemoryStructuredProvider(StructuredStorageProvider): - """This storage provider stores all data in memory under - self.storage. - This makes it ideal for testing and for small crawls where no persitence is required + """This storage provider stores all data in memory under self.storage. + This makes it ideal for testing and for small crawls where no persistence is required """ def __init__(self): @@ -39,3 +42,33 @@ def saved_visit_ids(self) -> List[Tuple[VisitId, bool]]: def shutdown(self) -> None: pass + + +class MemoryUnstructuredProvider(UnstructuredStorageProvider): + """This storage provider stores all data in memory under self.storage as a dict + from filename to content. + Use this provider for writing tests and for small crawls where no persistence is required + """ + + def __init__(self) -> None: + self.storage: Dict[str, bytes] = {} + + def store_blob( + self, + filename: str, + blob: bytes, + compressed: bool = True, + skip_if_exists: bool = True, + ) -> None: + if skip_if_exists and filename in self.storage: + return + if compressed: + bytesIO = self._compress(blob) + blob = bytesIO.getvalue() + self.storage[filename] = blob + + def flush_cache(self): + pass + + def shutdown(self): + pass diff --git a/automation/data_aggregator/storage_providers.py b/automation/data_aggregator/storage_providers.py index c94bec97d..f7a223024 100644 --- a/automation/data_aggregator/storage_providers.py +++ b/automation/data_aggregator/storage_providers.py @@ -1,3 +1,5 @@ +import gzip +import io from abc import ABC, abstractmethod from typing import Any, Dict, List, Tuple @@ -59,6 +61,24 @@ def saved_visit_ids(self) -> List[Tuple[VisitId, bool]]: class UnstructuredStorageProvider(StorageProvider): - def store_blob(self, blob: str) -> None: - """Stores the data passed in as an base64 encoded string""" + @abstractmethod + def store_blob( + self, + filename: str, + blob: bytes, + compressed: bool = True, + skip_if_exists: bool = True, + ) -> None: + """Stores the given bytes under the provided filename""" pass + + def _compress(self, blob: bytes) -> io.BytesIO: + """Takes a byte blob and compresses it with gzip + The returned BytesIO object is at stream position 0. + This means it can be treated like a zip file on disk. + """ + out_f = io.BytesIO() + with gzip.GzipFile(fileobj=out_f, mode="w") as writer: + writer.write(blob) + out_f.seek(0) + return out_f diff --git a/test/storage_providers/test_memory_storage_provider.py b/test/storage_providers/test_memory_storage_provider.py index ded3ad84d..14a65ebf1 100644 --- a/test/storage_providers/test_memory_storage_provider.py +++ b/test/storage_providers/test_memory_storage_provider.py @@ -1,17 +1,32 @@ import pytest -from automation.data_aggregator.in_memory_storage import MemoryStructuredProvider +from automation.data_aggregator.in_memory_storage import ( + MemoryStructuredProvider, + MemoryUnstructuredProvider, +) +from automation.types import VisitId pytestmark = pytest.mark.pyonly -def test_construction(): +def test_structured_construction(): assert MemoryStructuredProvider() -def test_basic_acess(): +def test_basic_access(): prov = MemoryStructuredProvider() - prov.store_record("test", {"visit_id": 2, "data": "test"}) - prov.run_visit_completion_tasks(2) + prov.store_record("test", VisitId(2), {"visit_id": 2, "data": "test"}) + prov.run_visit_completion_tasks(VisitId(2)) assert prov.saved_visit_ids() == [(2, False)] assert prov.storage == {"test": [{"visit_id": 2, "data": "test"}]} + + +def test_unstructured_construction(): + assert MemoryUnstructuredProvider() + + +def test_basic_unstructured_storing(): + test_string = "This is my test string" + blob = test_string.encode() + prov = MemoryUnstructuredProvider() + prov.store_blob("test", blob, compressed=False) From f6942540330ca79adee4de6a43a37031c0a5be29 Mon Sep 17 00:00:00 2001 From: vringar Date: Thu, 1 Oct 2020 18:07:20 +0200 Subject: [PATCH 008/139] Wrote a DataAggregator that starts and shuts down --- automation/data_aggregator/__init__.py | 144 ++++++++++++++---- .../storage_providers/test_data_aggregator.py | 13 ++ 2 files changed, 127 insertions(+), 30 deletions(-) diff --git a/automation/data_aggregator/__init__.py b/automation/data_aggregator/__init__.py index ae4dc25bc..4dfc1eb87 100644 --- a/automation/data_aggregator/__init__.py +++ b/automation/data_aggregator/__init__.py @@ -1,14 +1,17 @@ import base64 import logging import queue +import random import threading import time from typing import Any, Dict, List, Optional, Tuple from multiprocess import Queue +from automation.utilities.multiprocess_utils import Process + from ..SocketInterface import ServerSocket -from ..types import ManagerParams, VisitId +from ..types import BrowserId, ManagerParams, VisitId from .storage_providers import StructuredStorageProvider, UnstructuredStorageProvider RECORD_TYPE_CONTENT = "page_content" @@ -24,31 +27,6 @@ STATUS_UPDATE_INTERVAL = 5 # seconds -def listener_process_runner( - status_queue: Queue, - completion_queue: Queue, - shutdown_queue: Queue, - structured_storage: StructuredStorageProvider, - unstructured_storage: UnstructuredStorageProvider, -) -> None: - aggregator = DataAggregator( - structured_storage, - unstructured_storage, - status_queue=status_queue, - completion_queue=completion_queue, - shutdown_queue=shutdown_queue, - ) - aggregator.startup() - - while not aggregator.should_shutdown(): - aggregator.update_status_queue() - aggregator.save_batch_if_past_timeout() - aggregator.poll_queue() - - aggregator.drain_queue() - aggregator.shutdown() - - class DataAggregator: def __init__( self, @@ -84,6 +62,7 @@ def __init__( self.sock: Optional[ServerSocket] = None self.structured_storage = structured_storage self.unstructured_storage = unstructured_storage + self._last_record_received = None def startup(self): """Puts the DataAggregator into a runable state @@ -121,12 +100,12 @@ def poll_queue(self) -> None: self.unstructured_storage.store_blob(filename=content_hash, blob=content) return if record_type == RECORD_TYPE_META: - self.handle_meta(data) + self._handle_meta(data) return self.structured_storage.store_record - def handle_meta(self, data: Dict[str, Any]) -> None: + def _handle_meta(self, data: Dict[str, Any]) -> None: """ Messages for the table RECORD_TYPE_SPECIAL are metainformation communicated to the aggregator @@ -153,7 +132,7 @@ def handle_meta(self, data: Dict[str, Any]) -> None: "Unexpected meta " "information type: %s" % data["meta_type"] ) - def update_status_queue(self): + def update_status_queue(self) -> None: """Send manager process a status update.""" if (time.time() - self._last_update) < STATUS_UPDATE_INTERVAL: return @@ -165,6 +144,10 @@ def update_status_queue(self): ) self._last_update = time.time() + def update_completion_queue(self) -> None: + for pair in self.structured_storage.saved_visit_ids(): + self.completion_queue.put(pair) + def drain_queue(self) -> None: """ Ensures queue is empty before closing """ time.sleep(3) # TODO: the socket needs a better way of closing @@ -210,9 +193,110 @@ def save_batch_if_past_timeout(self): self._last_record_received = None +def listener_process_runner(aggregator: DataAggregator) -> None: + + aggregator.startup() + + while not aggregator.should_shutdown(): + aggregator.update_status_queue() + aggregator.update_completion_queue() + aggregator.save_batch_if_past_timeout() + aggregator.poll_queue() + + aggregator.drain_queue() + aggregator.shutdown() + + class DataAggregatorHandle: """This class contains all methods relevant for the TaskManager to interact with the DataAggregator """ - ... + def __init__( + self, + structured_storage: StructuredStorageProvider, + unstructured_storage: UnstructuredStorageProvider, + ) -> None: + + self.listener_address = None + self.listener_process = None + self.status_queue = Queue() + self.completion_queue = Queue() + self.shutdown_queue = Queue() + self._last_status = None + self._last_status_received = None + self.logger = logging.getLogger("openwpm") + self.aggregator = DataAggregator( + structured_storage, + unstructured_storage, + status_queue=self.status_queue, + completion_queue=self.completion_queue, + shutdown_queue=self.shutdown_queue, + ) + + def get_next_visit_id(self) -> VisitId: + """Generate visit id as randomly generated positive integer less than 2^53. + + Parquet can support integers up to 64 bits, but Javascript can only + represent integers up to 53 bits: + https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER + Thus, we cap these values at 53 bits. + """ + return VisitId(random.getrandbits(53)) + + def get_next_browser_id(self) -> BrowserId: + """Generate crawl id as randomly generated positive 32bit integer + + Note: Parquet's partitioned dataset reader only supports integer + partition columns up to 32 bits. + """ + return BrowserId(random.getrandbits(32)) + + def save_configuration(self, openwpm_version, browser_version): + # FIXME I need to find a solution for this + self.logger.error( + "Can't log config as of yet, because it's still not implemented" + ) + + def launch(self) -> None: + """Starts the data aggregator""" + self.listener_process = Process( + name="DataAggregator", + target=listener_process_runner, + args=(self.aggregator,), + ) + self.listener_process.daemon = True + self.listener_process.start() + + self.listener_address = self.status_queue.get() + + def get_new_completed_visits(self) -> List[Tuple[int, bool]]: + """ + Returns a list of all visit ids that have been processed since + the last time the method was called and whether or not they + have been interrupted. + + This method will return an empty list in case no visit ids have + been processed since the last time this method was called + """ + finished_visit_ids = list() + while not self.completion_queue.empty(): + finished_visit_ids.append(self.completion_queue.get()) + return finished_visit_ids + + def shutdown(self, relaxed: bool = True) -> None: + """ Terminate the aggregator listener process""" + assert isinstance(self.listener_process, Process) + self.logger.debug( + "Sending the shutdown signal to the %s listener process..." + % type(self).__name__ + ) + self.shutdown_queue.put((SHUTDOWN_SIGNAL, relaxed)) + start_time = time.time() + self.listener_process.join(300) + self.logger.debug( + "%s took %s seconds to close." + % (type(self).__name__, str(time.time() - start_time)) + ) + self.listener_address = None + self.listener_process = None diff --git a/test/storage_providers/test_data_aggregator.py b/test/storage_providers/test_data_aggregator.py index e69de29bb..7b6aa7119 100644 --- a/test/storage_providers/test_data_aggregator.py +++ b/test/storage_providers/test_data_aggregator.py @@ -0,0 +1,13 @@ +from automation.data_aggregator import DataAggregatorHandle +from automation.data_aggregator.in_memory_storage import ( + MemoryStructuredProvider, + MemoryUnstructuredProvider, +) + + +def test_construction(): + structured = MemoryStructuredProvider() + unstructured = MemoryUnstructuredProvider() + agg_handle = DataAggregatorHandle(structured, unstructured) + agg_handle.launch() + agg_handle.shutdown() From b7e9b1d34da0e9779f42e45d09ba1a407416213d Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 2 Oct 2020 12:49:46 +0200 Subject: [PATCH 009/139] Created tests and added more empty types --- automation/data_aggregator/arrow_storage.py | 2 +- .../data_aggregator/in_memory_storage.py | 9 ++- automation/data_aggregator/leveldb.py | 5 ++ automation/data_aggregator/local_storage.py | 10 +++ automation/data_aggregator/sql_provider.py | 5 ++ test/storage_providers/__init__.py | 1 + .../storage_providers/test_data_aggregator.py | 10 ++- .../test_memory_storage_provider.py | 79 +++++++++++++++---- 8 files changed, 99 insertions(+), 22 deletions(-) create mode 100644 automation/data_aggregator/leveldb.py create mode 100644 automation/data_aggregator/local_storage.py create mode 100644 automation/data_aggregator/sql_provider.py diff --git a/automation/data_aggregator/arrow_storage.py b/automation/data_aggregator/arrow_storage.py index 3c8a80ea0..dd62d9e28 100644 --- a/automation/data_aggregator/arrow_storage.py +++ b/automation/data_aggregator/arrow_storage.py @@ -10,7 +10,7 @@ from .storage_providers import StructuredStorageProvider -class ArrowAggregator(StructuredStorageProvider): +class ArrowProvider(StructuredStorageProvider): """This class implements a StructuredStorage provider that serializes records into the arrow format """ diff --git a/automation/data_aggregator/in_memory_storage.py b/automation/data_aggregator/in_memory_storage.py index e2d9af399..345b4b0a3 100644 --- a/automation/data_aggregator/in_memory_storage.py +++ b/automation/data_aggregator/in_memory_storage.py @@ -1,14 +1,11 @@ from collections import defaultdict from typing import Any, DefaultDict, Dict, List, Tuple -import pytest - from automation.types import VisitId +from .arrow_storage import ArrowProvider from .storage_providers import StructuredStorageProvider, UnstructuredStorageProvider -pytestmark = pytest.mark.pyonly - class MemoryStructuredProvider(StructuredStorageProvider): """This storage provider stores all data in memory under self.storage. @@ -72,3 +69,7 @@ def flush_cache(self): def shutdown(self): pass + + +class MemoryArrowProvider(ArrowProvider): + ... diff --git a/automation/data_aggregator/leveldb.py b/automation/data_aggregator/leveldb.py new file mode 100644 index 000000000..5850cc546 --- /dev/null +++ b/automation/data_aggregator/leveldb.py @@ -0,0 +1,5 @@ +from .storage_providers import UnstructuredStorageProvider + + +class LevelDbProvider(UnstructuredStorageProvider): + ... diff --git a/automation/data_aggregator/local_storage.py b/automation/data_aggregator/local_storage.py new file mode 100644 index 000000000..978299d8e --- /dev/null +++ b/automation/data_aggregator/local_storage.py @@ -0,0 +1,10 @@ +from .arrow_storage import ArrowProvider +from .storage_providers import UnstructuredStorageProvider + + +class LocalArrowProvider(ArrowProvider): + ... + + +class LocalGzipProvider(UnstructuredStorageProvider): + ... diff --git a/automation/data_aggregator/sql_provider.py b/automation/data_aggregator/sql_provider.py new file mode 100644 index 000000000..262d44efe --- /dev/null +++ b/automation/data_aggregator/sql_provider.py @@ -0,0 +1,5 @@ +from .storage_providers import StructuredStorageProvider + + +class SqlLiteStorageProvider(StructuredStorageProvider): + ... diff --git a/test/storage_providers/__init__.py b/test/storage_providers/__init__.py index e69de29bb..8b1378917 100644 --- a/test/storage_providers/__init__.py +++ b/test/storage_providers/__init__.py @@ -0,0 +1 @@ + diff --git a/test/storage_providers/test_data_aggregator.py b/test/storage_providers/test_data_aggregator.py index 7b6aa7119..5c7bd08b6 100644 --- a/test/storage_providers/test_data_aggregator.py +++ b/test/storage_providers/test_data_aggregator.py @@ -5,7 +5,15 @@ ) -def test_construction(): +def test_startup_and_shutdown(): + structured = MemoryStructuredProvider() + unstructured = MemoryUnstructuredProvider() + agg_handle = DataAggregatorHandle(structured, unstructured) + agg_handle.launch() + agg_handle.shutdown() + + +def test_startup_and_shutdown(): structured = MemoryStructuredProvider() unstructured = MemoryUnstructuredProvider() agg_handle = DataAggregatorHandle(structured, unstructured) diff --git a/test/storage_providers/test_memory_storage_provider.py b/test/storage_providers/test_memory_storage_provider.py index 14a65ebf1..bd7b57b1a 100644 --- a/test/storage_providers/test_memory_storage_provider.py +++ b/test/storage_providers/test_memory_storage_provider.py @@ -1,32 +1,79 @@ import pytest from automation.data_aggregator.in_memory_storage import ( + MemoryArrowProvider, MemoryStructuredProvider, MemoryUnstructuredProvider, ) +from automation.data_aggregator.leveldb import LevelDbProvider +from automation.data_aggregator.sql_provider import SqlLiteStorageProvider from automation.types import VisitId -pytestmark = pytest.mark.pyonly +from ..openwpmtest import OpenWPMTest +memory_structured = "memory_structured" +sqllite = "sqllite" +memory_arrow = "memory_arrow" -def test_structured_construction(): - assert MemoryStructuredProvider() +# Unstructured Providers +memory_unstructured = "memory_unstructured" +leveldb = "leveldb" -def test_basic_access(): - prov = MemoryStructuredProvider() - prov.store_record("test", VisitId(2), {"visit_id": 2, "data": "test"}) - prov.run_visit_completion_tasks(VisitId(2)) - assert prov.saved_visit_ids() == [(2, False)] - assert prov.storage == {"test": [{"visit_id": 2, "data": "test"}]} +structured_scenarios = [ + (memory_structured, {"structured_provider": memory_structured}), + (sqllite, {"structured_provider": sqllite}), + (memory_arrow, {"structured_provider": memory_arrow}), +] -def test_unstructured_construction(): - assert MemoryUnstructuredProvider() +@pytest.fixture +def structured_provider(request): + print("Fixture was executed") + if request.param == memory_structured: + return MemoryStructuredProvider() + elif request.param == sqllite: + return SqlLiteStorageProvider() + elif request.param == memory_arrow: + return MemoryArrowProvider() + else: + return MemoryStructuredProvider() + raise ValueError("invalid internal test config") -def test_basic_unstructured_storing(): - test_string = "This is my test string" - blob = test_string.encode() - prov = MemoryUnstructuredProvider() - prov.store_blob("test", blob, compressed=False) +def pytest_generate_tests(metafunc): + # Source: https://docs.pytest.org/en/latest/example/parametrize.html#a-quick-port-of-testscenarios # noqa + idlist = [] + argvalues = [] + for scenario in metafunc.cls.scenarios: + idlist.append(scenario[0]) + items = scenario[1].items() + argnames = [x[0] for x in items] + argvalues.append([x[1] for x in items]) + print("metafunc called") + metafunc.parametrize(argnames, argvalues, ids=idlist, scope="class", indirect=True) + + +class TestStructuredStorageProvider(OpenWPMTest): + scenarios = structured_scenarios + + def test_basic_access(self, structured_provider): + structured_provider.store_record( + "test", VisitId(2), {"visit_id": 2, "data": "test"} + ) + structured_provider.run_visit_completion_tasks(VisitId(2)) + structured_provider.flush_cache() + assert structured_provider.saved_visit_ids() == [(2, False)] + assert structured_provider.storage == { + "test": [{"visit_id": 2, "data": "test"}] + } + + +class TestUnstructuredStorageProvide(OpenWPMTest): + scenarios = [(memory_unstructured, {})] + + def test_basic_unstructured_storing(self): + test_string = "This is my test string" + blob = test_string.encode() + prov = MemoryUnstructuredProvider() + prov.store_blob("test", blob, compressed=False) From 8eaf7ce283a0c91adb18fcbbe1cd5db9a18891ce Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 2 Oct 2020 15:14:01 +0200 Subject: [PATCH 010/139] Got demo.py working --- automation/TaskManager.py | 61 +++++++++---------- automation/data_aggregator/__init__.py | 39 +++++++++++- automation/data_aggregator/s3_storage.py | 2 +- demo.py | 11 +++- .../storage_providers/test_data_aggregator.py | 8 --- .../test_memory_storage_provider.py | 3 - 6 files changed, 78 insertions(+), 46 deletions(-) diff --git a/automation/TaskManager.py b/automation/TaskManager.py index 50a4d7898..2351c7c26 100644 --- a/automation/TaskManager.py +++ b/automation/TaskManager.py @@ -15,6 +15,11 @@ from .BrowserManager import Browser from .Commands.utils.webdriver_utils import parse_neterror from .CommandSequence import CommandSequence +from .data_aggregator import DataAggregatorHandle +from .data_aggregator.storage_providers import ( + StructuredStorageProvider, + UnstructuredStorageProvider, +) from .DataAggregator import BaseAggregator, LocalAggregator, S3Aggregator from .DataAggregator.BaseAggregator import ACTION_TYPE_FINALIZE, RECORD_TYPE_SPECIAL from .Errors import CommandExecutionError @@ -67,6 +72,8 @@ def __init__( self, manager_params: ManagerParams, browser_params: List[BrowserParams], + structured_storage_provider: StructuredStorageProvider, + unstructured_storage_provider: UnstructuredStorageProvider, process_watchdog: bool = False, logger_kwargs: Dict[Any, Any] = {}, ) -> None: @@ -115,13 +122,7 @@ def __init__( if not os.path.exists(manager_params["source_dump_path"]): os.makedirs(manager_params["source_dump_path"]) - # Check size of parameter dictionary - self.num_browsers = manager_params["num_browsers"] - if len(browser_params) != self.num_browsers: - raise Exception( - "Number of dicts is not the same " - "as manager_params['num_browsers']" - ) + self.num_browsers = len(browser_params) # Parse and flesh out js_instrument_settings for a_browsers_params in self.browser_params: @@ -155,7 +156,9 @@ def __init__( self.logger = logging.getLogger("openwpm") # Initialize the data aggregators - self._launch_aggregators() + self._launch_aggregators( + structured_storage_provider, unstructured_storage_provider + ) # Sets up the BrowserManager(s) + associated queues self.browsers = self._initialize_browsers(browser_params) @@ -169,7 +172,7 @@ def __init__( # Save crawl config information to database openwpm_v, browser_v = get_version() - self.data_aggregator.save_configuration(openwpm_v, browser_v) + self.data_aggregator_handle.save_configuration(openwpm_v, browser_v) self.logger.info( get_configuration_string( self.manager_params, browser_params, (openwpm_v, browser_v) @@ -188,7 +191,9 @@ def _initialize_browsers( """ initialize the browser classes, each its unique set of params """ browsers = list() for i in range(self.num_browsers): - browser_params[i]["browser_id"] = self.data_aggregator.get_next_browser_id() + browser_params[i][ + "browser_id" + ] = self.data_aggregator_handle.get_next_browser_id() browsers.append(Browser(self.manager_params, browser_params[i])) return browsers @@ -279,25 +284,19 @@ def _manager_watchdog(self) -> None: ) kill_process_and_children(process, self.logger) - def _launch_aggregators(self) -> None: + def _launch_aggregators( + self, + structured_storage_provider: StructuredStorageProvider, + unstructured_storage_provider: UnstructuredStorageProvider, + ) -> None: """Launch the necessary data aggregators""" - self.data_aggregator: BaseAggregator.BaseAggregator - if self.manager_params["output_format"] == "local": - self.data_aggregator = LocalAggregator.LocalAggregator( - self.manager_params, self.browser_params - ) - elif self.manager_params["output_format"] == "s3": - self.data_aggregator = S3Aggregator.S3Aggregator( - self.manager_params, self.browser_params - ) - else: - raise Exception( - "Unrecognized output format: %s" % self.manager_params["output_format"] - ) - self.data_aggregator.launch() + self.data_aggregator_handle = DataAggregatorHandle( + structured_storage_provider, unstructured_storage_provider + ) + self.data_aggregator_handle.launch() self.manager_params[ "aggregator_address" - ] = self.data_aggregator.listener_address + ] = self.data_aggregator_handle.listener_address # open connection to aggregator for saving crawl details self.sock = ClientSocket(serialization="dill") @@ -334,7 +333,7 @@ def _shutdown_manager( browser.shutdown_browser(during_init, force=not relaxed) self.sock.close() # close socket to data aggregator - self.data_aggregator.shutdown(relaxed=relaxed) + self.data_aggregator_handle.shutdown(relaxed=relaxed) self.logging_server.close() if hasattr(self, "callback_thread"): self.callback_thread.join() @@ -383,7 +382,7 @@ def _start_thread( "Attempted to execute" " command on a closed TaskManager" ) self._check_failure_status() - visit_id = self.data_aggregator.get_next_visit_id() + visit_id = self.data_aggregator_handle.get_next_visit_id() browser.set_visit_id(visit_id) if command_sequence.callback: self.unsaved_command_sequences[visit_id] = command_sequence @@ -417,7 +416,7 @@ def _mark_command_sequences_complete(self) -> None: # we're shutting down and have no unprocessed callbacks break - visit_id_list = self.data_aggregator.get_new_completed_visits() + visit_id_list = self.data_aggregator_handle.get_new_completed_visits() if not visit_id_list: time.sleep(1) continue @@ -630,7 +629,7 @@ def execute_command_sequence( """ # Block if the aggregator queue is too large - agg_queue_size = self.data_aggregator.get_most_recent_status() + agg_queue_size = self.data_aggregator_handle.get_most_recent_status() if agg_queue_size >= AGGREGATOR_QUEUE_LIMIT: while agg_queue_size >= AGGREGATOR_QUEUE_LIMIT: self.logger.info( @@ -638,7 +637,7 @@ def execute_command_sequence( "is below the max queue size of %d. Current queue " "length %d. " % (AGGREGATOR_QUEUE_LIMIT, agg_queue_size) ) - agg_queue_size = self.data_aggregator.get_status() + agg_queue_size = self.data_aggregator_handle.get_status() # Distribute command if index is None: diff --git a/automation/data_aggregator/__init__.py b/automation/data_aggregator/__init__.py index 4dfc1eb87..75a661e24 100644 --- a/automation/data_aggregator/__init__.py +++ b/automation/data_aggregator/__init__.py @@ -62,7 +62,7 @@ def __init__( self.sock: Optional[ServerSocket] = None self.structured_storage = structured_storage self.unstructured_storage = unstructured_storage - self._last_record_received = None + self._last_record_received: Optional[float] = None def startup(self): """Puts the DataAggregator into a runable state @@ -219,7 +219,7 @@ def __init__( ) -> None: self.listener_address = None - self.listener_process = None + self.listener_process: Optional[Process] = None self.status_queue = Queue() self.completion_queue = Queue() self.shutdown_queue = Queue() @@ -300,3 +300,38 @@ def shutdown(self, relaxed: bool = True) -> None: ) self.listener_address = None self.listener_process = None + + def get_most_recent_status(self): + """Return the most recent queue size sent from the listener process""" + + # Block until we receive the first status update + if self._last_status is None: + return self.get_status() + + # Drain status queue until we receive most recent update + while not self.status_queue.empty(): + self._last_status = self.status_queue.get() + self._last_status_received = time.time() + + # Check last status signal + if (time.time() - self._last_status_received) > STATUS_TIMEOUT: + raise RuntimeError( + "No status update from DataAggregator listener process " + "for %d seconds." % (time.time() - self._last_status_received) + ) + + return self._last_status + + def get_status(self): + """Get listener process status. If the status queue is empty, block.""" + try: + self._last_status = self.status_queue.get( + block=True, timeout=STATUS_TIMEOUT + ) + self._last_status_received = time.time() + except queue.Empty: + raise RuntimeError( + "No status update from DataAggregator listener process " + "for %d seconds." % (time.time() - self._last_status_received) + ) + return self._last_status diff --git a/automation/data_aggregator/s3_storage.py b/automation/data_aggregator/s3_storage.py index f6e098783..a1f78c0b9 100644 --- a/automation/data_aggregator/s3_storage.py +++ b/automation/data_aggregator/s3_storage.py @@ -1,2 +1,2 @@ -from .arrow_storage import ArrowAggregator +from .arrow_storage import ArrowProvider from .storage_providers import UnstructuredStorageProvider diff --git a/demo.py b/demo.py index 821b9e135..3b2626170 100644 --- a/demo.py +++ b/demo.py @@ -1,4 +1,8 @@ from automation import CommandSequence, TaskManager +from automation.data_aggregator.in_memory_storage import ( + MemoryStructuredProvider, + MemoryUnstructuredProvider, +) # The list of sites that we wish to crawl NUM_BROWSERS = 1 @@ -37,7 +41,12 @@ # Instantiates the measurement platform # Commands time out by default after 60 seconds -manager = TaskManager.TaskManager(manager_params, browser_params) +manager = TaskManager.TaskManager( + manager_params, + browser_params, + MemoryStructuredProvider(), + MemoryUnstructuredProvider(), +) # Visits the sites for site in sites: diff --git a/test/storage_providers/test_data_aggregator.py b/test/storage_providers/test_data_aggregator.py index 5c7bd08b6..6535cee1c 100644 --- a/test/storage_providers/test_data_aggregator.py +++ b/test/storage_providers/test_data_aggregator.py @@ -5,14 +5,6 @@ ) -def test_startup_and_shutdown(): - structured = MemoryStructuredProvider() - unstructured = MemoryUnstructuredProvider() - agg_handle = DataAggregatorHandle(structured, unstructured) - agg_handle.launch() - agg_handle.shutdown() - - def test_startup_and_shutdown(): structured = MemoryStructuredProvider() unstructured = MemoryUnstructuredProvider() diff --git a/test/storage_providers/test_memory_storage_provider.py b/test/storage_providers/test_memory_storage_provider.py index bd7b57b1a..74c80600b 100644 --- a/test/storage_providers/test_memory_storage_provider.py +++ b/test/storage_providers/test_memory_storage_provider.py @@ -64,9 +64,6 @@ def test_basic_access(self, structured_provider): structured_provider.run_visit_completion_tasks(VisitId(2)) structured_provider.flush_cache() assert structured_provider.saved_visit_ids() == [(2, False)] - assert structured_provider.storage == { - "test": [{"visit_id": 2, "data": "test"}] - } class TestUnstructuredStorageProvide(OpenWPMTest): From d633f9510e8fed5c5288429590ced2a158f64750 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 2 Oct 2020 17:18:30 +0200 Subject: [PATCH 011/139] Created sql_provider --- automation/DataAggregator/LocalAggregator.py | 4 +- automation/TaskManager.py | 2 +- automation/data_aggregator/__init__.py | 21 +++-- .../data_aggregator/in_memory_storage.py | 5 -- .../schema.sql | 0 automation/data_aggregator/sql_provider.py | 87 ++++++++++++++++++- .../data_aggregator/storage_providers.py | 23 +++-- demo.py | 14 +-- .../test_memory_storage_provider.py | 16 ++-- 9 files changed, 136 insertions(+), 36 deletions(-) rename automation/{DataAggregator => data_aggregator}/schema.sql (100%) diff --git a/automation/DataAggregator/LocalAggregator.py b/automation/DataAggregator/LocalAggregator.py index 72e1ef2c5..b16a6c13d 100644 --- a/automation/DataAggregator/LocalAggregator.py +++ b/automation/DataAggregator/LocalAggregator.py @@ -19,7 +19,9 @@ SQL_BATCH_SIZE = 1000 LDB_BATCH_SIZE = 100 MIN_TIME = 5 # seconds -SCHEMA_FILE = os.path.join(os.path.dirname(__file__), "schema.sql") +SCHEMA_FILE = os.path.join( + os.path.dirname(__file__), "..", "data_aggregator", "schema.sql" +) LDB_NAME = "content.ldb" diff --git a/automation/TaskManager.py b/automation/TaskManager.py index 2351c7c26..f9df67f15 100644 --- a/automation/TaskManager.py +++ b/automation/TaskManager.py @@ -527,7 +527,7 @@ def _issue_command( { "browser_id": browser.browser_id, "visit_id": browser.curr_visit_id, - "command": type(command), + "command": type(command).__name__, "arguments": json.dumps( command.__dict__, default=lambda x: repr(x) ).encode("utf-8"), diff --git a/automation/data_aggregator/__init__.py b/automation/data_aggregator/__init__.py index 75a661e24..6c71717ec 100644 --- a/automation/data_aggregator/__init__.py +++ b/automation/data_aggregator/__init__.py @@ -31,7 +31,7 @@ class DataAggregator: def __init__( self, structured_storage: StructuredStorageProvider, - unstructured_storage: UnstructuredStorageProvider, + unstructured_storage: Optional[UnstructuredStorageProvider], status_queue: Queue, completion_queue: Queue, shutdown_queue: Queue, @@ -95,6 +95,12 @@ def poll_queue(self) -> None: if record_type == RECORD_TYPE_CONTENT: assert isinstance(data, tuple) assert len(data) == 2 + if self.unstructured_storage is None: + self.logger.error( + """Tried to save content while not having + provided any unstructured storage provider.""" + ) + return content, content_hash = data content = base64.b64decode(content) self.unstructured_storage.store_blob(filename=content_hash, blob=content) @@ -102,8 +108,11 @@ def poll_queue(self) -> None: if record_type == RECORD_TYPE_META: self._handle_meta(data) return - - self.structured_storage.store_record + visit_id = VisitId(data["visit_id"]) + self.curent_visit_ids.append(visit_id) + self.structured_storage.store_record( + table=record_type, visit_id=visit_id, record=data + ) def _handle_meta(self, data: Dict[str, Any]) -> None: """ @@ -147,6 +156,7 @@ def update_status_queue(self) -> None: def update_completion_queue(self) -> None: for pair in self.structured_storage.saved_visit_ids(): self.completion_queue.put(pair) + self.curent_visit_ids.remove(pair[0]) def drain_queue(self) -> None: """ Ensures queue is empty before closing """ @@ -157,9 +167,10 @@ def drain_queue(self) -> None: def shutdown(self) -> None: self.structured_storage.flush_cache() - self.unstructured_storage.flush_cache() self.structured_storage.shutdown() - self.unstructured_storage.shutdown() + if self.unstructured_storage is not None: + self.unstructured_storage.flush_cache() + self.unstructured_storage.shutdown() def should_shutdown(self): """Return `True` if the listener has received a shutdown signal diff --git a/automation/data_aggregator/in_memory_storage.py b/automation/data_aggregator/in_memory_storage.py index 345b4b0a3..96ac7b090 100644 --- a/automation/data_aggregator/in_memory_storage.py +++ b/automation/data_aggregator/in_memory_storage.py @@ -32,11 +32,6 @@ def run_visit_completion_tasks( self._completed_visit_ids.append((visit_id, interrupted)) pass - def saved_visit_ids(self) -> List[Tuple[VisitId, bool]]: - temp = self._completed_visit_ids - self._completed_visit_ids = list() - return temp - def shutdown(self) -> None: pass diff --git a/automation/DataAggregator/schema.sql b/automation/data_aggregator/schema.sql similarity index 100% rename from automation/DataAggregator/schema.sql rename to automation/data_aggregator/schema.sql diff --git a/automation/data_aggregator/sql_provider.py b/automation/data_aggregator/sql_provider.py index 262d44efe..f0f044852 100644 --- a/automation/data_aggregator/sql_provider.py +++ b/automation/data_aggregator/sql_provider.py @@ -1,5 +1,90 @@ +import json +import logging +import os +import sqlite3 +from os import PathLike +from sqlite3 import IntegrityError, InterfaceError, OperationalError, ProgrammingError +from typing import Any, Dict, List, Tuple + +from automation.types import VisitId + from .storage_providers import StructuredStorageProvider +SCHEMA_FILE = os.path.join(os.path.dirname(__file__), "schema.sql") + class SqlLiteStorageProvider(StructuredStorageProvider): - ... + def __init__(self, db_path: PathLike): + super().__init__() + self.db = sqlite3.connect(db_path, check_same_thread=False) + self.cur = self.db.cursor() + self._sql_counter = 0 + self._sql_commit_time = 0 + self.logger = logging.getLogger("openwpm") + self._create_tables() + + def _create_tables(self): + """Create tables (if this is a new database)""" + with open(SCHEMA_FILE, "r") as f: + self.db.executescript(f.read()) + self.db.commit() + + def flush_cache(self): + self.db.commit() + + def store_record( + self, table: str, visit_id: VisitId, record: Dict[str, Any] + ) -> None: + """Submit a record to be stored + The storing might not happen immediately + """ + statement, args = self._generate_insert(table=table, data=record) + for i in range(len(args)): + if isinstance(args[i], bytes): + args[i] = str(args[i], errors="ignore") + elif callable(args[i]): + args[i] = str(args[i]) + elif type(args[i]) == dict: + args[i] = json.dumps(args[i]) + try: + self.cur.execute(statement, args) + self._sql_counter += 1 + except ( + OperationalError, + ProgrammingError, + IntegrityError, + InterfaceError, + ) as e: + self.logger.error( + "Unsupported record:\n%s\n%s\n%s\n%s\n" + % (type(e), e, statement, repr(args)) + ) + + def _generate_insert( + self, table: str, data: Dict[str, Any] + ) -> Tuple[str, List[Any]]: + """Generate a SQL query from `record`""" + statement = "INSERT INTO %s (" % table + value_str = "VALUES (" + values = list() + first = True + for field, value in data.items(): + statement += "" if first else ", " + statement += field + value_str += "?" if first else ",?" + values.append(value) + first = False + statement = statement + ") " + value_str + ")" + return statement, values + + def run_visit_completion_tasks( + self, visit_id: VisitId, interrupted: bool = False + ) -> None: + if interrupted: + self.logger.warning("Visit with visit_id %d got interrupted", visit_id) + self.cur.execute("INSERT INTO incomplete_visits VALUES (?)", (visit_id,)) + self._completed_visit_ids.append((visit_id, interrupted)) + + def shutdown(self): + self.db.commit() + self.db.close() diff --git a/automation/data_aggregator/storage_providers.py b/automation/data_aggregator/storage_providers.py index f7a223024..e933b526e 100644 --- a/automation/data_aggregator/storage_providers.py +++ b/automation/data_aggregator/storage_providers.py @@ -32,6 +32,20 @@ def shutdown(self) -> None: class StructuredStorageProvider(StorageProvider): + def __init__(self): + super().__init__() + self._completed_visit_ids: List[Tuple[VisitId, bool]] = list() + + def saved_visit_ids(self) -> List[Tuple[VisitId, bool]]: + """Return the list of all visit_ids that have been saved to permanent storage + since the last time this method was called + + For each visit_id it's also noted whether they were interrupted + """ + temp = self._completed_visit_ids + self._completed_visit_ids = list() + return temp + @abstractmethod def store_record( self, table: str, visit_id: VisitId, record: Dict[str, Any] @@ -50,15 +64,6 @@ def run_visit_completion_tasks( """ pass - @abstractmethod - def saved_visit_ids(self) -> List[Tuple[VisitId, bool]]: - """Return the list of all visit_ids that have been saved to permanent storage - since the last time this method was called - - For each visit_id it's also noted whether they were interrupted - """ - pass - class UnstructuredStorageProvider(StorageProvider): @abstractmethod diff --git a/demo.py b/demo.py index 3b2626170..003a9da55 100644 --- a/demo.py +++ b/demo.py @@ -1,8 +1,8 @@ +import os + from automation import CommandSequence, TaskManager -from automation.data_aggregator.in_memory_storage import ( - MemoryStructuredProvider, - MemoryUnstructuredProvider, -) +from automation.data_aggregator.in_memory_storage import MemoryUnstructuredProvider +from automation.data_aggregator.sql_provider import SqlLiteStorageProvider # The list of sites that we wish to crawl NUM_BROWSERS = 1 @@ -44,8 +44,10 @@ manager = TaskManager.TaskManager( manager_params, browser_params, - MemoryStructuredProvider(), - MemoryUnstructuredProvider(), + SqlLiteStorageProvider( + os.path.expanduser(manager_params["data_directory"] + "crawl-data.sqlite") + ), + None, ) # Visits the sites diff --git a/test/storage_providers/test_memory_storage_provider.py b/test/storage_providers/test_memory_storage_provider.py index 74c80600b..8dc7d6b96 100644 --- a/test/storage_providers/test_memory_storage_provider.py +++ b/test/storage_providers/test_memory_storage_provider.py @@ -1,3 +1,5 @@ +from typing import Any, Dict, List, Tuple + import pytest from automation.data_aggregator.in_memory_storage import ( @@ -20,7 +22,7 @@ leveldb = "leveldb" -structured_scenarios = [ +structured_scenarios: List[Tuple[str, Dict[str, Any]]] = [ (memory_structured, {"structured_provider": memory_structured}), (sqllite, {"structured_provider": sqllite}), (memory_arrow, {"structured_provider": memory_arrow}), @@ -28,17 +30,15 @@ @pytest.fixture -def structured_provider(request): - print("Fixture was executed") +def structured_provider(request, tmp_path_factory): if request.param == memory_structured: return MemoryStructuredProvider() elif request.param == sqllite: - return SqlLiteStorageProvider() + tmp_path = tmp_path_factory.mktemp("sqllite") + return SqlLiteStorageProvider(tmp_path / "test_db.sqllite") elif request.param == memory_arrow: return MemoryArrowProvider() - else: - return MemoryStructuredProvider() - raise ValueError("invalid internal test config") + raise ValueError("invalid internal test config") def pytest_generate_tests(metafunc): @@ -67,7 +67,7 @@ def test_basic_access(self, structured_provider): class TestUnstructuredStorageProvide(OpenWPMTest): - scenarios = [(memory_unstructured, {})] + scenarios: List[Tuple[str, Dict[str, Any]]] = [(memory_unstructured, {})] def test_basic_unstructured_storing(self): test_string = "This is my test string" From 71ae8b06d412fe10f7a27454fb8ee59fc61b9326 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 9 Oct 2020 10:48:32 +0200 Subject: [PATCH 012/139] Cleaned up imports in TaskManager --- automation/TaskManager.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/automation/TaskManager.py b/automation/TaskManager.py index f9df67f15..c830e4fb6 100644 --- a/automation/TaskManager.py +++ b/automation/TaskManager.py @@ -15,13 +15,15 @@ from .BrowserManager import Browser from .Commands.utils.webdriver_utils import parse_neterror from .CommandSequence import CommandSequence -from .data_aggregator import DataAggregatorHandle +from .data_aggregator.storage_controller import ( + ACTION_TYPE_FINALIZE, + RECORD_TYPE_META, + DataAggregatorHandle, +) from .data_aggregator.storage_providers import ( StructuredStorageProvider, UnstructuredStorageProvider, ) -from .DataAggregator import BaseAggregator, LocalAggregator, S3Aggregator -from .DataAggregator.BaseAggregator import ACTION_TYPE_FINALIZE, RECORD_TYPE_SPECIAL from .Errors import CommandExecutionError from .js_instrumentation import clean_js_instrumentation_settings from .MPLogger import MPLogger @@ -542,7 +544,7 @@ def _issue_command( if command_status == "critical": self.sock.send( ( - RECORD_TYPE_SPECIAL, + RECORD_TYPE_META, { "browser_id": browser.browser_id, "success": False, @@ -579,7 +581,7 @@ def _issue_command( if browser.restart_required: self.sock.send( ( - RECORD_TYPE_SPECIAL, + RECORD_TYPE_META, { "browser_id": browser.browser_id, "success": False, From 6e11e1b60666f55fd05d49512f9ab429957d4397 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 9 Oct 2020 15:37:11 +0200 Subject: [PATCH 013/139] Added async --- automation/DataAggregator/S3Aggregator.py | 2 +- automation/SocketInterface.py | 30 +++- automation/TaskManager.py | 16 +-- automation/storage/__init__.py | 0 .../arrow_storage.py | 9 +- .../in_memory_storage.py | 54 +++++-- .../{data_aggregator => storage}/leveldb.py | 0 .../local_storage.py | 0 .../parquet_schema.py | 0 .../s3_storage.py | 0 .../{data_aggregator => storage}/schema.sql | 0 .../sql_provider.py | 13 +- .../storage_controller.py} | 134 ++++++++++-------- .../storage_providers.py | 27 ++-- demo.py | 4 +- environment.yaml | 14 +- scripts/environment-unpinned-dev.yaml | 2 +- .../storage_providers/test_data_aggregator.py | 8 +- .../test_memory_storage_provider.py | 28 ++-- test/test_s3_aggregator.py | 2 +- 20 files changed, 197 insertions(+), 146 deletions(-) create mode 100644 automation/storage/__init__.py rename automation/{data_aggregator => storage}/arrow_storage.py (88%) rename automation/{data_aggregator => storage}/in_memory_storage.py (54%) rename automation/{data_aggregator => storage}/leveldb.py (100%) rename automation/{data_aggregator => storage}/local_storage.py (100%) rename automation/{data_aggregator => storage}/parquet_schema.py (100%) rename automation/{data_aggregator => storage}/s3_storage.py (100%) rename automation/{data_aggregator => storage}/schema.sql (100%) rename automation/{data_aggregator => storage}/sql_provider.py (91%) rename automation/{data_aggregator/__init__.py => storage/storage_controller.py} (77%) rename automation/{data_aggregator => storage}/storage_providers.py (75%) diff --git a/automation/DataAggregator/S3Aggregator.py b/automation/DataAggregator/S3Aggregator.py index d750065e0..42dd035cd 100644 --- a/automation/DataAggregator/S3Aggregator.py +++ b/automation/DataAggregator/S3Aggregator.py @@ -18,7 +18,7 @@ from botocore.exceptions import ClientError, EndpointConnectionError from pyarrow.filesystem import S3FSWrapper # noqa -from ..data_aggregator.parquet_schema import PQ_SCHEMAS +from ..storage.parquet_schema import PQ_SCHEMAS from .BaseAggregator import ( RECORD_TYPE_CONTENT, RECORD_TYPE_CREATE, diff --git a/automation/SocketInterface.py b/automation/SocketInterface.py index f5ffbab4c..0a08ef663 100644 --- a/automation/SocketInterface.py +++ b/automation/SocketInterface.py @@ -1,3 +1,4 @@ +import asyncio import json import socket import struct @@ -47,9 +48,7 @@ def _accept(self): thread.start() except ConnectionAbortedError: # Workaround for #278 - print( - "A connection establish request was performed " "on a closed socket" - ) + print("A connection establish request was performed on a closed socket") return def _handle_conn(self, client, address): @@ -93,11 +92,15 @@ def _handle_conn(self, client, address): % (msg, traceback.format_exc(e)) ) continue - self.queue.put(msg) + self._put_into_queue(msg) except RuntimeError: if self.verbose: print("Client socket: " + str(address) + " closed") + def _put_into_queue(self, msg): + """Put the parsed message into a queue from where it can be read by consumers""" + self.queue.put(msg) + def receive_msg(self, client, msglen): msg = b"" while len(msg) < msglen: @@ -111,6 +114,25 @@ def close(self): self.sock.close() +class AsyncServerSocket(ServerSocket): + def __init__( + self, + queue: asyncio.Queue, + loop: asyncio.AbstractEventLoop, + name=None, + verbose=False, + ): + super().__init__(name=name, verbose=verbose) + self.queue = queue + self.loop = loop + + def _put_into_queue(self, msg): + async def callback(queue, msg): + queue.put_nowait(msg) + + asyncio.run_coroutine_threadsafe(callback(self.queue, msg), self.loop) + + class ClientSocket: """A client socket for sending messages""" diff --git a/automation/TaskManager.py b/automation/TaskManager.py index c830e4fb6..bbab5dee1 100644 --- a/automation/TaskManager.py +++ b/automation/TaskManager.py @@ -15,19 +15,19 @@ from .BrowserManager import Browser from .Commands.utils.webdriver_utils import parse_neterror from .CommandSequence import CommandSequence -from .data_aggregator.storage_controller import ( +from .Errors import CommandExecutionError +from .js_instrumentation import clean_js_instrumentation_settings +from .MPLogger import MPLogger +from .SocketInterface import ClientSocket +from .storage.storage_controller import ( ACTION_TYPE_FINALIZE, RECORD_TYPE_META, - DataAggregatorHandle, + StorageControllerHandle, ) -from .data_aggregator.storage_providers import ( +from .storage.storage_providers import ( StructuredStorageProvider, UnstructuredStorageProvider, ) -from .Errors import CommandExecutionError -from .js_instrumentation import clean_js_instrumentation_settings -from .MPLogger import MPLogger -from .SocketInterface import ClientSocket from .types import BrowserParams, ManagerParams from .utilities.multiprocess_utils import kill_process_and_children from .utilities.platform_utils import get_configuration_string, get_version @@ -292,7 +292,7 @@ def _launch_aggregators( unstructured_storage_provider: UnstructuredStorageProvider, ) -> None: """Launch the necessary data aggregators""" - self.data_aggregator_handle = DataAggregatorHandle( + self.data_aggregator_handle = StorageControllerHandle( structured_storage_provider, unstructured_storage_provider ) self.data_aggregator_handle.launch() diff --git a/automation/storage/__init__.py b/automation/storage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/automation/data_aggregator/arrow_storage.py b/automation/storage/arrow_storage.py similarity index 88% rename from automation/data_aggregator/arrow_storage.py rename to automation/storage/arrow_storage.py index dd62d9e28..8ebe96d9a 100644 --- a/automation/data_aggregator/arrow_storage.py +++ b/automation/storage/arrow_storage.py @@ -15,15 +15,12 @@ class ArrowProvider(StructuredStorageProvider): serializes records into the arrow format """ - def __init__(self): + def __init__(self) -> None: super().__init__() self._records: Dict[VisitId, Dict[str, List[Dict[str, Any]]]] = {} self._instance_id = random.getrandbits(32) - def flush_cache(self): - pass - - def store_record( + async def store_record( self, table: str, visit_id: VisitId, record: Dict[str, Any] ) -> None: records = self._records[visit_id] @@ -36,5 +33,5 @@ def store_record( records[table].append(record) @abstractmethod - def write_table(self, table: Table) -> None: + async def write_table(self, table: Table) -> None: """Write out the table to persistent storage""" diff --git a/automation/data_aggregator/in_memory_storage.py b/automation/storage/in_memory_storage.py similarity index 54% rename from automation/data_aggregator/in_memory_storage.py rename to automation/storage/in_memory_storage.py index 96ac7b090..5ce8c8c62 100644 --- a/automation/data_aggregator/in_memory_storage.py +++ b/automation/storage/in_memory_storage.py @@ -1,41 +1,67 @@ from collections import defaultdict from typing import Any, DefaultDict, Dict, List, Tuple +from multiprocess import Queue + from automation.types import VisitId from .arrow_storage import ArrowProvider from .storage_providers import StructuredStorageProvider, UnstructuredStorageProvider +""" +This module contains implementations for various kinds of storage providers +that store their results in memory. +These classes are designed to allow for easier parallel testing as there are +no shared resources between tests. It also makes it easier to verify results +by not having to do a round trip to a perstitent storage provider +""" + class MemoryStructuredProvider(StructuredStorageProvider): - """This storage provider stores all data in memory under self.storage. + """ + This storage provider passes all it's data to the MemoryStructuredProviderHandle in + process safe way. This makes it ideal for testing and for small crawls where no persistence is required """ - def __init__(self): + def __init__(self) -> None: super().__init__() - self.storage: DefaultDict[str, List[Any]] = defaultdict(list) - self._completed_visit_ids: List[Tuple[VisitId, bool]] = list() + self.queue = Queue() + self.handle = MemoryStructuredProviderHandle(self.queue) - def flush_cache(self) -> None: + async def flush_cache(self) -> None: pass - def store_record( + async def store_record( self, table: str, visit_id: VisitId, record: Dict[str, Any] ) -> None: - self.storage[table].append(record) - pass + self.queue.put((table, record)) - def run_visit_completion_tasks( + async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False ) -> None: - self._completed_visit_ids.append((visit_id, interrupted)) pass - def shutdown(self) -> None: + async def shutdown(self) -> None: pass +class MemoryStructuredProviderHandle: + """ + Call poll_queue to load all available data into the dict + at self.storage + """ + + def __init__(self, queue: Queue) -> None: + self.queue = queue + self.storage: DefaultDict[str, List[Any]] = defaultdict(list) + + def poll_queue(self) -> None: + while not self.queue.empty(): + table, record = self.queue.get() + self.storage[table].append(record) + + class MemoryUnstructuredProvider(UnstructuredStorageProvider): """This storage provider stores all data in memory under self.storage as a dict from filename to content. @@ -45,7 +71,7 @@ class MemoryUnstructuredProvider(UnstructuredStorageProvider): def __init__(self) -> None: self.storage: Dict[str, bytes] = {} - def store_blob( + async def store_blob( self, filename: str, blob: bytes, @@ -59,10 +85,10 @@ def store_blob( blob = bytesIO.getvalue() self.storage[filename] = blob - def flush_cache(self): + async def flush_cache(self) -> None: pass - def shutdown(self): + async def shutdown(self) -> None: pass diff --git a/automation/data_aggregator/leveldb.py b/automation/storage/leveldb.py similarity index 100% rename from automation/data_aggregator/leveldb.py rename to automation/storage/leveldb.py diff --git a/automation/data_aggregator/local_storage.py b/automation/storage/local_storage.py similarity index 100% rename from automation/data_aggregator/local_storage.py rename to automation/storage/local_storage.py diff --git a/automation/data_aggregator/parquet_schema.py b/automation/storage/parquet_schema.py similarity index 100% rename from automation/data_aggregator/parquet_schema.py rename to automation/storage/parquet_schema.py diff --git a/automation/data_aggregator/s3_storage.py b/automation/storage/s3_storage.py similarity index 100% rename from automation/data_aggregator/s3_storage.py rename to automation/storage/s3_storage.py diff --git a/automation/data_aggregator/schema.sql b/automation/storage/schema.sql similarity index 100% rename from automation/data_aggregator/schema.sql rename to automation/storage/schema.sql diff --git a/automation/data_aggregator/sql_provider.py b/automation/storage/sql_provider.py similarity index 91% rename from automation/data_aggregator/sql_provider.py rename to automation/storage/sql_provider.py index f0f044852..cd74711e1 100644 --- a/automation/data_aggregator/sql_provider.py +++ b/automation/storage/sql_provider.py @@ -14,7 +14,7 @@ class SqlLiteStorageProvider(StructuredStorageProvider): - def __init__(self, db_path: PathLike): + def __init__(self, db_path: PathLike) -> None: super().__init__() self.db = sqlite3.connect(db_path, check_same_thread=False) self.cur = self.db.cursor() @@ -23,16 +23,16 @@ def __init__(self, db_path: PathLike): self.logger = logging.getLogger("openwpm") self._create_tables() - def _create_tables(self): + def _create_tables(self) -> None: """Create tables (if this is a new database)""" with open(SCHEMA_FILE, "r") as f: self.db.executescript(f.read()) self.db.commit() - def flush_cache(self): + async def flush_cache(self) -> None: self.db.commit() - def store_record( + async def store_record( self, table: str, visit_id: VisitId, record: Dict[str, Any] ) -> None: """Submit a record to be stored @@ -77,14 +77,13 @@ def _generate_insert( statement = statement + ") " + value_str + ")" return statement, values - def run_visit_completion_tasks( + async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False ) -> None: if interrupted: self.logger.warning("Visit with visit_id %d got interrupted", visit_id) self.cur.execute("INSERT INTO incomplete_visits VALUES (?)", (visit_id,)) - self._completed_visit_ids.append((visit_id, interrupted)) - def shutdown(self): + async def shutdown(self) -> None: self.db.commit() self.db.close() diff --git a/automation/data_aggregator/__init__.py b/automation/storage/storage_controller.py similarity index 77% rename from automation/data_aggregator/__init__.py rename to automation/storage/storage_controller.py index 6c71717ec..bfb8ec0bb 100644 --- a/automation/data_aggregator/__init__.py +++ b/automation/storage/storage_controller.py @@ -1,3 +1,4 @@ +import asyncio import base64 import logging import queue @@ -10,7 +11,7 @@ from automation.utilities.multiprocess_utils import Process -from ..SocketInterface import ServerSocket +from ..SocketInterface import AsyncServerSocket from ..types import BrowserId, ManagerParams, VisitId from .storage_providers import StructuredStorageProvider, UnstructuredStorageProvider @@ -27,7 +28,7 @@ STATUS_UPDATE_INTERVAL = 5 # seconds -class DataAggregator: +class StorageController: def __init__( self, structured_storage: StructuredStorageProvider, @@ -58,31 +59,37 @@ def __init__( self._last_update = time.time() # last status update time self.record_queue: Queue = None # Initialized on `startup` self.logger = logging.getLogger("openwpm") - self.curent_visit_ids: List[VisitId] = list() # All visit_ids in flight - self.sock: Optional[ServerSocket] = None + self.current_visit_ids: List[VisitId] = list() # All visit_ids in flight + self.sock: Optional[AsyncServerSocket] = None self.structured_storage = structured_storage self.unstructured_storage = unstructured_storage self._last_record_received: Optional[float] = None - def startup(self): + async def startup(self) -> None: """Puts the DataAggregator into a runable state by starting up the ServerSocket""" - self.sock = ServerSocket(name=type(self).__name__) + self.record_queue = asyncio.Queue() + self.sock = AsyncServerSocket( + self.record_queue, asyncio.get_event_loop(), name=type(self).__name__ + ) self.status_queue.put(self.sock.sock.getsockname()) self.sock.start_accepting() - self.record_queue = self.sock.queue - def poll_queue(self) -> None: + async def poll_queue(self) -> None: + """Tries to get one record from the queue and processes it, if there is one""" assert self.record_queue is not None - try: - record: Tuple[str, Any] = self.record_queue.get(block=True, timeout=5) - except queue.Empty: + if self.record_queue.empty(): return + + record: Tuple[str, Any] = await self.record_queue.get() + if len(record) != 2: self.logger.error("Query is not the correct length %s", repr(record)) return + self._last_record_received = time.time() record_type, data = record + if record_type == RECORD_TYPE_CREATE: raise RuntimeError( f"""{RECORD_TYPE_CREATE} is no longer supported. @@ -92,6 +99,7 @@ def poll_queue(self) -> None: """ ) return + if record_type == RECORD_TYPE_CONTENT: assert isinstance(data, tuple) assert len(data) == 2 @@ -103,18 +111,22 @@ def poll_queue(self) -> None: return content, content_hash = data content = base64.b64decode(content) - self.unstructured_storage.store_blob(filename=content_hash, blob=content) + await self.unstructured_storage.store_blob( + filename=content_hash, blob=content + ) + return if record_type == RECORD_TYPE_META: self._handle_meta(data) return visit_id = VisitId(data["visit_id"]) - self.curent_visit_ids.append(visit_id) - self.structured_storage.store_record( + self.current_visit_ids.append(visit_id) + + await self.structured_storage.store_record( table=record_type, visit_id=visit_id, record=data ) - def _handle_meta(self, data: Dict[str, Any]) -> None: + async def _handle_meta(self, data: Dict[str, Any]) -> None: """ Messages for the table RECORD_TYPE_SPECIAL are metainformation communicated to the aggregator @@ -122,24 +134,27 @@ def _handle_meta(self, data: Dict[str, Any]) -> None: - finalize: A message sent by the extension to signal that a visit_id is complete. """ - if data["action"] == ACTION_TYPE_INITIALIZE: - self.curent_visit_ids.append(data["visit_id"]) - elif data["action"] == ACTION_TYPE_FINALIZE: + visit_id = VisitId(data["visit_id"]) + action = data["action"] + + if action == ACTION_TYPE_INITIALIZE: + self.current_visit_ids.append(visit_id) + elif action == ACTION_TYPE_FINALIZE: + success = data["success"] try: - self.curent_visit_ids.remove(data["visit_id"]) + self.current_visit_ids.remove(visit_id) except ValueError: self.logger.error( - "Trying to remove visit_id %i " "from current_visit_ids failed", - data["visit_id"], + "Trying to remove visit_id %i from current_visit_ids failed", + visit_id, ) - self.structured_storage.run_visit_completion_tasks( - data["visit_id"], interrupted=not data["success"] + await self.structured_storage.finalize_visit_id( + visit_id, interrupted=not success ) + self.completion_queue.put(visit_id, success) else: - raise ValueError( - "Unexpected meta " "information type: %s" % data["meta_type"] - ) + raise ValueError("Unexpected action: %s", action) def update_status_queue(self) -> None: """Send manager process a status update.""" @@ -153,26 +168,20 @@ def update_status_queue(self) -> None: ) self._last_update = time.time() - def update_completion_queue(self) -> None: - for pair in self.structured_storage.saved_visit_ids(): - self.completion_queue.put(pair) - self.curent_visit_ids.remove(pair[0]) - - def drain_queue(self) -> None: + async def drain_queue(self) -> None: """ Ensures queue is empty before closing """ - time.sleep(3) # TODO: the socket needs a better way of closing while not self.record_queue.empty(): - self.poll_queue() + await self.poll_queue() self.logger.info("Queue was flushed completely") - def shutdown(self) -> None: - self.structured_storage.flush_cache() - self.structured_storage.shutdown() + async def shutdown(self) -> None: + await self.structured_storage.flush_cache() + await self.structured_storage.shutdown() if self.unstructured_storage is not None: - self.unstructured_storage.flush_cache() - self.unstructured_storage.shutdown() + await self.unstructured_storage.flush_cache() + await self.unstructured_storage.shutdown() - def should_shutdown(self): + def should_shutdown(self) -> bool: """Return `True` if the listener has received a shutdown signal Sets `self._relaxed` and `self.shutdown_flag` `self._relaxed means this shutdown is @@ -187,7 +196,7 @@ def should_shutdown(self): return True return False - def save_batch_if_past_timeout(self): + async def save_batch_if_past_timeout(self) -> None: """Save the current batch of records if no new data has been received. If we aren't receiving new data for this batch we commit early @@ -197,28 +206,27 @@ def save_batch_if_past_timeout(self): if time.time() - self._last_record_received < BATCH_COMMIT_TIMEOUT: return self.logger.debug( - "Saving current record batches to S3 since no new data has " + "Saving current records since no new data has " "been written for %d seconds." % (time.time() - self._last_record_received) ) self.drain_queue() self._last_record_received = None + async def _run(self) -> None: + await self.startup() + while not self.should_shutdown(): + self.update_status_queue() + await self.save_batch_if_past_timeout() + await self.poll_queue() + await asyncio.sleep(3) + await self.drain_queue() + await self.shutdown() -def listener_process_runner(aggregator: DataAggregator) -> None: - - aggregator.startup() - - while not aggregator.should_shutdown(): - aggregator.update_status_queue() - aggregator.update_completion_queue() - aggregator.save_batch_if_past_timeout() - aggregator.poll_queue() - - aggregator.drain_queue() - aggregator.shutdown() + def run(self) -> None: + asyncio.run(self._run(), debug=True) -class DataAggregatorHandle: +class StorageControllerHandle: """This class contains all methods relevant for the TaskManager to interact with the DataAggregator """ @@ -235,9 +243,9 @@ def __init__( self.completion_queue = Queue() self.shutdown_queue = Queue() self._last_status = None - self._last_status_received = None + self._last_status_received: Optional[float] = None self.logger = logging.getLogger("openwpm") - self.aggregator = DataAggregator( + self.aggregator = StorageController( structured_storage, unstructured_storage, status_queue=self.status_queue, @@ -263,7 +271,7 @@ def get_next_browser_id(self) -> BrowserId: """ return BrowserId(random.getrandbits(32)) - def save_configuration(self, openwpm_version, browser_version): + def save_configuration(self, openwpm_version: str, browser_version: str) -> None: # FIXME I need to find a solution for this self.logger.error( "Can't log config as of yet, because it's still not implemented" @@ -272,8 +280,8 @@ def save_configuration(self, openwpm_version, browser_version): def launch(self) -> None: """Starts the data aggregator""" self.listener_process = Process( - name="DataAggregator", - target=listener_process_runner, + name="StorageController", + target=StorageController.run, args=(self.aggregator,), ) self.listener_process.daemon = True @@ -312,7 +320,7 @@ def shutdown(self, relaxed: bool = True) -> None: self.listener_address = None self.listener_process = None - def get_most_recent_status(self): + def get_most_recent_status(self) -> int: """Return the most recent queue size sent from the listener process""" # Block until we receive the first status update @@ -333,7 +341,7 @@ def get_most_recent_status(self): return self._last_status - def get_status(self): + def get_status(self) -> int: """Get listener process status. If the status queue is empty, block.""" try: self._last_status = self.status_queue.get( @@ -341,8 +349,10 @@ def get_status(self): ) self._last_status_received = time.time() except queue.Empty: + assert self._last_status_received is not None raise RuntimeError( "No status update from DataAggregator listener process " "for %d seconds." % (time.time() - self._last_status_received) ) + assert isinstance(self._last_status, int) return self._last_status diff --git a/automation/data_aggregator/storage_providers.py b/automation/storage/storage_providers.py similarity index 75% rename from automation/data_aggregator/storage_providers.py rename to automation/storage/storage_providers.py index e933b526e..459f3abd7 100644 --- a/automation/data_aggregator/storage_providers.py +++ b/automation/storage/storage_providers.py @@ -19,12 +19,12 @@ class StorageProvider(ABC): """ @abstractmethod - def flush_cache(self) -> None: + async def flush_cache(self) -> None: """ Blockingly write out any cached data to the respective storage """ pass @abstractmethod - def shutdown(self) -> None: + async def shutdown(self) -> None: """Close all open ressources After this method has been called no further calls should be made to the object """ @@ -32,22 +32,11 @@ def shutdown(self) -> None: class StructuredStorageProvider(StorageProvider): - def __init__(self): + def __init__(self) -> None: super().__init__() - self._completed_visit_ids: List[Tuple[VisitId, bool]] = list() - - def saved_visit_ids(self) -> List[Tuple[VisitId, bool]]: - """Return the list of all visit_ids that have been saved to permanent storage - since the last time this method was called - - For each visit_id it's also noted whether they were interrupted - """ - temp = self._completed_visit_ids - self._completed_visit_ids = list() - return temp @abstractmethod - def store_record( + async def store_record( self, table: str, visit_id: VisitId, record: Dict[str, Any] ) -> None: """Submit a record to be stored @@ -56,23 +45,25 @@ def store_record( pass @abstractmethod - def run_visit_completion_tasks( + async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False ) -> None: """This method is invoked to inform the StrucuturedStorageProvider that no more records for this visit_id will be submitted + It will only return once all records for this visit_id have been saved + to permanent storage. """ pass class UnstructuredStorageProvider(StorageProvider): @abstractmethod - def store_blob( + async def store_blob( self, filename: str, blob: bytes, compressed: bool = True, - skip_if_exists: bool = True, + overwrite: bool = False, ) -> None: """Stores the given bytes under the provided filename""" pass diff --git a/demo.py b/demo.py index 003a9da55..7a1a4b10e 100644 --- a/demo.py +++ b/demo.py @@ -1,8 +1,8 @@ import os from automation import CommandSequence, TaskManager -from automation.data_aggregator.in_memory_storage import MemoryUnstructuredProvider -from automation.data_aggregator.sql_provider import SqlLiteStorageProvider +from automation.storage.in_memory_storage import MemoryUnstructuredProvider +from automation.storage.sql_provider import SqlLiteStorageProvider # The list of sites that we wish to crawl NUM_BROWSERS = 1 diff --git a/environment.yaml b/environment.yaml index 4bc46a4c6..0ce0eb35f 100644 --- a/environment.yaml +++ b/environment.yaml @@ -3,28 +3,28 @@ channels: - main dependencies: - autopep8=1.5.4 -- beautifulsoup4=4.9.2 +- beautifulsoup4=4.9.3 - click=7.1.2 - codecov=2.1.9 - dill=0.3.2 -- flake8-isort=4.0.0 -- flake8=3.8.3 +- flake8=3.8.4 - geckodriver=0.27.0 - ipython=7.18.1 - leveldb=1.22 - localstack=0.11.1.1 - multiprocess=0.70.10 - mypy=0.782 -- nodejs=14.12.0 -- pandas=1.1.2 +- nodejs=14.13.1 +- pandas=1.1.3 - pillow=7.2.0 - pip=20.2.3 - pre-commit=2.7.1 - psutil=5.7.2 - pyarrow=1.0.1 +- pytest-asyncio=0.12.0 - pytest-cov=2.10.1 -- pytest=6.1.0 -- python=3.8.5 +- pytest=6.1.1 +- python=3.8.6 - pyvirtualdisplay=0.2.5 - redis-py=3.5.3 - s3fs=0.4.0 diff --git a/scripts/environment-unpinned-dev.yaml b/scripts/environment-unpinned-dev.yaml index eb61a75de..b4af8ccde 100644 --- a/scripts/environment-unpinned-dev.yaml +++ b/scripts/environment-unpinned-dev.yaml @@ -5,13 +5,13 @@ dependencies: - codecov - pytest-cov - flake8 - - flake8-isort - ipython - localstack==0.11.1.1 # See https://github.com/mozilla/OpenWPM/pull/682 - pip - pre-commit - pytest - mypy + - pytest-asyncio - pip: # Select depenedencies from localstack[full] that we need - amazon-kclpy diff --git a/test/storage_providers/test_data_aggregator.py b/test/storage_providers/test_data_aggregator.py index 6535cee1c..acd30608f 100644 --- a/test/storage_providers/test_data_aggregator.py +++ b/test/storage_providers/test_data_aggregator.py @@ -1,13 +1,13 @@ -from automation.data_aggregator import DataAggregatorHandle -from automation.data_aggregator.in_memory_storage import ( +from automation.storage.in_memory_storage import ( MemoryStructuredProvider, MemoryUnstructuredProvider, ) +from automation.storage.storage_controller import StorageControllerHandle -def test_startup_and_shutdown(): +def test_startup_and_shutdown() -> None: structured = MemoryStructuredProvider() unstructured = MemoryUnstructuredProvider() - agg_handle = DataAggregatorHandle(structured, unstructured) + agg_handle = StorageControllerHandle(structured, unstructured) agg_handle.launch() agg_handle.shutdown() diff --git a/test/storage_providers/test_memory_storage_provider.py b/test/storage_providers/test_memory_storage_provider.py index 8dc7d6b96..daf14b685 100644 --- a/test/storage_providers/test_memory_storage_provider.py +++ b/test/storage_providers/test_memory_storage_provider.py @@ -1,14 +1,16 @@ +import asyncio from typing import Any, Dict, List, Tuple import pytest -from automation.data_aggregator.in_memory_storage import ( +from automation.storage.in_memory_storage import ( MemoryArrowProvider, MemoryStructuredProvider, MemoryUnstructuredProvider, ) -from automation.data_aggregator.leveldb import LevelDbProvider -from automation.data_aggregator.sql_provider import SqlLiteStorageProvider +from automation.storage.leveldb import LevelDbProvider +from automation.storage.sql_provider import SqlLiteStorageProvider +from automation.storage.storage_providers import StructuredStorageProvider from automation.types import VisitId from ..openwpmtest import OpenWPMTest @@ -30,7 +32,9 @@ @pytest.fixture -def structured_provider(request, tmp_path_factory): +def structured_provider( + request: Any, tmp_path_factory: Any +) -> StructuredStorageProvider: if request.param == memory_structured: return MemoryStructuredProvider() elif request.param == sqllite: @@ -41,7 +45,7 @@ def structured_provider(request, tmp_path_factory): raise ValueError("invalid internal test config") -def pytest_generate_tests(metafunc): +def pytest_generate_tests(metafunc: Any) -> Any: # Source: https://docs.pytest.org/en/latest/example/parametrize.html#a-quick-port-of-testscenarios # noqa idlist = [] argvalues = [] @@ -54,22 +58,24 @@ def pytest_generate_tests(metafunc): metafunc.parametrize(argnames, argvalues, ids=idlist, scope="class", indirect=True) +@pytest.mark.asyncio class TestStructuredStorageProvider(OpenWPMTest): scenarios = structured_scenarios - def test_basic_access(self, structured_provider): - structured_provider.store_record( + async def test_basic_access( + self, structured_provider: StructuredStorageProvider + ) -> None: + await structured_provider.store_record( "test", VisitId(2), {"visit_id": 2, "data": "test"} ) - structured_provider.run_visit_completion_tasks(VisitId(2)) - structured_provider.flush_cache() - assert structured_provider.saved_visit_ids() == [(2, False)] + await structured_provider.finalize_visit_id(VisitId(2)) + await structured_provider.flush_cache() class TestUnstructuredStorageProvide(OpenWPMTest): scenarios: List[Tuple[str, Dict[str, Any]]] = [(memory_unstructured, {})] - def test_basic_unstructured_storing(self): + async def test_basic_unstructured_storing(self) -> None: test_string = "This is my test string" blob = test_string.encode() prov = MemoryUnstructuredProvider() diff --git a/test/test_s3_aggregator.py b/test/test_s3_aggregator.py index f29473401..f54136269 100644 --- a/test/test_s3_aggregator.py +++ b/test/test_s3_aggregator.py @@ -10,7 +10,7 @@ from automation import TaskManager from automation.CommandSequence import CommandSequence -from automation.data_aggregator.parquet_schema import PQ_SCHEMAS +from automation.storage.parquet_schema import PQ_SCHEMAS from .openwpmtest import OpenWPMTest from .utilities import BASE_TEST_URL, LocalS3Dataset, LocalS3Session, local_s3_bucket From 07db4ab2d003026da8285c3262386f34e811e9f4 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 9 Oct 2020 15:43:27 +0200 Subject: [PATCH 014/139] Fixed minor bugs --- automation/storage/storage_controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/storage/storage_controller.py b/automation/storage/storage_controller.py index bfb8ec0bb..b1f700f6f 100644 --- a/automation/storage/storage_controller.py +++ b/automation/storage/storage_controller.py @@ -117,7 +117,7 @@ async def poll_queue(self) -> None: return if record_type == RECORD_TYPE_META: - self._handle_meta(data) + await self._handle_meta(data) return visit_id = VisitId(data["visit_id"]) self.current_visit_ids.append(visit_id) @@ -152,7 +152,7 @@ async def _handle_meta(self, data: Dict[str, Any]) -> None: await self.structured_storage.finalize_visit_id( visit_id, interrupted=not success ) - self.completion_queue.put(visit_id, success) + self.completion_queue.put((visit_id, success)) else: raise ValueError("Unexpected action: %s", action) From f3857ac61320192770efce7300d4cbafecd25c7c Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 9 Oct 2020 17:33:40 +0200 Subject: [PATCH 015/139] First steps at porting arrow --- automation/storage/arrow_storage.py | 79 +++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/automation/storage/arrow_storage.py b/automation/storage/arrow_storage.py index 8ebe96d9a..010c3daa7 100644 --- a/automation/storage/arrow_storage.py +++ b/automation/storage/arrow_storage.py @@ -1,13 +1,21 @@ +import logging import random from abc import abstractmethod -from typing import Any, Dict, List +from typing import Any, DefaultDict, Dict, List +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq from pyarrow import Table from automation.types import VisitId from .parquet_schema import PQ_SCHEMAS -from .storage_providers import StructuredStorageProvider +from .storage_providers import StructuredStorageProvider, TableName + +SITE_VISITS_INDEX = TableName("_site_visits_index") +INCOMPLETE_VISITS = TableName("incomplete_visits") +CACHE_SIZE = 500 class ArrowProvider(StructuredStorageProvider): @@ -17,11 +25,23 @@ class ArrowProvider(StructuredStorageProvider): def __init__(self) -> None: super().__init__() - self._records: Dict[VisitId, Dict[str, List[Dict[str, Any]]]] = {} + self.logger = logging.getLogger("openwpm") + + def factory_function() -> DefaultDict[TableName, List[Dict[str, Any]]]: + return DefaultDict(list) + + # Raw records per VisitId and Table + self._records: DefaultDict[ + VisitId, DefaultDict[TableName, List[Dict[str, Any]]] + ] = DefaultDict(factory_function) + + # Record batches by TableName + self._batches: DefaultDict[TableName, List[pa.RecordBatch]] = DefaultDict(list) + self._instance_id = random.getrandbits(32) async def store_record( - self, table: str, visit_id: VisitId, record: Dict[str, Any] + self, table: TableName, visit_id: VisitId, record: Dict[str, Any] ) -> None: records = self._records[visit_id] # Add nulls @@ -32,6 +52,57 @@ async def store_record( record["instance_id"] = self._instance_id records[table].append(record) + def _create_batch(self, visit_id: VisitId) -> None: + """Create record batches for all records from `visit_id`""" + if visit_id not in self._records: + # The batch for this `visit_id` was already created, skip + return + for table_name, data in self._records[visit_id].items(): + try: + df = pd.DataFrame(data) + batch = pa.RecordBatch.from_pandas( + df, schema=PQ_SCHEMAS[table_name], preserve_index=False + ) + self._batches[table_name].append(batch) + self.logger.debug( + "Successfully created batch for table %s and " + "visit_id %s" % (table_name, visit_id) + ) + except pa.lib.ArrowInvalid: + self.logger.error( + "Error while creating record batch for table %s\n" % table_name, + exc_info=True, + ) + pass + # We construct a special index file from the site_visits data + # to make it easier to query the dataset + if table_name == "site_visits": + for item in data: + self._batches[SITE_VISITS_INDEX].append(item) + + del self._records[visit_id] + self._unsaved_visit_ids.add(visit_id) + @abstractmethod async def write_table(self, table: Table) -> None: """Write out the table to persistent storage""" + + def _is_cache_full(self) -> bool: + for batches in self._batches.values(): + if len(batches) > CACHE_SIZE: + should_send = True + + async def finalize_visit_id( + self, visit_id: VisitId, interrupted: bool = False + ) -> None: + if interrupted: + await self.store_record(INCOMPLETE_VISITS, {"visit_id": visit_id}, visit_id) + + self._create_batch(visit_id) + if self._is_cache_full(): + await self.flush_cache() + + raise NotImplementedError() + + async def flush_cache(self) -> None: + raise NotImplementedError() From ee5d5b6ed90a1c4d7e02ec2269ae2d0d8077d6a6 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 9 Oct 2020 17:34:24 +0200 Subject: [PATCH 016/139] Introduced TableName and different Task handling --- automation/storage/in_memory_storage.py | 8 +++- automation/storage/sql_provider.py | 6 +-- automation/storage/storage_controller.py | 45 ++++++++++++------- automation/storage/storage_providers.py | 6 ++- demo.py | 4 ++ .../test_memory_storage_provider.py | 4 +- 6 files changed, 49 insertions(+), 24 deletions(-) diff --git a/automation/storage/in_memory_storage.py b/automation/storage/in_memory_storage.py index 5ce8c8c62..5e689213b 100644 --- a/automation/storage/in_memory_storage.py +++ b/automation/storage/in_memory_storage.py @@ -6,7 +6,11 @@ from automation.types import VisitId from .arrow_storage import ArrowProvider -from .storage_providers import StructuredStorageProvider, UnstructuredStorageProvider +from .storage_providers import ( + StructuredStorageProvider, + TableName, + UnstructuredStorageProvider, +) """ This module contains implementations for various kinds of storage providers @@ -33,7 +37,7 @@ async def flush_cache(self) -> None: pass async def store_record( - self, table: str, visit_id: VisitId, record: Dict[str, Any] + self, table: TableName, visit_id: VisitId, record: Dict[str, Any] ) -> None: self.queue.put((table, record)) diff --git a/automation/storage/sql_provider.py b/automation/storage/sql_provider.py index cd74711e1..395696ff6 100644 --- a/automation/storage/sql_provider.py +++ b/automation/storage/sql_provider.py @@ -8,7 +8,7 @@ from automation.types import VisitId -from .storage_providers import StructuredStorageProvider +from .storage_providers import StructuredStorageProvider, TableName SCHEMA_FILE = os.path.join(os.path.dirname(__file__), "schema.sql") @@ -33,7 +33,7 @@ async def flush_cache(self) -> None: self.db.commit() async def store_record( - self, table: str, visit_id: VisitId, record: Dict[str, Any] + self, table: TableName, visit_id: VisitId, record: Dict[str, Any] ) -> None: """Submit a record to be stored The storing might not happen immediately @@ -61,7 +61,7 @@ async def store_record( ) def _generate_insert( - self, table: str, data: Dict[str, Any] + self, table: TableName, data: Dict[str, Any] ) -> Tuple[str, List[Any]]: """Generate a SQL query from `record`""" statement = "INSERT INTO %s (" % table diff --git a/automation/storage/storage_controller.py b/automation/storage/storage_controller.py index b1f700f6f..e6340e0c9 100644 --- a/automation/storage/storage_controller.py +++ b/automation/storage/storage_controller.py @@ -5,7 +5,7 @@ import random import threading import time -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, DefaultDict, Dict, List, Optional, Tuple from multiprocess import Queue @@ -13,7 +13,11 @@ from ..SocketInterface import AsyncServerSocket from ..types import BrowserId, ManagerParams, VisitId -from .storage_providers import StructuredStorageProvider, UnstructuredStorageProvider +from .storage_providers import ( + StructuredStorageProvider, + TableName, + UnstructuredStorageProvider, +) RECORD_TYPE_CONTENT = "page_content" RECORD_TYPE_META = "meta_information" @@ -59,7 +63,7 @@ def __init__( self._last_update = time.time() # last status update time self.record_queue: Queue = None # Initialized on `startup` self.logger = logging.getLogger("openwpm") - self.current_visit_ids: List[VisitId] = list() # All visit_ids in flight + self.current_tasks: DefaultDict[VisitId, List[asyncio.Task]] = DefaultDict(list) self.sock: Optional[AsyncServerSocket] = None self.structured_storage = structured_storage self.unstructured_storage = unstructured_storage @@ -120,10 +124,14 @@ async def poll_queue(self) -> None: await self._handle_meta(data) return visit_id = VisitId(data["visit_id"]) - self.current_visit_ids.append(visit_id) + table_name = TableName(record_type) - await self.structured_storage.store_record( - table=record_type, visit_id=visit_id, record=data + self.current_tasks[visit_id].append( + asyncio.create_task( + self.structured_storage.store_record( + table=table_name, visit_id=visit_id, record=data + ) + ) ) async def _handle_meta(self, data: Dict[str, Any]) -> None: @@ -137,22 +145,24 @@ async def _handle_meta(self, data: Dict[str, Any]) -> None: visit_id = VisitId(data["visit_id"]) action = data["action"] + self.logger.debug( + "Received meta message to %s for visit_id %d", action, visit_id + ) if action == ACTION_TYPE_INITIALIZE: - self.current_visit_ids.append(visit_id) + return elif action == ACTION_TYPE_FINALIZE: success = data["success"] - try: - self.current_visit_ids.remove(visit_id) - except ValueError: - self.logger.error( - "Trying to remove visit_id %i from current_visit_ids failed", - visit_id, - ) + for task in self.current_tasks[visit_id]: + await task + self.logger.debug( + "Awaited all tasks for visit_id %d while finalizing", visit_id + ) await self.structured_storage.finalize_visit_id( visit_id, interrupted=not success ) self.completion_queue.put((visit_id, success)) + del self.current_tasks[visit_id] else: raise ValueError("Unexpected action: %s", action) @@ -212,14 +222,19 @@ async def save_batch_if_past_timeout(self) -> None: self.drain_queue() self._last_record_received = None + async def finish_tasks(self) -> None: + for visit_id, tasks in self.current_tasks.items(): + for task in tasks: + await task + async def _run(self) -> None: await self.startup() while not self.should_shutdown(): self.update_status_queue() await self.save_batch_if_past_timeout() await self.poll_queue() - await asyncio.sleep(3) await self.drain_queue() + await self.finish_tasks() await self.shutdown() def run(self) -> None: diff --git a/automation/storage/storage_providers.py b/automation/storage/storage_providers.py index 459f3abd7..544ab9bd4 100644 --- a/automation/storage/storage_providers.py +++ b/automation/storage/storage_providers.py @@ -1,7 +1,7 @@ import gzip import io from abc import ABC, abstractmethod -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, NewType, Tuple from automation.types import VisitId @@ -11,6 +11,8 @@ without any changes to the rest of the code base """ +TableName = NewType("TableName", str) + class StorageProvider(ABC): """Base class that defines some general helper methods @@ -37,7 +39,7 @@ def __init__(self) -> None: @abstractmethod async def store_record( - self, table: str, visit_id: VisitId, record: Dict[str, Any] + self, table: TableName, visit_id: VisitId, record: Dict[str, Any] ) -> None: """Submit a record to be stored The storing might not happen immediately diff --git a/demo.py b/demo.py index 7a1a4b10e..5e65ab248 100644 --- a/demo.py +++ b/demo.py @@ -1,3 +1,4 @@ +import logging import os from automation import CommandSequence, TaskManager @@ -39,6 +40,8 @@ manager_params["data_directory"] = "~/Desktop/" manager_params["log_directory"] = "~/Desktop/" +logging_params = {"log_level_console": logging.DEBUG} + # Instantiates the measurement platform # Commands time out by default after 60 seconds manager = TaskManager.TaskManager( @@ -48,6 +51,7 @@ os.path.expanduser(manager_params["data_directory"] + "crawl-data.sqlite") ), None, + logger_kwargs=logging_params, ) # Visits the sites diff --git a/test/storage_providers/test_memory_storage_provider.py b/test/storage_providers/test_memory_storage_provider.py index daf14b685..a53edc797 100644 --- a/test/storage_providers/test_memory_storage_provider.py +++ b/test/storage_providers/test_memory_storage_provider.py @@ -10,7 +10,7 @@ ) from automation.storage.leveldb import LevelDbProvider from automation.storage.sql_provider import SqlLiteStorageProvider -from automation.storage.storage_providers import StructuredStorageProvider +from automation.storage.storage_providers import StructuredStorageProvider, TableName from automation.types import VisitId from ..openwpmtest import OpenWPMTest @@ -66,7 +66,7 @@ async def test_basic_access( self, structured_provider: StructuredStorageProvider ) -> None: await structured_provider.store_record( - "test", VisitId(2), {"visit_id": 2, "data": "test"} + TableName("test"), VisitId(2), {"visit_id": 2, "data": "test"} ) await structured_provider.finalize_visit_id(VisitId(2)) await structured_provider.flush_cache() From 9e06a5ee61b3c69b180d94e234535f25e75c92ca Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 9 Oct 2020 18:29:02 +0200 Subject: [PATCH 017/139] Added more failing tests --- automation/storage/storage_controller.py | 13 +++++++------ demo.py | 1 - ...ata_aggregator.py => test_storage_controller.py} | 8 ++++++++ 3 files changed, 15 insertions(+), 7 deletions(-) rename test/storage_providers/{test_data_aggregator.py => test_storage_controller.py} (59%) diff --git a/automation/storage/storage_controller.py b/automation/storage/storage_controller.py index e6340e0c9..1bbfd73d6 100644 --- a/automation/storage/storage_controller.py +++ b/automation/storage/storage_controller.py @@ -5,6 +5,7 @@ import random import threading import time +from collections import defaultdict from typing import Any, DefaultDict, Dict, List, Optional, Tuple from multiprocess import Queue @@ -12,7 +13,7 @@ from automation.utilities.multiprocess_utils import Process from ..SocketInterface import AsyncServerSocket -from ..types import BrowserId, ManagerParams, VisitId +from ..types import BrowserId, VisitId from .storage_providers import ( StructuredStorageProvider, TableName, @@ -63,7 +64,7 @@ def __init__( self._last_update = time.time() # last status update time self.record_queue: Queue = None # Initialized on `startup` self.logger = logging.getLogger("openwpm") - self.current_tasks: DefaultDict[VisitId, List[asyncio.Task]] = DefaultDict(list) + self.current_tasks: DefaultDict[VisitId, List[asyncio.Task]] = defaultdict(list) self.sock: Optional[AsyncServerSocket] = None self.structured_storage = structured_storage self.unstructured_storage = unstructured_storage @@ -86,7 +87,6 @@ async def poll_queue(self) -> None: return record: Tuple[str, Any] = await self.record_queue.get() - if len(record) != 2: self.logger.error("Query is not the correct length %s", repr(record)) return @@ -94,6 +94,8 @@ async def poll_queue(self) -> None: self._last_record_received = time.time() record_type, data = record + self.logger.info("Received record for record_type %s", record_type) + if record_type == RECORD_TYPE_CREATE: raise RuntimeError( f"""{RECORD_TYPE_CREATE} is no longer supported. @@ -102,7 +104,6 @@ async def poll_queue(self) -> None: launching the DataAggregator """ ) - return if record_type == RECORD_TYPE_CONTENT: assert isinstance(data, tuple) @@ -145,7 +146,7 @@ async def _handle_meta(self, data: Dict[str, Any]) -> None: visit_id = VisitId(data["visit_id"]) action = data["action"] - self.logger.debug( + self.logger.info( "Received meta message to %s for visit_id %d", action, visit_id ) if action == ACTION_TYPE_INITIALIZE: @@ -219,7 +220,7 @@ async def save_batch_if_past_timeout(self) -> None: "Saving current records since no new data has " "been written for %d seconds." % (time.time() - self._last_record_received) ) - self.drain_queue() + await self.drain_queue() self._last_record_received = None async def finish_tasks(self) -> None: diff --git a/demo.py b/demo.py index 5e65ab248..3b65d7000 100644 --- a/demo.py +++ b/demo.py @@ -51,7 +51,6 @@ os.path.expanduser(manager_params["data_directory"] + "crawl-data.sqlite") ), None, - logger_kwargs=logging_params, ) # Visits the sites diff --git a/test/storage_providers/test_data_aggregator.py b/test/storage_providers/test_storage_controller.py similarity index 59% rename from test/storage_providers/test_data_aggregator.py rename to test/storage_providers/test_storage_controller.py index acd30608f..bae2e5861 100644 --- a/test/storage_providers/test_data_aggregator.py +++ b/test/storage_providers/test_storage_controller.py @@ -1,3 +1,4 @@ +from automation.SocketInterface import ClientSocket from automation.storage.in_memory_storage import ( MemoryStructuredProvider, MemoryUnstructuredProvider, @@ -10,4 +11,11 @@ def test_startup_and_shutdown() -> None: unstructured = MemoryUnstructuredProvider() agg_handle = StorageControllerHandle(structured, unstructured) agg_handle.launch() + print(agg_handle.listener_address) + cs = ClientSocket() + cs.connect(*agg_handle.listener_address) + cs.send(("test", {"asd": "dfg"})) agg_handle.shutdown() + handle = structured.handle + handle.poll_queue() + assert handle.storage["test"] == {"asd": "dfg"} From 2737ec099818f2b67c7531db0a5c68eb5b27bb79 Mon Sep 17 00:00:00 2001 From: vringar Date: Tue, 13 Oct 2020 21:49:44 +0200 Subject: [PATCH 018/139] First first completes others don't --- .gitignore | 2 + .../Extension/firefox/feature.js/loggingdb.js | 2 +- automation/SocketInterface.py | 75 ++--- automation/storage/arrow_storage.py | 6 +- automation/storage/storage_controller.py | 257 +++++++++--------- demo.py | 6 +- environment.yaml | 6 +- 7 files changed, 184 insertions(+), 170 deletions(-) diff --git a/.gitignore b/.gitignore index 2a919d66a..b24c470af 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,5 @@ automation/Extension/firefox/dist automation/Extension/firefox/openwpm.xpi automation/Extension/firefox/src/content.js automation/Extension/firefox/src/feature.js + +datadir \ No newline at end of file diff --git a/automation/Extension/firefox/feature.js/loggingdb.js b/automation/Extension/firefox/feature.js/loggingdb.js index 87fdd8bfb..4626b34c5 100644 --- a/automation/Extension/firefox/feature.js/loggingdb.js +++ b/automation/Extension/firefox/feature.js/loggingdb.js @@ -64,7 +64,7 @@ export let open = async function(aggregatorAddress, logAddress, curr_crawlID) { if (aggregatorAddress != null) { dataAggregator = new socket.SendingSocket(); let rv = await dataAggregator.connect(aggregatorAddress[0], aggregatorAddress[1]); - console.log("sqliteSocket started?",rv); + console.log("StorageController started?",rv); } // Listen for incoming urls as visit ids diff --git a/automation/SocketInterface.py b/automation/SocketInterface.py index 0a08ef663..7a020d709 100644 --- a/automation/SocketInterface.py +++ b/automation/SocketInterface.py @@ -4,7 +4,9 @@ import struct import threading import traceback +from asyncio import IncompleteReadError from queue import Queue +from typing import Any import dill @@ -75,23 +77,14 @@ def _handle_conn(self, client, address): % (msglen, serialization) ) msg = self.receive_msg(client, msglen) - if serialization != b"n": - try: - if serialization == b"d": # dill serialization - msg = dill.loads(msg) - elif serialization == b"j": # json serialization - msg = json.loads(msg.decode("utf-8")) - elif serialization == b"u": # utf-8 serialization - msg = msg.decode("utf-8") - else: - print("Unrecognized serialization type: %r" % serialization) - continue - except (UnicodeDecodeError, ValueError) as e: - print( - "Error de-serializing message: %s \n %s" - % (msg, traceback.format_exc(e)) - ) - continue + try: + msg = _parse(serialization, msg) + except (UnicodeDecodeError, ValueError) as e: + print( + "Error de-serializing message: %s \n %s" + % (msg, traceback.format_exc(e)) + ) + continue self._put_into_queue(msg) except RuntimeError: if self.verbose: @@ -114,25 +107,6 @@ def close(self): self.sock.close() -class AsyncServerSocket(ServerSocket): - def __init__( - self, - queue: asyncio.Queue, - loop: asyncio.AbstractEventLoop, - name=None, - verbose=False, - ): - super().__init__(name=name, verbose=verbose) - self.queue = queue - self.loop = loop - - def _put_into_queue(self, msg): - async def callback(queue, msg): - queue.put_nowait(msg) - - asyncio.run_coroutine_threadsafe(callback(self.queue, msg), self.loop) - - class ClientSocket: """A client socket for sending messages""" @@ -188,6 +162,35 @@ def close(self): self.sock.close() +async def get_message_from_reader(reader: asyncio.StreamReader) -> Any: + msg = await get_n_bytes_from_reader(reader, 5) + msglen, serialization = struct.unpack(">Lc", msg) + msg = await get_n_bytes_from_reader(reader, msglen) + return _parse(serialization, msg) + + +async def get_n_bytes_from_reader(reader: asyncio.StreamReader, n: int) -> bytes: + b = b"" + while True: + try: + return await reader.readexactly(n) + except IncompleteReadError as e: + b += e.partial + n -= len(e.partial) + + +def _parse(serialization: bytes, msg: bytes) -> Any: + if serialization == b"n": + return msg + if serialization == b"d": # dill serialization + return dill.loads(msg) + if serialization == b"j": # json serialization + return json.loads(msg.decode("utf-8")) + if serialization == b"u": # utf-8 serialization + return msg.decode("utf-8") + raise ValueError("Unkown Encoding") + + def main(): import sys diff --git a/automation/storage/arrow_storage.py b/automation/storage/arrow_storage.py index 010c3daa7..07d5c8f76 100644 --- a/automation/storage/arrow_storage.py +++ b/automation/storage/arrow_storage.py @@ -81,7 +81,6 @@ def _create_batch(self, visit_id: VisitId) -> None: self._batches[SITE_VISITS_INDEX].append(item) del self._records[visit_id] - self._unsaved_visit_ids.add(visit_id) @abstractmethod async def write_table(self, table: Table) -> None: @@ -90,13 +89,14 @@ async def write_table(self, table: Table) -> None: def _is_cache_full(self) -> bool: for batches in self._batches.values(): if len(batches) > CACHE_SIZE: - should_send = True + return True + return False async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False ) -> None: if interrupted: - await self.store_record(INCOMPLETE_VISITS, {"visit_id": visit_id}, visit_id) + await self.store_record(INCOMPLETE_VISITS, visit_id, {"visit_id": visit_id}) self._create_batch(visit_id) if self._is_cache_full(): diff --git a/automation/storage/storage_controller.py b/automation/storage/storage_controller.py index 1bbfd73d6..99bfb93ee 100644 --- a/automation/storage/storage_controller.py +++ b/automation/storage/storage_controller.py @@ -3,16 +3,17 @@ import logging import queue import random +import socket import threading import time from collections import defaultdict -from typing import Any, DefaultDict, Dict, List, Optional, Tuple +from typing import Any, DefaultDict, Dict, List, NoReturn, Optional, Tuple from multiprocess import Queue from automation.utilities.multiprocess_utils import Process -from ..SocketInterface import AsyncServerSocket +from ..SocketInterface import get_message_from_reader from ..types import BrowserId, VisitId from .storage_providers import ( StructuredStorageProvider, @@ -61,79 +62,68 @@ def __init__( self.shutdown_queue = shutdown_queue self._shutdown_flag = False self._relaxed = False - self._last_update = time.time() # last status update time self.record_queue: Queue = None # Initialized on `startup` self.logger = logging.getLogger("openwpm") self.current_tasks: DefaultDict[VisitId, List[asyncio.Task]] = defaultdict(list) - self.sock: Optional[AsyncServerSocket] = None self.structured_storage = structured_storage self.unstructured_storage = unstructured_storage self._last_record_received: Optional[float] = None - async def startup(self) -> None: - """Puts the DataAggregator into a runable state - by starting up the ServerSocket""" - self.record_queue = asyncio.Queue() - self.sock = AsyncServerSocket( - self.record_queue, asyncio.get_event_loop(), name=type(self).__name__ - ) - self.status_queue.put(self.sock.sock.getsockname()) - self.sock.start_accepting() - - async def poll_queue(self) -> None: - """Tries to get one record from the queue and processes it, if there is one""" - assert self.record_queue is not None - if self.record_queue.empty(): - return - - record: Tuple[str, Any] = await self.record_queue.get() - if len(record) != 2: - self.logger.error("Query is not the correct length %s", repr(record)) - return - - self._last_record_received = time.time() - record_type, data = record - - self.logger.info("Received record for record_type %s", record_type) - - if record_type == RECORD_TYPE_CREATE: - raise RuntimeError( - f"""{RECORD_TYPE_CREATE} is no longer supported. - since the user now has access to the DB before it - goes into use, they should set up all schemas before - launching the DataAggregator - """ - ) + async def handler( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> NoReturn: + """Created for every new connection to the Server""" + while True: + record: Tuple[str, Any] = await get_message_from_reader(reader) + + if len(record) != 2: + self.logger.error("Query is not the correct length %s", repr(record)) + continue + + self._last_record_received = time.time() + record_type, data = record + + self.logger.info("Received record for record_type %s", record_type) + + if record_type == RECORD_TYPE_CREATE: + raise RuntimeError( + f"""{RECORD_TYPE_CREATE} is no longer supported. + since the user now has access to the DB before it + goes into use, they should set up all schemas before + launching the DataAggregator + """ + ) - if record_type == RECORD_TYPE_CONTENT: - assert isinstance(data, tuple) - assert len(data) == 2 - if self.unstructured_storage is None: - self.logger.error( - """Tried to save content while not having - provided any unstructured storage provider.""" + if record_type == RECORD_TYPE_CONTENT: + assert isinstance(data, tuple) + assert len(data) == 2 + if self.unstructured_storage is None: + self.logger.error( + """Tried to save content while not having + provided any unstructured storage provider.""" + ) + continue + content, content_hash = data + content = base64.b64decode(content) + await self.unstructured_storage.store_blob( + filename=content_hash, blob=content ) - return - content, content_hash = data - content = base64.b64decode(content) - await self.unstructured_storage.store_blob( - filename=content_hash, blob=content - ) + continue - return - if record_type == RECORD_TYPE_META: - await self._handle_meta(data) - return - visit_id = VisitId(data["visit_id"]) - table_name = TableName(record_type) + if record_type == RECORD_TYPE_META: + await self._handle_meta(data) + continue + + visit_id = VisitId(data["visit_id"]) + table_name = TableName(record_type) - self.current_tasks[visit_id].append( - asyncio.create_task( - self.structured_storage.store_record( - table=table_name, visit_id=visit_id, record=data + self.current_tasks[visit_id].append( + asyncio.create_task( + self.structured_storage.store_record( + table=table_name, visit_id=visit_id, record=data + ) ) ) - ) async def _handle_meta(self, data: Dict[str, Any]) -> None: """ @@ -146,12 +136,10 @@ async def _handle_meta(self, data: Dict[str, Any]) -> None: visit_id = VisitId(data["visit_id"]) action = data["action"] - self.logger.info( - "Received meta message to %s for visit_id %d", action, visit_id - ) if action == ACTION_TYPE_INITIALIZE: return elif action == ACTION_TYPE_FINALIZE: + self.logger.info("Awaiting all tasks for visit_id %d", visit_id) success = data["success"] for task in self.current_tasks[visit_id]: await task @@ -167,23 +155,25 @@ async def _handle_meta(self, data: Dict[str, Any]) -> None: else: raise ValueError("Unexpected action: %s", action) - def update_status_queue(self) -> None: + async def update_status_queue(self) -> NoReturn: """Send manager process a status update.""" - if (time.time() - self._last_update) < STATUS_UPDATE_INTERVAL: - return - qsize = self.record_queue.qsize() - self.status_queue.put(qsize) - self.logger.debug( - "Status update; current record queue size: %d. " - "current number of threads: %d." % (qsize, threading.active_count()) - ) - self._last_update = time.time() - - async def drain_queue(self) -> None: - """ Ensures queue is empty before closing """ - while not self.record_queue.empty(): - await self.poll_queue() - self.logger.info("Queue was flushed completely") + while True: + await asyncio.sleep(STATUS_UPDATE_INTERVAL) + visit_id_count = len(self.current_tasks.keys()) + task_count = 0 + for task_list in self.current_tasks.values(): + for task in task_list: + if not task.done(): + task_count += 1 + self.status_queue.put(task_count) + self.logger.debug( + ( + "StorageController status: There are currently %d scheduled tasks " + "for %d visit_ids" + ), + task_count, + visit_id_count, + ) async def shutdown(self) -> None: await self.structured_storage.flush_cache() @@ -192,49 +182,73 @@ async def shutdown(self) -> None: await self.unstructured_storage.flush_cache() await self.unstructured_storage.shutdown() - def should_shutdown(self) -> bool: - """Return `True` if the listener has received a shutdown signal - Sets `self._relaxed` and `self.shutdown_flag` - `self._relaxed means this shutdown is - happening after all visits have completed and - all data can be seen as complete - """ - if not self.shutdown_queue.empty(): - _, relaxed = self.shutdown_queue.get() - self._relaxed = relaxed - self._shutdown_flag = True - self.logger.info("Received shutdown signal!") - return True - return False - - async def save_batch_if_past_timeout(self) -> None: + async def should_shutdown(self) -> None: + """Returns when we should shut down""" + + while self.shutdown_queue.empty(): + await asyncio.sleep(STATUS_UPDATE_INTERVAL) + _, relaxed = self.shutdown_queue.get() + self._relaxed = relaxed + self._shutdown_flag = True + self.logger.info("Received shutdown signal!") + + async def save_batch_if_past_timeout(self) -> NoReturn: """Save the current batch of records if no new data has been received. If we aren't receiving new data for this batch we commit early regardless of the current batch size.""" - if self._last_record_received is None: - return - if time.time() - self._last_record_received < BATCH_COMMIT_TIMEOUT: - return - self.logger.debug( - "Saving current records since no new data has " - "been written for %d seconds." % (time.time() - self._last_record_received) - ) - await self.drain_queue() - self._last_record_received = None + while True: + if self._last_record_received is None: + await asyncio.sleep(BATCH_COMMIT_TIMEOUT) + continue + + time_until_timeout = ( + time.time() - self._last_record_received - BATCH_COMMIT_TIMEOUT + ) + if time_until_timeout > 0: + await asyncio.sleep(time_until_timeout) + continue + + self.logger.debug( + "Saving current records since no new data has " + "been written for %d seconds." + % (time.time() - self._last_record_received) + ) + await self.structured_storage.flush_cache() + if self.unstructured_storage: + await self.unstructured_storage.flush_cache() + self._last_record_received = None async def finish_tasks(self) -> None: + self.logger.info("Awaiting unfinished tasks before shutting down") for visit_id, tasks in self.current_tasks.items(): + self.logger.debug("Awaiting tasks for visit_id %d", visit_id) for task in tasks: await task async def _run(self) -> None: - await self.startup() - while not self.should_shutdown(): - self.update_status_queue() - await self.save_batch_if_past_timeout() - await self.poll_queue() - await self.drain_queue() + server: asyncio.AbstractServer = await asyncio.start_server( + self.handler, "localhost", 0, family=socket.AF_INET + ) + sockets = server.sockets + assert sockets is not None + assert len(sockets) == 1 + socketname = sockets[0].getsockname() + self.status_queue.put(socketname) + status_queue_update = asyncio.create_task( + self.update_status_queue(), name="StatusQueue" + ) + timeout_check = asyncio.create_task( + self.save_batch_if_past_timeout(), name="TimeoutCheck" + ) + # Blocks until we should shutdown + await self.should_shutdown() + + server.close() + status_queue_update.cancel() + timeout_check.cancel() + await server.wait_closed() + await self.finish_tasks() await self.shutdown() @@ -295,13 +309,13 @@ def save_configuration(self, openwpm_version: str, browser_version: str) -> None def launch(self) -> None: """Starts the data aggregator""" - self.listener_process = Process( + self.storage_controller = Process( name="StorageController", target=StorageController.run, args=(self.aggregator,), ) - self.listener_process.daemon = True - self.listener_process.start() + self.storage_controller.daemon = True + self.storage_controller.start() self.listener_address = self.status_queue.get() @@ -321,20 +335,15 @@ def get_new_completed_visits(self) -> List[Tuple[int, bool]]: def shutdown(self, relaxed: bool = True) -> None: """ Terminate the aggregator listener process""" - assert isinstance(self.listener_process, Process) - self.logger.debug( - "Sending the shutdown signal to the %s listener process..." - % type(self).__name__ - ) + assert isinstance(self.storage_controller, Process) + self.logger.debug("Sending the shutdown signal to the Storage Controller...") self.shutdown_queue.put((SHUTDOWN_SIGNAL, relaxed)) start_time = time.time() - self.listener_process.join(300) + self.storage_controller.join(300) self.logger.debug( "%s took %s seconds to close." % (type(self).__name__, str(time.time() - start_time)) ) - self.listener_address = None - self.listener_process = None def get_most_recent_status(self) -> int: """Return the most recent queue size sent from the listener process""" diff --git a/demo.py b/demo.py index 3b65d7000..7c468510e 100644 --- a/demo.py +++ b/demo.py @@ -34,11 +34,11 @@ # Launch only browser 0 headless -browser_params[0]["display_mode"] = "headless" +# browser_params[0]["display_mode"] = "headless" # Update TaskManager configuration (use this for crawl-wide settings) -manager_params["data_directory"] = "~/Desktop/" -manager_params["log_directory"] = "~/Desktop/" +manager_params["data_directory"] = "./datadir/" +manager_params["log_directory"] = "./datadir/" logging_params = {"log_level_console": logging.DEBUG} diff --git a/environment.yaml b/environment.yaml index 0ce0eb35f..ca187bcb9 100644 --- a/environment.yaml +++ b/environment.yaml @@ -5,7 +5,7 @@ dependencies: - autopep8=1.5.4 - beautifulsoup4=4.9.3 - click=7.1.2 -- codecov=2.1.9 +- codecov=2.1.10 - dill=0.3.2 - flake8=3.8.4 - geckodriver=0.27.0 @@ -13,7 +13,7 @@ dependencies: - leveldb=1.22 - localstack=0.11.1.1 - multiprocess=0.70.10 -- mypy=0.782 +- mypy=0.790 - nodejs=14.13.1 - pandas=1.1.3 - pillow=7.2.0 @@ -40,6 +40,6 @@ dependencies: - flask-cors==3.0.9 - jsonschema==3.2.0 - moto-ext==1.3.15.15 - - plyvel==1.2.0 + - plyvel==1.3.0 - subprocess32==3.5.4 name: openwpm From c63ae828cbca517d482b5c6859bbbc3cab41d87e Mon Sep 17 00:00:00 2001 From: vringar Date: Tue, 13 Oct 2020 22:36:40 +0200 Subject: [PATCH 019/139] It works --- automation/SocketInterface.py | 2 ++ automation/storage/storage_controller.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/automation/SocketInterface.py b/automation/SocketInterface.py index 7a020d709..a12a546f2 100644 --- a/automation/SocketInterface.py +++ b/automation/SocketInterface.py @@ -177,6 +177,8 @@ async def get_n_bytes_from_reader(reader: asyncio.StreamReader, n: int) -> bytes except IncompleteReadError as e: b += e.partial n -= len(e.partial) + if reader.at_eof(): + raise IOError("Socket Connection closed") def _parse(serialization: bytes, msg: bytes) -> Any: diff --git a/automation/storage/storage_controller.py b/automation/storage/storage_controller.py index 99bfb93ee..c8bbad325 100644 --- a/automation/storage/storage_controller.py +++ b/automation/storage/storage_controller.py @@ -71,10 +71,18 @@ def __init__( async def handler( self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter - ) -> NoReturn: + ) -> None: """Created for every new connection to the Server""" + self.logger.debug("Initializing new handler") while True: - record: Tuple[str, Any] = await get_message_from_reader(reader) + try: + record: Tuple[str, Any] = await get_message_from_reader(reader) + except IOError as e: + self.logger.debug( + "Terminatin handler, because the underlying socket closed", + exc_info=e, + ) + break if len(record) != 2: self.logger.error("Query is not the correct length %s", repr(record)) From 8177ed4b4b0dd72ce3995c8a57e6fb6fce2cc923 Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 14 Oct 2020 18:34:01 +0200 Subject: [PATCH 020/139] Started working on arrow_provider --- automation/storage/arrow_storage.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/automation/storage/arrow_storage.py b/automation/storage/arrow_storage.py index 07d5c8f76..339cc8d77 100644 --- a/automation/storage/arrow_storage.py +++ b/automation/storage/arrow_storage.py @@ -37,6 +37,7 @@ def factory_function() -> DefaultDict[TableName, List[Dict[str, Any]]]: # Record batches by TableName self._batches: DefaultDict[TableName, List[pa.RecordBatch]] = DefaultDict(list) + self._signals = list() self._instance_id = random.getrandbits(32) @@ -56,6 +57,10 @@ def _create_batch(self, visit_id: VisitId) -> None: """Create record batches for all records from `visit_id`""" if visit_id not in self._records: # The batch for this `visit_id` was already created, skip + self.logger.error( + "Trying to create batch for visit_id %d" "when one was already created", + visit_id, + ) return for table_name, data in self._records[visit_id].items(): try: From 50e1539e84c090a78e6970eda39f8d74835eb7e2 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 16 Oct 2020 11:35:00 +0200 Subject: [PATCH 021/139] Implemented ArrowProvider --- automation/storage/arrow_storage.py | 46 +++++++++++++++---- automation/storage/in_memory_storage.py | 18 ++++++-- automation/storage/storage_controller.py | 4 +- automation/types.py | 2 + .../test_storage_controller.py | 1 + 5 files changed, 56 insertions(+), 15 deletions(-) diff --git a/automation/storage/arrow_storage.py b/automation/storage/arrow_storage.py index 339cc8d77..583f96dda 100644 --- a/automation/storage/arrow_storage.py +++ b/automation/storage/arrow_storage.py @@ -1,3 +1,4 @@ +import asyncio import logging import random from abc import abstractmethod @@ -37,7 +38,7 @@ def factory_function() -> DefaultDict[TableName, List[Dict[str, Any]]]: # Record batches by TableName self._batches: DefaultDict[TableName, List[pa.RecordBatch]] = DefaultDict(list) - self._signals = list() + self.storing_condition = asyncio.Condition() self._instance_id = random.getrandbits(32) @@ -87,10 +88,6 @@ def _create_batch(self, visit_id: VisitId) -> None: del self._records[visit_id] - @abstractmethod - async def write_table(self, table: Table) -> None: - """Write out the table to persistent storage""" - def _is_cache_full(self) -> bool: for batches in self._batches.values(): if len(batches) > CACHE_SIZE: @@ -103,11 +100,40 @@ async def finalize_visit_id( if interrupted: await self.store_record(INCOMPLETE_VISITS, visit_id, {"visit_id": visit_id}) - self._create_batch(visit_id) - if self._is_cache_full(): - await self.flush_cache() + # This code is pretty tricky as there are a number of things going on + # 1. No finalize_visit_id shoudl return unless the visit has been saved to storage + # 2. No new batches should be created while saving out all the batches + async with self.storing_condition: + self._create_batch(visit_id) - raise NotImplementedError() + if self._is_cache_full(): + await self.flush_cache(self.storing_condition) + + await self.storing_condition.wait() - async def flush_cache(self) -> None: raise NotImplementedError() + + @abstractmethod + async def write_table(self, table_name: TableName, table: Table) -> None: + """Write out the table to persistent storage + This should only return once it's actually saved out + """ + + async def flush_cache(self, cond: asyncio.Condition = None) -> None: + """We need to hack around the fact that asyncio has no reentrant lock + and which prevents us from creating a reentrant condition + So we either grab the storing condition ourselves or the caller needs + to pass us the locked storing_condition + """ + assert cond is None or cond.locked() + _cond = cond + if not _cond: + _cond = self.storing_condition + _cond.acquire() + for table_name, batches in self._batches.items(): + table = pa.Table.from_batches(batches) + await self.write_table(table_name, table) + _cond.notify_all() + + if cond is None: + _cond.release() diff --git a/automation/storage/in_memory_storage.py b/automation/storage/in_memory_storage.py index 5e689213b..90790bed5 100644 --- a/automation/storage/in_memory_storage.py +++ b/automation/storage/in_memory_storage.py @@ -1,7 +1,9 @@ +import logging from collections import defaultdict from typing import Any, DefaultDict, Dict, List, Tuple from multiprocess import Queue +from pyarrow import Table from automation.types import VisitId @@ -31,7 +33,8 @@ class MemoryStructuredProvider(StructuredStorageProvider): def __init__(self) -> None: super().__init__() self.queue = Queue() - self.handle = MemoryStructuredProviderHandle(self.queue) + self.handle = MemoryProviderHandle(self.queue) + self.logger = logging.getLogger("openwpm") async def flush_cache(self) -> None: pass @@ -50,7 +53,7 @@ async def shutdown(self) -> None: pass -class MemoryStructuredProviderHandle: +class MemoryProviderHandle: """ Call poll_queue to load all available data into the dict at self.storage @@ -97,4 +100,13 @@ async def shutdown(self) -> None: class MemoryArrowProvider(ArrowProvider): - ... + def __init__(self) -> None: + super().__init__() + self.queue = Queue() + self.handle = MemoryProviderHandle(self.queue) + + async def write_table(self, table_name: TableName, table: Table) -> None: + self.queue.put((table_name, table)) + + async def shutdown(self) -> None: + pass diff --git a/automation/storage/storage_controller.py b/automation/storage/storage_controller.py index c8bbad325..f6f2cbe0c 100644 --- a/automation/storage/storage_controller.py +++ b/automation/storage/storage_controller.py @@ -79,7 +79,7 @@ async def handler( record: Tuple[str, Any] = await get_message_from_reader(reader) except IOError as e: self.logger.debug( - "Terminatin handler, because the underlying socket closed", + "Terminating handler, because the underlying socket closed", exc_info=e, ) break @@ -275,7 +275,7 @@ def __init__( unstructured_storage: UnstructuredStorageProvider, ) -> None: - self.listener_address = None + self.listener_address: Optional[Tuple[str, int]] = None self.listener_process: Optional[Process] = None self.status_queue = Queue() self.completion_queue = Queue() diff --git a/automation/types.py b/automation/types.py index f62b0d80c..6f26eb749 100644 --- a/automation/types.py +++ b/automation/types.py @@ -1,5 +1,7 @@ from typing import Any, Dict, NewType +from multiprocess import Queue + BrowserParams = NewType("BrowserParams", Dict[str, Any]) ManagerParams = NewType("ManagerParams", Dict[str, Any]) VisitId = NewType("VisitId", int) diff --git a/test/storage_providers/test_storage_controller.py b/test/storage_providers/test_storage_controller.py index bae2e5861..caee3e2ab 100644 --- a/test/storage_providers/test_storage_controller.py +++ b/test/storage_providers/test_storage_controller.py @@ -11,6 +11,7 @@ def test_startup_and_shutdown() -> None: unstructured = MemoryUnstructuredProvider() agg_handle = StorageControllerHandle(structured, unstructured) agg_handle.launch() + assert agg_handle.listener_address is not None print(agg_handle.listener_address) cs = ClientSocket() cs.connect(*agg_handle.listener_address) From 9e1d67e976350c718d040eda81405d2142b43811 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 16 Oct 2020 15:28:23 +0200 Subject: [PATCH 022/139] Added logger fixture --- automation/MPLogger.py | 5 ++- automation/storage/in_memory_storage.py | 3 ++ automation/storage/storage_controller.py | 37 +++++++++++++++---- test/openwpmtest.py | 2 +- .../test_memory_storage_provider.py | 1 - .../test_storage_controller.py | 25 +++++++++++-- 6 files changed, 58 insertions(+), 15 deletions(-) diff --git a/automation/MPLogger.py b/automation/MPLogger.py index 3b7256c1d..d7d6a1e84 100644 --- a/automation/MPLogger.py +++ b/automation/MPLogger.py @@ -18,6 +18,7 @@ from .Commands.utils.webdriver_utils import parse_neterror from .SocketInterface import ServerSocket +from .types import ManagerParams pickling_support.install() @@ -99,12 +100,12 @@ class MPLogger(object): def __init__( self, log_file, - crawl_context=None, + crawl_context: ManagerParams = None, log_level_console=logging.INFO, log_level_file=logging.DEBUG, log_level_sentry_breadcrumb=logging.DEBUG, log_level_sentry_event=logging.ERROR, - ): + ) -> None: self._crawl_context = crawl_context self._log_level_console = log_level_console self._log_level_file = log_level_file diff --git a/automation/storage/in_memory_storage.py b/automation/storage/in_memory_storage.py index 90790bed5..ed9791a1c 100644 --- a/automation/storage/in_memory_storage.py +++ b/automation/storage/in_memory_storage.py @@ -42,6 +42,9 @@ async def flush_cache(self) -> None: async def store_record( self, table: TableName, visit_id: VisitId, record: Dict[str, Any] ) -> None: + self.logger.debug( + "Saving into table %s for visit_id %d record %r", table, visit_id, record + ) self.queue.put((table, record)) async def finalize_visit_id( diff --git a/automation/storage/storage_controller.py b/automation/storage/storage_controller.py index f6f2cbe0c..50e325443 100644 --- a/automation/storage/storage_controller.py +++ b/automation/storage/storage_controller.py @@ -7,7 +7,7 @@ import threading import time from collections import defaultdict -from typing import Any, DefaultDict, Dict, List, NoReturn, Optional, Tuple +from typing import Any, DefaultDict, Dict, List, Literal, NoReturn, Optional, Tuple from multiprocess import Queue @@ -25,6 +25,7 @@ RECORD_TYPE_META = "meta_information" ACTION_TYPE_FINALIZE = "Finalize" ACTION_TYPE_INITIALIZE = "Initialize" + RECORD_TYPE_CREATE = "create_table" STATUS_TIMEOUT = 120 # seconds SHUTDOWN_SIGNAL = "SHUTDOWN" @@ -69,6 +70,20 @@ def __init__( self.unstructured_storage = unstructured_storage self._last_record_received: Optional[float] = None + async def _handler( + self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + ) -> None: + """This is a dirty hack around the fact that exceptions get swallowed by the asyncio.Server + and the coroutine just dies without any message. + By having this function be a wrapper we at least get a log message + """ + try: + await self.handler(reader, writer) + except Exception as e: + self.logger.error( + "An exception occured while listening for data", exc_info=e + ) + async def handler( self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter ) -> None: @@ -118,11 +133,18 @@ async def handler( ) continue - if record_type == RECORD_TYPE_META: - await self._handle_meta(data) + if not "visit_id" in data: + self.logger.error( + "Skipping record: No visit_id contained in record %r", record + ) continue visit_id = VisitId(data["visit_id"]) + + if record_type == RECORD_TYPE_META: + await self._handle_meta(visit_id, data) + continue + table_name = TableName(record_type) self.current_tasks[visit_id].append( @@ -133,7 +155,7 @@ async def handler( ) ) - async def _handle_meta(self, data: Dict[str, Any]) -> None: + async def _handle_meta(self, visit_id: VisitId, data: Dict[str, Any]) -> None: """ Messages for the table RECORD_TYPE_SPECIAL are metainformation communicated to the aggregator @@ -141,9 +163,7 @@ async def _handle_meta(self, data: Dict[str, Any]) -> None: - finalize: A message sent by the extension to signal that a visit_id is complete. """ - visit_id = VisitId(data["visit_id"]) - action = data["action"] - + action: str = data["action"] if action == ACTION_TYPE_INITIALIZE: return elif action == ACTION_TYPE_FINALIZE: @@ -236,7 +256,7 @@ async def finish_tasks(self) -> None: async def _run(self) -> None: server: asyncio.AbstractServer = await asyncio.start_server( - self.handler, "localhost", 0, family=socket.AF_INET + self._handler, "localhost", 0, family=socket.AF_INET ) sockets = server.sockets assert sockets is not None @@ -261,6 +281,7 @@ async def _run(self) -> None: await self.shutdown() def run(self) -> None: + logging.getLogger("asyncio").setLevel(logging.WARNING) asyncio.run(self._run(), debug=True) diff --git a/test/openwpmtest.py b/test/openwpmtest.py index 987e91cc7..17f8affde 100644 --- a/test/openwpmtest.py +++ b/test/openwpmtest.py @@ -10,7 +10,7 @@ from . import utilities -class OpenWPMTest(object): +class OpenWPMTest: NUM_BROWSERS = 1 @pytest.fixture(autouse=True) diff --git a/test/storage_providers/test_memory_storage_provider.py b/test/storage_providers/test_memory_storage_provider.py index a53edc797..21a5e5a57 100644 --- a/test/storage_providers/test_memory_storage_provider.py +++ b/test/storage_providers/test_memory_storage_provider.py @@ -54,7 +54,6 @@ def pytest_generate_tests(metafunc: Any) -> Any: items = scenario[1].items() argnames = [x[0] for x in items] argvalues.append([x[1] for x in items]) - print("metafunc called") metafunc.parametrize(argnames, argvalues, ids=idlist, scope="class", indirect=True) diff --git a/test/storage_providers/test_storage_controller.py b/test/storage_providers/test_storage_controller.py index caee3e2ab..0585d0d28 100644 --- a/test/storage_providers/test_storage_controller.py +++ b/test/storage_providers/test_storage_controller.py @@ -1,3 +1,9 @@ +import logging +import time + +import pytest + +from automation.MPLogger import MPLogger from automation.SocketInterface import ClientSocket from automation.storage.in_memory_storage import ( MemoryStructuredProvider, @@ -6,16 +12,29 @@ from automation.storage.storage_controller import StorageControllerHandle -def test_startup_and_shutdown() -> None: +@pytest.fixture(scope="session") +def logger() -> MPLogger: + """PyTest only captures logging events in the Main Process + so we need to log everything to console to have it show + up in our tests + """ + return MPLogger( + "/dev/null", + None, # We have no manager params here + log_level_console=logging.DEBUG, + ) + + +def test_startup_and_shutdown(logger: MPLogger) -> None: structured = MemoryStructuredProvider() unstructured = MemoryUnstructuredProvider() agg_handle = StorageControllerHandle(structured, unstructured) agg_handle.launch() assert agg_handle.listener_address is not None - print(agg_handle.listener_address) cs = ClientSocket() cs.connect(*agg_handle.listener_address) - cs.send(("test", {"asd": "dfg"})) + cs.send(("test", {"visit_id": 1, "asd": "dfg"})) + time.sleep(5) agg_handle.shutdown() handle = structured.handle handle.poll_queue() From fe2e977c5c91cd47986d309d108c828957d5bf71 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 16 Oct 2020 15:36:03 +0200 Subject: [PATCH 023/139] Fixed test_storage_controller --- test/storage_providers/test_storage_controller.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/storage_providers/test_storage_controller.py b/test/storage_providers/test_storage_controller.py index 0585d0d28..8c677cfc0 100644 --- a/test/storage_providers/test_storage_controller.py +++ b/test/storage_providers/test_storage_controller.py @@ -26,6 +26,7 @@ def logger() -> MPLogger: def test_startup_and_shutdown(logger: MPLogger) -> None: + data = {"visit_id": 1, "asd": "dfg"} structured = MemoryStructuredProvider() unstructured = MemoryUnstructuredProvider() agg_handle = StorageControllerHandle(structured, unstructured) @@ -33,9 +34,8 @@ def test_startup_and_shutdown(logger: MPLogger) -> None: assert agg_handle.listener_address is not None cs = ClientSocket() cs.connect(*agg_handle.listener_address) - cs.send(("test", {"visit_id": 1, "asd": "dfg"})) - time.sleep(5) + cs.send(("test", data)) agg_handle.shutdown() handle = structured.handle handle.poll_queue() - assert handle.storage["test"] == {"asd": "dfg"} + assert handle.storage["test"] == [data] From 157ee240e5608b8516305e89d8606e298487b85f Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 16 Oct 2020 16:35:26 +0200 Subject: [PATCH 024/139] Fixing OpenWPMTest.visit() --- test/conftest.py | 11 ++++------- test/openwpmtest.py | 10 +++++++++- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 9cf05681a..b6cc57213 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -31,10 +31,7 @@ def prepare_test_setup(request): print("Starting local_http_server") server, server_thread = utilities.start_server() - - def local_http_server_stop(): - print("\nClosing server thread...") - server.shutdown() - server_thread.join() - - request.addfinalizer(local_http_server_stop) + yield + print("\nClosing server thread...") + server.shutdown() + server_thread.join() diff --git a/test/openwpmtest.py b/test/openwpmtest.py index 17f8affde..7338f0bd4 100644 --- a/test/openwpmtest.py +++ b/test/openwpmtest.py @@ -1,3 +1,4 @@ +import logging import os from os.path import isfile, join from typing import List, Tuple @@ -5,6 +6,7 @@ import pytest from automation import TaskManager +from automation.storage.sql_provider import SqlLiteStorageProvider from automation.types import BrowserParams, ManagerParams from . import utilities @@ -25,7 +27,13 @@ def set_tmpdir(self, tmpdir): def visit(self, page_url, data_dir="", sleep_after=0): """Visit a test page with the given parameters.""" manager_params, browser_params = self.get_config(data_dir) - manager = TaskManager.TaskManager(manager_params, browser_params) + structured_provider = SqlLiteStorageProvider(manager_params["db"]) + manager = TaskManager.TaskManager( + manager_params, + browser_params, + structured_provider, + logger_kwargs={"log_level_console": logging.DEBUG}, + ) if not page_url.startswith("http"): page_url = utilities.BASE_TEST_URL + page_url manager.get(url=page_url, sleep=sleep_after) From 52c4ff6161ac4e6d244151c88c1af54251b89755 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 16 Oct 2020 16:37:35 +0200 Subject: [PATCH 025/139] Moved test/storage_providers to test/storage --- test/{storage_providers => storage}/__init__.py | 0 .../test_memory_storage_provider.py | 0 test/{storage_providers => storage}/test_storage_controller.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename test/{storage_providers => storage}/__init__.py (100%) rename test/{storage_providers => storage}/test_memory_storage_provider.py (100%) rename test/{storage_providers => storage}/test_storage_controller.py (100%) diff --git a/test/storage_providers/__init__.py b/test/storage/__init__.py similarity index 100% rename from test/storage_providers/__init__.py rename to test/storage/__init__.py diff --git a/test/storage_providers/test_memory_storage_provider.py b/test/storage/test_memory_storage_provider.py similarity index 100% rename from test/storage_providers/test_memory_storage_provider.py rename to test/storage/test_memory_storage_provider.py diff --git a/test/storage_providers/test_storage_controller.py b/test/storage/test_storage_controller.py similarity index 100% rename from test/storage_providers/test_storage_controller.py rename to test/storage/test_storage_controller.py From 1098727ab8b0d9457e37a7bab3b7209c9ca33a01 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 19 Oct 2020 14:01:24 +0200 Subject: [PATCH 026/139] Fixing up tests --- test/conftest.py | 19 +++++++++++++++++++ test/storage/test_memory_storage_provider.py | 10 +++++++++- test/test_storage_vectors.py | 8 ++------ 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index b6cc57213..9f804b03b 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,8 +1,12 @@ import os import subprocess +from typing import List, Tuple import pytest +from automation import TaskManager +from automation.types import BrowserParams, ManagerParams + from . import utilities EXTENSION_DIR = os.path.join( @@ -35,3 +39,18 @@ def prepare_test_setup(request): print("\nClosing server thread...") server.shutdown() server_thread.join() + + +@pytest.fixture() +def default_params(num_browsers, tmpdir) -> Tuple[ManagerParams, List[BrowserParams]]: + """Just a simple wrapper around TaskManager.load_default_params""" + data_dir = tmpdir + manager_params, browser_params = TaskManager.load_default_params(num_browsers) + manager_params["data_directory"] = data_dir + manager_params["log_directory"] = data_dir + for i in range(num_browsers): + browser_params[i]["display_mode"] = display_mode + manager_params["db"] = join( + manager_params["data_directory"], manager_params["database_name"] + ) + return manager_params, browser_params diff --git a/test/storage/test_memory_storage_provider.py b/test/storage/test_memory_storage_provider.py index 21a5e5a57..65f71f871 100644 --- a/test/storage/test_memory_storage_provider.py +++ b/test/storage/test_memory_storage_provider.py @@ -64,13 +64,21 @@ class TestStructuredStorageProvider(OpenWPMTest): async def test_basic_access( self, structured_provider: StructuredStorageProvider ) -> None: + data = { + "visit_id": 2, + "browser_id": 3, + "instance_id": 4, + "site_url": "https://example.com", + } + await structured_provider.store_record( - TableName("test"), VisitId(2), {"visit_id": 2, "data": "test"} + TableName("site_visits"), VisitId(2), data ) await structured_provider.finalize_visit_id(VisitId(2)) await structured_provider.flush_cache() +@pytest.mark.asyncio class TestUnstructuredStorageProvide(OpenWPMTest): scenarios: List[Tuple[str, Dict[str, Any]]] = [(memory_unstructured, {})] diff --git a/test/test_storage_vectors.py b/test/test_storage_vectors.py index b5ae5bbc6..8fc582f8b 100644 --- a/test/test_storage_vectors.py +++ b/test/test_storage_vectors.py @@ -28,15 +28,11 @@ class TestStorageVectors(OpenWPMTest): on to check for completeness and correctness. """ - def get_config(self, data_dir=""): - return self.get_test_config(data_dir) - - def test_js_profile_cookies(self): + def test_js_profile_cookies(self, default_params): """ Check that profile cookies set by JS are saved """ # Run the test crawl - manager_params, browser_params = self.get_config() browser_params[0]["cookie_instrument"] = True - manager = TaskManager.TaskManager(manager_params, browser_params) + manager = TaskManager.TaskManager(*default_params) url = utilities.BASE_TEST_URL + "/js_cookie.html" cs = CommandSequence.CommandSequence(url) cs.get(sleep=3, timeout=120) From d7e7268a51c41532fc3c42306f367da121a3570f Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 16 Nov 2020 12:42:21 +0100 Subject: [PATCH 027/139] Moved automation to openwpm --- {automation => openwpm}/BrowserManager.py | 0 {automation => openwpm}/CommandSequence.py | 0 {automation => openwpm}/Commands/Types.py | 0 {automation => openwpm}/Commands/__init__.py | 0 .../Commands/browser_commands.py | 0 .../Commands/command_executor.py | 0 .../Commands/profile_commands.py | 0 {automation => openwpm}/Commands/utils/XPathUtil.py | 0 {automation => openwpm}/Commands/utils/__init__.py | 0 .../Commands/utils/file_utils.py | 0 .../Commands/utils/firefox_profile.py | 0 .../Commands/utils/webdriver_utils.py | 0 .../DataAggregator/BaseAggregator.py | 0 .../DataAggregator/LocalAggregator.py | 0 .../DataAggregator/S3Aggregator.py | 0 {automation => openwpm}/DataAggregator/__init__.py | 0 {automation => openwpm}/DeployBrowsers/__init__.py | 0 .../DeployBrowsers/configure_firefox.py | 0 .../DeployBrowsers/deploy_browser.py | 0 .../DeployBrowsers/deploy_firefox.py | 0 .../firefox_extensions/disconnect-5.18.21.xpi | Bin .../ghostery/ghostery-7.3.3.7.xpi | Bin .../firefox_extensions/ghostery/storage.js | 0 .../https_everywhere-2017.10.4.xpi | Bin .../firefox_extensions/ublock_origin/storage.js | 0 .../ublock_origin/ublock_origin-1.14.10.xpi | Bin .../DeployBrowsers/screen_resolutions.txt | 0 .../DeployBrowsers/selenium_firefox.py | 0 .../DeployBrowsers/user_agent_strings.txt | 0 {automation => openwpm}/Errors.py | 0 {automation => openwpm}/Extension/firefox/.npmrc | 0 {automation => openwpm}/Extension/firefox/LICENSE | 0 {automation => openwpm}/Extension/firefox/README.md | 0 .../Extension/firefox/content.js/index.js | 0 .../firefox/feature.js/callstack-instrument.js | 0 .../Extension/firefox/feature.js/index.js | 0 .../Extension/firefox/feature.js/loggingdb.js | 0 .../Extension/firefox/feature.js/socket.js | 0 .../Extension/firefox/package-lock.json | 0 .../Extension/firefox/package.json | 0 .../Extension/firefox/src/manifest.json | 0 .../firefox/src/privileged/profileDirIO/api.js | 0 .../firefox/src/privileged/profileDirIO/schema.json | 0 .../Extension/firefox/src/privileged/sockets/api.js | 0 .../firefox/src/privileged/sockets/bufferpack.js | 0 .../firefox/src/privileged/sockets/schema.json | 0 .../privileged/stackDump/OpenWPMStackDumpChild.jsm | 0 .../privileged/stackDump/OpenWPMStackDumpParent.jsm | 0 .../firefox/src/privileged/stackDump/api.js | 0 .../firefox/src/privileged/stackDump/schema.json | 0 .../Extension/firefox/web-ext-config.js | 0 .../Extension/firefox/webpack.config.js | 0 .../Extension/webext-instrumentation/.auditignore | 0 .../Extension/webext-instrumentation/.editorconfig | 0 .../Extension/webext-instrumentation/.gitignore | 0 .../Extension/webext-instrumentation/.npmignore | 0 .../Extension/webext-instrumentation/.npmrc | 0 .../webext-instrumentation/.prettierignore | 0 .../Extension/webext-instrumentation/.publishrc | 0 .../webext-instrumentation/.versionrc.json | 0 .../Extension/webext-instrumentation/README.md | 0 .../webext-instrumentation/package-lock.json | 0 .../Extension/webext-instrumentation/package.json | 0 .../src/background/cookie-instrument.ts | 0 .../src/background/dns-instrument.ts | 0 .../src/background/http-instrument.ts | 0 .../src/background/javascript-instrument.ts | 0 .../src/background/navigation-instrument.ts | 0 .../content/javascript-instrument-content-scope.ts | 0 .../src/content/javascript-instrument-page-scope.ts | 0 .../Extension/webext-instrumentation/src/index.ts | 0 .../src/lib/extension-session-event-ordinal.ts | 0 .../src/lib/extension-session-uuid.ts | 0 .../src/lib/http-post-parser.ts | 0 .../src/lib/js-instruments.ts | 0 .../webext-instrumentation/src/lib/number.spec.ts | 0 .../webext-instrumentation/src/lib/number.ts | 0 .../src/lib/pending-navigation.ts | 0 .../src/lib/pending-request.ts | 0 .../src/lib/pending-response.ts | 0 .../src/lib/response-body-listener.ts | 0 .../webext-instrumentation/src/lib/sha256.ts | 0 .../webext-instrumentation/src/lib/string-utils.ts | 0 .../webext-instrumentation/src/lib/uuid.ts | 0 .../Extension/webext-instrumentation/src/schema.ts | 0 .../types/browser-web-navigation-event-details.ts | 0 .../src/types/browser-web-request-event-details.ts | 0 .../src/types/javascript-instrument.d.ts | 0 .../src/types/window.unimplemented.d.ts | 0 .../Extension/webext-instrumentation/tsconfig.json | 0 .../webext-instrumentation/tsconfig.module.json | 0 .../Extension/webext-instrumentation/tslint.json | 0 {automation => openwpm}/MPLogger.py | 0 {automation => openwpm}/SocketInterface.py | 0 {automation => openwpm}/TaskManager.py | 0 {automation => openwpm}/__init__.py | 0 {automation => openwpm}/default_browser_params.json | 0 {automation => openwpm}/default_manager_params.json | 0 {automation => openwpm}/js_instrumentation.py | 0 .../fingerprinting.json | 0 {automation => openwpm}/storage/__init__.py | 0 {automation => openwpm}/storage/arrow_storage.py | 0 .../storage/in_memory_storage.py | 0 {automation => openwpm}/storage/leveldb.py | 0 {automation => openwpm}/storage/local_storage.py | 0 {automation => openwpm}/storage/parquet_schema.py | 0 {automation => openwpm}/storage/s3_storage.py | 0 {automation => openwpm}/storage/schema.sql | 0 {automation => openwpm}/storage/sql_provider.py | 0 .../storage/storage_controller.py | 0 .../storage/storage_providers.py | 0 {automation => openwpm}/types.py | 0 {automation => openwpm}/utilities/Cookie.py | 0 {automation => openwpm}/utilities/__init__.py | 0 .../utilities/build_cookie_table.py | 0 {automation => openwpm}/utilities/db_utils.py | 0 .../utilities/multiprocess_utils.py | 0 {automation => openwpm}/utilities/platform_utils.py | 0 {automation => openwpm}/utilities/rediswq.py | 0 119 files changed, 0 insertions(+), 0 deletions(-) rename {automation => openwpm}/BrowserManager.py (100%) rename {automation => openwpm}/CommandSequence.py (100%) rename {automation => openwpm}/Commands/Types.py (100%) rename {automation => openwpm}/Commands/__init__.py (100%) rename {automation => openwpm}/Commands/browser_commands.py (100%) rename {automation => openwpm}/Commands/command_executor.py (100%) rename {automation => openwpm}/Commands/profile_commands.py (100%) rename {automation => openwpm}/Commands/utils/XPathUtil.py (100%) rename {automation => openwpm}/Commands/utils/__init__.py (100%) rename {automation => openwpm}/Commands/utils/file_utils.py (100%) rename {automation => openwpm}/Commands/utils/firefox_profile.py (100%) rename {automation => openwpm}/Commands/utils/webdriver_utils.py (100%) rename {automation => openwpm}/DataAggregator/BaseAggregator.py (100%) rename {automation => openwpm}/DataAggregator/LocalAggregator.py (100%) rename {automation => openwpm}/DataAggregator/S3Aggregator.py (100%) rename {automation => openwpm}/DataAggregator/__init__.py (100%) rename {automation => openwpm}/DeployBrowsers/__init__.py (100%) rename {automation => openwpm}/DeployBrowsers/configure_firefox.py (100%) rename {automation => openwpm}/DeployBrowsers/deploy_browser.py (100%) rename {automation => openwpm}/DeployBrowsers/deploy_firefox.py (100%) rename {automation => openwpm}/DeployBrowsers/firefox_extensions/disconnect-5.18.21.xpi (100%) rename {automation => openwpm}/DeployBrowsers/firefox_extensions/ghostery/ghostery-7.3.3.7.xpi (100%) rename {automation => openwpm}/DeployBrowsers/firefox_extensions/ghostery/storage.js (100%) rename {automation => openwpm}/DeployBrowsers/firefox_extensions/https_everywhere-2017.10.4.xpi (100%) rename {automation => openwpm}/DeployBrowsers/firefox_extensions/ublock_origin/storage.js (100%) rename {automation => openwpm}/DeployBrowsers/firefox_extensions/ublock_origin/ublock_origin-1.14.10.xpi (100%) rename {automation => openwpm}/DeployBrowsers/screen_resolutions.txt (100%) rename {automation => openwpm}/DeployBrowsers/selenium_firefox.py (100%) rename {automation => openwpm}/DeployBrowsers/user_agent_strings.txt (100%) rename {automation => openwpm}/Errors.py (100%) rename {automation => openwpm}/Extension/firefox/.npmrc (100%) rename {automation => openwpm}/Extension/firefox/LICENSE (100%) rename {automation => openwpm}/Extension/firefox/README.md (100%) rename {automation => openwpm}/Extension/firefox/content.js/index.js (100%) rename {automation => openwpm}/Extension/firefox/feature.js/callstack-instrument.js (100%) rename {automation => openwpm}/Extension/firefox/feature.js/index.js (100%) rename {automation => openwpm}/Extension/firefox/feature.js/loggingdb.js (100%) rename {automation => openwpm}/Extension/firefox/feature.js/socket.js (100%) rename {automation => openwpm}/Extension/firefox/package-lock.json (100%) rename {automation => openwpm}/Extension/firefox/package.json (100%) rename {automation => openwpm}/Extension/firefox/src/manifest.json (100%) rename {automation => openwpm}/Extension/firefox/src/privileged/profileDirIO/api.js (100%) rename {automation => openwpm}/Extension/firefox/src/privileged/profileDirIO/schema.json (100%) rename {automation => openwpm}/Extension/firefox/src/privileged/sockets/api.js (100%) rename {automation => openwpm}/Extension/firefox/src/privileged/sockets/bufferpack.js (100%) rename {automation => openwpm}/Extension/firefox/src/privileged/sockets/schema.json (100%) rename {automation => openwpm}/Extension/firefox/src/privileged/stackDump/OpenWPMStackDumpChild.jsm (100%) rename {automation => openwpm}/Extension/firefox/src/privileged/stackDump/OpenWPMStackDumpParent.jsm (100%) rename {automation => openwpm}/Extension/firefox/src/privileged/stackDump/api.js (100%) rename {automation => openwpm}/Extension/firefox/src/privileged/stackDump/schema.json (100%) rename {automation => openwpm}/Extension/firefox/web-ext-config.js (100%) rename {automation => openwpm}/Extension/firefox/webpack.config.js (100%) rename {automation => openwpm}/Extension/webext-instrumentation/.auditignore (100%) rename {automation => openwpm}/Extension/webext-instrumentation/.editorconfig (100%) rename {automation => openwpm}/Extension/webext-instrumentation/.gitignore (100%) rename {automation => openwpm}/Extension/webext-instrumentation/.npmignore (100%) rename {automation => openwpm}/Extension/webext-instrumentation/.npmrc (100%) rename {automation => openwpm}/Extension/webext-instrumentation/.prettierignore (100%) rename {automation => openwpm}/Extension/webext-instrumentation/.publishrc (100%) rename {automation => openwpm}/Extension/webext-instrumentation/.versionrc.json (100%) rename {automation => openwpm}/Extension/webext-instrumentation/README.md (100%) rename {automation => openwpm}/Extension/webext-instrumentation/package-lock.json (100%) rename {automation => openwpm}/Extension/webext-instrumentation/package.json (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/background/cookie-instrument.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/background/dns-instrument.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/background/http-instrument.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/background/javascript-instrument.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/background/navigation-instrument.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/content/javascript-instrument-content-scope.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/content/javascript-instrument-page-scope.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/index.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/lib/extension-session-event-ordinal.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/lib/extension-session-uuid.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/lib/http-post-parser.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/lib/js-instruments.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/lib/number.spec.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/lib/number.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/lib/pending-navigation.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/lib/pending-request.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/lib/pending-response.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/lib/response-body-listener.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/lib/sha256.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/lib/string-utils.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/lib/uuid.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/schema.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/types/browser-web-navigation-event-details.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/types/browser-web-request-event-details.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/types/javascript-instrument.d.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/src/types/window.unimplemented.d.ts (100%) rename {automation => openwpm}/Extension/webext-instrumentation/tsconfig.json (100%) rename {automation => openwpm}/Extension/webext-instrumentation/tsconfig.module.json (100%) rename {automation => openwpm}/Extension/webext-instrumentation/tslint.json (100%) rename {automation => openwpm}/MPLogger.py (100%) rename {automation => openwpm}/SocketInterface.py (100%) rename {automation => openwpm}/TaskManager.py (100%) rename {automation => openwpm}/__init__.py (100%) rename {automation => openwpm}/default_browser_params.json (100%) rename {automation => openwpm}/default_manager_params.json (100%) rename {automation => openwpm}/js_instrumentation.py (100%) rename {automation => openwpm}/js_instrumentation_collections/fingerprinting.json (100%) rename {automation => openwpm}/storage/__init__.py (100%) rename {automation => openwpm}/storage/arrow_storage.py (100%) rename {automation => openwpm}/storage/in_memory_storage.py (100%) rename {automation => openwpm}/storage/leveldb.py (100%) rename {automation => openwpm}/storage/local_storage.py (100%) rename {automation => openwpm}/storage/parquet_schema.py (100%) rename {automation => openwpm}/storage/s3_storage.py (100%) rename {automation => openwpm}/storage/schema.sql (100%) rename {automation => openwpm}/storage/sql_provider.py (100%) rename {automation => openwpm}/storage/storage_controller.py (100%) rename {automation => openwpm}/storage/storage_providers.py (100%) rename {automation => openwpm}/types.py (100%) rename {automation => openwpm}/utilities/Cookie.py (100%) rename {automation => openwpm}/utilities/__init__.py (100%) rename {automation => openwpm}/utilities/build_cookie_table.py (100%) rename {automation => openwpm}/utilities/db_utils.py (100%) rename {automation => openwpm}/utilities/multiprocess_utils.py (100%) rename {automation => openwpm}/utilities/platform_utils.py (100%) rename {automation => openwpm}/utilities/rediswq.py (100%) diff --git a/automation/BrowserManager.py b/openwpm/BrowserManager.py similarity index 100% rename from automation/BrowserManager.py rename to openwpm/BrowserManager.py diff --git a/automation/CommandSequence.py b/openwpm/CommandSequence.py similarity index 100% rename from automation/CommandSequence.py rename to openwpm/CommandSequence.py diff --git a/automation/Commands/Types.py b/openwpm/Commands/Types.py similarity index 100% rename from automation/Commands/Types.py rename to openwpm/Commands/Types.py diff --git a/automation/Commands/__init__.py b/openwpm/Commands/__init__.py similarity index 100% rename from automation/Commands/__init__.py rename to openwpm/Commands/__init__.py diff --git a/automation/Commands/browser_commands.py b/openwpm/Commands/browser_commands.py similarity index 100% rename from automation/Commands/browser_commands.py rename to openwpm/Commands/browser_commands.py diff --git a/automation/Commands/command_executor.py b/openwpm/Commands/command_executor.py similarity index 100% rename from automation/Commands/command_executor.py rename to openwpm/Commands/command_executor.py diff --git a/automation/Commands/profile_commands.py b/openwpm/Commands/profile_commands.py similarity index 100% rename from automation/Commands/profile_commands.py rename to openwpm/Commands/profile_commands.py diff --git a/automation/Commands/utils/XPathUtil.py b/openwpm/Commands/utils/XPathUtil.py similarity index 100% rename from automation/Commands/utils/XPathUtil.py rename to openwpm/Commands/utils/XPathUtil.py diff --git a/automation/Commands/utils/__init__.py b/openwpm/Commands/utils/__init__.py similarity index 100% rename from automation/Commands/utils/__init__.py rename to openwpm/Commands/utils/__init__.py diff --git a/automation/Commands/utils/file_utils.py b/openwpm/Commands/utils/file_utils.py similarity index 100% rename from automation/Commands/utils/file_utils.py rename to openwpm/Commands/utils/file_utils.py diff --git a/automation/Commands/utils/firefox_profile.py b/openwpm/Commands/utils/firefox_profile.py similarity index 100% rename from automation/Commands/utils/firefox_profile.py rename to openwpm/Commands/utils/firefox_profile.py diff --git a/automation/Commands/utils/webdriver_utils.py b/openwpm/Commands/utils/webdriver_utils.py similarity index 100% rename from automation/Commands/utils/webdriver_utils.py rename to openwpm/Commands/utils/webdriver_utils.py diff --git a/automation/DataAggregator/BaseAggregator.py b/openwpm/DataAggregator/BaseAggregator.py similarity index 100% rename from automation/DataAggregator/BaseAggregator.py rename to openwpm/DataAggregator/BaseAggregator.py diff --git a/automation/DataAggregator/LocalAggregator.py b/openwpm/DataAggregator/LocalAggregator.py similarity index 100% rename from automation/DataAggregator/LocalAggregator.py rename to openwpm/DataAggregator/LocalAggregator.py diff --git a/automation/DataAggregator/S3Aggregator.py b/openwpm/DataAggregator/S3Aggregator.py similarity index 100% rename from automation/DataAggregator/S3Aggregator.py rename to openwpm/DataAggregator/S3Aggregator.py diff --git a/automation/DataAggregator/__init__.py b/openwpm/DataAggregator/__init__.py similarity index 100% rename from automation/DataAggregator/__init__.py rename to openwpm/DataAggregator/__init__.py diff --git a/automation/DeployBrowsers/__init__.py b/openwpm/DeployBrowsers/__init__.py similarity index 100% rename from automation/DeployBrowsers/__init__.py rename to openwpm/DeployBrowsers/__init__.py diff --git a/automation/DeployBrowsers/configure_firefox.py b/openwpm/DeployBrowsers/configure_firefox.py similarity index 100% rename from automation/DeployBrowsers/configure_firefox.py rename to openwpm/DeployBrowsers/configure_firefox.py diff --git a/automation/DeployBrowsers/deploy_browser.py b/openwpm/DeployBrowsers/deploy_browser.py similarity index 100% rename from automation/DeployBrowsers/deploy_browser.py rename to openwpm/DeployBrowsers/deploy_browser.py diff --git a/automation/DeployBrowsers/deploy_firefox.py b/openwpm/DeployBrowsers/deploy_firefox.py similarity index 100% rename from automation/DeployBrowsers/deploy_firefox.py rename to openwpm/DeployBrowsers/deploy_firefox.py diff --git a/automation/DeployBrowsers/firefox_extensions/disconnect-5.18.21.xpi b/openwpm/DeployBrowsers/firefox_extensions/disconnect-5.18.21.xpi similarity index 100% rename from automation/DeployBrowsers/firefox_extensions/disconnect-5.18.21.xpi rename to openwpm/DeployBrowsers/firefox_extensions/disconnect-5.18.21.xpi diff --git a/automation/DeployBrowsers/firefox_extensions/ghostery/ghostery-7.3.3.7.xpi b/openwpm/DeployBrowsers/firefox_extensions/ghostery/ghostery-7.3.3.7.xpi similarity index 100% rename from automation/DeployBrowsers/firefox_extensions/ghostery/ghostery-7.3.3.7.xpi rename to openwpm/DeployBrowsers/firefox_extensions/ghostery/ghostery-7.3.3.7.xpi diff --git a/automation/DeployBrowsers/firefox_extensions/ghostery/storage.js b/openwpm/DeployBrowsers/firefox_extensions/ghostery/storage.js similarity index 100% rename from automation/DeployBrowsers/firefox_extensions/ghostery/storage.js rename to openwpm/DeployBrowsers/firefox_extensions/ghostery/storage.js diff --git a/automation/DeployBrowsers/firefox_extensions/https_everywhere-2017.10.4.xpi b/openwpm/DeployBrowsers/firefox_extensions/https_everywhere-2017.10.4.xpi similarity index 100% rename from automation/DeployBrowsers/firefox_extensions/https_everywhere-2017.10.4.xpi rename to openwpm/DeployBrowsers/firefox_extensions/https_everywhere-2017.10.4.xpi diff --git a/automation/DeployBrowsers/firefox_extensions/ublock_origin/storage.js b/openwpm/DeployBrowsers/firefox_extensions/ublock_origin/storage.js similarity index 100% rename from automation/DeployBrowsers/firefox_extensions/ublock_origin/storage.js rename to openwpm/DeployBrowsers/firefox_extensions/ublock_origin/storage.js diff --git a/automation/DeployBrowsers/firefox_extensions/ublock_origin/ublock_origin-1.14.10.xpi b/openwpm/DeployBrowsers/firefox_extensions/ublock_origin/ublock_origin-1.14.10.xpi similarity index 100% rename from automation/DeployBrowsers/firefox_extensions/ublock_origin/ublock_origin-1.14.10.xpi rename to openwpm/DeployBrowsers/firefox_extensions/ublock_origin/ublock_origin-1.14.10.xpi diff --git a/automation/DeployBrowsers/screen_resolutions.txt b/openwpm/DeployBrowsers/screen_resolutions.txt similarity index 100% rename from automation/DeployBrowsers/screen_resolutions.txt rename to openwpm/DeployBrowsers/screen_resolutions.txt diff --git a/automation/DeployBrowsers/selenium_firefox.py b/openwpm/DeployBrowsers/selenium_firefox.py similarity index 100% rename from automation/DeployBrowsers/selenium_firefox.py rename to openwpm/DeployBrowsers/selenium_firefox.py diff --git a/automation/DeployBrowsers/user_agent_strings.txt b/openwpm/DeployBrowsers/user_agent_strings.txt similarity index 100% rename from automation/DeployBrowsers/user_agent_strings.txt rename to openwpm/DeployBrowsers/user_agent_strings.txt diff --git a/automation/Errors.py b/openwpm/Errors.py similarity index 100% rename from automation/Errors.py rename to openwpm/Errors.py diff --git a/automation/Extension/firefox/.npmrc b/openwpm/Extension/firefox/.npmrc similarity index 100% rename from automation/Extension/firefox/.npmrc rename to openwpm/Extension/firefox/.npmrc diff --git a/automation/Extension/firefox/LICENSE b/openwpm/Extension/firefox/LICENSE similarity index 100% rename from automation/Extension/firefox/LICENSE rename to openwpm/Extension/firefox/LICENSE diff --git a/automation/Extension/firefox/README.md b/openwpm/Extension/firefox/README.md similarity index 100% rename from automation/Extension/firefox/README.md rename to openwpm/Extension/firefox/README.md diff --git a/automation/Extension/firefox/content.js/index.js b/openwpm/Extension/firefox/content.js/index.js similarity index 100% rename from automation/Extension/firefox/content.js/index.js rename to openwpm/Extension/firefox/content.js/index.js diff --git a/automation/Extension/firefox/feature.js/callstack-instrument.js b/openwpm/Extension/firefox/feature.js/callstack-instrument.js similarity index 100% rename from automation/Extension/firefox/feature.js/callstack-instrument.js rename to openwpm/Extension/firefox/feature.js/callstack-instrument.js diff --git a/automation/Extension/firefox/feature.js/index.js b/openwpm/Extension/firefox/feature.js/index.js similarity index 100% rename from automation/Extension/firefox/feature.js/index.js rename to openwpm/Extension/firefox/feature.js/index.js diff --git a/automation/Extension/firefox/feature.js/loggingdb.js b/openwpm/Extension/firefox/feature.js/loggingdb.js similarity index 100% rename from automation/Extension/firefox/feature.js/loggingdb.js rename to openwpm/Extension/firefox/feature.js/loggingdb.js diff --git a/automation/Extension/firefox/feature.js/socket.js b/openwpm/Extension/firefox/feature.js/socket.js similarity index 100% rename from automation/Extension/firefox/feature.js/socket.js rename to openwpm/Extension/firefox/feature.js/socket.js diff --git a/automation/Extension/firefox/package-lock.json b/openwpm/Extension/firefox/package-lock.json similarity index 100% rename from automation/Extension/firefox/package-lock.json rename to openwpm/Extension/firefox/package-lock.json diff --git a/automation/Extension/firefox/package.json b/openwpm/Extension/firefox/package.json similarity index 100% rename from automation/Extension/firefox/package.json rename to openwpm/Extension/firefox/package.json diff --git a/automation/Extension/firefox/src/manifest.json b/openwpm/Extension/firefox/src/manifest.json similarity index 100% rename from automation/Extension/firefox/src/manifest.json rename to openwpm/Extension/firefox/src/manifest.json diff --git a/automation/Extension/firefox/src/privileged/profileDirIO/api.js b/openwpm/Extension/firefox/src/privileged/profileDirIO/api.js similarity index 100% rename from automation/Extension/firefox/src/privileged/profileDirIO/api.js rename to openwpm/Extension/firefox/src/privileged/profileDirIO/api.js diff --git a/automation/Extension/firefox/src/privileged/profileDirIO/schema.json b/openwpm/Extension/firefox/src/privileged/profileDirIO/schema.json similarity index 100% rename from automation/Extension/firefox/src/privileged/profileDirIO/schema.json rename to openwpm/Extension/firefox/src/privileged/profileDirIO/schema.json diff --git a/automation/Extension/firefox/src/privileged/sockets/api.js b/openwpm/Extension/firefox/src/privileged/sockets/api.js similarity index 100% rename from automation/Extension/firefox/src/privileged/sockets/api.js rename to openwpm/Extension/firefox/src/privileged/sockets/api.js diff --git a/automation/Extension/firefox/src/privileged/sockets/bufferpack.js b/openwpm/Extension/firefox/src/privileged/sockets/bufferpack.js similarity index 100% rename from automation/Extension/firefox/src/privileged/sockets/bufferpack.js rename to openwpm/Extension/firefox/src/privileged/sockets/bufferpack.js diff --git a/automation/Extension/firefox/src/privileged/sockets/schema.json b/openwpm/Extension/firefox/src/privileged/sockets/schema.json similarity index 100% rename from automation/Extension/firefox/src/privileged/sockets/schema.json rename to openwpm/Extension/firefox/src/privileged/sockets/schema.json diff --git a/automation/Extension/firefox/src/privileged/stackDump/OpenWPMStackDumpChild.jsm b/openwpm/Extension/firefox/src/privileged/stackDump/OpenWPMStackDumpChild.jsm similarity index 100% rename from automation/Extension/firefox/src/privileged/stackDump/OpenWPMStackDumpChild.jsm rename to openwpm/Extension/firefox/src/privileged/stackDump/OpenWPMStackDumpChild.jsm diff --git a/automation/Extension/firefox/src/privileged/stackDump/OpenWPMStackDumpParent.jsm b/openwpm/Extension/firefox/src/privileged/stackDump/OpenWPMStackDumpParent.jsm similarity index 100% rename from automation/Extension/firefox/src/privileged/stackDump/OpenWPMStackDumpParent.jsm rename to openwpm/Extension/firefox/src/privileged/stackDump/OpenWPMStackDumpParent.jsm diff --git a/automation/Extension/firefox/src/privileged/stackDump/api.js b/openwpm/Extension/firefox/src/privileged/stackDump/api.js similarity index 100% rename from automation/Extension/firefox/src/privileged/stackDump/api.js rename to openwpm/Extension/firefox/src/privileged/stackDump/api.js diff --git a/automation/Extension/firefox/src/privileged/stackDump/schema.json b/openwpm/Extension/firefox/src/privileged/stackDump/schema.json similarity index 100% rename from automation/Extension/firefox/src/privileged/stackDump/schema.json rename to openwpm/Extension/firefox/src/privileged/stackDump/schema.json diff --git a/automation/Extension/firefox/web-ext-config.js b/openwpm/Extension/firefox/web-ext-config.js similarity index 100% rename from automation/Extension/firefox/web-ext-config.js rename to openwpm/Extension/firefox/web-ext-config.js diff --git a/automation/Extension/firefox/webpack.config.js b/openwpm/Extension/firefox/webpack.config.js similarity index 100% rename from automation/Extension/firefox/webpack.config.js rename to openwpm/Extension/firefox/webpack.config.js diff --git a/automation/Extension/webext-instrumentation/.auditignore b/openwpm/Extension/webext-instrumentation/.auditignore similarity index 100% rename from automation/Extension/webext-instrumentation/.auditignore rename to openwpm/Extension/webext-instrumentation/.auditignore diff --git a/automation/Extension/webext-instrumentation/.editorconfig b/openwpm/Extension/webext-instrumentation/.editorconfig similarity index 100% rename from automation/Extension/webext-instrumentation/.editorconfig rename to openwpm/Extension/webext-instrumentation/.editorconfig diff --git a/automation/Extension/webext-instrumentation/.gitignore b/openwpm/Extension/webext-instrumentation/.gitignore similarity index 100% rename from automation/Extension/webext-instrumentation/.gitignore rename to openwpm/Extension/webext-instrumentation/.gitignore diff --git a/automation/Extension/webext-instrumentation/.npmignore b/openwpm/Extension/webext-instrumentation/.npmignore similarity index 100% rename from automation/Extension/webext-instrumentation/.npmignore rename to openwpm/Extension/webext-instrumentation/.npmignore diff --git a/automation/Extension/webext-instrumentation/.npmrc b/openwpm/Extension/webext-instrumentation/.npmrc similarity index 100% rename from automation/Extension/webext-instrumentation/.npmrc rename to openwpm/Extension/webext-instrumentation/.npmrc diff --git a/automation/Extension/webext-instrumentation/.prettierignore b/openwpm/Extension/webext-instrumentation/.prettierignore similarity index 100% rename from automation/Extension/webext-instrumentation/.prettierignore rename to openwpm/Extension/webext-instrumentation/.prettierignore diff --git a/automation/Extension/webext-instrumentation/.publishrc b/openwpm/Extension/webext-instrumentation/.publishrc similarity index 100% rename from automation/Extension/webext-instrumentation/.publishrc rename to openwpm/Extension/webext-instrumentation/.publishrc diff --git a/automation/Extension/webext-instrumentation/.versionrc.json b/openwpm/Extension/webext-instrumentation/.versionrc.json similarity index 100% rename from automation/Extension/webext-instrumentation/.versionrc.json rename to openwpm/Extension/webext-instrumentation/.versionrc.json diff --git a/automation/Extension/webext-instrumentation/README.md b/openwpm/Extension/webext-instrumentation/README.md similarity index 100% rename from automation/Extension/webext-instrumentation/README.md rename to openwpm/Extension/webext-instrumentation/README.md diff --git a/automation/Extension/webext-instrumentation/package-lock.json b/openwpm/Extension/webext-instrumentation/package-lock.json similarity index 100% rename from automation/Extension/webext-instrumentation/package-lock.json rename to openwpm/Extension/webext-instrumentation/package-lock.json diff --git a/automation/Extension/webext-instrumentation/package.json b/openwpm/Extension/webext-instrumentation/package.json similarity index 100% rename from automation/Extension/webext-instrumentation/package.json rename to openwpm/Extension/webext-instrumentation/package.json diff --git a/automation/Extension/webext-instrumentation/src/background/cookie-instrument.ts b/openwpm/Extension/webext-instrumentation/src/background/cookie-instrument.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/background/cookie-instrument.ts rename to openwpm/Extension/webext-instrumentation/src/background/cookie-instrument.ts diff --git a/automation/Extension/webext-instrumentation/src/background/dns-instrument.ts b/openwpm/Extension/webext-instrumentation/src/background/dns-instrument.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/background/dns-instrument.ts rename to openwpm/Extension/webext-instrumentation/src/background/dns-instrument.ts diff --git a/automation/Extension/webext-instrumentation/src/background/http-instrument.ts b/openwpm/Extension/webext-instrumentation/src/background/http-instrument.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/background/http-instrument.ts rename to openwpm/Extension/webext-instrumentation/src/background/http-instrument.ts diff --git a/automation/Extension/webext-instrumentation/src/background/javascript-instrument.ts b/openwpm/Extension/webext-instrumentation/src/background/javascript-instrument.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/background/javascript-instrument.ts rename to openwpm/Extension/webext-instrumentation/src/background/javascript-instrument.ts diff --git a/automation/Extension/webext-instrumentation/src/background/navigation-instrument.ts b/openwpm/Extension/webext-instrumentation/src/background/navigation-instrument.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/background/navigation-instrument.ts rename to openwpm/Extension/webext-instrumentation/src/background/navigation-instrument.ts diff --git a/automation/Extension/webext-instrumentation/src/content/javascript-instrument-content-scope.ts b/openwpm/Extension/webext-instrumentation/src/content/javascript-instrument-content-scope.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/content/javascript-instrument-content-scope.ts rename to openwpm/Extension/webext-instrumentation/src/content/javascript-instrument-content-scope.ts diff --git a/automation/Extension/webext-instrumentation/src/content/javascript-instrument-page-scope.ts b/openwpm/Extension/webext-instrumentation/src/content/javascript-instrument-page-scope.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/content/javascript-instrument-page-scope.ts rename to openwpm/Extension/webext-instrumentation/src/content/javascript-instrument-page-scope.ts diff --git a/automation/Extension/webext-instrumentation/src/index.ts b/openwpm/Extension/webext-instrumentation/src/index.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/index.ts rename to openwpm/Extension/webext-instrumentation/src/index.ts diff --git a/automation/Extension/webext-instrumentation/src/lib/extension-session-event-ordinal.ts b/openwpm/Extension/webext-instrumentation/src/lib/extension-session-event-ordinal.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/lib/extension-session-event-ordinal.ts rename to openwpm/Extension/webext-instrumentation/src/lib/extension-session-event-ordinal.ts diff --git a/automation/Extension/webext-instrumentation/src/lib/extension-session-uuid.ts b/openwpm/Extension/webext-instrumentation/src/lib/extension-session-uuid.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/lib/extension-session-uuid.ts rename to openwpm/Extension/webext-instrumentation/src/lib/extension-session-uuid.ts diff --git a/automation/Extension/webext-instrumentation/src/lib/http-post-parser.ts b/openwpm/Extension/webext-instrumentation/src/lib/http-post-parser.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/lib/http-post-parser.ts rename to openwpm/Extension/webext-instrumentation/src/lib/http-post-parser.ts diff --git a/automation/Extension/webext-instrumentation/src/lib/js-instruments.ts b/openwpm/Extension/webext-instrumentation/src/lib/js-instruments.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/lib/js-instruments.ts rename to openwpm/Extension/webext-instrumentation/src/lib/js-instruments.ts diff --git a/automation/Extension/webext-instrumentation/src/lib/number.spec.ts b/openwpm/Extension/webext-instrumentation/src/lib/number.spec.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/lib/number.spec.ts rename to openwpm/Extension/webext-instrumentation/src/lib/number.spec.ts diff --git a/automation/Extension/webext-instrumentation/src/lib/number.ts b/openwpm/Extension/webext-instrumentation/src/lib/number.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/lib/number.ts rename to openwpm/Extension/webext-instrumentation/src/lib/number.ts diff --git a/automation/Extension/webext-instrumentation/src/lib/pending-navigation.ts b/openwpm/Extension/webext-instrumentation/src/lib/pending-navigation.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/lib/pending-navigation.ts rename to openwpm/Extension/webext-instrumentation/src/lib/pending-navigation.ts diff --git a/automation/Extension/webext-instrumentation/src/lib/pending-request.ts b/openwpm/Extension/webext-instrumentation/src/lib/pending-request.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/lib/pending-request.ts rename to openwpm/Extension/webext-instrumentation/src/lib/pending-request.ts diff --git a/automation/Extension/webext-instrumentation/src/lib/pending-response.ts b/openwpm/Extension/webext-instrumentation/src/lib/pending-response.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/lib/pending-response.ts rename to openwpm/Extension/webext-instrumentation/src/lib/pending-response.ts diff --git a/automation/Extension/webext-instrumentation/src/lib/response-body-listener.ts b/openwpm/Extension/webext-instrumentation/src/lib/response-body-listener.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/lib/response-body-listener.ts rename to openwpm/Extension/webext-instrumentation/src/lib/response-body-listener.ts diff --git a/automation/Extension/webext-instrumentation/src/lib/sha256.ts b/openwpm/Extension/webext-instrumentation/src/lib/sha256.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/lib/sha256.ts rename to openwpm/Extension/webext-instrumentation/src/lib/sha256.ts diff --git a/automation/Extension/webext-instrumentation/src/lib/string-utils.ts b/openwpm/Extension/webext-instrumentation/src/lib/string-utils.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/lib/string-utils.ts rename to openwpm/Extension/webext-instrumentation/src/lib/string-utils.ts diff --git a/automation/Extension/webext-instrumentation/src/lib/uuid.ts b/openwpm/Extension/webext-instrumentation/src/lib/uuid.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/lib/uuid.ts rename to openwpm/Extension/webext-instrumentation/src/lib/uuid.ts diff --git a/automation/Extension/webext-instrumentation/src/schema.ts b/openwpm/Extension/webext-instrumentation/src/schema.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/schema.ts rename to openwpm/Extension/webext-instrumentation/src/schema.ts diff --git a/automation/Extension/webext-instrumentation/src/types/browser-web-navigation-event-details.ts b/openwpm/Extension/webext-instrumentation/src/types/browser-web-navigation-event-details.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/types/browser-web-navigation-event-details.ts rename to openwpm/Extension/webext-instrumentation/src/types/browser-web-navigation-event-details.ts diff --git a/automation/Extension/webext-instrumentation/src/types/browser-web-request-event-details.ts b/openwpm/Extension/webext-instrumentation/src/types/browser-web-request-event-details.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/types/browser-web-request-event-details.ts rename to openwpm/Extension/webext-instrumentation/src/types/browser-web-request-event-details.ts diff --git a/automation/Extension/webext-instrumentation/src/types/javascript-instrument.d.ts b/openwpm/Extension/webext-instrumentation/src/types/javascript-instrument.d.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/types/javascript-instrument.d.ts rename to openwpm/Extension/webext-instrumentation/src/types/javascript-instrument.d.ts diff --git a/automation/Extension/webext-instrumentation/src/types/window.unimplemented.d.ts b/openwpm/Extension/webext-instrumentation/src/types/window.unimplemented.d.ts similarity index 100% rename from automation/Extension/webext-instrumentation/src/types/window.unimplemented.d.ts rename to openwpm/Extension/webext-instrumentation/src/types/window.unimplemented.d.ts diff --git a/automation/Extension/webext-instrumentation/tsconfig.json b/openwpm/Extension/webext-instrumentation/tsconfig.json similarity index 100% rename from automation/Extension/webext-instrumentation/tsconfig.json rename to openwpm/Extension/webext-instrumentation/tsconfig.json diff --git a/automation/Extension/webext-instrumentation/tsconfig.module.json b/openwpm/Extension/webext-instrumentation/tsconfig.module.json similarity index 100% rename from automation/Extension/webext-instrumentation/tsconfig.module.json rename to openwpm/Extension/webext-instrumentation/tsconfig.module.json diff --git a/automation/Extension/webext-instrumentation/tslint.json b/openwpm/Extension/webext-instrumentation/tslint.json similarity index 100% rename from automation/Extension/webext-instrumentation/tslint.json rename to openwpm/Extension/webext-instrumentation/tslint.json diff --git a/automation/MPLogger.py b/openwpm/MPLogger.py similarity index 100% rename from automation/MPLogger.py rename to openwpm/MPLogger.py diff --git a/automation/SocketInterface.py b/openwpm/SocketInterface.py similarity index 100% rename from automation/SocketInterface.py rename to openwpm/SocketInterface.py diff --git a/automation/TaskManager.py b/openwpm/TaskManager.py similarity index 100% rename from automation/TaskManager.py rename to openwpm/TaskManager.py diff --git a/automation/__init__.py b/openwpm/__init__.py similarity index 100% rename from automation/__init__.py rename to openwpm/__init__.py diff --git a/automation/default_browser_params.json b/openwpm/default_browser_params.json similarity index 100% rename from automation/default_browser_params.json rename to openwpm/default_browser_params.json diff --git a/automation/default_manager_params.json b/openwpm/default_manager_params.json similarity index 100% rename from automation/default_manager_params.json rename to openwpm/default_manager_params.json diff --git a/automation/js_instrumentation.py b/openwpm/js_instrumentation.py similarity index 100% rename from automation/js_instrumentation.py rename to openwpm/js_instrumentation.py diff --git a/automation/js_instrumentation_collections/fingerprinting.json b/openwpm/js_instrumentation_collections/fingerprinting.json similarity index 100% rename from automation/js_instrumentation_collections/fingerprinting.json rename to openwpm/js_instrumentation_collections/fingerprinting.json diff --git a/automation/storage/__init__.py b/openwpm/storage/__init__.py similarity index 100% rename from automation/storage/__init__.py rename to openwpm/storage/__init__.py diff --git a/automation/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py similarity index 100% rename from automation/storage/arrow_storage.py rename to openwpm/storage/arrow_storage.py diff --git a/automation/storage/in_memory_storage.py b/openwpm/storage/in_memory_storage.py similarity index 100% rename from automation/storage/in_memory_storage.py rename to openwpm/storage/in_memory_storage.py diff --git a/automation/storage/leveldb.py b/openwpm/storage/leveldb.py similarity index 100% rename from automation/storage/leveldb.py rename to openwpm/storage/leveldb.py diff --git a/automation/storage/local_storage.py b/openwpm/storage/local_storage.py similarity index 100% rename from automation/storage/local_storage.py rename to openwpm/storage/local_storage.py diff --git a/automation/storage/parquet_schema.py b/openwpm/storage/parquet_schema.py similarity index 100% rename from automation/storage/parquet_schema.py rename to openwpm/storage/parquet_schema.py diff --git a/automation/storage/s3_storage.py b/openwpm/storage/s3_storage.py similarity index 100% rename from automation/storage/s3_storage.py rename to openwpm/storage/s3_storage.py diff --git a/automation/storage/schema.sql b/openwpm/storage/schema.sql similarity index 100% rename from automation/storage/schema.sql rename to openwpm/storage/schema.sql diff --git a/automation/storage/sql_provider.py b/openwpm/storage/sql_provider.py similarity index 100% rename from automation/storage/sql_provider.py rename to openwpm/storage/sql_provider.py diff --git a/automation/storage/storage_controller.py b/openwpm/storage/storage_controller.py similarity index 100% rename from automation/storage/storage_controller.py rename to openwpm/storage/storage_controller.py diff --git a/automation/storage/storage_providers.py b/openwpm/storage/storage_providers.py similarity index 100% rename from automation/storage/storage_providers.py rename to openwpm/storage/storage_providers.py diff --git a/automation/types.py b/openwpm/types.py similarity index 100% rename from automation/types.py rename to openwpm/types.py diff --git a/automation/utilities/Cookie.py b/openwpm/utilities/Cookie.py similarity index 100% rename from automation/utilities/Cookie.py rename to openwpm/utilities/Cookie.py diff --git a/automation/utilities/__init__.py b/openwpm/utilities/__init__.py similarity index 100% rename from automation/utilities/__init__.py rename to openwpm/utilities/__init__.py diff --git a/automation/utilities/build_cookie_table.py b/openwpm/utilities/build_cookie_table.py similarity index 100% rename from automation/utilities/build_cookie_table.py rename to openwpm/utilities/build_cookie_table.py diff --git a/automation/utilities/db_utils.py b/openwpm/utilities/db_utils.py similarity index 100% rename from automation/utilities/db_utils.py rename to openwpm/utilities/db_utils.py diff --git a/automation/utilities/multiprocess_utils.py b/openwpm/utilities/multiprocess_utils.py similarity index 100% rename from automation/utilities/multiprocess_utils.py rename to openwpm/utilities/multiprocess_utils.py diff --git a/automation/utilities/platform_utils.py b/openwpm/utilities/platform_utils.py similarity index 100% rename from automation/utilities/platform_utils.py rename to openwpm/utilities/platform_utils.py diff --git a/automation/utilities/rediswq.py b/openwpm/utilities/rediswq.py similarity index 100% rename from automation/utilities/rediswq.py rename to openwpm/utilities/rediswq.py From ce5e901f5ebf7c144370bac6a2079876237358c3 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 16 Nov 2020 14:32:10 +0100 Subject: [PATCH 028/139] Readded datadir to .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index f09fbebd4..7eaa3a8dd 100644 --- a/.gitignore +++ b/.gitignore @@ -94,3 +94,5 @@ openwpm/Extension/firefox/dist openwpm/Extension/firefox/openwpm.xpi openwpm/Extension/firefox/src/content.js openwpm/Extension/firefox/src/feature.js + +datadir From 0631848b94ad10d848a2cd930d9345d89b1063e4 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 16 Nov 2020 15:04:43 +0100 Subject: [PATCH 029/139] Ran repin.sh --- environment.yaml | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/environment.yaml b/environment.yaml index 6be759dfa..e40df5392 100644 --- a/environment.yaml +++ b/environment.yaml @@ -2,34 +2,33 @@ channels: - conda-forge - main dependencies: -- autopep8=1.5.4 - beautifulsoup4=4.9.3 +- black=20.8b1 - click=7.1.2 - codecov=2.1.10 -- dill=0.3.2 -- flake8-isort=4.0.0 -- flake8=3.8.4 -- geckodriver=0.26.0 -- ipython=7.18.1 +- dill=0.3.3 +- geckodriver=0.28.0 +- ipython=7.19.0 - leveldb=1.22 - localstack=0.11.1.1 -- multiprocess=0.70.10 -- nodejs=14.13.0 -- pandas=1.1.3 -- pillow=7.2.0 -- pip=20.2.3 -- pre-commit=2.7.1 -- psutil=5.7.2 -- pyarrow=1.0.1 -- pytest-asyncio=0.12.0 +- multiprocess=0.70.11.1 +- mypy=0.790 +- nodejs=15.2.0 +- pandas=1.1.4 +- pillow=8.0.1 +- pip=20.2.4 +- pre-commit=2.8.2 +- psutil=5.7.3 +- pyarrow=2.0.0 +- pytest-asyncio=0.14.0 - pytest-cov=2.10.1 -- pytest=6.1.1 -- python=3.8.5 +- pytest=6.1.2 +- python=3.8.6 - pyvirtualdisplay=0.2.5 - redis-py=3.5.3 - s3fs=0.4.0 - selenium=3.141.0 -- sentry-sdk=0.18.0 +- sentry-sdk=0.19.3 - tabulate=0.8.7 - tblib=1.6.0 - wget=1.20.1 From 3d2d72021a8132564585c5dafe3291788cf1ff75 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 16 Nov 2020 15:04:56 +0100 Subject: [PATCH 030/139] Fixed formatting --- openwpm/storage/arrow_storage.py | 3 +-- openwpm/storage/in_memory_storage.py | 3 +-- openwpm/storage/storage_controller.py | 3 +-- test/storage/test_memory_storage_provider.py | 1 - test/storage/test_storage_controller.py | 1 - 5 files changed, 3 insertions(+), 8 deletions(-) diff --git a/openwpm/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py index 583f96dda..d23302fed 100644 --- a/openwpm/storage/arrow_storage.py +++ b/openwpm/storage/arrow_storage.py @@ -7,9 +7,8 @@ import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -from pyarrow import Table - from automation.types import VisitId +from pyarrow import Table from .parquet_schema import PQ_SCHEMAS from .storage_providers import StructuredStorageProvider, TableName diff --git a/openwpm/storage/in_memory_storage.py b/openwpm/storage/in_memory_storage.py index ed9791a1c..ec8464ee0 100644 --- a/openwpm/storage/in_memory_storage.py +++ b/openwpm/storage/in_memory_storage.py @@ -2,11 +2,10 @@ from collections import defaultdict from typing import Any, DefaultDict, Dict, List, Tuple +from automation.types import VisitId from multiprocess import Queue from pyarrow import Table -from automation.types import VisitId - from .arrow_storage import ArrowProvider from .storage_providers import ( StructuredStorageProvider, diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 50e325443..1dd3fd5ed 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -9,9 +9,8 @@ from collections import defaultdict from typing import Any, DefaultDict, Dict, List, Literal, NoReturn, Optional, Tuple -from multiprocess import Queue - from automation.utilities.multiprocess_utils import Process +from multiprocess import Queue from ..SocketInterface import get_message_from_reader from ..types import BrowserId, VisitId diff --git a/test/storage/test_memory_storage_provider.py b/test/storage/test_memory_storage_provider.py index 65f71f871..b68d5085d 100644 --- a/test/storage/test_memory_storage_provider.py +++ b/test/storage/test_memory_storage_provider.py @@ -2,7 +2,6 @@ from typing import Any, Dict, List, Tuple import pytest - from automation.storage.in_memory_storage import ( MemoryArrowProvider, MemoryStructuredProvider, diff --git a/test/storage/test_storage_controller.py b/test/storage/test_storage_controller.py index 8c677cfc0..84522c297 100644 --- a/test/storage/test_storage_controller.py +++ b/test/storage/test_storage_controller.py @@ -2,7 +2,6 @@ import time import pytest - from automation.MPLogger import MPLogger from automation.SocketInterface import ClientSocket from automation.storage.in_memory_storage import ( From d1678460a06b1f4569a3db2a9556a3105f13ceab Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 16 Nov 2020 16:27:42 +0100 Subject: [PATCH 031/139] Let's see if this works --- openwpm/storage/arrow_storage.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/openwpm/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py index d23302fed..a6a4a57a3 100644 --- a/openwpm/storage/arrow_storage.py +++ b/openwpm/storage/arrow_storage.py @@ -37,7 +37,10 @@ def factory_function() -> DefaultDict[TableName, List[Dict[str, Any]]]: # Record batches by TableName self._batches: DefaultDict[TableName, List[pa.RecordBatch]] = DefaultDict(list) + + # Used to synchronize the finalizing and the flushing self.storing_condition = asyncio.Condition() + self.is_flushing = False self._instance_id = random.getrandbits(32) @@ -100,15 +103,19 @@ async def finalize_visit_id( await self.store_record(INCOMPLETE_VISITS, visit_id, {"visit_id": visit_id}) # This code is pretty tricky as there are a number of things going on - # 1. No finalize_visit_id shoudl return unless the visit has been saved to storage + # 1. No finalize_visit_id shouldn't return unless the visit has been saved to storage # 2. No new batches should be created while saving out all the batches async with self.storing_condition: + if self.flushing: + await self.storing_condition.wait() self._create_batch(visit_id) if self._is_cache_full(): + self.flushing = True await self.flush_cache(self.storing_condition) - - await self.storing_condition.wait() + self.flushing = False + else: + await self.storing_condition.wait() raise NotImplementedError() @@ -124,15 +131,15 @@ async def flush_cache(self, cond: asyncio.Condition = None) -> None: So we either grab the storing condition ourselves or the caller needs to pass us the locked storing_condition """ - assert cond is None or cond.locked() - _cond = cond - if not _cond: - _cond = self.storing_condition - _cond.acquire() + got_cond = not not cond + if not got_cond: + cond = self.storing_condition + cond.acquire() + assert cond.locked() for table_name, batches in self._batches.items(): table = pa.Table.from_batches(batches) await self.write_table(table_name, table) - _cond.notify_all() + cond.notify_all() - if cond is None: - _cond.release() + if not got_cond: + cond.release() From 7f1597fd3728dc0732f60062b9372605b6b706e2 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 16 Nov 2020 16:30:09 +0100 Subject: [PATCH 032/139] Fixed imports --- openwpm/storage/in_memory_storage.py | 3 ++- openwpm/storage/sql_provider.py | 2 +- openwpm/storage/storage_controller.py | 3 ++- openwpm/storage/storage_providers.py | 2 +- test/conftest.py | 5 +++-- test/storage/test_memory_storage_provider.py | 11 ++++++----- test/storage/test_storage_controller.py | 9 +++++---- 7 files changed, 20 insertions(+), 15 deletions(-) diff --git a/openwpm/storage/in_memory_storage.py b/openwpm/storage/in_memory_storage.py index ec8464ee0..f960fddc3 100644 --- a/openwpm/storage/in_memory_storage.py +++ b/openwpm/storage/in_memory_storage.py @@ -2,10 +2,11 @@ from collections import defaultdict from typing import Any, DefaultDict, Dict, List, Tuple -from automation.types import VisitId from multiprocess import Queue from pyarrow import Table +from openwpm.types import VisitId + from .arrow_storage import ArrowProvider from .storage_providers import ( StructuredStorageProvider, diff --git a/openwpm/storage/sql_provider.py b/openwpm/storage/sql_provider.py index 395696ff6..43e3fb805 100644 --- a/openwpm/storage/sql_provider.py +++ b/openwpm/storage/sql_provider.py @@ -6,7 +6,7 @@ from sqlite3 import IntegrityError, InterfaceError, OperationalError, ProgrammingError from typing import Any, Dict, List, Tuple -from automation.types import VisitId +from openwpm.types import VisitId from .storage_providers import StructuredStorageProvider, TableName diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 1dd3fd5ed..d4756fc92 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -9,9 +9,10 @@ from collections import defaultdict from typing import Any, DefaultDict, Dict, List, Literal, NoReturn, Optional, Tuple -from automation.utilities.multiprocess_utils import Process from multiprocess import Queue +from openwpm.utilities.multiprocess_utils import Process + from ..SocketInterface import get_message_from_reader from ..types import BrowserId, VisitId from .storage_providers import ( diff --git a/openwpm/storage/storage_providers.py b/openwpm/storage/storage_providers.py index 544ab9bd4..642814449 100644 --- a/openwpm/storage/storage_providers.py +++ b/openwpm/storage/storage_providers.py @@ -3,7 +3,7 @@ from abc import ABC, abstractmethod from typing import Any, Dict, List, NewType, Tuple -from automation.types import VisitId +from openwpm.types import VisitId """ This module contains all base classes of the storage provider hierachy diff --git a/test/conftest.py b/test/conftest.py index b969d155e..be555ccfd 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -3,8 +3,9 @@ from typing import List, Tuple import pytest -from automation import TaskManager -from automation.types import BrowserParams, ManagerParams + +from openwpm import TaskManager +from openwpm.types import BrowserParams, ManagerParams from . import utilities diff --git a/test/storage/test_memory_storage_provider.py b/test/storage/test_memory_storage_provider.py index b68d5085d..47632cc40 100644 --- a/test/storage/test_memory_storage_provider.py +++ b/test/storage/test_memory_storage_provider.py @@ -2,15 +2,16 @@ from typing import Any, Dict, List, Tuple import pytest -from automation.storage.in_memory_storage import ( + +from openwpm.storage.in_memory_storage import ( MemoryArrowProvider, MemoryStructuredProvider, MemoryUnstructuredProvider, ) -from automation.storage.leveldb import LevelDbProvider -from automation.storage.sql_provider import SqlLiteStorageProvider -from automation.storage.storage_providers import StructuredStorageProvider, TableName -from automation.types import VisitId +from openwpm.storage.leveldb import LevelDbProvider +from openwpm.storage.sql_provider import SqlLiteStorageProvider +from openwpm.storage.storage_providers import StructuredStorageProvider, TableName +from openwpm.types import VisitId from ..openwpmtest import OpenWPMTest diff --git a/test/storage/test_storage_controller.py b/test/storage/test_storage_controller.py index 84522c297..45e29d763 100644 --- a/test/storage/test_storage_controller.py +++ b/test/storage/test_storage_controller.py @@ -2,13 +2,14 @@ import time import pytest -from automation.MPLogger import MPLogger -from automation.SocketInterface import ClientSocket -from automation.storage.in_memory_storage import ( + +from openwpm.MPLogger import MPLogger +from openwpm.SocketInterface import ClientSocket +from openwpm.storage.in_memory_storage import ( MemoryStructuredProvider, MemoryUnstructuredProvider, ) -from automation.storage.storage_controller import StorageControllerHandle +from openwpm.storage.storage_controller import StorageControllerHandle @pytest.fixture(scope="session") From 5de0822ff750500436bf667f05e3c8891732fbf4 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 16 Nov 2020 17:15:55 +0100 Subject: [PATCH 033/139] Got arrow_memory_provider working --- openwpm/Extension/firefox/package-lock.json | 10438 +++++++++++++- .../webext-instrumentation/package-lock.json | 11575 +++++++++++++++- openwpm/storage/arrow_storage.py | 23 +- test/conftest.py | 6 +- test/openwpm_jstest.py | 3 +- test/storage/test_memory_storage_provider.py | 9 +- 6 files changed, 21814 insertions(+), 240 deletions(-) diff --git a/openwpm/Extension/firefox/package-lock.json b/openwpm/Extension/firefox/package-lock.json index 21ce76433..e51305af0 100644 --- a/openwpm/Extension/firefox/package-lock.json +++ b/openwpm/Extension/firefox/package-lock.json @@ -1,7 +1,10036 @@ { "name": "OpenWPM", + "lockfileVersion": 2, "requires": true, - "lockfileVersion": 1, + "packages": { + "": { + "name": "OpenWPM", + "hasInstallScript": true, + "license": "MPL-2.0", + "dependencies": { + "openwpm-webext-instrumentation": "../webext-instrumentation" + }, + "devDependencies": { + "@typescript-eslint/parser": "^2.34.0", + "eslint": "^5.16.0", + "eslint-plugin-import": "^2.22.0", + "eslint-plugin-json": "^1.4.0", + "eslint-plugin-mozilla": "^0.14.0", + "eslint-plugin-no-unsanitized": "^3.1.2", + "npm-run-all": "^4.1.1", + "ts-loader": "^5.4.5", + "tslint": "^5.20.1", + "tslint-eslint-rules": "^5.4.0", + "typescript": "^3.9.7", + "web-ext": "^4.3.0", + "webpack": "^4.44.1", + "webpack-cli": "^3.3.12" + }, + "engines": { + "node": ">=8.11.1" + } + }, + "../webext-instrumentation": { + "name": "@openwpm/webext-instrumentation", + "license": "GPL-3.0-or-later", + "devDependencies": { + "@types/firefox-webext-browser": "^70.0.1", + "ava": "^3.12.1", + "codecov": "^3.7.2", + "commitizen": "^4.2.1", + "cz-conventional-changelog": "^2.1.0", + "gh-pages": "^2.1.1", + "npm-run-all": "^4.1.5", + "nyc": "^14.1.1", + "opn-cli": "^3.1.0", + "prettier": "^1.19.1", + "publish-please": "^5.5.1", + "standard-version": "github:conventional-changelog/standard-version#master", + "trash-cli": "^1.4.0", + "tslint": "^5.20.1", + "tslint-config-prettier": "^1.15.0", + "tslint-immutable": "^4.7.0", + "typedoc": "^0.17.8", + "typescript": "^3.9.7" + }, + "engines": { + "node": ">=8.9" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.0.0.tgz", + "integrity": "sha512-OfC2uemaknXr87bdLUkWog7nYuliM9Ij5HUcajsVcMCpQrcLmtxRbVFTIqmcSkSeYRBFBRxs2FiUqFJDLdiebA==", + "dev": true, + "dependencies": { + "@babel/highlight": "^7.0.0" + } + }, + "node_modules/@babel/highlight": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.0.0.tgz", + "integrity": "sha512-UFMC4ZeFC48Tpvj7C8UgLvtkaUuovQX+5xNWrsIoMG8o2z+XFKjKaN9iVmS84dPwVN00W4wPmqvYoZF3EGAsfw==", + "dev": true, + "dependencies": { + "chalk": "^2.0.0", + "esutils": "^2.0.2", + "js-tokens": "^4.0.0" + } + }, + "node_modules/@babel/polyfill": { + "version": "7.7.0", + "resolved": "https://registry.npmjs.org/@babel/polyfill/-/polyfill-7.7.0.tgz", + "integrity": "sha512-/TS23MVvo34dFmf8mwCisCbWGrfhbiWZSwBo6HkADTBhUa2Q/jWltyY/tpofz/b6/RIhqaqQcquptCirqIhOaQ==", + "dev": true, + "dependencies": { + "core-js": "^2.6.5", + "regenerator-runtime": "^0.13.2" + } + }, + "node_modules/@babel/runtime": { + "version": "7.7.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.7.7.tgz", + "integrity": "sha512-uCnC2JEVAu8AKB5do1WRIsvrdJ0flYx/A/9f/6chdacnEZ7LmavjdsDXr5ksYBegxtuTPR5Va9/+13QF/kFkCA==", + "dev": true, + "dependencies": { + "regenerator-runtime": "^0.13.2" + } + }, + "node_modules/@babel/runtime-corejs3": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.10.4.tgz", + "integrity": "sha512-BFlgP2SoLO9HJX9WBwN67gHWMBhDX/eDz64Jajd6mR/UAUzqrNMm99d4qHnVaKscAElZoFiPv+JpR/Siud5lXw==", + "dev": true, + "dependencies": { + "core-js-pure": "^3.0.0", + "regenerator-runtime": "^0.13.4" + } + }, + "node_modules/@cliqz-oss/firefox-client": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/@cliqz-oss/firefox-client/-/firefox-client-0.3.1.tgz", + "integrity": "sha512-RO+Tops/wGnBzWoZYkCraqyh2JqOejqJq5/a4b54HhmjTNSKdUPwAOK17EGg/zPb0nWqkuB7QyZsI9bo+ev8Kw==", + "dev": true, + "dependencies": { + "colors": "0.5.x", + "js-select": "~0.6.0" + } + }, + "node_modules/@cliqz-oss/node-firefox-connect": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@cliqz-oss/node-firefox-connect/-/node-firefox-connect-1.2.1.tgz", + "integrity": "sha512-O/IyiB5pfztCdmxQZg0/xeq5w+YiP3gtJz8d4We2EpLPKzbDVjOrtfLKYgVfm6Ya6mbvDge1uLkSRwaoVCWKnA==", + "dev": true, + "dependencies": { + "@cliqz-oss/firefox-client": "0.3.1", + "es6-promise": "^2.0.1" + } + }, + "node_modules/@sindresorhus/is": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", + "integrity": "sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==", + "dev": true + }, + "node_modules/@szmarczak/http-timer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-1.1.2.tgz", + "integrity": "sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA==", + "dev": true, + "dependencies": { + "defer-to-connect": "^1.0.1" + } + }, + "node_modules/@types/color-name": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", + "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==", + "dev": true + }, + "node_modules/@types/eslint-visitor-keys": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@types/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", + "integrity": "sha512-OCutwjDZ4aFS6PB1UZ988C4YgwlBHJd6wCeQqaLdmadZ/7e+w79+hbMUFC1QXDNCmdyoRfAFdm0RypzwR+Qpag==", + "dev": true + }, + "node_modules/@types/json-schema": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.5.tgz", + "integrity": "sha512-7+2BITlgjgDhH0vvwZU/HZJVyk+2XUlvxXe8dFMedNX/aMkaOq++rMAFXc0tM7ij15QaWlbdQASBR9dihi+bDQ==", + "dev": true + }, + "node_modules/@types/json5": { + "version": "0.0.29", + "resolved": "https://registry.npmjs.org/@types/json5/-/json5-0.0.29.tgz", + "integrity": "sha1-7ihweulOEdK4J7y+UnC86n8+ce4=", + "dev": true + }, + "node_modules/@types/minimatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", + "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==", + "dev": true + }, + "node_modules/@types/node": { + "version": "14.0.19", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.19.tgz", + "integrity": "sha512-yf3BP/NIXF37BjrK5klu//asUWitOEoUP5xE1mhSUjazotwJ/eJDgEmMQNlOeWOVv72j24QQ+3bqXHE++CFGag==", + "dev": true + }, + "node_modules/@typescript-eslint/experimental-utils": { + "version": "2.34.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-2.34.0.tgz", + "integrity": "sha512-eS6FTkq+wuMJ+sgtuNTtcqavWXqsflWcfBnlYhg/nS4aZ1leewkXGbvBhaapn1q6qf4M71bsR1tez5JTRMuqwA==", + "dev": true, + "dependencies": { + "@types/json-schema": "^7.0.3", + "@typescript-eslint/typescript-estree": "2.34.0", + "eslint-scope": "^5.0.0", + "eslint-utils": "^2.0.0" + } + }, + "node_modules/@typescript-eslint/experimental-utils/node_modules/eslint-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.0.0.tgz", + "integrity": "sha512-0HCPuJv+7Wv1bACm8y5/ECVfYdfsAm9xmVb7saeFlxjPYALefjhbYoCkBjPdPzGH8wWyTpAez82Fh3VKYEZ8OA==", + "dev": true, + "dependencies": { + "eslint-visitor-keys": "^1.1.0" + } + }, + "node_modules/@typescript-eslint/experimental-utils/node_modules/eslint-visitor-keys": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.2.0.tgz", + "integrity": "sha512-WFb4ihckKil6hu3Dp798xdzSfddwKKU3+nGniKF6HfeW6OLd2OUDEPP7TcHtB5+QXOKg2s6B2DaMPE1Nn/kxKQ==", + "dev": true + }, + "node_modules/@typescript-eslint/parser": { + "version": "2.34.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-2.34.0.tgz", + "integrity": "sha512-03ilO0ucSD0EPTw2X4PntSIRFtDPWjrVq7C3/Z3VQHRC7+13YB55rcJI3Jt+YgeHbjUdJPcPa7b23rXCBokuyA==", + "dev": true, + "dependencies": { + "@types/eslint-visitor-keys": "^1.0.0", + "@typescript-eslint/experimental-utils": "2.34.0", + "@typescript-eslint/typescript-estree": "2.34.0", + "eslint-visitor-keys": "^1.1.0" + } + }, + "node_modules/@typescript-eslint/parser/node_modules/eslint-visitor-keys": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.2.0.tgz", + "integrity": "sha512-WFb4ihckKil6hu3Dp798xdzSfddwKKU3+nGniKF6HfeW6OLd2OUDEPP7TcHtB5+QXOKg2s6B2DaMPE1Nn/kxKQ==", + "dev": true + }, + "node_modules/@typescript-eslint/typescript-estree": { + "version": "2.34.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-2.34.0.tgz", + "integrity": "sha512-OMAr+nJWKdlVM9LOqCqh3pQQPwxHAN7Du8DR6dmwCrAmxtiXQnhHJ6tBNtf+cggqfo51SG/FCwnKhXCIM7hnVg==", + "dev": true, + "dependencies": { + "debug": "^4.1.1", + "eslint-visitor-keys": "^1.1.0", + "glob": "^7.1.6", + "is-glob": "^4.0.1", + "lodash": "^4.17.15", + "semver": "^7.3.2", + "tsutils": "^3.17.1" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/eslint-visitor-keys": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.2.0.tgz", + "integrity": "sha512-WFb4ihckKil6hu3Dp798xdzSfddwKKU3+nGniKF6HfeW6OLd2OUDEPP7TcHtB5+QXOKg2s6B2DaMPE1Nn/kxKQ==", + "dev": true + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==", + "dev": true + }, + "node_modules/@typescript-eslint/typescript-estree/node_modules/tsutils": { + "version": "3.17.1", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.17.1.tgz", + "integrity": "sha512-kzeQ5B8H3w60nFY2g8cJIuH7JDpsALXySGtwGJ0p2LSjLgay3NdIpqq5SoOBe46bKDW2iq25irHCr8wjomUS2g==", + "dev": true, + "dependencies": { + "tslib": "^1.8.1" + } + }, + "node_modules/@webassemblyjs/ast": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz", + "integrity": "sha512-C6wW5L+b7ogSDVqymbkkvuW9kruN//YisMED04xzeBBqjHa2FYnmvOlS6Xj68xWQRgWvI9cIglsjFowH/RJyEA==", + "dev": true, + "dependencies": { + "@webassemblyjs/helper-module-context": "1.9.0", + "@webassemblyjs/helper-wasm-bytecode": "1.9.0", + "@webassemblyjs/wast-parser": "1.9.0" + } + }, + "node_modules/@webassemblyjs/floating-point-hex-parser": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.9.0.tgz", + "integrity": "sha512-TG5qcFsS8QB4g4MhrxK5TqfdNe7Ey/7YL/xN+36rRjl/BlGE/NcBvJcqsRgCP6Z92mRE+7N50pRIi8SmKUbcQA==", + "dev": true + }, + "node_modules/@webassemblyjs/helper-api-error": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.9.0.tgz", + "integrity": "sha512-NcMLjoFMXpsASZFxJ5h2HZRcEhDkvnNFOAKneP5RbKRzaWJN36NC4jqQHKwStIhGXu5mUWlUUk7ygdtrO8lbmw==", + "dev": true + }, + "node_modules/@webassemblyjs/helper-buffer": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.9.0.tgz", + "integrity": "sha512-qZol43oqhq6yBPx7YM3m9Bv7WMV9Eevj6kMi6InKOuZxhw+q9hOkvq5e/PpKSiLfyetpaBnogSbNCfBwyB00CA==", + "dev": true + }, + "node_modules/@webassemblyjs/helper-code-frame": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-code-frame/-/helper-code-frame-1.9.0.tgz", + "integrity": "sha512-ERCYdJBkD9Vu4vtjUYe8LZruWuNIToYq/ME22igL+2vj2dQ2OOujIZr3MEFvfEaqKoVqpsFKAGsRdBSBjrIvZA==", + "dev": true, + "dependencies": { + "@webassemblyjs/wast-printer": "1.9.0" + } + }, + "node_modules/@webassemblyjs/helper-fsm": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-fsm/-/helper-fsm-1.9.0.tgz", + "integrity": "sha512-OPRowhGbshCb5PxJ8LocpdX9Kl0uB4XsAjl6jH/dWKlk/mzsANvhwbiULsaiqT5GZGT9qinTICdj6PLuM5gslw==", + "dev": true + }, + "node_modules/@webassemblyjs/helper-module-context": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-module-context/-/helper-module-context-1.9.0.tgz", + "integrity": "sha512-MJCW8iGC08tMk2enck1aPW+BE5Cw8/7ph/VGZxwyvGbJwjktKkDK7vy7gAmMDx88D7mhDTCNKAW5tED+gZ0W8g==", + "dev": true, + "dependencies": { + "@webassemblyjs/ast": "1.9.0" + } + }, + "node_modules/@webassemblyjs/helper-wasm-bytecode": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.9.0.tgz", + "integrity": "sha512-R7FStIzyNcd7xKxCZH5lE0Bqy+hGTwS3LJjuv1ZVxd9O7eHCedSdrId/hMOd20I+v8wDXEn+bjfKDLzTepoaUw==", + "dev": true + }, + "node_modules/@webassemblyjs/helper-wasm-section": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.9.0.tgz", + "integrity": "sha512-XnMB8l3ek4tvrKUUku+IVaXNHz2YsJyOOmz+MMkZvh8h1uSJpSen6vYnw3IoQ7WwEuAhL8Efjms1ZWjqh2agvw==", + "dev": true, + "dependencies": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/helper-buffer": "1.9.0", + "@webassemblyjs/helper-wasm-bytecode": "1.9.0", + "@webassemblyjs/wasm-gen": "1.9.0" + } + }, + "node_modules/@webassemblyjs/ieee754": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.9.0.tgz", + "integrity": "sha512-dcX8JuYU/gvymzIHc9DgxTzUUTLexWwt8uCTWP3otys596io0L5aW02Gb1RjYpx2+0Jus1h4ZFqjla7umFniTg==", + "dev": true, + "dependencies": { + "@xtuc/ieee754": "^1.2.0" + } + }, + "node_modules/@webassemblyjs/leb128": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.9.0.tgz", + "integrity": "sha512-ENVzM5VwV1ojs9jam6vPys97B/S65YQtv/aanqnU7D8aSoHFX8GyhGg0CMfyKNIHBuAVjy3tlzd5QMMINa7wpw==", + "dev": true, + "dependencies": { + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/utf8": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.9.0.tgz", + "integrity": "sha512-GZbQlWtopBTP0u7cHrEx+73yZKrQoBMpwkGEIqlacljhXCkVM1kMQge/Mf+csMJAjEdSwhOyLAS0AoR3AG5P8w==", + "dev": true + }, + "node_modules/@webassemblyjs/wasm-edit": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.9.0.tgz", + "integrity": "sha512-FgHzBm80uwz5M8WKnMTn6j/sVbqilPdQXTWraSjBwFXSYGirpkSWE2R9Qvz9tNiTKQvoKILpCuTjBKzOIm0nxw==", + "dev": true, + "dependencies": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/helper-buffer": "1.9.0", + "@webassemblyjs/helper-wasm-bytecode": "1.9.0", + "@webassemblyjs/helper-wasm-section": "1.9.0", + "@webassemblyjs/wasm-gen": "1.9.0", + "@webassemblyjs/wasm-opt": "1.9.0", + "@webassemblyjs/wasm-parser": "1.9.0", + "@webassemblyjs/wast-printer": "1.9.0" + } + }, + "node_modules/@webassemblyjs/wasm-gen": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.9.0.tgz", + "integrity": "sha512-cPE3o44YzOOHvlsb4+E9qSqjc9Qf9Na1OO/BHFy4OI91XDE14MjFN4lTMezzaIWdPqHnsTodGGNP+iRSYfGkjA==", + "dev": true, + "dependencies": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/helper-wasm-bytecode": "1.9.0", + "@webassemblyjs/ieee754": "1.9.0", + "@webassemblyjs/leb128": "1.9.0", + "@webassemblyjs/utf8": "1.9.0" + } + }, + "node_modules/@webassemblyjs/wasm-opt": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.9.0.tgz", + "integrity": "sha512-Qkjgm6Anhm+OMbIL0iokO7meajkzQD71ioelnfPEj6r4eOFuqm4YC3VBPqXjFyyNwowzbMD+hizmprP/Fwkl2A==", + "dev": true, + "dependencies": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/helper-buffer": "1.9.0", + "@webassemblyjs/wasm-gen": "1.9.0", + "@webassemblyjs/wasm-parser": "1.9.0" + } + }, + "node_modules/@webassemblyjs/wasm-parser": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.9.0.tgz", + "integrity": "sha512-9+wkMowR2AmdSWQzsPEjFU7njh8HTO5MqO8vjwEHuM+AMHioNqSBONRdr0NQQ3dVQrzp0s8lTcYqzUdb7YgELA==", + "dev": true, + "dependencies": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/helper-api-error": "1.9.0", + "@webassemblyjs/helper-wasm-bytecode": "1.9.0", + "@webassemblyjs/ieee754": "1.9.0", + "@webassemblyjs/leb128": "1.9.0", + "@webassemblyjs/utf8": "1.9.0" + } + }, + "node_modules/@webassemblyjs/wast-parser": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-parser/-/wast-parser-1.9.0.tgz", + "integrity": "sha512-qsqSAP3QQ3LyZjNC/0jBJ/ToSxfYJ8kYyuiGvtn/8MK89VrNEfwj7BPQzJVHi0jGTRK2dGdJ5PRqhtjzoww+bw==", + "dev": true, + "dependencies": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/floating-point-hex-parser": "1.9.0", + "@webassemblyjs/helper-api-error": "1.9.0", + "@webassemblyjs/helper-code-frame": "1.9.0", + "@webassemblyjs/helper-fsm": "1.9.0", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@webassemblyjs/wast-printer": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.9.0.tgz", + "integrity": "sha512-2J0nE95rHXHyQ24cWjMKJ1tqB/ds8z/cyeOZxJhcb+rW+SQASVjuznUSmdz5GpVJTzU8JkhYut0D3siFDD6wsA==", + "dev": true, + "dependencies": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/wast-parser": "1.9.0", + "@xtuc/long": "4.2.2" + } + }, + "node_modules/@xtuc/ieee754": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", + "dev": true + }, + "node_modules/@xtuc/long": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", + "dev": true + }, + "node_modules/acorn": { + "version": "7.3.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.3.1.tgz", + "integrity": "sha512-tLc0wSnatxAQHVHUapaHdz72pi9KUyHjq5KyHjGg9Y8Ifdc79pTh2XvI6I1/chZbnM7QtNKzh66ooDogPZSleA==", + "dev": true + }, + "node_modules/acorn-jsx": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.0.1.tgz", + "integrity": "sha512-HJ7CfNHrfJLlNTzIEUTj43LNWGkqpRLxm3YjAlcD0ACydk9XynzYsCBHxut+iqt+1aBXkx9UP/w/ZqMr13XIzg==", + "dev": true + }, + "node_modules/adbkit": { + "version": "2.11.1", + "resolved": "https://registry.npmjs.org/adbkit/-/adbkit-2.11.1.tgz", + "integrity": "sha512-hDTiRg9NX3HQt7WoDAPCplUpvzr4ZzQa2lq7BdTTJ/iOZ6O7YNAs6UYD8sFAiBEcYHDRIyq3cm9sZP6uZnhvXw==", + "dev": true, + "dependencies": { + "adbkit-logcat": "^1.1.0", + "adbkit-monkey": "~1.0.1", + "bluebird": "~2.9.24", + "commander": "^2.3.0", + "debug": "~2.6.3", + "node-forge": "^0.7.1", + "split": "~0.3.3" + } + }, + "node_modules/adbkit-logcat": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/adbkit-logcat/-/adbkit-logcat-1.1.0.tgz", + "integrity": "sha1-Adf5sM75CTowvLOwB+//MBUIli8=", + "dev": true + }, + "node_modules/adbkit-monkey": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/adbkit-monkey/-/adbkit-monkey-1.0.1.tgz", + "integrity": "sha1-8pG+cBou/FZ6Y/x6pq/N7TFDC+E=", + "dev": true, + "dependencies": { + "async": "~0.2.9" + } + }, + "node_modules/adbkit/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/adbkit/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "node_modules/addons-linter": { + "version": "1.26.0", + "resolved": "https://registry.npmjs.org/addons-linter/-/addons-linter-1.26.0.tgz", + "integrity": "sha512-PKytX6qxbZapc076auO0LBhAGuw2z7eyPnYusMgNBPbY72MAXzUCt3AhSbwGhZ43d5Tn/3At5H0xPi31VXG2Mg==", + "dev": true, + "dependencies": { + "@babel/runtime": "7.10.2", + "ajv": "6.12.2", + "ajv-merge-patch": "4.1.0", + "chalk": "4.1.0", + "cheerio": "1.0.0-rc.3", + "columnify": "1.5.4", + "common-tags": "1.8.0", + "deepmerge": "4.2.2", + "dispensary": "0.51.2", + "es6-promisify": "6.1.1", + "eslint": "5.16.0", + "eslint-plugin-no-unsanitized": "3.1.2", + "eslint-visitor-keys": "1.2.0", + "espree": "6.2.1", + "esprima": "4.0.1", + "first-chunk-stream": "3.0.0", + "fluent-syntax": "0.13.0", + "fsevents": "2.1.3", + "glob": "7.1.6", + "is-mergeable-object": "1.1.1", + "jed": "1.1.1", + "mdn-browser-compat-data": "1.0.25", + "os-locale": "5.0.0", + "pino": "6.3.2", + "postcss": "7.0.32", + "probe-image-size": "5.0.0", + "relaxed-json": "1.0.3", + "semver": "7.3.2", + "source-map-support": "0.5.19", + "strip-bom-stream": "4.0.0", + "tosource": "1.0.0", + "upath": "1.2.0", + "whatwg-url": "8.1.0", + "yargs": "15.3.1", + "yauzl": "2.10.0" + }, + "optionalDependencies": { + "fsevents": "2.1.3" + } + }, + "node_modules/addons-linter/node_modules/@babel/runtime": { + "version": "7.10.2", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.10.2.tgz", + "integrity": "sha512-6sF3uQw2ivImfVIl62RZ7MXhO2tap69WeWK57vAaimT6AZbE4FbqjdEJIN1UqoD6wI6B+1n9UiagafH1sxjOtg==", + "dev": true, + "dependencies": { + "regenerator-runtime": "^0.13.4" + } + }, + "node_modules/addons-linter/node_modules/acorn-jsx": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.2.0.tgz", + "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==", + "dev": true + }, + "node_modules/addons-linter/node_modules/ajv": { + "version": "6.12.2", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.2.tgz", + "integrity": "sha512-k+V+hzjm5q/Mr8ef/1Y9goCmlsK4I6Sm74teeyGvFk1XrOsbsKLjEdrvny42CZ+a8sXbk8KWpY/bDwS+FLL2UQ==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "node_modules/addons-linter/node_modules/ansi-styles": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", + "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "dev": true, + "dependencies": { + "@types/color-name": "^1.1.1", + "color-convert": "^2.0.1" + } + }, + "node_modules/addons-linter/node_modules/chalk": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "node_modules/addons-linter/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + } + }, + "node_modules/addons-linter/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/addons-linter/node_modules/eslint-visitor-keys": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.2.0.tgz", + "integrity": "sha512-WFb4ihckKil6hu3Dp798xdzSfddwKKU3+nGniKF6HfeW6OLd2OUDEPP7TcHtB5+QXOKg2s6B2DaMPE1Nn/kxKQ==", + "dev": true + }, + "node_modules/addons-linter/node_modules/espree": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-6.2.1.tgz", + "integrity": "sha512-ysCxRQY3WaXJz9tdbWOwuWr5Y/XrPTGX9Kiz3yoUXwW0VZ4w30HTkQLaGx/+ttFjF8i+ACbArnB4ce68a9m5hw==", + "dev": true, + "dependencies": { + "acorn": "^7.1.1", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.1.0" + } + }, + "node_modules/addons-linter/node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true + }, + "node_modules/addons-linter/node_modules/glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "node_modules/addons-linter/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "node_modules/addons-linter/node_modules/semver": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==", + "dev": true + }, + "node_modules/addons-linter/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "node_modules/addons-linter/node_modules/source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "dev": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/addons-linter/node_modules/supports-color": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", + "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + } + }, + "node_modules/adm-zip": { + "version": "0.4.16", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.4.16.tgz", + "integrity": "sha512-TFi4HBKSGfIKsK5YCkKaaFG2m4PEDyViZmEwof3MTIgzimHLto6muaHVpbrljdIvIrFZzEq/p4nafOeLcYegrg==", + "dev": true + }, + "node_modules/ajv": { + "version": "6.12.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.0.tgz", + "integrity": "sha512-D6gFiFA0RRLyUbvijN74DWAjXSFxWKaWP7mldxkVhyhAV3+SWA9HEJPHQ2c9soIeTFJqcSdFDGFgdqs1iUU2Hw==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "node_modules/ajv-errors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ajv-errors/-/ajv-errors-1.0.1.tgz", + "integrity": "sha512-DCRfO/4nQ+89p/RK43i8Ezd41EqdGIU4ld7nGF8OQ14oc/we5rEntLCUa7+jrn3nn83BosfwZA0wb4pon2o8iQ==", + "dev": true + }, + "node_modules/ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", + "dev": true + }, + "node_modules/ajv-merge-patch": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ajv-merge-patch/-/ajv-merge-patch-4.1.0.tgz", + "integrity": "sha512-0mAYXMSauA8RZ7r+B4+EAOYcZEcO9OK5EiQCR7W7Cv4E44pJj56ZnkKLJ9/PAcOc0dT+LlV9fdDcq2TxVJfOYw==", + "dev": true, + "dependencies": { + "fast-json-patch": "^2.0.6", + "json-merge-patch": "^0.2.3" + } + }, + "node_modules/ajv/node_modules/fast-deep-equal": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.1.tgz", + "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==", + "dev": true + }, + "node_modules/ansi-align": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.0.tgz", + "integrity": "sha512-ZpClVKqXN3RGBmKibdfWzqCY4lnjEuoNzU5T0oEFpfd/z5qJHVarukridD4juLO2FXMiwUQxr9WqQtaYa8XRYw==", + "dev": true, + "dependencies": { + "string-width": "^3.0.0" + } + }, + "node_modules/ansi-align/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "node_modules/ansi-align/node_modules/string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "dependencies": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "node_modules/ansi-align/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "dependencies": { + "ansi-regex": "^4.1.0" + } + }, + "node_modules/ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true + }, + "node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + } + }, + "node_modules/any-promise": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/any-promise/-/any-promise-1.3.0.tgz", + "integrity": "sha1-q8av7tzqUugJzcA3au0845Y10X8=", + "dev": true + }, + "node_modules/anymatch": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-2.0.0.tgz", + "integrity": "sha512-5teOsQWABXHHBFP9y3skS5P3d/WfWXpv3FUpy+LorMrNYaT9pI4oLMQX7jzQ2KklNpGpWHzdCXTDT2Y3XGlZBw==", + "dev": true, + "dependencies": { + "micromatch": "^3.1.4", + "normalize-path": "^2.1.1" + } + }, + "node_modules/aproba": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", + "dev": true + }, + "node_modules/archiver": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/archiver/-/archiver-2.1.1.tgz", + "integrity": "sha1-/2YrSnggFJSj7lRNOjP+dJZQnrw=", + "dev": true, + "dependencies": { + "archiver-utils": "^1.3.0", + "async": "^2.0.0", + "buffer-crc32": "^0.2.1", + "glob": "^7.0.0", + "lodash": "^4.8.0", + "readable-stream": "^2.0.0", + "tar-stream": "^1.5.0", + "zip-stream": "^1.2.0" + } + }, + "node_modules/archiver-utils": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/archiver-utils/-/archiver-utils-1.3.0.tgz", + "integrity": "sha1-5QtMCccL89aA4y/xt5lOn52JUXQ=", + "dev": true, + "dependencies": { + "glob": "^7.0.0", + "graceful-fs": "^4.1.0", + "lazystream": "^1.0.0", + "lodash": "^4.8.0", + "normalize-path": "^2.0.0", + "readable-stream": "^2.0.0" + } + }, + "node_modules/archiver/node_modules/async": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", + "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", + "dev": true, + "dependencies": { + "lodash": "^4.17.14" + } + }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/arr-diff": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", + "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA=", + "dev": true + }, + "node_modules/arr-flatten": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", + "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", + "dev": true + }, + "node_modules/arr-union": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", + "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=", + "dev": true + }, + "node_modules/array-differ": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/array-differ/-/array-differ-3.0.0.tgz", + "integrity": "sha512-THtfYS6KtME/yIAhKjZ2ul7XI96lQGHRputJQHO80LAWQnuGP4iCIN8vdMRboGbIEYBwU33q8Tch1os2+X0kMg==", + "dev": true + }, + "node_modules/array-filter": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/array-filter/-/array-filter-0.0.1.tgz", + "integrity": "sha1-fajPLiZijtcygDWB/SH2fKzS7uw=", + "dev": true + }, + "node_modules/array-includes": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.1.tgz", + "integrity": "sha512-c2VXaCHl7zPsvpkFsw4nxvFie4fh1ur9bpcgsVkIjqn0H/Xwdg+7fv3n2r/isyS8EBj5b06M9kHyZuIr4El6WQ==", + "dev": true, + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0", + "is-string": "^1.0.5" + } + }, + "node_modules/array-includes/node_modules/es-abstract": { + "version": "1.17.6", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", + "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==", + "dev": true, + "dependencies": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.0", + "is-regex": "^1.1.0", + "object-inspect": "^1.7.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.0", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + }, + "node_modules/array-includes/node_modules/es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "dev": true, + "dependencies": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, + "node_modules/array-includes/node_modules/has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", + "dev": true + }, + "node_modules/array-includes/node_modules/is-callable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", + "integrity": "sha512-pyVD9AaGLxtg6srb2Ng6ynWJqkHU9bEM087AKck0w8QwDarTfNcpIYoU8x8Hv2Icm8u6kFJM18Dag8lyqGkviw==", + "dev": true + }, + "node_modules/array-includes/node_modules/is-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.0.tgz", + "integrity": "sha512-iI97M8KTWID2la5uYXlkbSDQIg4F6o1sYboZKKTDpnDQMLtUL86zxhgDet3Q2SriaYsyGqZ6Mn2SjbRKeLHdqw==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.1" + } + }, + "node_modules/array-includes/node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true + }, + "node_modules/array-map": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/array-map/-/array-map-0.0.0.tgz", + "integrity": "sha1-iKK6tz0c97zVwbEYoAP2b2ZfpmI=", + "dev": true + }, + "node_modules/array-reduce": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/array-reduce/-/array-reduce-0.0.0.tgz", + "integrity": "sha1-FziZ0//Rx9k4PkR5Ul2+J4yrXys=", + "dev": true + }, + "node_modules/array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true + }, + "node_modules/array-unique": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", + "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=", + "dev": true + }, + "node_modules/array.prototype.flat": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.2.3.tgz", + "integrity": "sha512-gBlRZV0VSmfPIeWfuuy56XZMvbVfbEUnOXUvt3F/eUUUSyzlgLxhEX4YAEpxNAogRGehPSnfXyPtYyKAhkzQhQ==", + "dev": true, + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1" + } + }, + "node_modules/array.prototype.flat/node_modules/es-abstract": { + "version": "1.17.6", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", + "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==", + "dev": true, + "dependencies": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.0", + "is-regex": "^1.1.0", + "object-inspect": "^1.7.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.0", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + }, + "node_modules/array.prototype.flat/node_modules/es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "dev": true, + "dependencies": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, + "node_modules/array.prototype.flat/node_modules/has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", + "dev": true + }, + "node_modules/array.prototype.flat/node_modules/is-callable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", + "integrity": "sha512-pyVD9AaGLxtg6srb2Ng6ynWJqkHU9bEM087AKck0w8QwDarTfNcpIYoU8x8Hv2Icm8u6kFJM18Dag8lyqGkviw==", + "dev": true + }, + "node_modules/array.prototype.flat/node_modules/is-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.0.tgz", + "integrity": "sha512-iI97M8KTWID2la5uYXlkbSDQIg4F6o1sYboZKKTDpnDQMLtUL86zxhgDet3Q2SriaYsyGqZ6Mn2SjbRKeLHdqw==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.1" + } + }, + "node_modules/array.prototype.flat/node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true + }, + "node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "dev": true + }, + "node_modules/asn1": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "dev": true, + "dependencies": { + "safer-buffer": "~2.1.0" + } + }, + "node_modules/asn1.js": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", + "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", + "dev": true, + "dependencies": { + "bn.js": "^4.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "safer-buffer": "^2.1.0" + } + }, + "node_modules/asn1.js/node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", + "dev": true + }, + "node_modules/assert": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/assert/-/assert-1.5.0.tgz", + "integrity": "sha512-EDsgawzwoun2CZkCgtxJbv392v4nbk9XDD06zI+kQYoBM/3RBWLlEyJARDOmhAAosBjWACEkKL6S+lIZtcAubA==", + "dev": true, + "dependencies": { + "object-assign": "^4.1.1", + "util": "0.10.3" + } + }, + "node_modules/assert-plus": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", + "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=", + "dev": true + }, + "node_modules/assert/node_modules/inherits": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz", + "integrity": "sha1-sX0I0ya0Qj5Wjv9xn5GwscvfafE=", + "dev": true + }, + "node_modules/assert/node_modules/util": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz", + "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=", + "dev": true, + "dependencies": { + "inherits": "2.0.1" + } + }, + "node_modules/assign-symbols": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", + "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=", + "dev": true + }, + "node_modules/astral-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", + "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", + "dev": true + }, + "node_modules/async": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/async/-/async-0.2.10.tgz", + "integrity": "sha1-trvgsGdLnXGXCMo43owjfLUmw9E=", + "dev": true + }, + "node_modules/async-each": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/async-each/-/async-each-1.0.3.tgz", + "integrity": "sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==", + "dev": true + }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", + "dev": true + }, + "node_modules/atob": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", + "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + "dev": true + }, + "node_modules/atomic-sleep": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", + "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", + "dev": true + }, + "node_modules/aws-sign2": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz", + "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=", + "dev": true + }, + "node_modules/aws4": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.10.0.tgz", + "integrity": "sha512-3YDiu347mtVtjpyV3u5kVqQLP242c06zwDOgpeRnybmXlYYsLbtTrUBUm8i8srONt+FWobl5aibnU1030PeeuA==", + "dev": true + }, + "node_modules/balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true + }, + "node_modules/base": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", + "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", + "dev": true, + "dependencies": { + "cache-base": "^1.0.1", + "class-utils": "^0.3.5", + "component-emitter": "^1.2.1", + "define-property": "^1.0.0", + "isobject": "^3.0.1", + "mixin-deep": "^1.2.0", + "pascalcase": "^0.1.1" + } + }, + "node_modules/base/node_modules/define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.0" + } + }, + "node_modules/base/node_modules/is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + } + }, + "node_modules/base/node_modules/is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + } + }, + "node_modules/base/node_modules/is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + } + }, + "node_modules/base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==", + "dev": true + }, + "node_modules/bcrypt-pbkdf": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", + "dev": true, + "dependencies": { + "tweetnacl": "^0.14.3" + } + }, + "node_modules/big.js": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", + "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==", + "dev": true + }, + "node_modules/binary-extensions": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-1.13.1.tgz", + "integrity": "sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw==", + "dev": true + }, + "node_modules/bl": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz", + "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", + "dev": true, + "dependencies": { + "readable-stream": "^2.3.5", + "safe-buffer": "^5.1.1" + } + }, + "node_modules/bluebird": { + "version": "2.9.34", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.9.34.tgz", + "integrity": "sha1-L3tOyAIWMoqf3evfacjUlC/v99g=", + "dev": true + }, + "node_modules/bn.js": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.1.3.tgz", + "integrity": "sha512-GkTiFpjFtUzU9CbMeJ5iazkCzGL3jrhzerzZIuqLABjbwRaFt33I9tUdSNryIptM+RxDet6OKm2WnLXzW51KsQ==", + "dev": true + }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=", + "dev": true + }, + "node_modules/boxen": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-4.2.0.tgz", + "integrity": "sha512-eB4uT9RGzg2odpER62bBwSLvUeGC+WbRjjyyFhGsKnc8wp/m0+hQsMUvUe3H2V0D5vw0nBdO1hCJoZo5mKeuIQ==", + "dev": true, + "dependencies": { + "ansi-align": "^3.0.0", + "camelcase": "^5.3.1", + "chalk": "^3.0.0", + "cli-boxes": "^2.2.0", + "string-width": "^4.1.0", + "term-size": "^2.1.0", + "type-fest": "^0.8.1", + "widest-line": "^3.1.0" + } + }, + "node_modules/boxen/node_modules/ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "node_modules/boxen/node_modules/ansi-styles": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", + "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "dev": true, + "dependencies": { + "@types/color-name": "^1.1.1", + "color-convert": "^2.0.1" + } + }, + "node_modules/boxen/node_modules/chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "node_modules/boxen/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + } + }, + "node_modules/boxen/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/boxen/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/boxen/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "node_modules/boxen/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "node_modules/boxen/node_modules/string-width": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", + "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + } + }, + "node_modules/boxen/node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.0" + } + }, + "node_modules/boxen/node_modules/supports-color": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", + "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz", + "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==", + "dev": true, + "dependencies": { + "arr-flatten": "^1.1.0", + "array-unique": "^0.3.2", + "extend-shallow": "^2.0.1", + "fill-range": "^4.0.0", + "isobject": "^3.0.1", + "repeat-element": "^1.1.2", + "snapdragon": "^0.8.1", + "snapdragon-node": "^2.0.1", + "split-string": "^3.0.2", + "to-regex": "^3.0.1" + } + }, + "node_modules/braces/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + } + }, + "node_modules/brorand": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", + "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=", + "dev": true + }, + "node_modules/browserify-aes": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", + "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==", + "dev": true, + "dependencies": { + "buffer-xor": "^1.0.3", + "cipher-base": "^1.0.0", + "create-hash": "^1.1.0", + "evp_bytestokey": "^1.0.3", + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/browserify-cipher": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz", + "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==", + "dev": true, + "dependencies": { + "browserify-aes": "^1.0.4", + "browserify-des": "^1.0.0", + "evp_bytestokey": "^1.0.0" + } + }, + "node_modules/browserify-des": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.2.tgz", + "integrity": "sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==", + "dev": true, + "dependencies": { + "cipher-base": "^1.0.1", + "des.js": "^1.0.0", + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "node_modules/browserify-rsa": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz", + "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=", + "dev": true, + "dependencies": { + "bn.js": "^4.1.0", + "randombytes": "^2.0.1" + } + }, + "node_modules/browserify-rsa/node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", + "dev": true + }, + "node_modules/browserify-sign": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz", + "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==", + "dev": true, + "dependencies": { + "bn.js": "^5.1.1", + "browserify-rsa": "^4.0.1", + "create-hash": "^1.2.0", + "create-hmac": "^1.1.7", + "elliptic": "^6.5.3", + "inherits": "^2.0.4", + "parse-asn1": "^5.1.5", + "readable-stream": "^3.6.0", + "safe-buffer": "^5.2.0" + } + }, + "node_modules/browserify-sign/node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/browserify-sign/node_modules/readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "node_modules/browserify-sign/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + }, + "node_modules/browserify-zlib": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz", + "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==", + "dev": true, + "dependencies": { + "pako": "~1.0.5" + } + }, + "node_modules/buffer": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", + "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", + "dev": true, + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4" + } + }, + "node_modules/buffer-alloc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", + "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", + "dev": true, + "dependencies": { + "buffer-alloc-unsafe": "^1.1.0", + "buffer-fill": "^1.0.0" + } + }, + "node_modules/buffer-alloc-unsafe": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", + "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==", + "dev": true + }, + "node_modules/buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=", + "dev": true + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha1-+OcRMvf/5uAaXJaXpMbz5I1cyBk=", + "dev": true + }, + "node_modules/buffer-fill": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", + "integrity": "sha1-+PeLdniYiO858gXNY39o5wISKyw=", + "dev": true + }, + "node_modules/buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", + "dev": true + }, + "node_modules/buffer-xor": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", + "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=", + "dev": true + }, + "node_modules/builtin-modules": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", + "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=", + "dev": true + }, + "node_modules/builtin-status-codes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", + "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=", + "dev": true + }, + "node_modules/bunyan": { + "version": "1.8.12", + "resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.12.tgz", + "integrity": "sha1-8VDw9nSKvdcq6uhPBEA74u8RN5c=", + "dev": true, + "dependencies": { + "dtrace-provider": "~0.8", + "moment": "^2.10.6", + "mv": "~2", + "safe-json-stringify": "~1" + }, + "optionalDependencies": { + "dtrace-provider": "~0.8", + "moment": "^2.10.6", + "mv": "~2", + "safe-json-stringify": "~1" + } + }, + "node_modules/cacache": { + "version": "12.0.4", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-12.0.4.tgz", + "integrity": "sha512-a0tMB40oefvuInr4Cwb3GerbL9xTj1D5yg0T5xrjGCGyfvbxseIXX7BAO/u/hIXdafzOI5JC3wDwHyf24buOAQ==", + "dev": true, + "dependencies": { + "bluebird": "^3.5.5", + "chownr": "^1.1.1", + "figgy-pudding": "^3.5.1", + "glob": "^7.1.4", + "graceful-fs": "^4.1.15", + "infer-owner": "^1.0.3", + "lru-cache": "^5.1.1", + "mississippi": "^3.0.0", + "mkdirp": "^0.5.1", + "move-concurrently": "^1.0.1", + "promise-inflight": "^1.0.1", + "rimraf": "^2.6.3", + "ssri": "^6.0.1", + "unique-filename": "^1.1.1", + "y18n": "^4.0.0" + } + }, + "node_modules/cacache/node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==", + "dev": true + }, + "node_modules/cacache/node_modules/glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "node_modules/cache-base": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", + "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", + "dev": true, + "dependencies": { + "collection-visit": "^1.0.0", + "component-emitter": "^1.2.1", + "get-value": "^2.0.6", + "has-value": "^1.0.0", + "isobject": "^3.0.1", + "set-value": "^2.0.0", + "to-object-path": "^0.3.0", + "union-value": "^1.0.0", + "unset-value": "^1.0.0" + } + }, + "node_modules/cacheable-request": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", + "integrity": "sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg==", + "dev": true, + "dependencies": { + "clone-response": "^1.0.2", + "get-stream": "^5.1.0", + "http-cache-semantics": "^4.0.0", + "keyv": "^3.0.0", + "lowercase-keys": "^2.0.0", + "normalize-url": "^4.1.0", + "responselike": "^1.0.2" + } + }, + "node_modules/cacheable-request/node_modules/lowercase-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", + "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==", + "dev": true + }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true + }, + "node_modules/caseless": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", + "integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw=", + "dev": true + }, + "node_modules/chalk": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz", + "integrity": "sha512-ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "node_modules/chardet": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", + "dev": true + }, + "node_modules/cheerio": { + "version": "1.0.0-rc.3", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz", + "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==", + "dev": true, + "dependencies": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.1", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash": "^4.15.0", + "parse5": "^3.0.1" + } + }, + "node_modules/cheerio/node_modules/dom-serializer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", + "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "dev": true, + "dependencies": { + "domelementtype": "^1.3.0", + "entities": "^1.1.1" + } + }, + "node_modules/chokidar": { + "version": "2.1.8", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.1.8.tgz", + "integrity": "sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==", + "dev": true, + "dependencies": { + "anymatch": "^2.0.0", + "async-each": "^1.0.1", + "braces": "^2.3.2", + "fsevents": "^1.2.7", + "glob-parent": "^3.1.0", + "inherits": "^2.0.3", + "is-binary-path": "^1.0.0", + "is-glob": "^4.0.0", + "normalize-path": "^3.0.0", + "path-is-absolute": "^1.0.0", + "readdirp": "^2.2.1", + "upath": "^1.1.1" + }, + "optionalDependencies": { + "fsevents": "^1.2.7" + } + }, + "node_modules/chokidar/node_modules/fsevents": { + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-1.2.12.tgz", + "integrity": "sha512-Ggd/Ktt7E7I8pxZRbGIs7vwqAPscSESMrCSkx2FtWeqmheJgCo2R74fTsZFCifr0VTPwqRpPv17+6b8Zp7th0Q==", + "bundleDependencies": [ + "node-pre-gyp" + ], + "dev": true, + "optional": true, + "dependencies": { + "nan": "^2.12.1", + "node-pre-gyp": "*" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/abbrev": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", + "extraneous": true, + "inBundle": true, + "license": "ISC" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/aproba": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", + "extraneous": true, + "inBundle": true, + "license": "ISC" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/are-we-there-yet": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "delegates": "^1.0.0", + "readable-stream": "^2.0.6" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "extraneous": true, + "inBundle": true, + "license": "MIT" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "extraneous": true, + "inBundle": true, + "license": "ISC" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "extraneous": true, + "inBundle": true, + "license": "MIT" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/console-control-strings": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", + "extraneous": true, + "inBundle": true, + "license": "ISC" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", + "extraneous": true, + "inBundle": true, + "license": "MIT" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/delegates": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", + "extraneous": true, + "inBundle": true, + "license": "MIT" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/detect-libc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", + "extraneous": true, + "inBundle": true, + "license": "Apache-2.0", + "bin": { + "detect-libc": "bin/detect-libc.js" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/fs-minipass": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz", + "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "minipass": "^2.6.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "extraneous": true, + "inBundle": true, + "license": "ISC" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/gauge": { + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "aproba": "^1.0.3", + "console-control-strings": "^1.0.0", + "has-unicode": "^2.0.0", + "object-assign": "^4.1.0", + "signal-exit": "^3.0.0", + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wide-align": "^1.1.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/has-unicode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", + "extraneous": true, + "inBundle": true, + "license": "ISC" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/ignore-walk": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", + "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "minimatch": "^3.0.4" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "extraneous": true, + "inBundle": true, + "license": "ISC" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/ini": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "engines": { + "node": "*" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "dependencies": { + "number-is-nan": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "extraneous": true, + "inBundle": true, + "license": "MIT" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "extraneous": true, + "inBundle": true, + "license": "MIT" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/minipass": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz", + "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "safe-buffer": "^5.1.2", + "yallist": "^3.0.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/minizlib": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz", + "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "dependencies": { + "minipass": "^2.9.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/mkdirp": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.3.tgz", + "integrity": "sha512-P+2gwrFqx8lhew375MQHHeTlY8AuOJSrGf0R5ddkEndUkmwpgUob/vQuBD1V22/Cw1/lJr4x+EjllSezBThzBg==", + "deprecated": "Legacy versions of mkdirp are no longer supported. Please update to mkdirp 1.x. (Note that the API surface has changed to use Promises in 1.x.)", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "dependencies": { + "minimist": "^1.2.5" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "extraneous": true, + "inBundle": true, + "license": "MIT" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/needle": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/needle/-/needle-2.3.3.tgz", + "integrity": "sha512-EkY0GeSq87rWp1hoq/sH/wnTWgFVhYlnIkbJ0YJFfRgEFlz2RraCjBpFQ+vrEgEdp0ThfyHADmkChEhcb7PKyw==", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "dependencies": { + "debug": "^3.2.6", + "iconv-lite": "^0.4.4", + "sax": "^1.2.4" + }, + "bin": { + "needle": "bin/needle" + }, + "engines": { + "node": ">= 4.4.x" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/node-pre-gyp": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.14.0.tgz", + "integrity": "sha512-+CvDC7ZttU/sSt9rFjix/P05iS43qHCOOGzcr3Ry99bXG7VX953+vFyEuph/tfqoYu8dttBkE86JSKBO2OzcxA==", + "extraneous": true, + "inBundle": true, + "license": "BSD-3-Clause", + "dependencies": { + "detect-libc": "^1.0.2", + "mkdirp": "^0.5.1", + "needle": "^2.2.1", + "nopt": "^4.0.1", + "npm-packlist": "^1.1.6", + "npmlog": "^4.0.2", + "rc": "^1.2.7", + "rimraf": "^2.6.1", + "semver": "^5.3.0", + "tar": "^4.4.2" + }, + "bin": { + "node-pre-gyp": "bin/node-pre-gyp" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/nopt": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", + "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "abbrev": "1", + "osenv": "^0.1.4" + }, + "bin": { + "nopt": "bin/nopt.js" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/npm-bundled": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.1.tgz", + "integrity": "sha512-gqkfgGePhTpAEgUsGEgcq1rqPXA+tv/aVBlgEzfXwA1yiUJF7xtEt3CtVwOjNYQOVknDk0F20w58Fnm3EtG0fA==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "npm-normalize-package-bin": "^1.0.1" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/npm-normalize-package-bin": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz", + "integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA==", + "extraneous": true, + "inBundle": true, + "license": "ISC" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/npm-packlist": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.8.tgz", + "integrity": "sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "ignore-walk": "^3.0.1", + "npm-bundled": "^1.0.1", + "npm-normalize-package-bin": "^1.0.1" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/npmlog": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "are-we-there-yet": "~1.1.2", + "console-control-strings": "~1.1.0", + "gauge": "~2.7.3", + "set-blocking": "~2.0.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/osenv": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "os-homedir": "^1.0.0", + "os-tmpdir": "^1.0.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "extraneous": true, + "inBundle": true, + "license": "MIT" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "extraneous": true, + "inBundle": true, + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "extraneous": true, + "inBundle": true, + "license": "MIT" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "extraneous": true, + "inBundle": true, + "license": "MIT" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", + "extraneous": true, + "inBundle": true, + "license": "ISC" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", + "extraneous": true, + "inBundle": true, + "license": "ISC" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/signal-exit": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", + "extraneous": true, + "inBundle": true, + "license": "ISC" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "dependencies": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "extraneous": true, + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/tar": { + "version": "4.4.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz", + "integrity": "sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "chownr": "^1.1.1", + "fs-minipass": "^1.2.5", + "minipass": "^2.8.6", + "minizlib": "^1.2.1", + "mkdirp": "^0.5.0", + "safe-buffer": "^5.1.2", + "yallist": "^3.0.3" + }, + "engines": { + "node": ">=4.5" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", + "extraneous": true, + "inBundle": true, + "license": "MIT" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/wide-align": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", + "extraneous": true, + "inBundle": true, + "license": "ISC", + "dependencies": { + "string-width": "^1.0.2 || 2" + } + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "extraneous": true, + "inBundle": true, + "license": "ISC" + }, + "node_modules/chokidar/node_modules/fsevents/node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "extraneous": true, + "inBundle": true, + "license": "ISC" + }, + "node_modules/chokidar/node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true + }, + "node_modules/chrome-launcher": { + "version": "0.13.3", + "resolved": "https://registry.npmjs.org/chrome-launcher/-/chrome-launcher-0.13.3.tgz", + "integrity": "sha512-ovrDuFXgXS96lzeDqFPQRsczkxla+6QMvzsF+1u0mKlD1KE8EuhjdLwiDfIFedb0FSLz18RK3y6IbKu8oqA0qw==", + "dev": true, + "dependencies": { + "@types/node": "*", + "escape-string-regexp": "^1.0.5", + "is-wsl": "^2.2.0", + "lighthouse-logger": "^1.0.0", + "mkdirp": "^0.5.3", + "rimraf": "^3.0.2" + } + }, + "node_modules/chrome-launcher/node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + } + }, + "node_modules/chrome-trace-event": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/chrome-trace-event/-/chrome-trace-event-1.0.2.tgz", + "integrity": "sha512-9e/zx1jw7B4CO+c/RXoCsfg/x1AfUBioy4owYH0bJprEYAx5hRFLRhWBqHAG57D0ZM4H7vxbP7bPe0VwhQRYDQ==", + "dev": true, + "dependencies": { + "tslib": "^1.9.0" + } + }, + "node_modules/ci-info": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", + "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==", + "dev": true + }, + "node_modules/cipher-base": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz", + "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==", + "dev": true, + "dependencies": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/class-utils": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", + "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", + "dev": true, + "dependencies": { + "arr-union": "^3.1.0", + "define-property": "^0.2.5", + "isobject": "^3.0.0", + "static-extend": "^0.1.1" + } + }, + "node_modules/class-utils/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + } + }, + "node_modules/cli-boxes": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.0.tgz", + "integrity": "sha512-gpaBrMAizVEANOpfZp/EEUixTXDyGt7DFzdK5hU+UbWt/J0lB0w20ncZj59Z9a93xHb9u12zF5BS6i9RKbtg4w==", + "dev": true + }, + "node_modules/cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", + "dev": true, + "dependencies": { + "restore-cursor": "^2.0.0" + } + }, + "node_modules/cli-width": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz", + "integrity": "sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk=", + "dev": true + }, + "node_modules/cliui": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", + "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^6.2.0" + } + }, + "node_modules/cliui/node_modules/ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "node_modules/cliui/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/cliui/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "node_modules/cliui/node_modules/string-width": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", + "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + } + }, + "node_modules/cliui/node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.0" + } + }, + "node_modules/clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=", + "dev": true + }, + "node_modules/clone-response": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", + "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=", + "dev": true, + "dependencies": { + "mimic-response": "^1.0.0" + } + }, + "node_modules/collection-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", + "integrity": "sha1-S8A3PBZLwykbTTaMgpzxqApZ3KA=", + "dev": true, + "dependencies": { + "map-visit": "^1.0.0", + "object-visit": "^1.0.0" + } + }, + "node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "node_modules/colors": { + "version": "0.5.1", + "resolved": "https://registry.npmjs.org/colors/-/colors-0.5.1.tgz", + "integrity": "sha1-fQAj6usVTo7p/Oddy5I9DtFmd3Q=", + "dev": true + }, + "node_modules/columnify": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/columnify/-/columnify-1.5.4.tgz", + "integrity": "sha1-Rzfd8ce2mop8NAVweC6UfuyOeLs=", + "dev": true, + "dependencies": { + "strip-ansi": "^3.0.0", + "wcwidth": "^1.0.0" + } + }, + "node_modules/columnify/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true + }, + "node_modules/columnify/node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, + "dependencies": { + "ansi-regex": "^2.0.0" + } + }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "dependencies": { + "delayed-stream": "~1.0.0" + } + }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true + }, + "node_modules/common-tags": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.0.tgz", + "integrity": "sha512-6P6g0uetGpW/sdyUy/iQQCbFF0kWVMSIVSyYz7Zgjcgh8mgw8PQzDNZeyZ5DQ2gM7LBoZPHmnjz8rUthkBG5tw==", + "dev": true + }, + "node_modules/commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=", + "dev": true + }, + "node_modules/component-emitter": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.2.1.tgz", + "integrity": "sha1-E3kY1teCg/ffemt8WmPhQOaUJeY=", + "dev": true + }, + "node_modules/compress-commons": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/compress-commons/-/compress-commons-1.2.2.tgz", + "integrity": "sha1-UkqfEJA/OoEzibAiXSfEi7dRiQ8=", + "dev": true, + "dependencies": { + "buffer-crc32": "^0.2.1", + "crc32-stream": "^2.0.0", + "normalize-path": "^2.0.0", + "readable-stream": "^2.0.0" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true + }, + "node_modules/concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "dev": true, + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + } + }, + "node_modules/configstore": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz", + "integrity": "sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==", + "dev": true, + "dependencies": { + "dot-prop": "^5.2.0", + "graceful-fs": "^4.1.2", + "make-dir": "^3.0.0", + "unique-string": "^2.0.0", + "write-file-atomic": "^3.0.0", + "xdg-basedir": "^4.0.0" + } + }, + "node_modules/console-browserify": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.2.0.tgz", + "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==", + "dev": true + }, + "node_modules/constants-browserify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz", + "integrity": "sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U=", + "dev": true + }, + "node_modules/contains-path": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/contains-path/-/contains-path-0.1.0.tgz", + "integrity": "sha1-/ozxhP9mcLa67wGp1IYaXL7EEgo=", + "dev": true + }, + "node_modules/copy-concurrently": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/copy-concurrently/-/copy-concurrently-1.0.5.tgz", + "integrity": "sha512-f2domd9fsVDFtaFcbaRZuYXwtdmnzqbADSwhSWYxYB/Q8zsdUUFMXVRwXGDMWmbEzAn1kdRrtI1T/KTFOL4X2A==", + "dev": true, + "dependencies": { + "aproba": "^1.1.1", + "fs-write-stream-atomic": "^1.0.8", + "iferr": "^0.1.5", + "mkdirp": "^0.5.1", + "rimraf": "^2.5.4", + "run-queue": "^1.0.0" + } + }, + "node_modules/copy-descriptor": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", + "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40=", + "dev": true + }, + "node_modules/core-js": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.11.tgz", + "integrity": "sha512-5wjnpaT/3dV+XB4borEsnAYQchn00XSgTAWKDkEqv+K8KevjbzmofK6hfJ9TZIlpj2N0xQpazy7PiRQiWHqzWg==", + "dev": true + }, + "node_modules/core-js-pure": { + "version": "3.6.5", + "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.6.5.tgz", + "integrity": "sha512-lacdXOimsiD0QyNf9BC/mxivNJ/ybBGJXQFKzRekp1WTHoVUWsUHEn+2T8GJAzzIhyOuXA+gOxCVN3l+5PLPUA==", + "dev": true + }, + "node_modules/core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", + "dev": true + }, + "node_modules/crc": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/crc/-/crc-3.8.0.tgz", + "integrity": "sha512-iX3mfgcTMIq3ZKLIsVFAbv7+Mc10kxabAGQb8HvjA1o3T1PIYprbakQ65d3I+2HGHt6nSKkM9PYjgoJO2KcFBQ==", + "dev": true, + "dependencies": { + "buffer": "^5.1.0" + } + }, + "node_modules/crc32-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/crc32-stream/-/crc32-stream-2.0.0.tgz", + "integrity": "sha1-483TtN8xaN10494/u8t7KX/pCPQ=", + "dev": true, + "dependencies": { + "crc": "^3.4.4", + "readable-stream": "^2.0.0" + } + }, + "node_modules/create-ecdh": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz", + "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==", + "dev": true, + "dependencies": { + "bn.js": "^4.1.0", + "elliptic": "^6.5.3" + } + }, + "node_modules/create-ecdh/node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", + "dev": true + }, + "node_modules/create-hash": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", + "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==", + "dev": true, + "dependencies": { + "cipher-base": "^1.0.1", + "inherits": "^2.0.1", + "md5.js": "^1.3.4", + "ripemd160": "^2.0.1", + "sha.js": "^2.4.0" + } + }, + "node_modules/create-hmac": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", + "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==", + "dev": true, + "dependencies": { + "cipher-base": "^1.0.3", + "create-hash": "^1.1.0", + "inherits": "^2.0.1", + "ripemd160": "^2.0.0", + "safe-buffer": "^5.0.1", + "sha.js": "^2.4.8" + } + }, + "node_modules/cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "dev": true, + "dependencies": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + } + }, + "node_modules/crypto-browserify": { + "version": "3.12.0", + "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz", + "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==", + "dev": true, + "dependencies": { + "browserify-cipher": "^1.0.0", + "browserify-sign": "^4.0.0", + "create-ecdh": "^4.0.0", + "create-hash": "^1.1.0", + "create-hmac": "^1.1.0", + "diffie-hellman": "^5.0.0", + "inherits": "^2.0.1", + "pbkdf2": "^3.0.3", + "public-encrypt": "^4.0.0", + "randombytes": "^2.0.0", + "randomfill": "^1.0.3" + } + }, + "node_modules/crypto-random-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz", + "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==", + "dev": true + }, + "node_modules/css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", + "dev": true, + "dependencies": { + "boolbase": "~1.0.0", + "css-what": "2.1", + "domutils": "1.5.1", + "nth-check": "~1.0.1" + } + }, + "node_modules/css-select/node_modules/domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "dev": true, + "dependencies": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, + "node_modules/css-what": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz", + "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==", + "dev": true + }, + "node_modules/cyclist": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/cyclist/-/cyclist-1.0.1.tgz", + "integrity": "sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk=", + "dev": true + }, + "node_modules/dashdash": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", + "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", + "dev": true, + "dependencies": { + "assert-plus": "^1.0.0" + } + }, + "node_modules/debounce": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/debounce/-/debounce-1.2.0.tgz", + "integrity": "sha512-mYtLl1xfZLi1m4RtQYlZgJUNQjl4ZxVnHzIR8nLLgi4q1YT8o/WM+MK/f8yfcc9s5Ir5zRaPZyZU6xs1Syoocg==", + "dev": true + }, + "node_modules/debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "dev": true, + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/decamelize": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-3.2.0.tgz", + "integrity": "sha512-4TgkVUsmmu7oCSyGBm5FvfMoACuoh9EOidm7V5/J2X2djAwwt57qb3F2KMP2ITqODTCSwb+YRV+0Zqrv18k/hw==", + "dev": true, + "dependencies": { + "xregexp": "^4.2.4" + } + }, + "node_modules/decode-uri-component": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", + "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=", + "dev": true + }, + "node_modules/decompress-response": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", + "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", + "dev": true, + "dependencies": { + "mimic-response": "^1.0.0" + } + }, + "node_modules/deep-equal": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.1.tgz", + "integrity": "sha512-yd9c5AdiqVcR+JjcwUQb9DkhJc8ngNr0MahEBGvDiJw8puWab2yZlh+nkasOnZP+EGTAP6rRp2JzJhJZzvNF8g==", + "dev": true, + "dependencies": { + "is-arguments": "^1.0.4", + "is-date-object": "^1.0.1", + "is-regex": "^1.0.4", + "object-is": "^1.0.1", + "object-keys": "^1.1.1", + "regexp.prototype.flags": "^1.2.0" + } + }, + "node_modules/deep-equal/node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true + }, + "node_modules/deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", + "dev": true + }, + "node_modules/deepcopy": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/deepcopy/-/deepcopy-2.0.0.tgz", + "integrity": "sha512-d5ZK7pJw7F3k6M5vqDjGiiUS9xliIyWkdzBjnPhnSeRGjkYOGZMCFkdKVwV/WiHOe0NwzB8q+iDo7afvSf0arA==", + "dev": true, + "dependencies": { + "type-detect": "^4.0.8" + } + }, + "node_modules/deepmerge": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.2.2.tgz", + "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==", + "dev": true + }, + "node_modules/defaults": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz", + "integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=", + "dev": true, + "dependencies": { + "clone": "^1.0.2" + } + }, + "node_modules/defer-to-connect": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz", + "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==", + "dev": true + }, + "node_modules/define-properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "dev": true, + "dependencies": { + "object-keys": "^1.0.12" + } + }, + "node_modules/define-property": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", + "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.2", + "isobject": "^3.0.1" + } + }, + "node_modules/define-property/node_modules/is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + } + }, + "node_modules/define-property/node_modules/is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + } + }, + "node_modules/define-property/node_modules/is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + } + }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", + "dev": true + }, + "node_modules/des.js": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.0.1.tgz", + "integrity": "sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0" + } + }, + "node_modules/detect-file": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/detect-file/-/detect-file-1.0.0.tgz", + "integrity": "sha1-8NZtA2cqglyxtzvbP+YjEMjlUrc=", + "dev": true + }, + "node_modules/diff": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.1.tgz", + "integrity": "sha512-s2+XdvhPCOF01LRQBC8hf4vhbVmI2CGS5aZnxLJlT5FtdhPCDFq80q++zK2KlrVorVDdL5BOGZ/VfLrVtYNF+Q==", + "dev": true + }, + "node_modules/diffie-hellman": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", + "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", + "dev": true, + "dependencies": { + "bn.js": "^4.1.0", + "miller-rabin": "^4.0.0", + "randombytes": "^2.0.0" + } + }, + "node_modules/diffie-hellman/node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", + "dev": true + }, + "node_modules/dispensary": { + "version": "0.51.2", + "resolved": "https://registry.npmjs.org/dispensary/-/dispensary-0.51.2.tgz", + "integrity": "sha512-liUDx/g1xFEBFoOL6308Vr0aYAZlGAyXGcOvuXVa/6qVBZT4QZrv4pVNeb5QOeD5C/Flta+A+qTnLkLnhgs40g==", + "dev": true, + "dependencies": { + "async": "~3.2.0", + "natural-compare-lite": "~1.4.0", + "pino": "~6.0.0", + "request": "~2.88.0", + "sha.js": "~2.4.4", + "source-map-support": "~0.5.4", + "yargs": "~15.3.0" + } + }, + "node_modules/dispensary/node_modules/async": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz", + "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==", + "dev": true + }, + "node_modules/dispensary/node_modules/pino": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-6.0.0.tgz", + "integrity": "sha512-3RfX2L76o7v230FP1fZ3Fo/WX7Su+P1Ld+pvBm2j+MyUjtA/KqDYxMkzBqzcX3R00zbC7Gf/HqIzyuu3tgvi9Q==", + "dev": true, + "dependencies": { + "fast-redact": "^2.0.0", + "fast-safe-stringify": "^2.0.7", + "flatstr": "^1.0.12", + "pino-std-serializers": "^2.4.2", + "quick-format-unescaped": "^4.0.1", + "sonic-boom": "^1.0.0" + } + }, + "node_modules/doctrine": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-1.5.0.tgz", + "integrity": "sha1-N53Ocw9hZvds76TmcHoVmwLFpvo=", + "dev": true, + "dependencies": { + "esutils": "^2.0.2", + "isarray": "^1.0.0" + } + }, + "node_modules/dom-serializer": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz", + "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=", + "dev": true, + "dependencies": { + "domelementtype": "~1.1.1", + "entities": "~1.1.1" + } + }, + "node_modules/dom-serializer/node_modules/domelementtype": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz", + "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=", + "dev": true + }, + "node_modules/domain-browser": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-1.2.0.tgz", + "integrity": "sha512-jnjyiM6eRyZl2H+W8Q/zLMA481hzi0eszAaBUzIVnmYVDBbnLxVNnfu1HgEBvCbL+71FrxMl3E6lpKH7Ge3OXA==", + "dev": true + }, + "node_modules/domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==", + "dev": true + }, + "node_modules/domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "dev": true, + "dependencies": { + "domelementtype": "1" + } + }, + "node_modules/domutils": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.7.0.tgz", + "integrity": "sha512-Lgd2XcJ/NjEw+7tFvfKxOzCYKZsdct5lczQ2ZaQY8Djz7pfAD3Gbp8ySJWtreII/vDlMVmxwa6pHmdxIYgttDg==", + "dev": true, + "dependencies": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, + "node_modules/dot-prop": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz", + "integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==", + "dev": true, + "dependencies": { + "is-obj": "^2.0.0" + } + }, + "node_modules/dtrace-provider": { + "version": "0.8.8", + "resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.8.8.tgz", + "integrity": "sha512-b7Z7cNtHPhH9EJhNNbbeqTcXB8LGFFZhq1PGgEvpeHlzd36bhbdTWoE/Ba/YguqpBSlAPKnARWhVlhunCMwfxg==", + "dev": true, + "optional": true, + "dependencies": { + "nan": "^2.14.0" + } + }, + "node_modules/duplexer3": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", + "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=", + "dev": true + }, + "node_modules/duplexify": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", + "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", + "dev": true, + "dependencies": { + "end-of-stream": "^1.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.0.0", + "stream-shift": "^1.0.0" + } + }, + "node_modules/ecc-jsbn": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", + "dev": true, + "dependencies": { + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dev": true, + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/elliptic": { + "version": "6.5.3", + "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.3.tgz", + "integrity": "sha512-IMqzv5wNQf+E6aHeIqATs0tOLeOTwj1QKbRcS3jBbYkl5oLAserA8yJTT7/VyHUYG91PRmPyeQDObKLPpeS4dw==", + "dev": true, + "dependencies": { + "bn.js": "^4.4.0", + "brorand": "^1.0.1", + "hash.js": "^1.0.0", + "hmac-drbg": "^1.0.0", + "inherits": "^2.0.1", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.0" + } + }, + "node_modules/elliptic/node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", + "dev": true + }, + "node_modules/emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "node_modules/emojis-list": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-2.1.0.tgz", + "integrity": "sha1-TapNnbAPmBmIDHn6RXrlsJof04k=", + "dev": true + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dev": true, + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/enhanced-resolve": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.1.0.tgz", + "integrity": "sha512-F/7vkyTtyc/llOIn8oWclcB25KdRaiPBpZYDgJHgh/UHtpgT2p2eldQgtQnLtUvfMKPKxbRaQM/hHkvLHt1Vng==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "memory-fs": "^0.4.0", + "tapable": "^1.0.0" + } + }, + "node_modules/entities": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==", + "dev": true + }, + "node_modules/errno": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/errno/-/errno-0.1.7.tgz", + "integrity": "sha512-MfrRBDWzIWifgq6tJj60gkAwtLNb6sQPlcFrSOflcP1aFmmruKQ2wRnze/8V6kgyz7H3FF8Npzv78mZ7XLLflg==", + "dev": true, + "dependencies": { + "prr": "~1.0.1" + } + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/es-abstract": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.13.0.tgz", + "integrity": "sha512-vDZfg/ykNxQVwup/8E1BZhVzFfBxs9NqMzGcvIJrqg5k2/5Za2bWo40dK2J1pgLngZ7c+Shh8lwYtLGyrwPutg==", + "dev": true, + "dependencies": { + "es-to-primitive": "^1.2.0", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "is-callable": "^1.1.4", + "is-regex": "^1.0.4", + "object-keys": "^1.0.12" + } + }, + "node_modules/es-to-primitive": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.0.tgz", + "integrity": "sha512-qZryBOJjV//LaxLTV6UC//WewneB3LcXOL9NP++ozKVXsIIIpm/2c13UDiD9Jp2eThsecw9m3jPqDwTyobcdbg==", + "dev": true, + "dependencies": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "dev": true + }, + "node_modules/es6-promise": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-2.3.0.tgz", + "integrity": "sha1-lu258v2wGZWCKyY92KratnSBgbw=", + "dev": true + }, + "node_modules/es6-promisify": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-6.1.1.tgz", + "integrity": "sha512-HBL8I3mIki5C1Cc9QjKUenHtnG0A5/xA8Q/AllRcfiwl2CZFXGK7ddBiCoRwAix4i2KxcQfjtIVcrVbB3vbmwg==", + "dev": true + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true + }, + "node_modules/eslint": { + "version": "5.16.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-5.16.0.tgz", + "integrity": "sha512-S3Rz11i7c8AA5JPv7xAH+dOyq/Cu/VXHiHXBPOU1k/JAM5dXqQPt3qcrhpHSorXmrpu2g0gkIBVXAqCpzfoZIg==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.0.0", + "ajv": "^6.9.1", + "chalk": "^2.1.0", + "cross-spawn": "^6.0.5", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "eslint-scope": "^4.0.3", + "eslint-utils": "^1.3.1", + "eslint-visitor-keys": "^1.0.0", + "espree": "^5.0.1", + "esquery": "^1.0.1", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob": "^7.1.2", + "globals": "^11.7.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "inquirer": "^6.2.2", + "js-yaml": "^3.13.0", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.3.0", + "lodash": "^4.17.11", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "natural-compare": "^1.4.0", + "optionator": "^0.8.2", + "path-is-inside": "^1.0.2", + "progress": "^2.0.0", + "regexpp": "^2.0.1", + "semver": "^5.5.1", + "strip-ansi": "^4.0.0", + "strip-json-comments": "^2.0.1", + "table": "^5.2.3", + "text-table": "^0.2.0" + } + }, + "node_modules/eslint-import-resolver-node": { + "version": "0.3.4", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.4.tgz", + "integrity": "sha512-ogtf+5AB/O+nM6DIeBUNr2fuT7ot9Qg/1harBfBtaP13ekEWFQEEMP94BCB7zaNW3gyY+8SHYF00rnqYwXKWOA==", + "dev": true, + "dependencies": { + "debug": "^2.6.9", + "resolve": "^1.13.1" + } + }, + "node_modules/eslint-import-resolver-node/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/eslint-import-resolver-node/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "node_modules/eslint-module-utils": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.6.0.tgz", + "integrity": "sha512-6j9xxegbqe8/kZY8cYpcp0xhbK0EgJlg3g9mib3/miLaExuuwc3n5UEfSnU6hWMbT0FAYVvDbL9RrRgpUeQIvA==", + "dev": true, + "dependencies": { + "debug": "^2.6.9", + "pkg-dir": "^2.0.0" + } + }, + "node_modules/eslint-module-utils/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/eslint-module-utils/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "node_modules/eslint-plugin-import": { + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.22.0.tgz", + "integrity": "sha512-66Fpf1Ln6aIS5Gr/55ts19eUuoDhAbZgnr6UxK5hbDx6l/QgQgx61AePq+BV4PP2uXQFClgMVzep5zZ94qqsxg==", + "dev": true, + "dependencies": { + "array-includes": "^3.1.1", + "array.prototype.flat": "^1.2.3", + "contains-path": "^0.1.0", + "debug": "^2.6.9", + "doctrine": "1.5.0", + "eslint-import-resolver-node": "^0.3.3", + "eslint-module-utils": "^2.6.0", + "has": "^1.0.3", + "minimatch": "^3.0.4", + "object.values": "^1.1.1", + "read-pkg-up": "^2.0.0", + "resolve": "^1.17.0", + "tsconfig-paths": "^3.9.0" + } + }, + "node_modules/eslint-plugin-import/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/eslint-plugin-import/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "node_modules/eslint-plugin-import/node_modules/resolve": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.17.0.tgz", + "integrity": "sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w==", + "dev": true, + "dependencies": { + "path-parse": "^1.0.6" + } + }, + "node_modules/eslint-plugin-json": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-json/-/eslint-plugin-json-1.4.0.tgz", + "integrity": "sha512-CECvgRAWtUzuepdlPWd+VA7fhyF9HT183pZnl8wQw5x699Mk/MbME/q8xtULBfooi3LUbj6fToieNmsvUcDxWA==", + "dev": true, + "dependencies": { + "vscode-json-languageservice": "^3.2.1" + } + }, + "node_modules/eslint-plugin-mozilla": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-mozilla/-/eslint-plugin-mozilla-0.14.0.tgz", + "integrity": "sha512-H0hb49FJP4qKFgT2l89tlm7QYIsBpKC0cXvFY+mT9Or+7lqlovsYexYtzl9qfT2PFKg37ay7TB2C0U3N7rlp1Q==", + "dev": true, + "dependencies": { + "htmlparser2": "3.9.2", + "ini-parser": "0.0.2", + "sax": "1.2.4" + } + }, + "node_modules/eslint-plugin-no-unsanitized": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-no-unsanitized/-/eslint-plugin-no-unsanitized-3.1.2.tgz", + "integrity": "sha512-KPShfliA3Uy9qqwQx35P1fwIOeJjZkb0FbMMUFztRYRposzaynsM8JCEb952fqkidROl1kpqY80uSvn+TcWkQQ==", + "dev": true + }, + "node_modules/eslint-scope": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.0.tgz", + "integrity": "sha512-iiGRvtxWqgtx5m8EyQUJihBloE4EnYeGE/bz1wSPwJE6tZuJUtHlhqDM4Xj2ukE8Dyy1+HCZ4hE0fzIVMzb58w==", + "dev": true, + "dependencies": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + }, + "node_modules/eslint-utils": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-1.4.2.tgz", + "integrity": "sha512-eAZS2sEUMlIeCjBeubdj45dmBHQwPHWyBcT1VSYB7o9x9WRRqKxyUoiXlRjyAwzN7YEzHJlYg0NmzDRWx6GP4Q==", + "dev": true, + "dependencies": { + "eslint-visitor-keys": "^1.0.0" + } + }, + "node_modules/eslint-visitor-keys": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.0.0.tgz", + "integrity": "sha512-qzm/XxIbxm/FHyH341ZrbnMUpe+5Bocte9xkmFMzPMjRaZMcXww+MpBptFvtU+79L362nqiLhekCxCxDPaUMBQ==", + "dev": true + }, + "node_modules/eslint/node_modules/ajv": { + "version": "6.10.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", + "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "node_modules/eslint/node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "dependencies": { + "esutils": "^2.0.2" + } + }, + "node_modules/eslint/node_modules/eslint-scope": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", + "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", + "dev": true, + "dependencies": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + }, + "node_modules/eslint/node_modules/file-entry-cache": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", + "integrity": "sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g==", + "dev": true, + "dependencies": { + "flat-cache": "^2.0.1" + } + }, + "node_modules/eslint/node_modules/flat-cache": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", + "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", + "dev": true, + "dependencies": { + "flatted": "^2.0.0", + "rimraf": "2.6.3", + "write": "1.0.3" + } + }, + "node_modules/eslint/node_modules/js-yaml": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, + "node_modules/eslint/node_modules/mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "dependencies": { + "minimist": "^1.2.5" + } + }, + "node_modules/eslint/node_modules/write": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz", + "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==", + "dev": true, + "dependencies": { + "mkdirp": "^0.5.1" + } + }, + "node_modules/espree": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-5.0.1.tgz", + "integrity": "sha512-qWAZcWh4XE/RwzLJejfcofscgMc9CamR6Tn1+XRXNzrvUSSbiAjGOI/fggztjIi7y9VLPqnICMIPiGyr8JaZ0A==", + "dev": true, + "dependencies": { + "acorn": "^6.0.7", + "acorn-jsx": "^5.0.0", + "eslint-visitor-keys": "^1.0.0" + } + }, + "node_modules/espree/node_modules/acorn": { + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", + "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", + "dev": true + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true + }, + "node_modules/esquery": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.0.1.tgz", + "integrity": "sha512-SmiyZ5zIWH9VM+SRUReLS5Q8a7GxtRdxEBVZpm98rJM7Sb+A9DVCndXfkeFUd3byderg+EbDkfnevfCwynWaNA==", + "dev": true, + "dependencies": { + "estraverse": "^4.0.0" + } + }, + "node_modules/esrecurse": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.2.1.tgz", + "integrity": "sha512-64RBB++fIOAXPw3P9cy89qfMlvZEXZkqqJkjqqXIvzP5ezRZjW+lPWjw35UX/3EhUPFYbg5ER4JYgDw4007/DQ==", + "dev": true, + "dependencies": { + "estraverse": "^4.1.0" + } + }, + "node_modules/estraverse": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.2.0.tgz", + "integrity": "sha1-De4/7TH81GlhjOc0IJn8GvoL2xM=", + "dev": true + }, + "node_modules/esutils": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.2.tgz", + "integrity": "sha1-Cr9PHKpbyx96nYrMbepPqqBLrJs=", + "dev": true + }, + "node_modules/event-to-promise": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/event-to-promise/-/event-to-promise-0.8.0.tgz", + "integrity": "sha1-S4TxF3K28l93Uvx02XFTGsb1tiY=", + "dev": true + }, + "node_modules/events": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.2.0.tgz", + "integrity": "sha512-/46HWwbfCX2xTawVfkKLGxMifJYQBWMwY1mjywRtb4c9x8l5NP3KoJtnIOiL1hfdRkIuYhETxQlo62IF8tcnlg==", + "dev": true + }, + "node_modules/evp_bytestokey": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz", + "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==", + "dev": true, + "dependencies": { + "md5.js": "^1.3.4", + "safe-buffer": "^5.1.1" + } + }, + "node_modules/execa": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/execa/-/execa-4.0.3.tgz", + "integrity": "sha512-WFDXGHckXPWZX19t1kCsXzOpqX9LWYNqn4C+HqZlk/V0imTkzJZqf87ZBhvpHaftERYknpk0fjSylnXVlVgI0A==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.0", + "get-stream": "^5.0.0", + "human-signals": "^1.1.1", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.0", + "onetime": "^5.1.0", + "signal-exit": "^3.0.2", + "strip-final-newline": "^2.0.0" + } + }, + "node_modules/execa/node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dev": true, + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + } + }, + "node_modules/execa/node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true + }, + "node_modules/execa/node_modules/onetime": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz", + "integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==", + "dev": true, + "dependencies": { + "mimic-fn": "^2.1.0" + } + }, + "node_modules/execa/node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true + }, + "node_modules/execa/node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "dependencies": { + "shebang-regex": "^3.0.0" + } + }, + "node_modules/execa/node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true + }, + "node_modules/execa/node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + } + }, + "node_modules/expand-brackets": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz", + "integrity": "sha1-t3c14xXOMPa27/D4OwQVGiJEliI=", + "dev": true, + "dependencies": { + "debug": "^2.3.3", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "posix-character-classes": "^0.1.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + } + }, + "node_modules/expand-brackets/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/expand-brackets/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + } + }, + "node_modules/expand-brackets/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + } + }, + "node_modules/expand-brackets/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "node_modules/expand-tilde": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz", + "integrity": "sha1-l+gBqgUt8CRU3kawK/YhZCzchQI=", + "dev": true, + "dependencies": { + "homedir-polyfill": "^1.0.1" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "dev": true + }, + "node_modules/extend-shallow": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", + "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=", + "dev": true, + "dependencies": { + "assign-symbols": "^1.0.0", + "is-extendable": "^1.0.1" + } + }, + "node_modules/extend-shallow/node_modules/is-extendable": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", + "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "dev": true, + "dependencies": { + "is-plain-object": "^2.0.4" + } + }, + "node_modules/external-editor": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.0.3.tgz", + "integrity": "sha512-bn71H9+qWoOQKyZDo25mOMVpSmXROAsTJVVVYzrrtol3d4y+AsKjf4Iwl2Q+IuT0kFSQ1qo166UuIwqYq7mGnA==", + "dev": true, + "dependencies": { + "chardet": "^0.7.0", + "iconv-lite": "^0.4.24", + "tmp": "^0.0.33" + } + }, + "node_modules/extglob": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz", + "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==", + "dev": true, + "dependencies": { + "array-unique": "^0.3.2", + "define-property": "^1.0.0", + "expand-brackets": "^2.1.4", + "extend-shallow": "^2.0.1", + "fragment-cache": "^0.2.1", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + } + }, + "node_modules/extglob/node_modules/define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.0" + } + }, + "node_modules/extglob/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + } + }, + "node_modules/extglob/node_modules/is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + } + }, + "node_modules/extglob/node_modules/is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + } + }, + "node_modules/extglob/node_modules/is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + } + }, + "node_modules/extsprintf": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", + "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=", + "dev": true + }, + "node_modules/fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=", + "dev": true + }, + "node_modules/fast-json-patch": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/fast-json-patch/-/fast-json-patch-2.2.1.tgz", + "integrity": "sha512-4j5uBaTnsYAV5ebkidvxiLUYOwjQ+JSFljeqfTxCrH9bDmlCQaOJFS84oDJ2rAXZq2yskmk3ORfoP9DCwqFNig==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^2.0.1" + } + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz", + "integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I=", + "dev": true + }, + "node_modules/fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", + "dev": true + }, + "node_modules/fast-redact": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-2.0.0.tgz", + "integrity": "sha512-zxpkULI9W9MNTK2sJ3BpPQrTEXFNESd2X6O1tXMFpK/XM0G5c5Rll2EVYZH2TqI3xRGK/VaJ+eEOt7pnENJpeA==", + "dev": true + }, + "node_modules/fast-safe-stringify": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.0.7.tgz", + "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==", + "dev": true + }, + "node_modules/fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=", + "dev": true, + "dependencies": { + "pend": "~1.2.0" + } + }, + "node_modules/figgy-pudding": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/figgy-pudding/-/figgy-pudding-3.5.2.tgz", + "integrity": "sha512-0btnI/H8f2pavGMN8w40mlSKOfTK2SVJmBfBeVIj3kNw0swwgzyRq0d5TJVOwodFmtvpPeWPN/MCcfuWF0Ezbw==", + "dev": true + }, + "node_modules/figures": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", + "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", + "dev": true, + "dependencies": { + "escape-string-regexp": "^1.0.5" + } + }, + "node_modules/fill-range": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", + "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=", + "dev": true, + "dependencies": { + "extend-shallow": "^2.0.1", + "is-number": "^3.0.0", + "repeat-string": "^1.6.1", + "to-regex-range": "^2.1.0" + } + }, + "node_modules/fill-range/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + } + }, + "node_modules/find-cache-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.1.0.tgz", + "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==", + "dev": true, + "dependencies": { + "commondir": "^1.0.1", + "make-dir": "^2.0.0", + "pkg-dir": "^3.0.0" + } + }, + "node_modules/find-cache-dir/node_modules/find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "dependencies": { + "locate-path": "^3.0.0" + } + }, + "node_modules/find-cache-dir/node_modules/locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "node_modules/find-cache-dir/node_modules/make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "dependencies": { + "pify": "^4.0.1", + "semver": "^5.6.0" + } + }, + "node_modules/find-cache-dir/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + } + }, + "node_modules/find-cache-dir/node_modules/p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "dependencies": { + "p-limit": "^2.0.0" + } + }, + "node_modules/find-cache-dir/node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true + }, + "node_modules/find-cache-dir/node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true + }, + "node_modules/find-cache-dir/node_modules/pkg-dir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", + "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "dev": true, + "dependencies": { + "find-up": "^3.0.0" + } + }, + "node_modules/find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dev": true, + "dependencies": { + "locate-path": "^2.0.0" + } + }, + "node_modules/findup-sync": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/findup-sync/-/findup-sync-3.0.0.tgz", + "integrity": "sha512-YbffarhcicEhOrm4CtrwdKBdCuz576RLdhJDsIfvNtxUuhdRet1qZcsMjqbePtAseKdAnDyM/IyXbu7PRPRLYg==", + "dev": true, + "dependencies": { + "detect-file": "^1.0.0", + "is-glob": "^4.0.0", + "micromatch": "^3.0.4", + "resolve-dir": "^1.0.1" + } + }, + "node_modules/firefox-profile": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/firefox-profile/-/firefox-profile-1.3.1.tgz", + "integrity": "sha512-8q7DnwVIXvuJuBm1shr5ivRh0Ih2ytWwOIMwHInDSlVyrjQVXy7Ik0frItDdWb/P5CIpQFcMk9fPsUwNqi2lyQ==", + "dev": true, + "dependencies": { + "adm-zip": "~0.4.x", + "archiver": "~2.1.0", + "async": "~2.5.0", + "fs-extra": "~4.0.2", + "ini": "~1.3.3", + "jetpack-id": "1.0.0", + "lazystream": "~1.0.0", + "lodash": "~4.17.2", + "minimist": "^1.1.1", + "uuid": "^3.0.0", + "xml2js": "~0.4.4" + } + }, + "node_modules/firefox-profile/node_modules/async": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/async/-/async-2.5.0.tgz", + "integrity": "sha512-e+lJAJeNWuPCNyxZKOBdaJGyLGHugXVQtrAwtuAe2vhxTYxFTKE73p8JuTmdH0qdQZtDvI4dhJwjZc5zsfIsYw==", + "dev": true, + "dependencies": { + "lodash": "^4.14.0" + } + }, + "node_modules/first-chunk-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/first-chunk-stream/-/first-chunk-stream-3.0.0.tgz", + "integrity": "sha512-LNRvR4hr/S8cXXkIY5pTgVP7L3tq6LlYWcg9nWBuW7o1NMxKZo6oOVa/6GIekMGI0Iw7uC+HWimMe9u/VAeKqw==", + "dev": true + }, + "node_modules/flatstr": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/flatstr/-/flatstr-1.0.12.tgz", + "integrity": "sha512-4zPxDyhCyiN2wIAtSLI6gc82/EjqZc1onI4Mz/l0pWrAlsSfYH/2ZIcU+e3oA2wDwbzIWNKwa23F8rh6+DRWkw==", + "dev": true + }, + "node_modules/flatted": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.0.tgz", + "integrity": "sha512-R+H8IZclI8AAkSBRQJLVOsxwAoHd6WC40b4QTNWIjzAa6BXOBfQcM587MXDTVPeYaopFNWHUFLx7eNmHDSxMWg==", + "dev": true + }, + "node_modules/fluent-syntax": { + "version": "0.13.0", + "resolved": "https://registry.npmjs.org/fluent-syntax/-/fluent-syntax-0.13.0.tgz", + "integrity": "sha512-0Bk1AsliuYB550zr4JV9AYhsETsD3ELXUQzdXGJfIc1Ni/ukAfBdQInDhVMYJUaT2QxoamNslwkYF7MlOrPUwg==", + "dev": true + }, + "node_modules/flush-write-stream": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.1.1.tgz", + "integrity": "sha512-3Z4XhFZ3992uIq0XOqb9AreonueSYphE6oYbpt5+3u06JWklbsPkNv3ZKkP9Bz/r+1MWCaMoSQ28P85+1Yc77w==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "readable-stream": "^2.3.6" + } + }, + "node_modules/for-in": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", + "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=", + "dev": true + }, + "node_modules/forever-agent": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", + "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=", + "dev": true + }, + "node_modules/form-data": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" + } + }, + "node_modules/fragment-cache": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", + "integrity": "sha1-QpD60n8T6Jvn8zeZxrxaCr//DRk=", + "dev": true, + "dependencies": { + "map-cache": "^0.2.2" + } + }, + "node_modules/from2": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/from2/-/from2-2.3.0.tgz", + "integrity": "sha1-i/tVAr3kpNNs/e6gB/zKIdfjgq8=", + "dev": true, + "dependencies": { + "inherits": "^2.0.1", + "readable-stream": "^2.0.0" + } + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true + }, + "node_modules/fs-extra": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-4.0.3.tgz", + "integrity": "sha512-q6rbdDd1o2mAnQreO7YADIxf/Whx4AHBiRf6d+/cVT8h44ss+lHgxf1FemcqDnQt9X3ct4McHr+JMGlYSsK7Cg==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + } + }, + "node_modules/fs-write-stream-atomic": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/fs-write-stream-atomic/-/fs-write-stream-atomic-1.0.10.tgz", + "integrity": "sha1-tH31NJPvkR33VzHnCp3tAYnbQMk=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "iferr": "^0.1.5", + "imurmurhash": "^0.1.4", + "readable-stream": "1 || 2" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true + }, + "node_modules/fsevents": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.3.tgz", + "integrity": "sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==", + "dev": true, + "optional": true + }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "node_modules/functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "dev": true + }, + "node_modules/fx-runner": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/fx-runner/-/fx-runner-1.0.12.tgz", + "integrity": "sha512-SLATlfKI2lyIcQsU8Sgfcwrni6PpC1VMTgp3aRomK/6azrzSQ3r63HqoTRliE/6JP8WjqVkIdCOGWk1ZqhfceA==", + "dev": true, + "dependencies": { + "commander": "2.9.0", + "shell-quote": "1.6.1", + "spawn-sync": "1.0.15", + "when": "3.7.7", + "which": "1.2.4", + "winreg": "0.0.12" + } + }, + "node_modules/fx-runner/node_modules/commander": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz", + "integrity": "sha1-nJkJQXbhIkDLItbFFGCYQA/g99Q=", + "dev": true, + "dependencies": { + "graceful-readlink": ">= 1.0.0" + } + }, + "node_modules/fx-runner/node_modules/isexe": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-1.1.2.tgz", + "integrity": "sha1-NvPiLmB1CSD15yQaR2qMakInWtA=", + "dev": true + }, + "node_modules/fx-runner/node_modules/which": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/which/-/which-1.2.4.tgz", + "integrity": "sha1-FVf5YIBgTlsRs1meufRbUKnv1yI=", + "dev": true, + "dependencies": { + "is-absolute": "^0.1.7", + "isexe": "^1.1.1" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true + }, + "node_modules/get-stream": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz", + "integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==", + "dev": true, + "dependencies": { + "pump": "^3.0.0" + } + }, + "node_modules/get-value": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", + "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=", + "dev": true + }, + "node_modules/getpass": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", + "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", + "dev": true, + "dependencies": { + "assert-plus": "^1.0.0" + } + }, + "node_modules/git-rev-sync": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/git-rev-sync/-/git-rev-sync-2.0.0.tgz", + "integrity": "sha512-vnHFv2eocTmt/wHqZm3ksxtVshK4vptT0cEoumk6hAYRFx3do6Qo7xHBTBCv29+r3ZZCQOQ1i328MUCsYF7AUw==", + "dev": true, + "dependencies": { + "escape-string-regexp": "1.0.5", + "graceful-fs": "4.1.15", + "shelljs": "0.7.7" + } + }, + "node_modules/glob": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "node_modules/glob-parent": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-3.1.0.tgz", + "integrity": "sha1-nmr2KZ2NO9K9QEMIMr0RPfkGxa4=", + "dev": true, + "dependencies": { + "is-glob": "^3.1.0", + "path-dirname": "^1.0.0" + } + }, + "node_modules/glob-parent/node_modules/is-glob": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-3.1.0.tgz", + "integrity": "sha1-e6WuJCF4BKxwcHuWkiVnSGzD6Eo=", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.0" + } + }, + "node_modules/global-dirs": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-2.0.1.tgz", + "integrity": "sha512-5HqUqdhkEovj2Of/ms3IeS/EekcO54ytHRLV4PEY2rhRwrHXLQjeVEES0Lhka0xwNDtGYn58wyC4s5+MHsOO6A==", + "dev": true, + "dependencies": { + "ini": "^1.3.5" + } + }, + "node_modules/global-modules": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/global-modules/-/global-modules-2.0.0.tgz", + "integrity": "sha512-NGbfmJBp9x8IxyJSd1P+otYK8vonoJactOogrVfFRIAEY1ukil8RSKDz2Yo7wh1oihl51l/r6W4epkeKJHqL8A==", + "dev": true, + "dependencies": { + "global-prefix": "^3.0.0" + } + }, + "node_modules/global-modules/node_modules/global-prefix": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-3.0.0.tgz", + "integrity": "sha512-awConJSVCHVGND6x3tmMaKcQvwXLhjdkmomy2W+Goaui8YPgYgXJZewhg3fWC+DlfqqQuWg8AwqjGTD2nAPVWg==", + "dev": true, + "dependencies": { + "ini": "^1.3.5", + "kind-of": "^6.0.2", + "which": "^1.3.1" + } + }, + "node_modules/global-prefix": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-1.0.2.tgz", + "integrity": "sha1-2/dDxsFJklk8ZVVoy2btMsASLr4=", + "dev": true, + "dependencies": { + "expand-tilde": "^2.0.2", + "homedir-polyfill": "^1.0.1", + "ini": "^1.3.4", + "is-windows": "^1.0.1", + "which": "^1.2.14" + } + }, + "node_modules/globals": { + "version": "11.9.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.9.0.tgz", + "integrity": "sha512-5cJVtyXWH8PiJPVLZzzoIizXx944O4OmRro5MWKx5fT4MgcN7OfaMutPeaTdJCCURwbWdhhcCWcKIffPnmTzBg==", + "dev": true + }, + "node_modules/got": { + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", + "integrity": "sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==", + "dev": true, + "dependencies": { + "@sindresorhus/is": "^0.14.0", + "@szmarczak/http-timer": "^1.1.2", + "cacheable-request": "^6.0.0", + "decompress-response": "^3.3.0", + "duplexer3": "^0.1.4", + "get-stream": "^4.1.0", + "lowercase-keys": "^1.0.1", + "mimic-response": "^1.0.1", + "p-cancelable": "^1.0.0", + "to-readable-stream": "^1.0.0", + "url-parse-lax": "^3.0.0" + } + }, + "node_modules/got/node_modules/get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dev": true, + "dependencies": { + "pump": "^3.0.0" + } + }, + "node_modules/graceful-fs": { + "version": "4.1.15", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.15.tgz", + "integrity": "sha512-6uHUhOPEBgQ24HM+r6b/QwWfZq+yiFcipKFrOFiBEnWdy5sdzYoi+pJeQaPI5qOLRFqWmAXUPQNsielzdLoecA==", + "dev": true + }, + "node_modules/graceful-readlink": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz", + "integrity": "sha1-TK+tdrxi8C+gObL5Tpo906ORpyU=", + "dev": true + }, + "node_modules/growly": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/growly/-/growly-1.3.0.tgz", + "integrity": "sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE=", + "dev": true + }, + "node_modules/har-schema": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz", + "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=", + "dev": true + }, + "node_modules/har-validator": { + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", + "dev": true, + "dependencies": { + "ajv": "^6.5.5", + "har-schema": "^2.0.0" + } + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1" + } + }, + "node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true + }, + "node_modules/has-symbols": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.0.tgz", + "integrity": "sha1-uhqPGvKg/DllD1yFA2dwQSIGO0Q=", + "dev": true + }, + "node_modules/has-value": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", + "integrity": "sha1-GLKB2lhbHFxR3vJMkw7SmgvmsXc=", + "dev": true, + "dependencies": { + "get-value": "^2.0.6", + "has-values": "^1.0.0", + "isobject": "^3.0.0" + } + }, + "node_modules/has-values": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz", + "integrity": "sha1-lbC2P+whRmGab+V/51Yo1aOe/k8=", + "dev": true, + "dependencies": { + "is-number": "^3.0.0", + "kind-of": "^4.0.0" + } + }, + "node_modules/has-values/node_modules/kind-of": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz", + "integrity": "sha1-IIE989cSkosgc3hpGkUGb65y3Vc=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + } + }, + "node_modules/has-yarn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", + "integrity": "sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw==", + "dev": true + }, + "node_modules/hash-base": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.1.0.tgz", + "integrity": "sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.4", + "readable-stream": "^3.6.0", + "safe-buffer": "^5.2.0" + } + }, + "node_modules/hash-base/node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/hash-base/node_modules/readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, + "node_modules/hash-base/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + }, + "node_modules/hash.js": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", + "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "minimalistic-assert": "^1.0.1" + } + }, + "node_modules/hmac-drbg": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", + "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=", + "dev": true, + "dependencies": { + "hash.js": "^1.0.3", + "minimalistic-assert": "^1.0.0", + "minimalistic-crypto-utils": "^1.0.1" + } + }, + "node_modules/homedir-polyfill": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz", + "integrity": "sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA==", + "dev": true, + "dependencies": { + "parse-passwd": "^1.0.0" + } + }, + "node_modules/hosted-git-info": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.7.1.tgz", + "integrity": "sha512-7T/BxH19zbcCTa8XkMlbK5lTo1WtgkFi3GvdWEyNuc4Vex7/9Dqbnpsf4JMydcfj9HCg4zUWFTL3Za6lapg5/w==", + "dev": true + }, + "node_modules/htmlparser2": { + "version": "3.9.2", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.9.2.tgz", + "integrity": "sha1-G9+HrMoPP55T+k/M6w9LTLsAszg=", + "dev": true, + "dependencies": { + "domelementtype": "^1.3.0", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^2.0.2" + } + }, + "node_modules/http-cache-semantics": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", + "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==", + "dev": true + }, + "node_modules/http-signature": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", + "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", + "dev": true, + "dependencies": { + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" + } + }, + "node_modules/https-browserify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", + "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=", + "dev": true + }, + "node_modules/human-signals": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-1.1.1.tgz", + "integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==", + "dev": true + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "node_modules/ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==", + "dev": true + }, + "node_modules/iferr": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/iferr/-/iferr-0.1.5.tgz", + "integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE=", + "dev": true + }, + "node_modules/ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + }, + "node_modules/import-fresh": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.0.0.tgz", + "integrity": "sha512-pOnA9tfM3Uwics+SaBLCNyZZZbK+4PTu0OPZtLlMIrv17EdBoC15S9Kn8ckJ9TZTyKb3ywNE5y1yeDxxGA7nTQ==", + "dev": true, + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + } + }, + "node_modules/import-fresh/node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true + }, + "node_modules/import-lazy": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz", + "integrity": "sha1-BWmOPUXIjo1+nZLLBYTnfwlvPkM=", + "dev": true + }, + "node_modules/import-local": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-2.0.0.tgz", + "integrity": "sha512-b6s04m3O+s3CGSbqDIyP4R6aAwAeYlVq9+WUWep6iHa8ETRf9yei1U48C5MmfJmV9AiLYYBKPMq/W+/WRpQmCQ==", + "dev": true, + "dependencies": { + "pkg-dir": "^3.0.0", + "resolve-cwd": "^2.0.0" + } + }, + "node_modules/import-local/node_modules/find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "dependencies": { + "locate-path": "^3.0.0" + } + }, + "node_modules/import-local/node_modules/locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "node_modules/import-local/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + } + }, + "node_modules/import-local/node_modules/p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "dependencies": { + "p-limit": "^2.0.0" + } + }, + "node_modules/import-local/node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true + }, + "node_modules/import-local/node_modules/pkg-dir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", + "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "dev": true, + "dependencies": { + "find-up": "^3.0.0" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "dev": true + }, + "node_modules/infer-owner": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz", + "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==", + "dev": true + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", + "dev": true + }, + "node_modules/ini": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "dev": true + }, + "node_modules/ini-parser": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/ini-parser/-/ini-parser-0.0.2.tgz", + "integrity": "sha1-+kF4flZ3Y7P/Zdel2alO23QHh+8=", + "dev": true + }, + "node_modules/inquirer": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.3.1.tgz", + "integrity": "sha512-MmL624rfkFt4TG9y/Jvmt8vdmOo836U7Y0Hxr2aFk3RelZEGX4Igk0KabWrcaaZaTv9uzglOqWh1Vly+FAWAXA==", + "dev": true, + "dependencies": { + "ansi-escapes": "^3.2.0", + "chalk": "^2.4.2", + "cli-cursor": "^2.1.0", + "cli-width": "^2.0.0", + "external-editor": "^3.0.3", + "figures": "^2.0.0", + "lodash": "^4.17.11", + "mute-stream": "0.0.7", + "run-async": "^2.2.0", + "rxjs": "^6.4.0", + "string-width": "^2.1.0", + "strip-ansi": "^5.1.0", + "through": "^2.3.6" + } + }, + "node_modules/inquirer/node_modules/ansi-escapes": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", + "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", + "dev": true + }, + "node_modules/inquirer/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "node_modules/inquirer/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "node_modules/inquirer/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "dependencies": { + "ansi-regex": "^4.1.0" + } + }, + "node_modules/interpret": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", + "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", + "dev": true + }, + "node_modules/invert-kv": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-3.0.1.tgz", + "integrity": "sha512-CYdFeFexxhv/Bcny+Q0BfOV+ltRlJcd4BBZBYFX/O0u4npJrgZtIcjokegtiSMAvlMTJ+Koq0GBCc//3bueQxw==", + "dev": true + }, + "node_modules/is-absolute": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/is-absolute/-/is-absolute-0.1.7.tgz", + "integrity": "sha1-hHSREZ/MtftDYhfMc39/qtUPYD8=", + "dev": true, + "dependencies": { + "is-relative": "^0.1.0" + } + }, + "node_modules/is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + } + }, + "node_modules/is-accessor-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + } + }, + "node_modules/is-arguments": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.0.4.tgz", + "integrity": "sha512-xPh0Rmt8NE65sNzvyUmWgI1tz3mKq74lGA0mL8LYZcoIzKOzDh6HmrYm3d18k60nHerC8A9Km8kYu87zfSFnLA==", + "dev": true + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=", + "dev": true + }, + "node_modules/is-binary-path": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-1.0.1.tgz", + "integrity": "sha1-dfFmQrSA8YenEcgUFh/TpKdlWJg=", + "dev": true, + "dependencies": { + "binary-extensions": "^1.0.0" + } + }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "dev": true + }, + "node_modules/is-builtin-module": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz", + "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=", + "dev": true, + "dependencies": { + "builtin-modules": "^1.0.0" + } + }, + "node_modules/is-callable": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.4.tgz", + "integrity": "sha512-r5p9sxJjYnArLjObpjA4xu5EKI3CuKHkJXMhT7kwbpUyIFD1n5PMAsoPvWnvtZiNz7LjkYDRZhd7FlI0eMijEA==", + "dev": true + }, + "node_modules/is-ci": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz", + "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==", + "dev": true, + "dependencies": { + "ci-info": "^2.0.0" + } + }, + "node_modules/is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + } + }, + "node_modules/is-data-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + } + }, + "node_modules/is-date-object": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.1.tgz", + "integrity": "sha1-mqIOtq7rv/d/vTPnTKAbM1gdOhY=", + "dev": true + }, + "node_modules/is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + } + }, + "node_modules/is-descriptor/node_modules/kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true + }, + "node_modules/is-docker": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.0.0.tgz", + "integrity": "sha512-pJEdRugimx4fBMra5z2/5iRdZ63OhYV0vr0Dwm5+xtW4D1FvRkB8hamMIhnWfyJeDdyr/aa7BDyNbtG38VxgoQ==", + "dev": true + }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=", + "dev": true + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "dev": true + }, + "node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "node_modules/is-glob": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", + "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + } + }, + "node_modules/is-installed-globally": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.3.2.tgz", + "integrity": "sha512-wZ8x1js7Ia0kecP/CHM/3ABkAmujX7WPvQk6uu3Fly/Mk44pySulQpnHG46OMjHGXApINnV4QhY3SWnECO2z5g==", + "dev": true, + "dependencies": { + "global-dirs": "^2.0.1", + "is-path-inside": "^3.0.1" + } + }, + "node_modules/is-mergeable-object": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-mergeable-object/-/is-mergeable-object-1.1.1.tgz", + "integrity": "sha512-CPduJfuGg8h8vW74WOxHtHmtQutyQBzR+3MjQ6iDHIYdbOnm1YC7jv43SqCoU8OPGTJD4nibmiryA4kmogbGrA==", + "dev": true + }, + "node_modules/is-npm": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-4.0.0.tgz", + "integrity": "sha512-96ECIfh9xtDDlPylNPXhzjsykHsMJZ18ASpaWzQyBr4YRTcVjUvzaHayDAES2oU/3KpljhHUjtSRNiDwi0F0ig==", + "dev": true + }, + "node_modules/is-number": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", + "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + } + }, + "node_modules/is-number/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + } + }, + "node_modules/is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "dev": true + }, + "node_modules/is-path-inside": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.2.tgz", + "integrity": "sha512-/2UGPSgmtqwo1ktx8NDHjuPwZWmHhO+gj0f93EkhLB5RgW9RZevWYYlIkS6zePc6U2WpOdQYIwHe9YC4DWEBVg==", + "dev": true + }, + "node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "dependencies": { + "isobject": "^3.0.1" + } + }, + "node_modules/is-promise": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.1.0.tgz", + "integrity": "sha1-eaKp7OfwlugPNtKy87wWwf9L8/o=", + "dev": true + }, + "node_modules/is-regex": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.4.tgz", + "integrity": "sha1-VRdIm1RwkbCTDglWVM7SXul+lJE=", + "dev": true, + "dependencies": { + "has": "^1.0.1" + } + }, + "node_modules/is-relative": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/is-relative/-/is-relative-0.1.3.tgz", + "integrity": "sha1-kF/uiuhvRbPsYUvDwVyGnfCHboI=", + "dev": true + }, + "node_modules/is-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", + "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==", + "dev": true + }, + "node_modules/is-string": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.5.tgz", + "integrity": "sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ==", + "dev": true + }, + "node_modules/is-symbol": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.2.tgz", + "integrity": "sha512-HS8bZ9ox60yCJLH9snBpIwv9pYUAkcuLhSA1oero1UB5y9aiQpRA8y2ex945AOtCZL1lJDeIk3G5LthswI46Lw==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.0" + } + }, + "node_modules/is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", + "dev": true + }, + "node_modules/is-utf8": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", + "integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI=", + "dev": true + }, + "node_modules/is-windows": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", + "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "dev": true + }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "dev": true, + "dependencies": { + "is-docker": "^2.0.0" + } + }, + "node_modules/is-yarn-global": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.3.0.tgz", + "integrity": "sha512-VjSeb/lHmkoyd8ryPVIKvOCn4D1koMqY+vqyjjUfc3xyKtP4dYOxM44sZrnqQSzSds3xyOrUTLTC9LVCVgLngw==", + "dev": true + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "dev": true + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=", + "dev": true + }, + "node_modules/isstream": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", + "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=", + "dev": true + }, + "node_modules/jed": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/jed/-/jed-1.1.1.tgz", + "integrity": "sha1-elSbvZ/+FYWwzQoZHiAwVb7ldLQ=", + "dev": true + }, + "node_modules/jetpack-id": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/jetpack-id/-/jetpack-id-1.0.0.tgz", + "integrity": "sha1-LPn7rkbYB0/Ba33gBxyO/rykc6Y=", + "dev": true + }, + "node_modules/js-select": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/js-select/-/js-select-0.6.0.tgz", + "integrity": "sha1-woTiKCTVknrsli3N8kcXSu+w0ZA=", + "dev": true, + "dependencies": { + "JSONSelect": "0.2.1", + "traverse": "0.4.x" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "node_modules/js-yaml": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, + "node_modules/jsbn": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", + "dev": true + }, + "node_modules/json-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", + "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=", + "dev": true + }, + "node_modules/json-merge-patch": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-merge-patch/-/json-merge-patch-0.2.3.tgz", + "integrity": "sha1-+ixrWvh9p3uuKWalidUuI+2B/kA=", + "dev": true, + "dependencies": { + "deep-equal": "^1.0.0" + } + }, + "node_modules/json-parse-better-errors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "dev": true + }, + "node_modules/json-schema": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz", + "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=", + "dev": true + }, + "node_modules/json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "node_modules/json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", + "dev": true + }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", + "dev": true + }, + "node_modules/json5": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", + "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "dev": true, + "dependencies": { + "minimist": "^1.2.0" + } + }, + "node_modules/json5/node_modules/minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "node_modules/jsonc-parser": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-2.1.0.tgz", + "integrity": "sha512-n9GrT8rrr2fhvBbANa1g+xFmgGK5X91KFeDwlKQ3+SJfmH5+tKv/M/kahx/TXOMflfWHKGKqKyfHQaLKTNzJ6w==", + "dev": true + }, + "node_modules/jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/jsonify": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/jsonify/-/jsonify-0.0.0.tgz", + "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=", + "dev": true + }, + "node_modules/JSONSelect": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/JSONSelect/-/JSONSelect-0.2.1.tgz", + "integrity": "sha1-QVQYpSbTP+MddLTe+jyDbUhewgM=", + "dev": true + }, + "node_modules/jsonwebtoken": { + "version": "8.5.1", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz", + "integrity": "sha512-XjwVfRS6jTMsqYs0EsuJ4LGxXV14zQybNd4L2r0UvbVnSF9Af8x7p5MzbJ90Ioz/9TI41/hTCvznF/loiSzn8w==", + "dev": true, + "dependencies": { + "jws": "^3.2.2", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^5.6.0" + } + }, + "node_modules/jsprim": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz", + "integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=", + "dev": true, + "dependencies": { + "assert-plus": "1.0.0", + "extsprintf": "1.3.0", + "json-schema": "0.2.3", + "verror": "1.10.0" + } + }, + "node_modules/jszip": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-2.6.1.tgz", + "integrity": "sha1-uI86ey5noqBIFSmCx6N1bZxIKPA=", + "dev": true, + "dependencies": { + "pako": "~1.0.2" + } + }, + "node_modules/jwa": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", + "dev": true, + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "dev": true, + "dependencies": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/keyv": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz", + "integrity": "sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==", + "dev": true, + "dependencies": { + "json-buffer": "3.0.0" + } + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true + }, + "node_modules/latest-version": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-5.1.0.tgz", + "integrity": "sha512-weT+r0kTkRQdCdYCNtkMwWXQTMEswKrFBkm4ckQOMVhhqhIMI1UT2hMj+1iigIhgSZm5gTmrRXBNoGUgaTY1xA==", + "dev": true, + "dependencies": { + "package-json": "^6.3.0" + } + }, + "node_modules/lazystream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/lazystream/-/lazystream-1.0.0.tgz", + "integrity": "sha1-9plf4PggOS9hOWvolGJAe7dxaOQ=", + "dev": true, + "dependencies": { + "readable-stream": "^2.0.5" + } + }, + "node_modules/lcid": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/lcid/-/lcid-3.1.1.tgz", + "integrity": "sha512-M6T051+5QCGLBQb8id3hdvIW8+zeFV2FyBGFS9IEK5H9Wt4MueD4bW1eWikpHgZp+5xR3l5c8pZUkQsIA0BFZg==", + "dev": true, + "dependencies": { + "invert-kv": "^3.0.0" + } + }, + "node_modules/levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha1-OwmSTt+fCDwEkP3UwLxEIeBHZO4=", + "dev": true, + "dependencies": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + } + }, + "node_modules/lighthouse-logger": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/lighthouse-logger/-/lighthouse-logger-1.2.0.tgz", + "integrity": "sha512-wzUvdIeJZhRsG6gpZfmSCfysaxNEr43i+QT+Hie94wvHDKFLi4n7C2GqZ4sTC+PH5b5iktmXJvU87rWvhP3lHw==", + "dev": true, + "dependencies": { + "debug": "^2.6.8", + "marky": "^1.2.0" + } + }, + "node_modules/lighthouse-logger/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/lighthouse-logger/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "node_modules/lines-and-columns": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.1.6.tgz", + "integrity": "sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA=", + "dev": true + }, + "node_modules/load-json-file": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-2.0.0.tgz", + "integrity": "sha1-eUfkIUmvgNaWy/eXvKq8/h/inKg=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "parse-json": "^2.2.0", + "pify": "^2.0.0", + "strip-bom": "^3.0.0" + } + }, + "node_modules/loader-runner": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-2.4.0.tgz", + "integrity": "sha512-Jsmr89RcXGIwivFY21FcRrisYZfvLMTWx5kOLc+JTxtpBOG6xML0vzbc6SEQG2FO9/4Fc3wW4LVcB5DmGflaRw==", + "dev": true + }, + "node_modules/loader-utils": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.2.3.tgz", + "integrity": "sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA==", + "dev": true, + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^2.0.0", + "json5": "^1.0.1" + } + }, + "node_modules/locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dev": true, + "dependencies": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + } + }, + "node_modules/lodash": { + "version": "4.17.19", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", + "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==", + "dev": true + }, + "node_modules/lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha1-YLuYqHy5I8aMoeUTJUgzFISfVT8=", + "dev": true + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha1-bC4XHbKiV82WgC/UOwGyDV9YcPY=", + "dev": true + }, + "node_modules/lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha1-YZwK89A/iwTDH1iChAt3sRzWg0M=", + "dev": true + }, + "node_modules/lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha1-POdoEMWSjQM1IwGsKHMX8RwLH/w=", + "dev": true + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha1-fFJqUtibRcRcxpC4gWO+BJf1UMs=", + "dev": true + }, + "node_modules/lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha1-1SfftUVuynzJu5XV2ur4i6VKVFE=", + "dev": true + }, + "node_modules/lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha1-DdOXEhPHxW34gJd9UEyI+0cal6w=", + "dev": true + }, + "node_modules/lodash.sortby": { + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", + "integrity": "sha1-7dFMgk4sycHgsKG0K7UhBRakJDg=", + "dev": true + }, + "node_modules/lowercase-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", + "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==", + "dev": true + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, + "dependencies": { + "semver": "^6.0.0" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + }, + "node_modules/map-age-cleaner": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz", + "integrity": "sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==", + "dev": true, + "dependencies": { + "p-defer": "^1.0.0" + } + }, + "node_modules/map-cache": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", + "integrity": "sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8=", + "dev": true + }, + "node_modules/map-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", + "integrity": "sha1-7Nyo8TFE5mDxtb1B8S80edmN+48=", + "dev": true, + "dependencies": { + "object-visit": "^1.0.0" + } + }, + "node_modules/marky": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/marky/-/marky-1.2.1.tgz", + "integrity": "sha512-md9k+Gxa3qLH6sUKpeC2CNkJK/Ld+bEz5X96nYwloqphQE0CKCVEKco/6jxEZixinqNdz5RFi/KaCyfbMDMAXQ==", + "dev": true + }, + "node_modules/md5.js": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", + "integrity": "sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==", + "dev": true, + "dependencies": { + "hash-base": "^3.0.0", + "inherits": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "node_modules/mdn-browser-compat-data": { + "version": "1.0.25", + "resolved": "https://registry.npmjs.org/mdn-browser-compat-data/-/mdn-browser-compat-data-1.0.25.tgz", + "integrity": "sha512-4klqILpitRnmWRai5Ols/GXP1eGDYMluAcBRoNZnGNkV2OnkDmpA9hUlM+9pTFym5FGDO5TAm3HweVSVc7ziiQ==", + "dev": true, + "dependencies": { + "extend": "3.0.2" + } + }, + "node_modules/mem": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/mem/-/mem-5.1.1.tgz", + "integrity": "sha512-qvwipnozMohxLXG1pOqoLiZKNkC4r4qqRucSoDwXowsNGDSULiqFTRUF05vcZWnwJSG22qTsynQhxbaMtnX9gw==", + "dev": true, + "dependencies": { + "map-age-cleaner": "^0.1.3", + "mimic-fn": "^2.1.0", + "p-is-promise": "^2.1.0" + } + }, + "node_modules/mem/node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true + }, + "node_modules/memory-fs": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.4.1.tgz", + "integrity": "sha1-OpoguEYlI+RHz7x+i7gO1me/xVI=", + "dev": true, + "dependencies": { + "errno": "^0.1.3", + "readable-stream": "^2.0.1" + } + }, + "node_modules/memorystream": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/memorystream/-/memorystream-0.3.1.tgz", + "integrity": "sha1-htcJCzDORV1j+64S3aUaR93K+bI=", + "dev": true + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true + }, + "node_modules/micromatch": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", + "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==", + "dev": true, + "dependencies": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "braces": "^2.3.1", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "extglob": "^2.0.4", + "fragment-cache": "^0.2.1", + "kind-of": "^6.0.2", + "nanomatch": "^1.2.9", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.2" + } + }, + "node_modules/miller-rabin": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/miller-rabin/-/miller-rabin-4.0.1.tgz", + "integrity": "sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA==", + "dev": true, + "dependencies": { + "bn.js": "^4.0.0", + "brorand": "^1.0.1" + } + }, + "node_modules/miller-rabin/node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", + "dev": true + }, + "node_modules/mime-db": { + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.44.0.tgz", + "integrity": "sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==", + "dev": true + }, + "node_modules/mime-types": { + "version": "2.1.27", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.27.tgz", + "integrity": "sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==", + "dev": true, + "dependencies": { + "mime-db": "1.44.0" + } + }, + "node_modules/mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "dev": true + }, + "node_modules/mimic-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", + "dev": true + }, + "node_modules/minimalistic-assert": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", + "dev": true + }, + "node_modules/minimalistic-crypto-utils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", + "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=", + "dev": true + }, + "node_modules/minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + } + }, + "node_modules/minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "node_modules/mississippi": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mississippi/-/mississippi-3.0.0.tgz", + "integrity": "sha512-x471SsVjUtBRtcvd4BzKE9kFC+/2TeWgKCgw0bZcw1b9l2X3QX5vCWgF+KaZaYm87Ss//rHnWryupDrgLvmSkA==", + "dev": true, + "dependencies": { + "concat-stream": "^1.5.0", + "duplexify": "^3.4.2", + "end-of-stream": "^1.1.0", + "flush-write-stream": "^1.0.0", + "from2": "^2.1.0", + "parallel-transform": "^1.1.0", + "pump": "^3.0.0", + "pumpify": "^1.3.3", + "stream-each": "^1.1.0", + "through2": "^2.0.0" + } + }, + "node_modules/mixin-deep": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", + "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", + "dev": true, + "dependencies": { + "for-in": "^1.0.2", + "is-extendable": "^1.0.1" + } + }, + "node_modules/mixin-deep/node_modules/is-extendable": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", + "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "dev": true, + "dependencies": { + "is-plain-object": "^2.0.4" + } + }, + "node_modules/mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "dependencies": { + "minimist": "^1.2.5" + } + }, + "node_modules/moment": { + "version": "2.27.0", + "resolved": "https://registry.npmjs.org/moment/-/moment-2.27.0.tgz", + "integrity": "sha512-al0MUK7cpIcglMv3YF13qSgdAIqxHTO7brRtaz3DlSULbqfazqkc5kEjNrLDOM7fsjshoFIihnU8snrP7zUvhQ==", + "dev": true, + "optional": true + }, + "node_modules/move-concurrently": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz", + "integrity": "sha1-viwAX9oy4LKa8fBdfEszIUxwH5I=", + "dev": true, + "dependencies": { + "aproba": "^1.1.1", + "copy-concurrently": "^1.0.0", + "fs-write-stream-atomic": "^1.0.8", + "mkdirp": "^0.5.1", + "rimraf": "^2.5.4", + "run-queue": "^1.0.3" + } + }, + "node_modules/ms": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.1.tgz", + "integrity": "sha512-tgp+dl5cGk28utYktBsrFqA7HKgrhgPsg6Z/EfhWI4gl1Hwq8B/GmY/0oXZ6nF8hDVesS/FpnYaD/kOWhYQvyg==", + "dev": true + }, + "node_modules/multimatch": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/multimatch/-/multimatch-4.0.0.tgz", + "integrity": "sha512-lDmx79y1z6i7RNx0ZGCPq1bzJ6ZoDDKbvh7jxr9SJcWLkShMzXrHbYVpTdnhNM5MXpDUxCQ4DgqVttVXlBgiBQ==", + "dev": true, + "dependencies": { + "@types/minimatch": "^3.0.3", + "array-differ": "^3.0.0", + "array-union": "^2.1.0", + "arrify": "^2.0.1", + "minimatch": "^3.0.4" + } + }, + "node_modules/mute-stream": { + "version": "0.0.7", + "resolved": "http://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", + "dev": true + }, + "node_modules/mv": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", + "integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=", + "dev": true, + "optional": true, + "dependencies": { + "mkdirp": "~0.5.1", + "ncp": "~2.0.0", + "rimraf": "~2.4.0" + } + }, + "node_modules/mv/node_modules/glob": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", + "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=", + "dev": true, + "optional": true, + "dependencies": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "node_modules/mv/node_modules/rimraf": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", + "integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=", + "dev": true, + "optional": true, + "dependencies": { + "glob": "^6.0.1" + } + }, + "node_modules/mz": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", + "integrity": "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==", + "dev": true, + "dependencies": { + "any-promise": "^1.0.0", + "object-assign": "^4.0.1", + "thenify-all": "^1.0.0" + } + }, + "node_modules/nan": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz", + "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==", + "dev": true, + "optional": true + }, + "node_modules/nanomatch": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", + "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", + "dev": true, + "dependencies": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "fragment-cache": "^0.2.1", + "is-windows": "^1.0.2", + "kind-of": "^6.0.2", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + } + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", + "dev": true + }, + "node_modules/natural-compare-lite": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare-lite/-/natural-compare-lite-1.4.0.tgz", + "integrity": "sha1-F7CVgZiJef3a/gIB6TG6kzyWy7Q=", + "dev": true + }, + "node_modules/ncp": { + "version": "2.0.0", + "resolved": "http://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", + "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=", + "dev": true, + "optional": true + }, + "node_modules/neo-async": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.1.tgz", + "integrity": "sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw==", + "dev": true + }, + "node_modules/next-tick": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", + "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==", + "dev": true + }, + "node_modules/nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", + "dev": true + }, + "node_modules/node-forge": { + "version": "0.7.6", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.6.tgz", + "integrity": "sha512-sol30LUpz1jQFBjOKwbjxijiE3b6pjd74YwfD0fJOKPjF+fONKb2Yg8rYgS6+bK6VDl+/wfr4IYpC7jDzLUIfw==", + "dev": true + }, + "node_modules/node-libs-browser": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz", + "integrity": "sha512-h/zcD8H9kaDZ9ALUWwlBUDo6TKF8a7qBSCSEGfjTVIYeqsioSKaAX+BN7NgiMGp6iSIXZ3PxgCu8KS3b71YK5Q==", + "dev": true, + "dependencies": { + "assert": "^1.1.1", + "browserify-zlib": "^0.2.0", + "buffer": "^4.3.0", + "console-browserify": "^1.1.0", + "constants-browserify": "^1.0.0", + "crypto-browserify": "^3.11.0", + "domain-browser": "^1.1.1", + "events": "^3.0.0", + "https-browserify": "^1.0.0", + "os-browserify": "^0.3.0", + "path-browserify": "0.0.1", + "process": "^0.11.10", + "punycode": "^1.2.4", + "querystring-es3": "^0.2.0", + "readable-stream": "^2.3.3", + "stream-browserify": "^2.0.1", + "stream-http": "^2.7.2", + "string_decoder": "^1.0.0", + "timers-browserify": "^2.0.4", + "tty-browserify": "0.0.0", + "url": "^0.11.0", + "util": "^0.11.0", + "vm-browserify": "^1.0.1" + } + }, + "node_modules/node-libs-browser/node_modules/buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "dev": true, + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "node_modules/node-libs-browser/node_modules/punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", + "dev": true + }, + "node_modules/node-notifier": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/node-notifier/-/node-notifier-6.0.0.tgz", + "integrity": "sha512-SVfQ/wMw+DesunOm5cKqr6yDcvUTDl/yc97ybGHMrteNEY6oekXpNpS3lZwgLlwz0FLgHoiW28ZpmBHUDg37cw==", + "dev": true, + "dependencies": { + "growly": "^1.3.0", + "is-wsl": "^2.1.1", + "semver": "^6.3.0", + "shellwords": "^0.1.1", + "which": "^1.3.1" + } + }, + "node_modules/node-notifier/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + }, + "node_modules/normalize-package-data": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz", + "integrity": "sha512-9jjUFbTPfEy3R/ad/2oNbKtW9Hgovl5O1FvFWKkKblNXoN/Oou6+9+KKohPK13Yc3/TyunyWhJp6gvRNR/PPAw==", + "dev": true, + "dependencies": { + "hosted-git-info": "^2.1.4", + "is-builtin-module": "^1.0.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } + }, + "node_modules/normalize-path": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-2.1.1.tgz", + "integrity": "sha1-GrKLVW4Zg2Oowab35vogE3/mrtk=", + "dev": true, + "dependencies": { + "remove-trailing-separator": "^1.0.1" + } + }, + "node_modules/normalize-url": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz", + "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==", + "dev": true + }, + "node_modules/npm-run-all": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/npm-run-all/-/npm-run-all-4.1.5.tgz", + "integrity": "sha512-Oo82gJDAVcaMdi3nuoKFavkIHBRVqQ1qvMb+9LHk/cF4P6B2m8aP04hGf7oL6wZ9BuGwX1onlLhpuoofSyoQDQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "chalk": "^2.4.1", + "cross-spawn": "^6.0.5", + "memorystream": "^0.3.1", + "minimatch": "^3.0.4", + "pidtree": "^0.3.0", + "read-pkg": "^3.0.0", + "shell-quote": "^1.6.1", + "string.prototype.padend": "^3.0.0" + } + }, + "node_modules/npm-run-all/node_modules/load-json-file": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "parse-json": "^4.0.0", + "pify": "^3.0.0", + "strip-bom": "^3.0.0" + } + }, + "node_modules/npm-run-all/node_modules/parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", + "dev": true, + "dependencies": { + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1" + } + }, + "node_modules/npm-run-all/node_modules/path-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", + "dev": true, + "dependencies": { + "pify": "^3.0.0" + } + }, + "node_modules/npm-run-all/node_modules/pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true + }, + "node_modules/npm-run-all/node_modules/read-pkg": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", + "dev": true, + "dependencies": { + "load-json-file": "^4.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^3.0.0" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "dependencies": { + "path-key": "^3.0.0" + } + }, + "node_modules/npm-run-path/node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true + }, + "node_modules/nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "dev": true, + "dependencies": { + "boolbase": "~1.0.0" + } + }, + "node_modules/oauth-sign": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==", + "dev": true + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", + "dev": true + }, + "node_modules/object-copy": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", + "integrity": "sha1-fn2Fi3gb18mRpBupde04EnVOmYw=", + "dev": true, + "dependencies": { + "copy-descriptor": "^0.1.0", + "define-property": "^0.2.5", + "kind-of": "^3.0.3" + } + }, + "node_modules/object-copy/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + } + }, + "node_modules/object-copy/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + } + }, + "node_modules/object-inspect": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.8.0.tgz", + "integrity": "sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA==", + "dev": true + }, + "node_modules/object-is": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.2.tgz", + "integrity": "sha512-5lHCz+0uufF6wZ7CRFWJN3hp8Jqblpgve06U5CMQ3f//6iDjPr2PEo9MWCjEssDsa+UZEL4PkFpr+BMop6aKzQ==", + "dev": true, + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" + } + }, + "node_modules/object-is/node_modules/es-abstract": { + "version": "1.17.6", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", + "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==", + "dev": true, + "dependencies": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.0", + "is-regex": "^1.1.0", + "object-inspect": "^1.7.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.0", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + }, + "node_modules/object-is/node_modules/es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "dev": true, + "dependencies": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, + "node_modules/object-is/node_modules/has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", + "dev": true + }, + "node_modules/object-is/node_modules/is-callable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", + "integrity": "sha512-pyVD9AaGLxtg6srb2Ng6ynWJqkHU9bEM087AKck0w8QwDarTfNcpIYoU8x8Hv2Icm8u6kFJM18Dag8lyqGkviw==", + "dev": true + }, + "node_modules/object-is/node_modules/is-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.0.tgz", + "integrity": "sha512-iI97M8KTWID2la5uYXlkbSDQIg4F6o1sYboZKKTDpnDQMLtUL86zxhgDet3Q2SriaYsyGqZ6Mn2SjbRKeLHdqw==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.1" + } + }, + "node_modules/object-is/node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true + }, + "node_modules/object-keys": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.0.12.tgz", + "integrity": "sha512-FTMyFUm2wBcGHnH2eXmz7tC6IwlqQZ6mVZ+6dm6vZ4IQIHjs6FdNsQBuKGPuUUUY6NfJw2PshC08Tn6LzLDOag==", + "dev": true + }, + "node_modules/object-visit": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", + "integrity": "sha1-95xEk68MU3e1n+OdOV5BBC3QRbs=", + "dev": true, + "dependencies": { + "isobject": "^3.0.0" + } + }, + "node_modules/object.assign": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", + "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", + "dev": true, + "dependencies": { + "define-properties": "^1.1.2", + "function-bind": "^1.1.1", + "has-symbols": "^1.0.0", + "object-keys": "^1.0.11" + } + }, + "node_modules/object.pick": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz", + "integrity": "sha1-h6EKxMFpS9Lhy/U1kaZhQftd10c=", + "dev": true, + "dependencies": { + "isobject": "^3.0.1" + } + }, + "node_modules/object.values": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.1.tgz", + "integrity": "sha512-WTa54g2K8iu0kmS/us18jEmdv1a4Wi//BZ/DTVYEcH0XhLM5NYdpDHja3gt57VrZLcNAO2WGA+KpWsDBaHt6eA==", + "dev": true, + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1", + "function-bind": "^1.1.1", + "has": "^1.0.3" + } + }, + "node_modules/object.values/node_modules/es-abstract": { + "version": "1.17.6", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", + "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==", + "dev": true, + "dependencies": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.0", + "is-regex": "^1.1.0", + "object-inspect": "^1.7.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.0", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + }, + "node_modules/object.values/node_modules/es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "dev": true, + "dependencies": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, + "node_modules/object.values/node_modules/has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", + "dev": true + }, + "node_modules/object.values/node_modules/is-callable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", + "integrity": "sha512-pyVD9AaGLxtg6srb2Ng6ynWJqkHU9bEM087AKck0w8QwDarTfNcpIYoU8x8Hv2Icm8u6kFJM18Dag8lyqGkviw==", + "dev": true + }, + "node_modules/object.values/node_modules/is-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.0.tgz", + "integrity": "sha512-iI97M8KTWID2la5uYXlkbSDQIg4F6o1sYboZKKTDpnDQMLtUL86zxhgDet3Q2SriaYsyGqZ6Mn2SjbRKeLHdqw==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.1" + } + }, + "node_modules/object.values/node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", + "dev": true, + "dependencies": { + "mimic-fn": "^1.0.0" + } + }, + "node_modules/open": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/open/-/open-7.0.0.tgz", + "integrity": "sha512-K6EKzYqnwQzk+/dzJAQSBORub3xlBTxMz+ntpZpH/LyCa1o6KjXhuN+2npAaI9jaSmU3R1Q8NWf4KUWcyytGsQ==", + "dev": true, + "dependencies": { + "is-wsl": "^2.1.0" + } + }, + "node_modules/openwpm-webext-instrumentation": { + "resolved": "../webext-instrumentation", + "link": true + }, + "node_modules/optionator": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.2.tgz", + "integrity": "sha1-NkxeQJ0/TWMB1sC0wFu6UBgK62Q=", + "dev": true, + "dependencies": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.4", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "wordwrap": "~1.0.0" + } + }, + "node_modules/os-browserify": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz", + "integrity": "sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc=", + "dev": true + }, + "node_modules/os-locale": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-5.0.0.tgz", + "integrity": "sha512-tqZcNEDAIZKBEPnHPlVDvKrp7NzgLi7jRmhKiUoa2NUmhl13FtkAGLUVR+ZsYvApBQdBfYm43A4tXXQ4IrYLBA==", + "dev": true, + "dependencies": { + "execa": "^4.0.0", + "lcid": "^3.0.0", + "mem": "^5.0.0" + } + }, + "node_modules/os-shim": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/os-shim/-/os-shim-0.1.3.tgz", + "integrity": "sha1-a2LDeRz3kJ6jXtRuF2WLtBfLORc=", + "dev": true + }, + "node_modules/os-tmpdir": { + "version": "1.0.2", + "resolved": "http://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", + "dev": true + }, + "node_modules/p-cancelable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz", + "integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==", + "dev": true + }, + "node_modules/p-defer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", + "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=", + "dev": true + }, + "node_modules/p-is-promise": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/p-is-promise/-/p-is-promise-2.1.0.tgz", + "integrity": "sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg==", + "dev": true + }, + "node_modules/p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "dependencies": { + "p-try": "^1.0.0" + } + }, + "node_modules/p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "dev": true, + "dependencies": { + "p-limit": "^1.1.0" + } + }, + "node_modules/p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true + }, + "node_modules/package-json": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-6.5.0.tgz", + "integrity": "sha512-k3bdm2n25tkyxcjSKzB5x8kfVxlMdgsbPr0GkZcwHsLpba6cBjqCt1KlcChKEvxHIcTB1FVMuwoijZ26xex5MQ==", + "dev": true, + "dependencies": { + "got": "^9.6.0", + "registry-auth-token": "^4.0.0", + "registry-url": "^5.0.0", + "semver": "^6.2.0" + } + }, + "node_modules/package-json/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + }, + "node_modules/pako": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", + "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==", + "dev": true + }, + "node_modules/parallel-transform": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/parallel-transform/-/parallel-transform-1.2.0.tgz", + "integrity": "sha512-P2vSmIu38uIlvdcU7fDkyrxj33gTUy/ABO5ZUbGowxNCopBq/OoD42bP4UmMrJoPyk4Uqf0mu3mtWBhHCZD8yg==", + "dev": true, + "dependencies": { + "cyclist": "^1.0.1", + "inherits": "^2.0.3", + "readable-stream": "^2.1.5" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "dependencies": { + "callsites": "^3.0.0" + } + }, + "node_modules/parent-module/node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true + }, + "node_modules/parse-asn1": { + "version": "5.1.6", + "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", + "integrity": "sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==", + "dev": true, + "dependencies": { + "asn1.js": "^5.2.0", + "browserify-aes": "^1.0.0", + "evp_bytestokey": "^1.0.0", + "pbkdf2": "^3.0.3", + "safe-buffer": "^5.1.1" + } + }, + "node_modules/parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", + "dev": true, + "dependencies": { + "error-ex": "^1.2.0" + } + }, + "node_modules/parse-passwd": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/parse-passwd/-/parse-passwd-1.0.0.tgz", + "integrity": "sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY=", + "dev": true + }, + "node_modules/parse5": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", + "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/pascalcase": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", + "integrity": "sha1-s2PlXoAGym/iF4TS2yK9FdeRfxQ=", + "dev": true + }, + "node_modules/path-browserify": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-0.0.1.tgz", + "integrity": "sha512-BapA40NHICOS+USX9SN4tyhq+A2RrN/Ws5F0Z5aMHDp98Fl86lX8Oti8B7uN93L4Ifv4fHOEA+pQw87gmMO/lQ==", + "dev": true + }, + "node_modules/path-dirname": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-dirname/-/path-dirname-1.0.2.tgz", + "integrity": "sha1-zDPSTVJeCZpTiMAzbG4yuRYGCeA=", + "dev": true + }, + "node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true + }, + "node_modules/path-is-inside": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/path-is-inside/-/path-is-inside-1.0.2.tgz", + "integrity": "sha1-NlQX3t5EQw0cEa9hAn+s8HS9/FM=", + "dev": true + }, + "node_modules/path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", + "dev": true + }, + "node_modules/path-parse": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "dev": true + }, + "node_modules/path-type": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-2.0.0.tgz", + "integrity": "sha1-8BLMuEFbcJb8LaoQVMPXI4lZTHM=", + "dev": true, + "dependencies": { + "pify": "^2.0.0" + } + }, + "node_modules/pbkdf2": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.1.tgz", + "integrity": "sha512-4Ejy1OPxi9f2tt1rRV7Go7zmfDQ+ZectEQz3VGUQhgq62HtIRPDyG/JtnwIxs6x3uNMwo2V7q1fMvKjb+Tnpqg==", + "dev": true, + "dependencies": { + "create-hash": "^1.1.2", + "create-hmac": "^1.1.4", + "ripemd160": "^2.0.1", + "safe-buffer": "^5.0.1", + "sha.js": "^2.4.8" + } + }, + "node_modules/pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=", + "dev": true + }, + "node_modules/performance-now": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", + "integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns=", + "dev": true + }, + "node_modules/picomatch": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", + "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==", + "dev": true, + "optional": true + }, + "node_modules/pidtree": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/pidtree/-/pidtree-0.3.0.tgz", + "integrity": "sha512-9CT4NFlDcosssyg8KVFltgokyKZIFjoBxw8CTGy+5F38Y1eQWrt8tRayiUOXE+zVKQnYu5BR8JjCtvK3BcnBhg==", + "dev": true + }, + "node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true + }, + "node_modules/pino": { + "version": "6.3.2", + "resolved": "https://registry.npmjs.org/pino/-/pino-6.3.2.tgz", + "integrity": "sha512-EiP3L1hoFw19KPocWimjnfXeysld0ne89ZRQ+bf8nAeA2TyuLoggNlibAi+Kla67GvQBopLdIZOsh1z/Lruo5Q==", + "dev": true, + "dependencies": { + "fast-redact": "^2.0.0", + "fast-safe-stringify": "^2.0.7", + "flatstr": "^1.0.12", + "pino-std-serializers": "^2.4.2", + "quick-format-unescaped": "^4.0.1", + "sonic-boom": "^1.0.0" + } + }, + "node_modules/pino-std-serializers": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-2.4.2.tgz", + "integrity": "sha512-WaL504dO8eGs+vrK+j4BuQQq6GLKeCCcHaMB2ItygzVURcL1CycwNEUHTD/lHFHs/NL5qAz2UKrjYWXKSf4aMQ==", + "dev": true + }, + "node_modules/pkg-dir": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-2.0.0.tgz", + "integrity": "sha1-9tXREJ4Z1j7fQo4L1X4Sd3YVM0s=", + "dev": true, + "dependencies": { + "find-up": "^2.1.0" + } + }, + "node_modules/posix-character-classes": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz", + "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=", + "dev": true + }, + "node_modules/postcss": { + "version": "7.0.32", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-7.0.32.tgz", + "integrity": "sha512-03eXong5NLnNCD05xscnGKGDZ98CyzoqPSMjOe6SuoQY7Z2hIj0Ld1g/O/UQRuOle2aRtiIRDg9tDcTGAkLfKw==", + "dev": true, + "dependencies": { + "chalk": "^2.4.2", + "source-map": "^0.6.1", + "supports-color": "^6.1.0" + } + }, + "node_modules/postcss/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "node_modules/postcss/node_modules/chalk/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + } + }, + "node_modules/postcss/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "node_modules/postcss/node_modules/supports-color": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", + "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + } + }, + "node_modules/prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha1-IZMqVJ9eUv/ZqCf1cOBL5iqX2lQ=", + "dev": true + }, + "node_modules/prepend-http": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", + "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=", + "dev": true + }, + "node_modules/probe-image-size": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/probe-image-size/-/probe-image-size-5.0.0.tgz", + "integrity": "sha512-V6uBYw5eBc5UVIE7MUZD6Nxg0RYuGDWLDenEn0B1WC6PcTvn1xdQ6HLDDuznefsiExC6rNrCz7mFRBo0f3Xekg==", + "dev": true, + "dependencies": { + "deepmerge": "^4.0.0", + "inherits": "^2.0.3", + "next-tick": "^1.0.0", + "request": "^2.83.0", + "stream-parser": "~0.3.1" + } + }, + "node_modules/process": { + "version": "0.11.10", + "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", + "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI=", + "dev": true + }, + "node_modules/process-nextick-args": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", + "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", + "dev": true + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true + }, + "node_modules/promise-inflight": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", + "integrity": "sha1-mEcocL8igTL8vdhoEputEsPAKeM=", + "dev": true + }, + "node_modules/prr": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", + "integrity": "sha1-0/wRS6BplaRexok/SEzrHXj19HY=", + "dev": true + }, + "node_modules/psl": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", + "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==", + "dev": true + }, + "node_modules/public-encrypt": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz", + "integrity": "sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==", + "dev": true, + "dependencies": { + "bn.js": "^4.1.0", + "browserify-rsa": "^4.0.0", + "create-hash": "^1.1.0", + "parse-asn1": "^5.0.0", + "randombytes": "^2.0.1", + "safe-buffer": "^5.1.2" + } + }, + "node_modules/public-encrypt/node_modules/bn.js": { + "version": "4.11.9", + "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.11.9.tgz", + "integrity": "sha512-E6QoYqCKZfgatHTdHzs1RRKP7ip4vvm+EyRUeE2RF0NblwVvb0p6jSVeNTOFxPn26QXN2o6SMfNxKp6kU8zQaw==", + "dev": true + }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/pumpify": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.5.1.tgz", + "integrity": "sha512-oClZI37HvuUJJxSKKrC17bZ9Cu0ZYhEAGPsPUy9KlMUmv9dKX2o77RUmq7f3XjIxbwyGwYzbzQ1L2Ks8sIradQ==", + "dev": true, + "dependencies": { + "duplexify": "^3.6.0", + "inherits": "^2.0.3", + "pump": "^2.0.0" + } + }, + "node_modules/pumpify/node_modules/pump": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", + "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", + "dev": true, + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "dev": true + }, + "node_modules/qs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==", + "dev": true + }, + "node_modules/querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "dev": true + }, + "node_modules/querystring-es3": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/querystring-es3/-/querystring-es3-0.2.1.tgz", + "integrity": "sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM=", + "dev": true + }, + "node_modules/quick-format-unescaped": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.1.tgz", + "integrity": "sha512-RyYpQ6Q5/drsJyOhrWHYMWTedvjTIat+FTwv0K4yoUxzvekw2aRHMQJLlnvt8UantkZg2++bEzD9EdxXqkWf4A==", + "dev": true + }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dev": true, + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, + "node_modules/randomfill": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/randomfill/-/randomfill-1.0.4.tgz", + "integrity": "sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==", + "dev": true, + "dependencies": { + "randombytes": "^2.0.5", + "safe-buffer": "^5.1.0" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + } + }, + "node_modules/read-pkg": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-2.0.0.tgz", + "integrity": "sha1-jvHAYjxqbbDcZxPEv6xGMysjaPg=", + "dev": true, + "dependencies": { + "load-json-file": "^2.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^2.0.0" + } + }, + "node_modules/read-pkg-up": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-2.0.0.tgz", + "integrity": "sha1-a3KoBImE4MQeeVEP1en6mbO1Sb4=", + "dev": true, + "dependencies": { + "find-up": "^2.0.0", + "read-pkg": "^2.0.0" + } + }, + "node_modules/readable-stream": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", + "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/readdirp": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-2.2.1.tgz", + "integrity": "sha512-1JU/8q+VgFZyxwrJ+SVIOsh+KywWGpds3NTqikiKpDMZWScmAYyKIgqkO+ARvNWJfXeXR1zxz7aHF4u4CyH6vQ==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.11", + "micromatch": "^3.1.10", + "readable-stream": "^2.0.2" + } + }, + "node_modules/rechoir": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", + "integrity": "sha1-hSBLVNuoLVdC4oyWdW70OvUOM4Q=", + "dev": true, + "dependencies": { + "resolve": "^1.1.6" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.13.5", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.5.tgz", + "integrity": "sha512-ZS5w8CpKFinUzOwW3c83oPeVXoNsrLsaCoLtJvAClH135j/R77RuymhiSErhm2lKcwSCIpmvIWSbDkIfAqKQlA==", + "dev": true + }, + "node_modules/regex-not": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", + "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", + "dev": true, + "dependencies": { + "extend-shallow": "^3.0.2", + "safe-regex": "^1.1.0" + } + }, + "node_modules/regexp.prototype.flags": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.3.0.tgz", + "integrity": "sha512-2+Q0C5g951OlYlJz6yu5/M33IcsESLlLfsyIaLJaG4FA2r4yP8MvVMJUUP/fVBkSpbbbZlS5gynbEWLipiiXiQ==", + "dev": true, + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1" + } + }, + "node_modules/regexp.prototype.flags/node_modules/es-abstract": { + "version": "1.17.6", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", + "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==", + "dev": true, + "dependencies": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.0", + "is-regex": "^1.1.0", + "object-inspect": "^1.7.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.0", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + }, + "node_modules/regexp.prototype.flags/node_modules/es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "dev": true, + "dependencies": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, + "node_modules/regexp.prototype.flags/node_modules/has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", + "dev": true + }, + "node_modules/regexp.prototype.flags/node_modules/is-callable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", + "integrity": "sha512-pyVD9AaGLxtg6srb2Ng6ynWJqkHU9bEM087AKck0w8QwDarTfNcpIYoU8x8Hv2Icm8u6kFJM18Dag8lyqGkviw==", + "dev": true + }, + "node_modules/regexp.prototype.flags/node_modules/is-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.0.tgz", + "integrity": "sha512-iI97M8KTWID2la5uYXlkbSDQIg4F6o1sYboZKKTDpnDQMLtUL86zxhgDet3Q2SriaYsyGqZ6Mn2SjbRKeLHdqw==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.1" + } + }, + "node_modules/regexp.prototype.flags/node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true + }, + "node_modules/regexpp": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-2.0.1.tgz", + "integrity": "sha512-lv0M6+TkDVniA3aD1Eg0DVpfU/booSu7Eev3TDO/mZKHBfVjgCGTV4t4buppESEYDtkArYFOxTJWv6S5C+iaNw==", + "dev": true + }, + "node_modules/registry-auth-token": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-4.1.1.tgz", + "integrity": "sha512-9bKS7nTl9+/A1s7tnPeGrUpRcVY+LUh7bfFgzpndALdPfXQBfQV77rQVtqgUV3ti4vc/Ik81Ex8UJDWDQ12zQA==", + "dev": true, + "dependencies": { + "rc": "^1.2.8" + } + }, + "node_modules/registry-url": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-5.1.0.tgz", + "integrity": "sha512-8acYXXTI0AkQv6RAOjE3vOaIXZkT9wo4LOFbBKYQEEnnMNBpKqdUrI6S4NT0KPIo/WVvJ5tE/X5LF/TQUf0ekw==", + "dev": true, + "dependencies": { + "rc": "^1.2.8" + } + }, + "node_modules/relaxed-json": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/relaxed-json/-/relaxed-json-1.0.3.tgz", + "integrity": "sha512-b7wGPo7o2KE/g7SqkJDDbav6zmrEeP4TK2VpITU72J/M949TLe/23y/ZHJo+pskcGM52xIfFoT9hydwmgr1AEg==", + "dev": true, + "dependencies": { + "chalk": "^2.4.2", + "commander": "^2.6.0" + } + }, + "node_modules/relaxed-json/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "node_modules/remove-trailing-separator": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/remove-trailing-separator/-/remove-trailing-separator-1.1.0.tgz", + "integrity": "sha1-wkvOKig62tW8P1jg1IJJuSN52O8=", + "dev": true + }, + "node_modules/repeat-element": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.3.tgz", + "integrity": "sha512-ahGq0ZnV5m5XtZLMb+vP76kcAM5nkLqk0lpqAuojSKGgQtn4eRi4ZZGm2olo2zKFH+sMsWaqOCW1dqAnOru72g==", + "dev": true + }, + "node_modules/repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=", + "dev": true + }, + "node_modules/request": { + "version": "2.88.2", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "dev": true, + "dependencies": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.3", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.5.0", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "dev": true + }, + "node_modules/require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", + "dev": true + }, + "node_modules/resolve": { + "version": "1.14.2", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.14.2.tgz", + "integrity": "sha512-EjlOBLBO1kxsUxsKjLt7TAECyKW6fOh1VRkykQkKGzcBbjjPIxBqGh0jf7GJ3k/f5mxMqW3htMD3WdTUVtW8HQ==", + "dev": true, + "dependencies": { + "path-parse": "^1.0.6" + } + }, + "node_modules/resolve-cwd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-2.0.0.tgz", + "integrity": "sha1-AKn3OHVW4nA46uIyyqNypqWbZlo=", + "dev": true, + "dependencies": { + "resolve-from": "^3.0.0" + } + }, + "node_modules/resolve-cwd/node_modules/resolve-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-3.0.0.tgz", + "integrity": "sha1-six699nWiBvItuZTM17rywoYh0g=", + "dev": true + }, + "node_modules/resolve-dir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/resolve-dir/-/resolve-dir-1.0.1.tgz", + "integrity": "sha1-eaQGRMNivoLybv/nOcm7U4IEb0M=", + "dev": true, + "dependencies": { + "expand-tilde": "^2.0.0", + "global-modules": "^1.0.0" + } + }, + "node_modules/resolve-dir/node_modules/global-modules": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/global-modules/-/global-modules-1.0.0.tgz", + "integrity": "sha512-sKzpEkf11GpOFuw0Zzjzmt4B4UZwjOcG757PPvrfhxcLFbq0wpsgpOqxpxtxFiCG4DtG93M6XRVbF2oGdev7bg==", + "dev": true, + "dependencies": { + "global-prefix": "^1.0.1", + "is-windows": "^1.0.1", + "resolve-dir": "^1.0.0" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true + }, + "node_modules/resolve-url": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", + "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=", + "dev": true + }, + "node_modules/responselike": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz", + "integrity": "sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec=", + "dev": true, + "dependencies": { + "lowercase-keys": "^1.0.0" + } + }, + "node_modules/restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", + "dev": true, + "dependencies": { + "onetime": "^2.0.0", + "signal-exit": "^3.0.2" + } + }, + "node_modules/ret": { + "version": "0.1.15", + "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", + "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "dev": true + }, + "node_modules/rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + } + }, + "node_modules/ripemd160": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.2.tgz", + "integrity": "sha512-ii4iagi25WusVoiC4B4lq7pbXfAp3D9v5CwfkY33vffw2+pkDjY1D8GaN7spsxvCSx8dkPqOZCEZyfxcmJG2IA==", + "dev": true, + "dependencies": { + "hash-base": "^3.0.0", + "inherits": "^2.0.1" + } + }, + "node_modules/run-async": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.3.0.tgz", + "integrity": "sha1-A3GrSuC91yDUFm19/aZP96RFpsA=", + "dev": true, + "dependencies": { + "is-promise": "^2.1.0" + } + }, + "node_modules/run-queue": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz", + "integrity": "sha1-6Eg5bwV9Ij8kOGkkYY4laUFh7Ec=", + "dev": true, + "dependencies": { + "aproba": "^1.1.1" + } + }, + "node_modules/rxjs": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.2.tgz", + "integrity": "sha512-HUb7j3kvb7p7eCUHE3FqjoDsC1xfZQ4AHFWfTKSpZ+sAhhz5X1WX0ZuUqWbzB2QhSLp3DoLUG+hMdEDKqWo2Zg==", + "dev": true, + "dependencies": { + "tslib": "^1.9.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "node_modules/safe-json-stringify": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.2.0.tgz", + "integrity": "sha512-gH8eh2nZudPQO6TytOvbxnuhYBOvDBBLW52tz5q6X58lJcd/tkmqFR+5Z9adS8aJtURSXWThWy/xJtJwixErvg==", + "dev": true, + "optional": true + }, + "node_modules/safe-regex": { + "version": "1.1.0", + "resolved": "http://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=", + "dev": true, + "dependencies": { + "ret": "~0.1.10" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, + "node_modules/sax": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", + "dev": true + }, + "node_modules/schema-utils": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz", + "integrity": "sha512-i27Mic4KovM/lnGsy8whRCHhc7VicJajAjTrYg11K9zfZXnYIt4k5F+kZkwjnrhKzLic/HLU4j11mjsz2G/75g==", + "dev": true, + "dependencies": { + "ajv": "^6.1.0", + "ajv-errors": "^1.0.0", + "ajv-keywords": "^3.1.0" + } + }, + "node_modules/semver": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==", + "dev": true + }, + "node_modules/semver-diff": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-3.1.1.tgz", + "integrity": "sha512-GX0Ix/CJcHyB8c4ykpHGIAvLyOwOobtM/8d+TQkAd81/bEjgPHrfba41Vpesr7jX/t8Uh+R3EX9eAS5be+jQYg==", + "dev": true, + "dependencies": { + "semver": "^6.3.0" + } + }, + "node_modules/semver-diff/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true + }, + "node_modules/serialize-javascript": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-4.0.0.tgz", + "integrity": "sha512-GaNA54380uFefWghODBWEGisLZFj00nS5ACs6yHa9nLqlLpVLO8ChDGeKRjZnV4Nh4n0Qi7nhYZD/9fCPzEqkw==", + "dev": true, + "dependencies": { + "randombytes": "^2.1.0" + } + }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", + "dev": true + }, + "node_modules/set-value": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", + "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", + "dev": true, + "dependencies": { + "extend-shallow": "^2.0.1", + "is-extendable": "^0.1.1", + "is-plain-object": "^2.0.3", + "split-string": "^3.0.1" + } + }, + "node_modules/set-value/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + } + }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=", + "dev": true + }, + "node_modules/sha.js": { + "version": "2.4.11", + "resolved": "http://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", + "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==", + "dev": true, + "dependencies": { + "inherits": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "dev": true, + "dependencies": { + "shebang-regex": "^1.0.0" + } + }, + "node_modules/shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", + "dev": true + }, + "node_modules/shell-quote": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.6.1.tgz", + "integrity": "sha1-9HgZSczkAmlxJ0MOo7PFR29IF2c=", + "dev": true, + "dependencies": { + "array-filter": "~0.0.0", + "array-map": "~0.0.0", + "array-reduce": "~0.0.0", + "jsonify": "~0.0.0" + } + }, + "node_modules/shelljs": { + "version": "0.7.7", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.7.7.tgz", + "integrity": "sha1-svXHfvlxSPS09uImguELuoZnz/E=", + "dev": true, + "dependencies": { + "glob": "^7.0.0", + "interpret": "^1.0.0", + "rechoir": "^0.6.2" + } + }, + "node_modules/shellwords": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/shellwords/-/shellwords-0.1.1.tgz", + "integrity": "sha512-vFwSUfQvqybiICwZY5+DAWIPLKsWO31Q91JSKl3UYv+K5c2QRPzn0qzec6QPu1Qc9eHYItiP3NdJqNVqetYAww==", + "dev": true + }, + "node_modules/sign-addon": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/sign-addon/-/sign-addon-2.0.5.tgz", + "integrity": "sha512-dVjIWe1VJ2VQCdScREWXWECmJhgjpJMqwPKkW+L78PPx2Jyr/t+//kNHqG1hYrmIsvQN7vGjAjv9s7ix0vw0zA==", + "dev": true, + "dependencies": { + "common-tags": "1.8.0", + "core-js": "3.6.4", + "deepcopy": "2.0.0", + "es6-error": "4.1.1", + "es6-promisify": "6.0.2", + "jsonwebtoken": "8.5.1", + "mz": "2.7.0", + "request": "2.88.0", + "source-map-support": "0.5.16", + "stream-to-promise": "2.2.0" + } + }, + "node_modules/sign-addon/node_modules/core-js": { + "version": "3.6.4", + "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.6.4.tgz", + "integrity": "sha512-4paDGScNgZP2IXXilaffL9X7968RuvwlkK3xWtZRVqgd8SYNiVKRJvkFd1aqqEuPfN7E68ZHEp9hDj6lHj4Hyw==", + "dev": true + }, + "node_modules/sign-addon/node_modules/es6-promisify": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/es6-promisify/-/es6-promisify-6.0.2.tgz", + "integrity": "sha512-eO6vFm0JvqGzjWIQA6QVKjxpmELfhWbDUWHm1rPfIbn55mhKPiAa5xpLmQWJrNa629ZIeQ8ZvMAi13kvrjK6Mg==", + "dev": true + }, + "node_modules/sign-addon/node_modules/punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=", + "dev": true + }, + "node_modules/sign-addon/node_modules/request": { + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", + "dev": true, + "dependencies": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, + "node_modules/sign-addon/node_modules/tough-cookie": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", + "dev": true, + "dependencies": { + "psl": "^1.1.24", + "punycode": "^1.4.1" + } + }, + "node_modules/signal-exit": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", + "dev": true + }, + "node_modules/slice-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", + "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.0", + "astral-regex": "^1.0.0", + "is-fullwidth-code-point": "^2.0.0" + } + }, + "node_modules/snapdragon": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", + "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", + "dev": true, + "dependencies": { + "base": "^0.11.1", + "debug": "^2.2.0", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "map-cache": "^0.2.2", + "source-map": "^0.5.6", + "source-map-resolve": "^0.5.0", + "use": "^3.1.0" + } + }, + "node_modules/snapdragon-node": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", + "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", + "dev": true, + "dependencies": { + "define-property": "^1.0.0", + "isobject": "^3.0.0", + "snapdragon-util": "^3.0.1" + } + }, + "node_modules/snapdragon-node/node_modules/define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.0" + } + }, + "node_modules/snapdragon-node/node_modules/is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + } + }, + "node_modules/snapdragon-node/node_modules/is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + } + }, + "node_modules/snapdragon-node/node_modules/is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + } + }, + "node_modules/snapdragon-util": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", + "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", + "dev": true, + "dependencies": { + "kind-of": "^3.2.0" + } + }, + "node_modules/snapdragon-util/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + } + }, + "node_modules/snapdragon/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/snapdragon/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + } + }, + "node_modules/snapdragon/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + } + }, + "node_modules/snapdragon/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "node_modules/sonic-boom": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-1.0.1.tgz", + "integrity": "sha512-o9tx+bonVEXSaPtptyXQXpP8l6UV9Bi3im2geZskvWw2a/o/hrbWI7EBbbv+rOx6Hubnzun9GgH4WfbgEA3MFQ==", + "dev": true, + "dependencies": { + "atomic-sleep": "^1.0.0", + "flatstr": "^1.0.12" + } + }, + "node_modules/source-list-map": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/source-list-map/-/source-list-map-2.0.1.tgz", + "integrity": "sha512-qnQ7gVMxGNxsiL4lEuJwe/To8UnK7fAnmbGEEH8RpLouuKbeEm0lhbQVFIrNSuB+G7tVrAlVsZgETT5nljf+Iw==", + "dev": true + }, + "node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", + "dev": true + }, + "node_modules/source-map-resolve": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.2.tgz", + "integrity": "sha512-MjqsvNwyz1s0k81Goz/9vRBe9SZdB09Bdw+/zYyO+3CuPk6fouTaxscHkgtE8jKvf01kVfl8riHzERQ/kefaSA==", + "dev": true, + "dependencies": { + "atob": "^2.1.1", + "decode-uri-component": "^0.2.0", + "resolve-url": "^0.2.1", + "source-map-url": "^0.4.0", + "urix": "^0.1.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.16.tgz", + "integrity": "sha512-efyLRJDr68D9hBBNIPWFjhpFzURh+KJykQwvMyW5UiZzYwoF6l4YMMDIJJEyFWxWCqfyxLzz6tSfUFR+kXXsVQ==", + "dev": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/source-map-support/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "node_modules/source-map-url": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz", + "integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=", + "dev": true + }, + "node_modules/spawn-sync": { + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/spawn-sync/-/spawn-sync-1.0.15.tgz", + "integrity": "sha1-sAeZVX63+wyDdsKdROih6mfldHY=", + "dev": true, + "dependencies": { + "concat-stream": "^1.4.7", + "os-shim": "^0.1.2" + } + }, + "node_modules/spdx-correct": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz", + "integrity": "sha512-lr2EZCctC2BNR7j7WzJ2FpDznxky1sjfxvvYEyzxNyb6lZXHODmEoJeFu4JupYlkfha1KZpJyoqiJ7pgA1qq8Q==", + "dev": true, + "dependencies": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-exceptions": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.2.0.tgz", + "integrity": "sha512-2XQACfElKi9SlVb1CYadKDXvoajPgBVPn/gOQLrTvHdElaVhr7ZEbqJaRnJLVNeaI4cMEAgVCeBMKF6MWRDCRA==", + "dev": true + }, + "node_modules/spdx-expression-parse": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz", + "integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==", + "dev": true, + "dependencies": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-license-ids": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.3.tgz", + "integrity": "sha512-uBIcIl3Ih6Phe3XHK1NqboJLdGfwr1UN3k6wSD1dZpmPsIkb8AGNbZYJ1fOBk834+Gxy8rpfDxrS6XLEMZMY2g==", + "dev": true + }, + "node_modules/split": { + "version": "0.3.3", + "resolved": "http://registry.npmjs.org/split/-/split-0.3.3.tgz", + "integrity": "sha1-zQ7qXmOiEd//frDwkcQTPi0N0o8=", + "dev": true, + "dependencies": { + "through": "2" + } + }, + "node_modules/split-string": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", + "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", + "dev": true, + "dependencies": { + "extend-shallow": "^3.0.0" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "http://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + }, + "node_modules/sshpk": { + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", + "dev": true, + "dependencies": { + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" + } + }, + "node_modules/ssri": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-6.0.1.tgz", + "integrity": "sha512-3Wge10hNcT1Kur4PDFwEieXSCMCJs/7WvSACcrMYrNp+b8kDL1/0wJch5Ni2WrtwEa2IO8OsVfeKIciKCDx/QA==", + "dev": true, + "dependencies": { + "figgy-pudding": "^3.5.1" + } + }, + "node_modules/static-extend": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", + "integrity": "sha1-YICcOcv/VTNyJv1eC1IPNB8ftcY=", + "dev": true, + "dependencies": { + "define-property": "^0.2.5", + "object-copy": "^0.1.0" + } + }, + "node_modules/static-extend/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + } + }, + "node_modules/stream-browserify": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz", + "integrity": "sha512-nX6hmklHs/gr2FuxYDltq8fJA1GDlxKQCz8O/IM4atRqBH8OORmBNgfvW5gG10GT/qQ9u0CzIvr2X5Pkt6ntqg==", + "dev": true, + "dependencies": { + "inherits": "~2.0.1", + "readable-stream": "^2.0.2" + } + }, + "node_modules/stream-each": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/stream-each/-/stream-each-1.2.3.tgz", + "integrity": "sha512-vlMC2f8I2u/bZGqkdfLQW/13Zihpej/7PmSiMQsbYddxuTsJp8vRe2x2FvVExZg7FaOds43ROAuFJwPR4MTZLw==", + "dev": true, + "dependencies": { + "end-of-stream": "^1.1.0", + "stream-shift": "^1.0.0" + } + }, + "node_modules/stream-http": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/stream-http/-/stream-http-2.8.3.tgz", + "integrity": "sha512-+TSkfINHDo4J+ZobQLWiMouQYB+UVYFttRA94FpEzzJ7ZdqcL4uUUQ7WkdkI4DSozGmgBUE/a47L+38PenXhUw==", + "dev": true, + "dependencies": { + "builtin-status-codes": "^3.0.0", + "inherits": "^2.0.1", + "readable-stream": "^2.3.6", + "to-arraybuffer": "^1.0.0", + "xtend": "^4.0.0" + } + }, + "node_modules/stream-parser": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/stream-parser/-/stream-parser-0.3.1.tgz", + "integrity": "sha1-FhhUhpRCACGhGC/wrxkRwSl2F3M=", + "dev": true, + "dependencies": { + "debug": "2" + } + }, + "node_modules/stream-parser/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/stream-parser/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "node_modules/stream-shift": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", + "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==", + "dev": true + }, + "node_modules/stream-to-array": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/stream-to-array/-/stream-to-array-2.3.0.tgz", + "integrity": "sha1-u/azn19D7DC8cbq8s3VXrOzzQ1M=", + "dev": true, + "dependencies": { + "any-promise": "^1.1.0" + } + }, + "node_modules/stream-to-promise": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/stream-to-promise/-/stream-to-promise-2.2.0.tgz", + "integrity": "sha1-se2y4cjLESidG1A8CNPyrvUeZQ8=", + "dev": true, + "dependencies": { + "any-promise": "~1.3.0", + "end-of-stream": "~1.1.0", + "stream-to-array": "~2.3.0" + } + }, + "node_modules/stream-to-promise/node_modules/end-of-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.1.0.tgz", + "integrity": "sha1-6TUyWLqpEIll78QcsO+K3i88+wc=", + "dev": true, + "dependencies": { + "once": "~1.3.0" + } + }, + "node_modules/stream-to-promise/node_modules/once": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/once/-/once-1.3.3.tgz", + "integrity": "sha1-suJhVXzkwxTsgwTz+oJmPkKXyiA=", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "dependencies": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + } + }, + "node_modules/string.prototype.padend": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/string.prototype.padend/-/string.prototype.padend-3.0.0.tgz", + "integrity": "sha1-86rvfBcZ8XDF6rHDK/eA2W4h8vA=", + "dev": true, + "dependencies": { + "define-properties": "^1.1.2", + "es-abstract": "^1.4.3", + "function-bind": "^1.0.2" + } + }, + "node_modules/string.prototype.trimend": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz", + "integrity": "sha512-LRPxFUaTtpqYsTeNKaFOw3R4bxIzWOnbQ837QfBylo8jIxtcbK/A/sMV7Q+OAV/vWo+7s25pOE10KYSjaSO06g==", + "dev": true, + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" + } + }, + "node_modules/string.prototype.trimend/node_modules/es-abstract": { + "version": "1.17.6", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", + "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==", + "dev": true, + "dependencies": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.0", + "is-regex": "^1.1.0", + "object-inspect": "^1.7.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.0", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + }, + "node_modules/string.prototype.trimend/node_modules/es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "dev": true, + "dependencies": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, + "node_modules/string.prototype.trimend/node_modules/has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", + "dev": true + }, + "node_modules/string.prototype.trimend/node_modules/is-callable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", + "integrity": "sha512-pyVD9AaGLxtg6srb2Ng6ynWJqkHU9bEM087AKck0w8QwDarTfNcpIYoU8x8Hv2Icm8u6kFJM18Dag8lyqGkviw==", + "dev": true + }, + "node_modules/string.prototype.trimend/node_modules/is-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.0.tgz", + "integrity": "sha512-iI97M8KTWID2la5uYXlkbSDQIg4F6o1sYboZKKTDpnDQMLtUL86zxhgDet3Q2SriaYsyGqZ6Mn2SjbRKeLHdqw==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.1" + } + }, + "node_modules/string.prototype.trimend/node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true + }, + "node_modules/string.prototype.trimstart": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz", + "integrity": "sha512-XxZn+QpvrBI1FOcg6dIpxUPgWCPuNXvMD72aaRaUQv1eD4e/Qy8i/hFTe0BUmD60p/QA6bh1avmuPTfNjqVWRw==", + "dev": true, + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" + } + }, + "node_modules/string.prototype.trimstart/node_modules/es-abstract": { + "version": "1.17.6", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", + "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==", + "dev": true, + "dependencies": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.0", + "is-regex": "^1.1.0", + "object-inspect": "^1.7.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.0", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + } + }, + "node_modules/string.prototype.trimstart/node_modules/es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "dev": true, + "dependencies": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + } + }, + "node_modules/string.prototype.trimstart/node_modules/has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", + "dev": true + }, + "node_modules/string.prototype.trimstart/node_modules/is-callable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", + "integrity": "sha512-pyVD9AaGLxtg6srb2Ng6ynWJqkHU9bEM087AKck0w8QwDarTfNcpIYoU8x8Hv2Icm8u6kFJM18Dag8lyqGkviw==", + "dev": true + }, + "node_modules/string.prototype.trimstart/node_modules/is-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.0.tgz", + "integrity": "sha512-iI97M8KTWID2la5uYXlkbSDQIg4F6o1sYboZKKTDpnDQMLtUL86zxhgDet3Q2SriaYsyGqZ6Mn2SjbRKeLHdqw==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.1" + } + }, + "node_modules/string.prototype.trimstart/node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true + }, + "node_modules/strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "dependencies": { + "ansi-regex": "^3.0.0" + } + }, + "node_modules/strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "dev": true + }, + "node_modules/strip-bom-buf": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-bom-buf/-/strip-bom-buf-2.0.0.tgz", + "integrity": "sha512-gLFNHucd6gzb8jMsl5QmZ3QgnUJmp7qn4uUSHNwEXumAp7YizoGYw19ZUVfuq4aBOQUtyn2k8X/CwzWB73W2lQ==", + "dev": true, + "dependencies": { + "is-utf8": "^0.2.1" + } + }, + "node_modules/strip-bom-stream": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom-stream/-/strip-bom-stream-4.0.0.tgz", + "integrity": "sha512-0ApK3iAkHv6WbgLICw/J4nhwHeDZsBxIIsOD+gHgZICL6SeJ0S9f/WZqemka9cjkTyMN5geId6e8U5WGFAn3cQ==", + "dev": true, + "dependencies": { + "first-chunk-stream": "^3.0.0", + "strip-bom-buf": "^2.0.0" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "dev": true + }, + "node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + } + }, + "node_modules/table": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/table/-/table-5.4.0.tgz", + "integrity": "sha512-nHFDrxmbrkU7JAFKqKbDJXfzrX2UBsWmrieXFTGxiI5e4ncg3VqsZeI4EzNmX0ncp4XNGVeoxIWJXfCIXwrsvw==", + "dev": true, + "dependencies": { + "ajv": "^6.9.1", + "lodash": "^4.17.11", + "slice-ansi": "^2.1.0", + "string-width": "^3.0.0" + } + }, + "node_modules/table/node_modules/ajv": { + "version": "6.10.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", + "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "node_modules/table/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "node_modules/table/node_modules/string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "dependencies": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "node_modules/table/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "dependencies": { + "ansi-regex": "^4.1.0" + } + }, + "node_modules/tapable": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-1.1.3.tgz", + "integrity": "sha512-4WK/bYZmj8xLr+HUCODHGF1ZFzsYffasLUgEiMBY4fgtltdO6B4WJtlSbPaDTLpYTcGVwM2qLnFTICEcNxs3kA==", + "dev": true + }, + "node_modules/tar-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", + "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", + "dev": true, + "dependencies": { + "bl": "^1.0.0", + "buffer-alloc": "^1.2.0", + "end-of-stream": "^1.0.0", + "fs-constants": "^1.0.0", + "readable-stream": "^2.3.0", + "to-buffer": "^1.1.1", + "xtend": "^4.0.0" + } + }, + "node_modules/term-size": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.0.tgz", + "integrity": "sha512-a6sumDlzyHVJWb8+YofY4TW112G6p2FCPEAFk+59gIYHv3XHRhm9ltVQ9kli4hNWeQBwSpe8cRN25x0ROunMOw==", + "dev": true + }, + "node_modules/terser": { + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/terser/-/terser-4.8.0.tgz", + "integrity": "sha512-EAPipTNeWsb/3wLPeup1tVPaXfIaU68xMnVdPafIL1TV05OhASArYyIfFvnvJCNrR2NIOvDVNNTFRa+Re2MWyw==", + "dev": true, + "dependencies": { + "commander": "^2.20.0", + "source-map": "~0.6.1", + "source-map-support": "~0.5.12" + } + }, + "node_modules/terser-webpack-plugin": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/terser-webpack-plugin/-/terser-webpack-plugin-1.4.5.tgz", + "integrity": "sha512-04Rfe496lN8EYruwi6oPQkG0vo8C+HT49X687FZnpPF0qMAIHONI6HEXYPKDOE8e5HjXTyKfqRd/agHtH0kOtw==", + "dev": true, + "dependencies": { + "cacache": "^12.0.2", + "find-cache-dir": "^2.1.0", + "is-wsl": "^1.1.0", + "schema-utils": "^1.0.0", + "serialize-javascript": "^4.0.0", + "source-map": "^0.6.1", + "terser": "^4.1.2", + "webpack-sources": "^1.4.0", + "worker-farm": "^1.7.0" + } + }, + "node_modules/terser-webpack-plugin/node_modules/is-wsl": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-1.1.0.tgz", + "integrity": "sha1-HxbkqiKwTRM2tmGIpmrzxgDDpm0=", + "dev": true + }, + "node_modules/terser-webpack-plugin/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "node_modules/terser/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "node_modules/text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", + "dev": true + }, + "node_modules/thenify": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", + "integrity": "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==", + "dev": true, + "dependencies": { + "any-promise": "^1.0.0" + } + }, + "node_modules/thenify-all": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/thenify-all/-/thenify-all-1.6.0.tgz", + "integrity": "sha1-GhkY1ALY/D+Y+/I02wvMjMEOlyY=", + "dev": true, + "dependencies": { + "thenify": ">= 3.1.0 < 4" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", + "dev": true + }, + "node_modules/through2": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "dev": true, + "dependencies": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + }, + "node_modules/timers-browserify": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.11.tgz", + "integrity": "sha512-60aV6sgJ5YEbzUdn9c8kYGIqOubPoUdqQCul3SBAsRCZ40s6Y5cMcrW4dt3/k/EsbLVJNl9n6Vz3fTc+k2GeKQ==", + "dev": true, + "dependencies": { + "setimmediate": "^1.0.4" + } + }, + "node_modules/tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "dev": true, + "dependencies": { + "os-tmpdir": "~1.0.2" + } + }, + "node_modules/to-arraybuffer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/to-arraybuffer/-/to-arraybuffer-1.0.1.tgz", + "integrity": "sha1-fSKbH8xjfkZsoIEYCDanqr/4P0M=", + "dev": true + }, + "node_modules/to-buffer": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", + "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==", + "dev": true + }, + "node_modules/to-object-path": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz", + "integrity": "sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68=", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + } + }, + "node_modules/to-object-path/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + } + }, + "node_modules/to-readable-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz", + "integrity": "sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q==", + "dev": true + }, + "node_modules/to-regex": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", + "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", + "dev": true, + "dependencies": { + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "regex-not": "^1.0.2", + "safe-regex": "^1.1.0" + } + }, + "node_modules/to-regex-range": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz", + "integrity": "sha1-fIDBe53+vlmeJzZ+DU3VWQFB2zg=", + "dev": true, + "dependencies": { + "is-number": "^3.0.0", + "repeat-string": "^1.6.1" + } + }, + "node_modules/tosource": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/tosource/-/tosource-1.0.0.tgz", + "integrity": "sha1-QtiN0RZhi88A1hBt1URvNCeQL/E=", + "dev": true + }, + "node_modules/tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dev": true, + "dependencies": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } + }, + "node_modules/tr46": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-2.0.2.tgz", + "integrity": "sha512-3n1qG+/5kg+jrbTzwAykB5yRYtQCTqOGKq5U5PE3b0a1/mzo6snDhjGS0zJVJunO0NrT3Dg1MLy5TjWP/UJppg==", + "dev": true, + "dependencies": { + "punycode": "^2.1.1" + } + }, + "node_modules/traverse": { + "version": "0.4.6", + "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.4.6.tgz", + "integrity": "sha1-0EsigOTHkqWBVCnve4tgxkyczDQ=", + "dev": true + }, + "node_modules/ts-loader": { + "version": "5.4.5", + "resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-5.4.5.tgz", + "integrity": "sha512-XYsjfnRQCBum9AMRZpk2rTYSVpdZBpZK+kDh0TeT3kxmQNBDVIeUjdPjY5RZry4eIAb8XHc4gYSUiUWPYvzSRw==", + "dev": true, + "dependencies": { + "chalk": "^2.3.0", + "enhanced-resolve": "^4.0.0", + "loader-utils": "^1.0.2", + "micromatch": "^3.1.4", + "semver": "^5.0.1" + } + }, + "node_modules/tsconfig-paths": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.9.0.tgz", + "integrity": "sha512-dRcuzokWhajtZWkQsDVKbWyY+jgcLC5sqJhg2PSgf4ZkH2aHPvaOY8YWGhmjb68b5qqTfasSsDO9k7RUiEmZAw==", + "dev": true, + "dependencies": { + "@types/json5": "^0.0.29", + "json5": "^1.0.1", + "minimist": "^1.2.0", + "strip-bom": "^3.0.0" + } + }, + "node_modules/tslib": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.3.tgz", + "integrity": "sha512-4krF8scpejhaOgqzBEcGM7yDIEfi0/8+8zDRZhNZZ2kjmHJ4hv3zCbQWxoJGz1iw5U0Jl0nma13xzHXcncMavQ==", + "dev": true + }, + "node_modules/tslint": { + "version": "5.20.1", + "resolved": "https://registry.npmjs.org/tslint/-/tslint-5.20.1.tgz", + "integrity": "sha512-EcMxhzCFt8k+/UP5r8waCf/lzmeSyVlqxqMEDQE7rWYiQky8KpIBz1JAoYXfROHrPZ1XXd43q8yQnULOLiBRQg==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.0.0", + "builtin-modules": "^1.1.1", + "chalk": "^2.3.0", + "commander": "^2.12.1", + "diff": "^4.0.1", + "glob": "^7.1.1", + "js-yaml": "^3.13.1", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "resolve": "^1.3.2", + "semver": "^5.3.0", + "tslib": "^1.8.0", + "tsutils": "^2.29.0" + } + }, + "node_modules/tslint-eslint-rules": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/tslint-eslint-rules/-/tslint-eslint-rules-5.4.0.tgz", + "integrity": "sha512-WlSXE+J2vY/VPgIcqQuijMQiel+UtmXS+4nvK4ZzlDiqBfXse8FAvkNnTcYhnQyOTW5KFM+uRRGXxYhFpuBc6w==", + "dev": true, + "dependencies": { + "doctrine": "0.7.2", + "tslib": "1.9.0", + "tsutils": "^3.0.0" + } + }, + "node_modules/tslint-eslint-rules/node_modules/doctrine": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-0.7.2.tgz", + "integrity": "sha1-fLhgNZujvpDgQLJrcpzkv6ZUxSM=", + "dev": true, + "dependencies": { + "esutils": "^1.1.6", + "isarray": "0.0.1" + } + }, + "node_modules/tslint-eslint-rules/node_modules/esutils": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-1.1.6.tgz", + "integrity": "sha1-wBzKqa5LiXxtDD4hCuUvPHqEQ3U=", + "dev": true + }, + "node_modules/tslint-eslint-rules/node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=", + "dev": true + }, + "node_modules/tslint-eslint-rules/node_modules/tslib": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.9.0.tgz", + "integrity": "sha512-f/qGG2tUkrISBlQZEjEqoZ3B2+npJjIf04H1wuAv9iA8i04Icp+61KRXxFdha22670NJopsZCIjhC3SnjPRKrQ==", + "dev": true + }, + "node_modules/tslint-eslint-rules/node_modules/tsutils": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-3.6.0.tgz", + "integrity": "sha512-hCG3lZz+uRmmiC4brr/kY6Yuypnl20PNe8t49DO4OUGlbxWkxYHF63EeG2XPSd0JcKiWmp9p55yQyrkxqSS5Dg==", + "dev": true, + "dependencies": { + "tslib": "^1.8.1" + } + }, + "node_modules/tsutils": { + "version": "2.29.0", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-2.29.0.tgz", + "integrity": "sha512-g5JVHCIJwzfISaXpXE1qvNalca5Jwob6FjI4AoPlqMusJ6ftFE7IkkFoMhVLRgK+4Kx3gkzb8UZK5t5yTTvEmA==", + "dev": true, + "dependencies": { + "tslib": "^1.8.1" + } + }, + "node_modules/tty-browserify": { + "version": "0.0.0", + "resolved": "http://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.0.tgz", + "integrity": "sha1-oVe6QC2iTpv5V/mqadUk7tQpAaY=", + "dev": true + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", + "dev": true, + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/tweetnacl": { + "version": "0.14.5", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", + "dev": true + }, + "node_modules/type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha1-WITKtRLPHTVeP7eE8wgEsrUg23I=", + "dev": true, + "dependencies": { + "prelude-ls": "~1.1.2" + } + }, + "node_modules/type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true + }, + "node_modules/type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "dev": true + }, + "node_modules/typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=", + "dev": true + }, + "node_modules/typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "dev": true, + "dependencies": { + "is-typedarray": "^1.0.0" + } + }, + "node_modules/typescript": { + "version": "3.9.7", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.7.tgz", + "integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==", + "dev": true + }, + "node_modules/union-value": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", + "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", + "dev": true, + "dependencies": { + "arr-union": "^3.1.0", + "get-value": "^2.0.6", + "is-extendable": "^0.1.1", + "set-value": "^2.0.1" + } + }, + "node_modules/unique-filename": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", + "integrity": "sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==", + "dev": true, + "dependencies": { + "unique-slug": "^2.0.0" + } + }, + "node_modules/unique-slug": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-2.0.2.tgz", + "integrity": "sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w==", + "dev": true, + "dependencies": { + "imurmurhash": "^0.1.4" + } + }, + "node_modules/unique-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz", + "integrity": "sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==", + "dev": true, + "dependencies": { + "crypto-random-string": "^2.0.0" + } + }, + "node_modules/universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "dev": true + }, + "node_modules/unset-value": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", + "integrity": "sha1-g3aHP30jNRef+x5vw6jtDfyKtVk=", + "dev": true, + "dependencies": { + "has-value": "^0.3.1", + "isobject": "^3.0.0" + } + }, + "node_modules/unset-value/node_modules/has-value": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/has-value/-/has-value-0.3.1.tgz", + "integrity": "sha1-ex9YutpiyoJ+wKIHgCVlSEWZXh8=", + "dev": true, + "dependencies": { + "get-value": "^2.0.3", + "has-values": "^0.1.4", + "isobject": "^2.0.0" + } + }, + "node_modules/unset-value/node_modules/has-value/node_modules/isobject": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz", + "integrity": "sha1-8GVWEJaj8dou9GJy+BXIQNh+DIk=", + "dev": true, + "dependencies": { + "isarray": "1.0.0" + } + }, + "node_modules/unset-value/node_modules/has-values": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/has-values/-/has-values-0.1.4.tgz", + "integrity": "sha1-bWHeldkd/Km5oCCJrThL/49it3E=", + "dev": true + }, + "node_modules/upath": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz", + "integrity": "sha512-aZwGpamFO61g3OlfT7OQCHqhGnW43ieH9WZeP7QxN/G/jS4jfqUkZxoryvJgVPEcrl5NL/ggHsSmLMHuH64Lhg==", + "dev": true + }, + "node_modules/update-notifier": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-4.0.0.tgz", + "integrity": "sha512-p9zf71hWt5GVXM4iEBujpUgx8mK9AWiCCapEJm/O1z5ntCim83Z1ATqzZFBHFYqx03laMqv8LiDgs/7ikXjf/g==", + "dev": true, + "dependencies": { + "boxen": "^4.2.0", + "chalk": "^3.0.0", + "configstore": "^5.0.0", + "has-yarn": "^2.1.0", + "import-lazy": "^2.1.0", + "is-ci": "^2.0.0", + "is-installed-globally": "^0.3.1", + "is-npm": "^4.0.0", + "is-yarn-global": "^0.3.0", + "latest-version": "^5.0.0", + "semver-diff": "^3.1.1", + "xdg-basedir": "^4.0.0" + } + }, + "node_modules/update-notifier/node_modules/ansi-styles": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", + "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "dev": true, + "dependencies": { + "@types/color-name": "^1.1.1", + "color-convert": "^2.0.1" + } + }, + "node_modules/update-notifier/node_modules/chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "node_modules/update-notifier/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + } + }, + "node_modules/update-notifier/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/update-notifier/node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "node_modules/update-notifier/node_modules/supports-color": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", + "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + } + }, + "node_modules/uri-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "dev": true, + "dependencies": { + "punycode": "^2.1.0" + } + }, + "node_modules/urix": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", + "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=", + "dev": true + }, + "node_modules/url": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", + "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=", + "dev": true, + "dependencies": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "node_modules/url-parse-lax": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", + "integrity": "sha1-FrXK/Afb42dsGxmZF3gj1lA6yww=", + "dev": true, + "dependencies": { + "prepend-http": "^2.0.0" + } + }, + "node_modules/url/node_modules/punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=", + "dev": true + }, + "node_modules/use": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", + "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", + "dev": true + }, + "node_modules/util": { + "version": "0.11.1", + "resolved": "https://registry.npmjs.org/util/-/util-0.11.1.tgz", + "integrity": "sha512-HShAsny+zS2TZfaXxD9tYj4HQGlBezXZMZuM/S5PKLLoZkShZiGk9o5CzukI1LVHZvjdvZ2Sj1aW/Ndn2NB/HQ==", + "dev": true, + "dependencies": { + "inherits": "2.0.3" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", + "dev": true + }, + "node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "dev": true + }, + "node_modules/v8-compile-cache": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz", + "integrity": "sha512-8OQ9CL+VWyt3JStj7HX7/ciTL2V3Rl1Wf5OL+SNTm0yK1KvtReVulksyeRnCANHHuUxHlQig+JJDlUhBt1NQDQ==", + "dev": true + }, + "node_modules/validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "dev": true, + "dependencies": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "node_modules/verror": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", + "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", + "dev": true, + "dependencies": { + "assert-plus": "^1.0.0", + "core-util-is": "1.0.2", + "extsprintf": "^1.2.0" + } + }, + "node_modules/vm-browserify": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", + "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==", + "dev": true + }, + "node_modules/vscode-json-languageservice": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/vscode-json-languageservice/-/vscode-json-languageservice-3.3.0.tgz", + "integrity": "sha512-upq1PhwDItazdtRJ/R7uU0Fgrf9iaYa1xLK4WFLExR0DgbPojd0YgMpfyknVyXGlxsg3fJQ0H7J++QeByXHh9w==", + "dev": true, + "dependencies": { + "jsonc-parser": "^2.1.0", + "vscode-languageserver-types": "^3.15.0-next.2", + "vscode-nls": "^4.1.1", + "vscode-uri": "^2.0.1" + } + }, + "node_modules/vscode-json-languageservice/node_modules/vscode-languageserver-types": { + "version": "3.15.0-next.2", + "resolved": "https://registry.npmjs.org/vscode-languageserver-types/-/vscode-languageserver-types-3.15.0-next.2.tgz", + "integrity": "sha512-2JkrMWWUi2rlVLSo9OFR2PIGUzdiowEM8NgNYiwLKnXTjpwpjjIrJbNNxDik7Rv4oo9KtikcFQZKXbrKilL/MQ==", + "dev": true + }, + "node_modules/vscode-nls": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/vscode-nls/-/vscode-nls-4.1.1.tgz", + "integrity": "sha512-4R+2UoUUU/LdnMnFjePxfLqNhBS8lrAFyX7pjb2ud/lqDkrUavFUTcG7wR0HBZFakae0Q6KLBFjMS6W93F403A==", + "dev": true + }, + "node_modules/vscode-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-2.0.1.tgz", + "integrity": "sha512-s/k0zsYr6y+tsocFyxT/+G5aq8mEdpDZuph3LZ+UmCs7LNhx/xomiCy5kyP+jOAKC7RMCUvb6JbPD1/TgAvq0g==", + "dev": true + }, + "node_modules/watchpack": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.6.1.tgz", + "integrity": "sha512-+IF9hfUFOrYOOaKyfaI7h7dquUIOgyEMoQMLA7OP5FxegKA2+XdXThAZ9TU2kucfhDH7rfMHs1oPYziVGWRnZA==", + "dev": true, + "dependencies": { + "chokidar": "^2.1.8", + "graceful-fs": "^4.1.2", + "neo-async": "^2.5.0" + } + }, + "node_modules/watchpack-chokidar2": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/watchpack-chokidar2/-/watchpack-chokidar2-2.0.0.tgz", + "integrity": "sha512-9TyfOyN/zLUbA288wZ8IsMZ+6cbzvsNyEzSBp6e/zkifi6xxbl8SmQ/CxQq32k8NNqrdVEVUVSEf56L4rQ/ZxA==", + "dev": true, + "optional": true, + "dependencies": { + "chokidar": "^2.1.8" + } + }, + "node_modules/wcwidth": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", + "integrity": "sha1-8LDc+RW8X/FSivrbLA4XtTLaL+g=", + "dev": true, + "dependencies": { + "defaults": "^1.0.3" + } + }, + "node_modules/web-ext": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/web-ext/-/web-ext-4.3.0.tgz", + "integrity": "sha512-1BCFeXuMY3QHnlkiUqgCV4ODNN84X3mX4GJk+Gb8tFv0Z8Grj4LneYa4A/0txoxpsz5E05THnKtb31t0XPJ8EQ==", + "dev": true, + "dependencies": { + "@babel/polyfill": "7.7.0", + "@babel/runtime": "7.7.7", + "@cliqz-oss/firefox-client": "0.3.1", + "@cliqz-oss/node-firefox-connect": "1.2.1", + "adbkit": "2.11.1", + "addons-linter": "1.26.0", + "bunyan": "1.8.12", + "camelcase": "5.3.1", + "chrome-launcher": "0.13.3", + "debounce": "1.2.0", + "decamelize": "3.2.0", + "es6-error": "4.1.1", + "event-to-promise": "0.8.0", + "firefox-profile": "1.3.1", + "fx-runner": "1.0.12", + "git-rev-sync": "2.0.0", + "import-fresh": "3.2.1", + "mkdirp": "1.0.4", + "multimatch": "4.0.0", + "mz": "2.7.0", + "node-notifier": "6.0.0", + "open": "7.0.0", + "parse-json": "5.0.0", + "sign-addon": "2.0.5", + "source-map-support": "0.5.19", + "stream-to-promise": "2.2.0", + "strip-bom": "4.0.0", + "strip-json-comments": "3.0.1", + "tmp": "0.1.0", + "update-notifier": "4.0.0", + "watchpack": "1.6.1", + "ws": "7.2.3", + "yargs": "15.3.1", + "zip-dir": "1.0.2" + } + }, + "node_modules/web-ext/node_modules/import-fresh": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", + "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", + "dev": true, + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + } + }, + "node_modules/web-ext/node_modules/mkdirp": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", + "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", + "dev": true + }, + "node_modules/web-ext/node_modules/parse-json": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.0.0.tgz", + "integrity": "sha512-OOY5b7PAEFV0E2Fir1KOkxchnZNCdowAJgQ5NuxjpBKTRP3pQhwkrkxqQjeoKJ+fO7bCpmIZaogI4eZGDMEGOw==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1", + "lines-and-columns": "^1.1.6" + } + }, + "node_modules/web-ext/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "node_modules/web-ext/node_modules/source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "dev": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/web-ext/node_modules/strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", + "dev": true + }, + "node_modules/web-ext/node_modules/strip-json-comments": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.0.1.tgz", + "integrity": "sha512-VTyMAUfdm047mwKl+u79WIdrZxtFtn+nBxHeb844XBQ9uMNTuTHdx2hc5RiAJYqwTj3wc/xe5HLSdJSkJ+WfZw==", + "dev": true + }, + "node_modules/web-ext/node_modules/tmp": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.1.0.tgz", + "integrity": "sha512-J7Z2K08jbGcdA1kkQpJSqLF6T0tdQqpR2pnSUXsIchbPdTI9v3e85cLW0d6WDhwuAleOV71j2xWs8qMPfK7nKw==", + "dev": true, + "dependencies": { + "rimraf": "^2.6.3" + } + }, + "node_modules/webidl-conversions": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-5.0.0.tgz", + "integrity": "sha512-VlZwKPCkYKxQgeSbH5EyngOmRp7Ww7I9rQLERETtf5ofd9pGeswWiOtogpEO850jziPRarreGxn5QIiTqpb2wA==", + "dev": true + }, + "node_modules/webpack": { + "version": "4.44.1", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-4.44.1.tgz", + "integrity": "sha512-4UOGAohv/VGUNQJstzEywwNxqX417FnjZgZJpJQegddzPmTvph37eBIRbRTfdySXzVtJXLJfbMN3mMYhM6GdmQ==", + "dev": true, + "dependencies": { + "@webassemblyjs/ast": "1.9.0", + "@webassemblyjs/helper-module-context": "1.9.0", + "@webassemblyjs/wasm-edit": "1.9.0", + "@webassemblyjs/wasm-parser": "1.9.0", + "acorn": "^6.4.1", + "ajv": "^6.10.2", + "ajv-keywords": "^3.4.1", + "chrome-trace-event": "^1.0.2", + "enhanced-resolve": "^4.3.0", + "eslint-scope": "^4.0.3", + "json-parse-better-errors": "^1.0.2", + "loader-runner": "^2.4.0", + "loader-utils": "^1.2.3", + "memory-fs": "^0.4.1", + "micromatch": "^3.1.10", + "mkdirp": "^0.5.3", + "neo-async": "^2.6.1", + "node-libs-browser": "^2.2.1", + "schema-utils": "^1.0.0", + "tapable": "^1.1.3", + "terser-webpack-plugin": "^1.4.3", + "watchpack": "^1.7.4", + "webpack-sources": "^1.4.1" + } + }, + "node_modules/webpack-cli": { + "version": "3.3.12", + "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-3.3.12.tgz", + "integrity": "sha512-NVWBaz9k839ZH/sinurM+HcDvJOTXwSjYp1ku+5XKeOC03z8v5QitnK/x+lAxGXFyhdayoIf/GOpv85z3/xPag==", + "dev": true, + "dependencies": { + "chalk": "^2.4.2", + "cross-spawn": "^6.0.5", + "enhanced-resolve": "^4.1.1", + "findup-sync": "^3.0.0", + "global-modules": "^2.0.0", + "import-local": "^2.0.0", + "interpret": "^1.4.0", + "loader-utils": "^1.4.0", + "supports-color": "^6.1.0", + "v8-compile-cache": "^2.1.1", + "yargs": "^13.3.2" + } + }, + "node_modules/webpack-cli/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "node_modules/webpack-cli/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "node_modules/webpack-cli/node_modules/chalk/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + } + }, + "node_modules/webpack-cli/node_modules/cliui": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", + "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", + "dev": true, + "dependencies": { + "string-width": "^3.1.0", + "strip-ansi": "^5.2.0", + "wrap-ansi": "^5.1.0" + } + }, + "node_modules/webpack-cli/node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true + }, + "node_modules/webpack-cli/node_modules/emojis-list": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-3.0.0.tgz", + "integrity": "sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==", + "dev": true + }, + "node_modules/webpack-cli/node_modules/enhanced-resolve": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.2.0.tgz", + "integrity": "sha512-S7eiFb/erugyd1rLb6mQ3Vuq+EXHv5cpCkNqqIkYkBgN2QdFnyCZzFBleqwGEx4lgNGYij81BWnCrFNK7vxvjQ==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "memory-fs": "^0.5.0", + "tapable": "^1.0.0" + } + }, + "node_modules/webpack-cli/node_modules/find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "dependencies": { + "locate-path": "^3.0.0" + } + }, + "node_modules/webpack-cli/node_modules/loader-utils": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.0.tgz", + "integrity": "sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==", + "dev": true, + "dependencies": { + "big.js": "^5.2.2", + "emojis-list": "^3.0.0", + "json5": "^1.0.1" + } + }, + "node_modules/webpack-cli/node_modules/locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + } + }, + "node_modules/webpack-cli/node_modules/memory-fs": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.5.0.tgz", + "integrity": "sha512-jA0rdU5KoQMC0e6ppoNRtpp6vjFq6+NY7r8hywnC7V+1Xj/MtHwGIbB1QaK/dunyjWteJzmkpd7ooeWg10T7GA==", + "dev": true, + "dependencies": { + "errno": "^0.1.3", + "readable-stream": "^2.0.1" + } + }, + "node_modules/webpack-cli/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + } + }, + "node_modules/webpack-cli/node_modules/p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "dependencies": { + "p-limit": "^2.0.0" + } + }, + "node_modules/webpack-cli/node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true + }, + "node_modules/webpack-cli/node_modules/string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "dependencies": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "node_modules/webpack-cli/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "dependencies": { + "ansi-regex": "^4.1.0" + } + }, + "node_modules/webpack-cli/node_modules/supports-color": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", + "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + } + }, + "node_modules/webpack-cli/node_modules/wrap-ansi": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", + "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.0", + "string-width": "^3.0.0", + "strip-ansi": "^5.0.0" + } + }, + "node_modules/webpack-cli/node_modules/yargs": { + "version": "13.3.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.3.2.tgz", + "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", + "dev": true, + "dependencies": { + "cliui": "^5.0.0", + "find-up": "^3.0.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^3.0.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^13.1.2" + } + }, + "node_modules/webpack-cli/node_modules/yargs-parser": { + "version": "13.1.2", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz", + "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", + "dev": true, + "dependencies": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } + }, + "node_modules/webpack-sources": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/webpack-sources/-/webpack-sources-1.4.3.tgz", + "integrity": "sha512-lgTS3Xhv1lCOKo7SA5TjKXMjpSM4sBjNV5+q2bqesbSPs5FjGmU6jjtBSkX9b4qW87vDIsCIlUPOEhbZrMdjeQ==", + "dev": true, + "dependencies": { + "source-list-map": "^2.0.0", + "source-map": "~0.6.1" + } + }, + "node_modules/webpack-sources/node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true + }, + "node_modules/webpack/node_modules/acorn": { + "version": "6.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-6.4.1.tgz", + "integrity": "sha512-ZVA9k326Nwrj3Cj9jlh3wGFutC2ZornPNARZwsNYqQYgN0EsV2d53w5RN/co65Ohn4sUAUtb1rSUAOD6XN9idA==", + "dev": true + }, + "node_modules/webpack/node_modules/anymatch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", + "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==", + "dev": true, + "optional": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + } + }, + "node_modules/webpack/node_modules/binary-extensions": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz", + "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==", + "dev": true, + "optional": true + }, + "node_modules/webpack/node_modules/braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "optional": true, + "dependencies": { + "fill-range": "^7.0.1" + } + }, + "node_modules/webpack/node_modules/chokidar": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.2.tgz", + "integrity": "sha512-IZHaDeBeI+sZJRX7lGcXsdzgvZqKv6sECqsbErJA4mHWfpRrD8B97kSFN4cQz6nGBGiuFia1MKR4d6c1o8Cv7A==", + "dev": true, + "optional": true, + "dependencies": { + "anymatch": "~3.1.1", + "braces": "~3.0.2", + "fsevents": "~2.1.2", + "glob-parent": "~5.1.0", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.4.0" + } + }, + "node_modules/webpack/node_modules/enhanced-resolve": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-4.3.0.tgz", + "integrity": "sha512-3e87LvavsdxyoCfGusJnrZ5G8SLPOFeHSNpZI/ATL9a5leXo2k0w6MKnbqhdBad9qTobSfB20Ld7UmgoNbAZkQ==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "memory-fs": "^0.5.0", + "tapable": "^1.0.0" + } + }, + "node_modules/webpack/node_modules/enhanced-resolve/node_modules/memory-fs": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/memory-fs/-/memory-fs-0.5.0.tgz", + "integrity": "sha512-jA0rdU5KoQMC0e6ppoNRtpp6vjFq6+NY7r8hywnC7V+1Xj/MtHwGIbB1QaK/dunyjWteJzmkpd7ooeWg10T7GA==", + "dev": true, + "dependencies": { + "errno": "^0.1.3", + "readable-stream": "^2.0.1" + } + }, + "node_modules/webpack/node_modules/eslint-scope": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-4.0.3.tgz", + "integrity": "sha512-p7VutNr1O/QrxysMo3E45FjYDTeXBy0iTltPFNSqKAIfjDSXC+4dj+qfyuD8bfAXrW/y6lW3O76VaYNPKfpKrg==", + "dev": true, + "dependencies": { + "esrecurse": "^4.1.0", + "estraverse": "^4.1.1" + } + }, + "node_modules/webpack/node_modules/fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "optional": true, + "dependencies": { + "to-regex-range": "^5.0.1" + } + }, + "node_modules/webpack/node_modules/glob-parent": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", + "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "dev": true, + "optional": true, + "dependencies": { + "is-glob": "^4.0.1" + } + }, + "node_modules/webpack/node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "optional": true, + "dependencies": { + "binary-extensions": "^2.0.0" + } + }, + "node_modules/webpack/node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "optional": true + }, + "node_modules/webpack/node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "optional": true + }, + "node_modules/webpack/node_modules/readdirp": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz", + "integrity": "sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ==", + "dev": true, + "optional": true, + "dependencies": { + "picomatch": "^2.2.1" + } + }, + "node_modules/webpack/node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "optional": true, + "dependencies": { + "is-number": "^7.0.0" + } + }, + "node_modules/webpack/node_modules/watchpack": { + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-1.7.4.tgz", + "integrity": "sha512-aWAgTW4MoSJzZPAicljkO1hsi1oKj/RRq/OJQh2PKI2UKL04c2Bs+MBOB+BBABHTXJpf9mCwHN7ANCvYsvY2sg==", + "dev": true, + "dependencies": { + "chokidar": "^3.4.1", + "graceful-fs": "^4.1.2", + "neo-async": "^2.5.0", + "watchpack-chokidar2": "^2.0.0" + }, + "optionalDependencies": { + "chokidar": "^3.4.1", + "watchpack-chokidar2": "^2.0.0" + } + }, + "node_modules/whatwg-url": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-8.1.0.tgz", + "integrity": "sha512-vEIkwNi9Hqt4TV9RdnaBPNt+E2Sgmo3gePebCRgZ1R7g6d23+53zCTnuB0amKI4AXq6VM8jj2DUAa0S1vjJxkw==", + "dev": true, + "dependencies": { + "lodash.sortby": "^4.7.0", + "tr46": "^2.0.2", + "webidl-conversions": "^5.0.0" + } + }, + "node_modules/when": { + "version": "3.7.7", + "resolved": "https://registry.npmjs.org/when/-/when-3.7.7.tgz", + "integrity": "sha1-q6A/w7tzbWyIsJHQE9io5ZDYRxg=", + "dev": true + }, + "node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + } + }, + "node_modules/which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", + "dev": true + }, + "node_modules/widest-line": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-3.1.0.tgz", + "integrity": "sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==", + "dev": true, + "dependencies": { + "string-width": "^4.0.0" + } + }, + "node_modules/widest-line/node_modules/ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "node_modules/widest-line/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/widest-line/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "node_modules/widest-line/node_modules/string-width": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", + "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + } + }, + "node_modules/widest-line/node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.0" + } + }, + "node_modules/winreg": { + "version": "0.0.12", + "resolved": "https://registry.npmjs.org/winreg/-/winreg-0.0.12.tgz", + "integrity": "sha1-BxBVVLoanQiXklHRKUdb/64wBrc=", + "dev": true + }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=", + "dev": true + }, + "node_modules/worker-farm": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/worker-farm/-/worker-farm-1.7.0.tgz", + "integrity": "sha512-rvw3QTZc8lAxyVrqcSGVm5yP/IJ2UcB3U0graE3LCFoZ0Yn2x4EoVSqJKdB/T5M+FLcRPjz4TDacRf3OCfNUzw==", + "dev": true, + "dependencies": { + "errno": "~0.1.7" + } + }, + "node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", + "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "dev": true, + "dependencies": { + "@types/color-name": "^1.1.1", + "color-convert": "^2.0.1" + } + }, + "node_modules/wrap-ansi/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + } + }, + "node_modules/wrap-ansi/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/wrap-ansi/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/wrap-ansi/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "node_modules/wrap-ansi/node_modules/string-width": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", + "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + } + }, + "node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.0" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + }, + "node_modules/write-file-atomic": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", + "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==", + "dev": true, + "dependencies": { + "imurmurhash": "^0.1.4", + "is-typedarray": "^1.0.0", + "signal-exit": "^3.0.2", + "typedarray-to-buffer": "^3.1.5" + } + }, + "node_modules/ws": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.3.tgz", + "integrity": "sha512-HTDl9G9hbkNDk98naoR/cHDws7+EyYMOdL1BmjsZXRUjf7d+MficC4B7HLUPlSiho0vg+CWKrGIt/VJBd1xunQ==", + "dev": true + }, + "node_modules/xdg-basedir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz", + "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==", + "dev": true + }, + "node_modules/xml2js": { + "version": "0.4.23", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.23.tgz", + "integrity": "sha512-ySPiMjM0+pLDftHgXY4By0uswI3SPKLDw/i3UXbnO8M/p28zqexCUoPmQFrYD+/1BzhGJSs2i1ERWKJAtiLrug==", + "dev": true, + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "dev": true + }, + "node_modules/xregexp": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-4.3.0.tgz", + "integrity": "sha512-7jXDIFXh5yJ/orPn4SXjuVrWWoi4Cr8jfV1eHv9CixKSbU+jY4mxfrBwAuDvupPNKpMUY+FeIqsVw/JLT9+B8g==", + "dev": true, + "dependencies": { + "@babel/runtime-corejs3": "^7.8.3" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "dev": true + }, + "node_modules/y18n": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", + "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==", + "dev": true + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true + }, + "node_modules/yargs": { + "version": "15.3.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.3.1.tgz", + "integrity": "sha512-92O1HWEjw27sBfgmXiixJWT5hRBp2eobqXicLtPBIDBhYB+1HpwZlXmbW2luivBJHBzki+7VyCLRtAkScbTBQA==", + "dev": true, + "dependencies": { + "cliui": "^6.0.0", + "decamelize": "^1.2.0", + "find-up": "^4.1.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^4.2.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^18.1.1" + } + }, + "node_modules/yargs-parser": { + "version": "18.1.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", + "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", + "dev": true, + "dependencies": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } + }, + "node_modules/yargs-parser/node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true + }, + "node_modules/yargs/node_modules/ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "node_modules/yargs/node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true + }, + "node_modules/yargs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/yargs/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "node_modules/yargs/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true + }, + "node_modules/yargs/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "dependencies": { + "p-locate": "^4.1.0" + } + }, + "node_modules/yargs/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + } + }, + "node_modules/yargs/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "dependencies": { + "p-limit": "^2.2.0" + } + }, + "node_modules/yargs/node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true + }, + "node_modules/yargs/node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true + }, + "node_modules/yargs/node_modules/string-width": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", + "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + } + }, + "node_modules/yargs/node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.0" + } + }, + "node_modules/yauzl": { + "version": "2.10.0", + "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", + "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=", + "dev": true, + "dependencies": { + "buffer-crc32": "~0.2.3", + "fd-slicer": "~1.1.0" + } + }, + "node_modules/zip-dir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/zip-dir/-/zip-dir-1.0.2.tgz", + "integrity": "sha1-JT+QeurWKiGs2HIdi4gDKyQRwFE=", + "dev": true, + "dependencies": { + "async": "^1.5.2", + "jszip": "^2.4.0" + } + }, + "node_modules/zip-dir/node_modules/async": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz", + "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=", + "dev": true + }, + "node_modules/zip-stream": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/zip-stream/-/zip-stream-1.2.0.tgz", + "integrity": "sha1-qLxF9MG0lpnGuQGYuqyqzbzUugQ=", + "dev": true, + "dependencies": { + "archiver-utils": "^1.3.0", + "compress-commons": "^1.2.0", + "lodash": "^4.8.0", + "readable-stream": "^2.0.0" + } + } + }, "dependencies": { "@babel/code-frame": { "version": "7.0.0", @@ -411,12 +10440,6 @@ "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", "dev": true }, - "JSONSelect": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/JSONSelect/-/JSONSelect-0.2.1.tgz", - "integrity": "sha1-QVQYpSbTP+MddLTe+jyDbUhewgM=", - "dev": true - }, "acorn": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.3.1.tgz", @@ -1770,27 +11793,31 @@ "dependencies": { "abbrev": { "version": "1.1.1", + "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", + "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "ansi-regex": { "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "aproba": { "version": "1.2.0", + "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "are-we-there-yet": { "version": "1.1.5", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", + "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "delegates": "^1.0.0", "readable-stream": "^2.0.6" @@ -1798,15 +11825,17 @@ }, "balanced-match": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "brace-expansion": { "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -1814,81 +11843,93 @@ }, "chownr": { "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "code-point-at": { "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "concat-map": { "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "console-control-strings": { "version": "1.1.0", + "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "core-util-is": { "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "debug": { "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "ms": "^2.1.1" } }, "deep-extend": { "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "delegates": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", + "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "detect-libc": { "version": "1.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "fs-minipass": { "version": "1.2.7", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz", + "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "minipass": "^2.6.0" } }, "fs.realpath": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "gauge": { "version": "2.7.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", + "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "aproba": "^1.0.3", "console-control-strings": "^1.0.0", @@ -1902,9 +11943,10 @@ }, "glob": { "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1916,33 +11958,37 @@ }, "has-unicode": { "version": "2.0.1", + "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "iconv-lite": { "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "safer-buffer": ">= 2.1.2 < 3" } }, "ignore-walk": { "version": "3.0.3", + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", + "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "minimatch": "^3.0.4" } }, "inflight": { "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -1950,51 +11996,58 @@ }, "inherits": { "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "ini": { "version": "1.3.5", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "is-fullwidth-code-point": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "number-is-nan": "^1.0.0" } }, "isarray": { "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "minimatch": { "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "brace-expansion": "^1.1.7" } }, "minimist": { "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "minipass": { "version": "2.9.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz", + "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "safe-buffer": "^5.1.2", "yallist": "^3.0.0" @@ -2002,33 +12055,37 @@ }, "minizlib": { "version": "1.3.3", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz", + "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "minipass": "^2.9.0" } }, "mkdirp": { "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.3.tgz", + "integrity": "sha512-P+2gwrFqx8lhew375MQHHeTlY8AuOJSrGf0R5ddkEndUkmwpgUob/vQuBD1V22/Cw1/lJr4x+EjllSezBThzBg==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "minimist": "^1.2.5" } }, "ms": { "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "needle": { "version": "2.3.3", + "resolved": "https://registry.npmjs.org/needle/-/needle-2.3.3.tgz", + "integrity": "sha512-EkY0GeSq87rWp1hoq/sH/wnTWgFVhYlnIkbJ0YJFfRgEFlz2RraCjBpFQ+vrEgEdp0ThfyHADmkChEhcb7PKyw==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "debug": "^3.2.6", "iconv-lite": "^0.4.4", @@ -2037,9 +12094,10 @@ }, "node-pre-gyp": { "version": "0.14.0", + "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.14.0.tgz", + "integrity": "sha512-+CvDC7ZttU/sSt9rFjix/P05iS43qHCOOGzcr3Ry99bXG7VX953+vFyEuph/tfqoYu8dttBkE86JSKBO2OzcxA==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "detect-libc": "^1.0.2", "mkdirp": "^0.5.1", @@ -2055,9 +12113,10 @@ }, "nopt": { "version": "4.0.3", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.3.tgz", + "integrity": "sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "abbrev": "1", "osenv": "^0.1.4" @@ -2065,24 +12124,27 @@ }, "npm-bundled": { "version": "1.1.1", + "resolved": "https://registry.npmjs.org/npm-bundled/-/npm-bundled-1.1.1.tgz", + "integrity": "sha512-gqkfgGePhTpAEgUsGEgcq1rqPXA+tv/aVBlgEzfXwA1yiUJF7xtEt3CtVwOjNYQOVknDk0F20w58Fnm3EtG0fA==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "npm-normalize-package-bin": "^1.0.1" } }, "npm-normalize-package-bin": { "version": "1.0.1", + "resolved": "https://registry.npmjs.org/npm-normalize-package-bin/-/npm-normalize-package-bin-1.0.1.tgz", + "integrity": "sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA==", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "npm-packlist": { "version": "1.4.8", + "resolved": "https://registry.npmjs.org/npm-packlist/-/npm-packlist-1.4.8.tgz", + "integrity": "sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "ignore-walk": "^3.0.1", "npm-bundled": "^1.0.1", @@ -2091,9 +12153,10 @@ }, "npmlog": { "version": "4.1.2", + "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", + "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "are-we-there-yet": "~1.1.2", "console-control-strings": "~1.1.0", @@ -2103,42 +12166,48 @@ }, "number-is-nan": { "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "object-assign": { "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "once": { "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "wrappy": "1" } }, "os-homedir": { "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "os-tmpdir": { "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "osenv": { "version": "0.1.5", + "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "os-homedir": "^1.0.0", "os-tmpdir": "^1.0.0" @@ -2146,21 +12215,24 @@ }, "path-is-absolute": { "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "process-nextick-args": { "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "rc": { "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -2170,9 +12242,10 @@ }, "readable-stream": { "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", @@ -2185,89 +12258,101 @@ }, "rimraf": { "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "glob": "^7.1.3" } }, "safe-buffer": { "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "safer-buffer": { "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "sax": { "version": "1.2.4", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", + "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "semver": { "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "set-blocking": { "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "signal-exit": { "version": "3.0.2", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.2.tgz", + "integrity": "sha1-tf3AjxKH6hF4Yo5BXiUTK3NkbG0=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "bundled": true, + "extraneous": true, + "requires": { + "safe-buffer": "~5.1.0" + } }, "string-width": { "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", "strip-ansi": "^3.0.0" } }, - "string_decoder": { - "version": "1.1.1", - "bundled": true, - "dev": true, - "optional": true, - "requires": { - "safe-buffer": "~5.1.0" - } - }, "strip-ansi": { "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "ansi-regex": "^2.0.0" } }, "strip-json-comments": { "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "tar": { "version": "4.4.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz", + "integrity": "sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "chownr": "^1.1.1", "fs-minipass": "^1.2.5", @@ -2280,30 +12365,34 @@ }, "util-deprecate": { "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "wide-align": { "version": "1.1.3", + "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", + "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", "bundled": true, - "dev": true, - "optional": true, + "extraneous": true, "requires": { "string-width": "^1.0.2 || 2" } }, "wrappy": { "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", "bundled": true, - "dev": true, - "optional": true + "extraneous": true }, "yallist": { "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", "bundled": true, - "dev": true, - "optional": true + "extraneous": true } } }, @@ -5318,6 +15407,12 @@ "integrity": "sha1-LHS27kHZPKUbe1qu6PUDYx0lKnM=", "dev": true }, + "JSONSelect": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/JSONSelect/-/JSONSelect-0.2.1.tgz", + "integrity": "sha1-QVQYpSbTP+MddLTe+jyDbUhewgM=", + "dev": true + }, "jsonwebtoken": { "version": "8.5.1", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz", @@ -6429,20 +16524,25 @@ }, "openwpm-webext-instrumentation": { "version": "file:../webext-instrumentation", - "dependencies": { - "extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" - }, - "mdn-browser-compat-data": { - "version": "1.0.20", - "resolved": "https://registry.npmjs.org/mdn-browser-compat-data/-/mdn-browser-compat-data-1.0.20.tgz", - "integrity": "sha512-6sS4ydJ6+/ZnSwKUN8Jtu7NzD2pobXRolYRjHupKtRBDmx+8WZG1yUdOfF9QfCNPkWKrf3GcSFEurIrnJC56Qw==", - "requires": { - "extend": "3.0.2" - } - } + "requires": { + "@types/firefox-webext-browser": "^70.0.1", + "ava": "^3.12.1", + "codecov": "^3.7.2", + "commitizen": "^4.2.1", + "cz-conventional-changelog": "^2.1.0", + "gh-pages": "^2.1.1", + "npm-run-all": "^4.1.5", + "nyc": "^14.1.1", + "opn-cli": "^3.1.0", + "prettier": "^1.19.1", + "publish-please": "^5.5.1", + "standard-version": "github:conventional-changelog/standard-version#master", + "trash-cli": "^1.4.0", + "tslint": "^5.20.1", + "tslint-config-prettier": "^1.15.0", + "tslint-immutable": "^4.7.0", + "typedoc": "^0.17.8", + "typescript": "^3.9.7" } }, "optionator": { @@ -8020,6 +18120,15 @@ } } }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.0" + } + }, "string-width": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", @@ -8179,15 +18288,6 @@ } } }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, - "requires": { - "safe-buffer": "~5.1.0" - } - }, "strip-ansi": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", diff --git a/openwpm/Extension/webext-instrumentation/package-lock.json b/openwpm/Extension/webext-instrumentation/package-lock.json index bd2cadae7..36bb9884a 100644 --- a/openwpm/Extension/webext-instrumentation/package-lock.json +++ b/openwpm/Extension/webext-instrumentation/package-lock.json @@ -1,7 +1,11493 @@ { "name": "@openwpm/webext-instrumentation", + "lockfileVersion": 2, "requires": true, - "lockfileVersion": 1, + "packages": { + "": { + "name": "@openwpm/webext-instrumentation", + "license": "GPL-3.0-or-later", + "devDependencies": { + "@types/firefox-webext-browser": "^70.0.1", + "ava": "^3.12.1", + "codecov": "^3.7.2", + "commitizen": "^4.2.1", + "cz-conventional-changelog": "^2.1.0", + "gh-pages": "^2.1.1", + "npm-run-all": "^4.1.5", + "nyc": "^14.1.1", + "opn-cli": "^3.1.0", + "prettier": "^1.19.1", + "publish-please": "^5.5.1", + "standard-version": "github:conventional-changelog/standard-version#master", + "trash-cli": "^1.4.0", + "tslint": "^5.20.1", + "tslint-config-prettier": "^1.15.0", + "tslint-immutable": "^4.7.0", + "typedoc": "^0.17.8", + "typescript": "^3.9.7" + }, + "engines": { + "node": ">=8.9" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.10.4.tgz", + "integrity": "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==", + "dev": true, + "dependencies": { + "@babel/highlight": "^7.10.4" + } + }, + "node_modules/@babel/generator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.10.4.tgz", + "integrity": "sha512-toLIHUIAgcQygFZRAQcsLQV3CBuX6yOIru1kJk/qqqvcRmZrYe6WavZTSG+bB8MxhnL9YPf+pKQfuiP161q7ng==", + "dev": true, + "dependencies": { + "@babel/types": "^7.10.4", + "jsesc": "^2.5.1", + "lodash": "^4.17.13", + "source-map": "^0.5.0" + } + }, + "node_modules/@babel/generator/node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@babel/helper-function-name": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.10.4.tgz", + "integrity": "sha512-YdaSyz1n8gY44EmN7x44zBn9zQ1Ry2Y+3GTA+3vH6Mizke1Vw0aWDM66FOYEPw8//qKkmqOckrGgTYa+6sceqQ==", + "dev": true, + "dependencies": { + "@babel/helper-get-function-arity": "^7.10.4", + "@babel/template": "^7.10.4", + "@babel/types": "^7.10.4" + } + }, + "node_modules/@babel/helper-get-function-arity": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.10.4.tgz", + "integrity": "sha512-EkN3YDB+SRDgiIUnNgcmiD361ti+AVbL3f3Henf6dqqUyr5dMsorno0lJWJuLhDhkI5sYEpgj6y9kB8AOU1I2A==", + "dev": true, + "dependencies": { + "@babel/types": "^7.10.4" + } + }, + "node_modules/@babel/helper-split-export-declaration": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.10.4.tgz", + "integrity": "sha512-pySBTeoUff56fL5CBU2hWm9TesA4r/rOkI9DyJLvvgz09MB9YtfIYe3iBriVaYNaPe+Alua0vBIOVOLs2buWhg==", + "dev": true, + "dependencies": { + "@babel/types": "^7.10.4" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.10.4.tgz", + "integrity": "sha512-3U9y+43hz7ZM+rzG24Qe2mufW5KhvFg/NhnNph+i9mgCtdTCtMJuI1TMkrIUiK7Ix4PYlRF9I5dhqaLYA/ADXw==", + "dev": true + }, + "node_modules/@babel/highlight": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.10.4.tgz", + "integrity": "sha512-i6rgnR/YgPEQzZZnbTHHuZdlE8qyoBNalD6F+q4vAFlcMEcqmkoG+mPqJYJCo63qPf74+Y1UZsl3l6f7/RIkmA==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.10.4", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + } + }, + "node_modules/@babel/highlight/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/@babel/highlight/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "node_modules/@babel/highlight/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/highlight/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@babel/parser": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.10.4.tgz", + "integrity": "sha512-8jHII4hf+YVDsskTF6WuMB3X4Eh+PsUkC2ljq22so5rHvH+T8BzyL94VOdyFLNR8tBSVXOTbNHOKpR4TfRxVtA==", + "dev": true, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/template": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.10.4.tgz", + "integrity": "sha512-ZCjD27cGJFUB6nmCB1Enki3r+L5kJveX9pq1SvAUKoICy6CZ9yD8xO086YXdYhvNjBdnekm4ZnaP5yC8Cs/1tA==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.10.4", + "@babel/parser": "^7.10.4", + "@babel/types": "^7.10.4" + } + }, + "node_modules/@babel/traverse": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.10.4.tgz", + "integrity": "sha512-aSy7p5THgSYm4YyxNGz6jZpXf+Ok40QF3aA2LyIONkDHpAcJzDUqlCKXv6peqYUs2gmic849C/t2HKw2a2K20Q==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.10.4", + "@babel/generator": "^7.10.4", + "@babel/helper-function-name": "^7.10.4", + "@babel/helper-split-export-declaration": "^7.10.4", + "@babel/parser": "^7.10.4", + "@babel/types": "^7.10.4", + "debug": "^4.1.0", + "globals": "^11.1.0", + "lodash": "^4.17.13" + } + }, + "node_modules/@babel/types": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.10.4.tgz", + "integrity": "sha512-UTCFOxC3FsFHb7lkRMVvgLzaRVamXuAs2Tz4wajva4WxtVY82eZeaUBtC2Zt95FU9TiznuC0Zk35tsim8jeVpg==", + "dev": true, + "dependencies": { + "@babel/helper-validator-identifier": "^7.10.4", + "lodash": "^4.17.13", + "to-fast-properties": "^2.0.0" + } + }, + "node_modules/@commitlint/execute-rule": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/@commitlint/execute-rule/-/execute-rule-9.1.2.tgz", + "integrity": "sha512-NGbeo0KCVYo1yj9vVPFHv6RGFpIF6wcQxpFYUKGIzZVV9Vz1WyiKS689JXa99Dt1aN0cZlEJJLnTNDIgYls0Vg==", + "dev": true, + "optional": true, + "engines": { + "node": ">=v8.17.0" + } + }, + "node_modules/@commitlint/load": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/@commitlint/load/-/load-9.1.2.tgz", + "integrity": "sha512-FPL82xBuF7J3EJ57kLVoligQP4BFRwrknooP+vNT787AXmQ/Fddc/iYYwHwy67pNkk5N++/51UyDl/CqiHb6nA==", + "dev": true, + "optional": true, + "dependencies": { + "@commitlint/execute-rule": "^9.1.2", + "@commitlint/resolve-extends": "^9.1.2", + "@commitlint/types": "^9.1.2", + "chalk": "4.1.0", + "cosmiconfig": "^6.0.0", + "lodash": "^4.17.19", + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=v8.17.0" + } + }, + "node_modules/@commitlint/resolve-extends": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/@commitlint/resolve-extends/-/resolve-extends-9.1.2.tgz", + "integrity": "sha512-HcoL+qFGmWEu9VM4fY0HI+VzF4yHcg3x+9Hx6pYFZ+r2wLbnKs964y0v68oyMO/mS/46MVoLNXZGR8U3adpadg==", + "dev": true, + "optional": true, + "dependencies": { + "import-fresh": "^3.0.0", + "lodash": "^4.17.19", + "resolve-from": "^5.0.0", + "resolve-global": "^1.0.0" + }, + "engines": { + "node": ">=v8.17.0" + } + }, + "node_modules/@commitlint/types": { + "version": "9.1.2", + "resolved": "https://registry.npmjs.org/@commitlint/types/-/types-9.1.2.tgz", + "integrity": "sha512-r3fwVbVH+M8W0qYlBBZFsUwKe6NT5qvz+EmU7sr8VeN1cQ63z+3cfXyTo7WGGEMEgKiT0jboNAK3b1FZp8k9LQ==", + "dev": true, + "optional": true, + "engines": { + "node": ">=v8.17.0" + } + }, + "node_modules/@concordance/react": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@concordance/react/-/react-2.0.0.tgz", + "integrity": "sha512-huLSkUuM2/P+U0uy2WwlKuixMsTODD8p4JVQBI4VKeopkiN0C7M3N9XYVawb4M+4spN5RrO/eLhk7KoQX6nsfA==", + "dev": true, + "dependencies": { + "arrify": "^1.0.1" + }, + "engines": { + "node": ">=6.12.3 <7 || >=8.9.4 <9 || >=10.0.0" + } + }, + "node_modules/@concordance/react/node_modules/arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz", + "integrity": "sha512-eGmwYQn3gxo4r7jdQnkrrN6bY478C3P+a/y72IJukF8LjB6ZHeB3c+Ehacj3sYeSmUXGlnA67/PmbM9CVwL7Dw==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "2.0.3", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.3.tgz", + "integrity": "sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.4.tgz", + "integrity": "sha512-1V9XOY4rDW0rehzbrcqAmHnz8e7SKvX27gh8Gt2WgB0+pdzdiLV83p72kZPU+jvMbS1qU5mauP2iOvO8rhmurQ==", + "dev": true, + "dependencies": { + "@nodelib/fs.scandir": "2.1.3", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@sindresorhus/df": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/df/-/df-2.1.0.tgz", + "integrity": "sha1-0gjPJ+BvC7R20U197M19cm6ao4k=", + "dev": true, + "dependencies": { + "execa": "^0.2.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/@sindresorhus/df/node_modules/execa": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/execa/-/execa-0.2.2.tgz", + "integrity": "sha1-4urUcsLDGq1vc/GslW7vReEjIMs=", + "dev": true, + "dependencies": { + "cross-spawn-async": "^2.1.1", + "npm-run-path": "^1.0.0", + "object-assign": "^4.0.1", + "path-key": "^1.0.0", + "strip-eof": "^1.0.0" + }, + "engines": { + "node": ">=0.12" + } + }, + "node_modules/@sindresorhus/df/node_modules/npm-run-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-1.0.0.tgz", + "integrity": "sha1-9cMr9ZX+ga6Sfa7FLoL4sACsPI8=", + "dev": true, + "dependencies": { + "path-key": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@sindresorhus/df/node_modules/path-key": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-1.0.0.tgz", + "integrity": "sha1-XVPVeAGWRsDWiADbThRua9wqx68=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@sindresorhus/is": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-0.14.0.tgz", + "integrity": "sha512-9NET910DNaIPngYnLLPeg+Ogzqsi9uM4mSboU5y6p8S5DzMTVEsJZrawi+BoDNUVBa2DhJqQYUFvMDfgU062LQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/@szmarczak/http-timer": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-1.1.2.tgz", + "integrity": "sha512-XIB2XbzHTN6ieIjfIMV9hlVcfPU26s2vafYWQcZHWXHOxiaRZYEDKEwdl129Zyg50+foYV2jCgtrqSA6qNuNSA==", + "dev": true, + "dependencies": { + "defer-to-connect": "^1.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/@types/color-name": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz", + "integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ==", + "dev": true + }, + "node_modules/@types/firefox-webext-browser": { + "version": "70.0.1", + "resolved": "https://registry.npmjs.org/@types/firefox-webext-browser/-/firefox-webext-browser-70.0.1.tgz", + "integrity": "sha512-hjHsTR9vKs+yikWbNS/s7TVCx15M/MEn+VYx47wtT/W/wORsIZDD75gfUfP7lkzi+IxRvKMQBB/5/wMFlfgvgQ==", + "dev": true + }, + "node_modules/@types/glob": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-SEYeGAIQIQX8NN6LDKprLjbrd5dARM5EXsd8GI/A5l0apYI1fGMWgPHSe4ZKL4eozlAyI+doUE9XbYS4xCkQ1w==", + "dev": true, + "dependencies": { + "@types/minimatch": "*", + "@types/node": "*" + } + }, + "node_modules/@types/minimatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", + "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==", + "dev": true + }, + "node_modules/@types/minimist": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.0.tgz", + "integrity": "sha1-aaI6OtKcrwCX8G7aWbNh7i8GOfY=", + "dev": true + }, + "node_modules/@types/node": { + "version": "14.6.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.6.0.tgz", + "integrity": "sha512-mikldZQitV94akrc4sCcSjtJfsTKt4p+e/s0AGscVA6XArQ9kFclP+ZiYUMnq987rc6QlYxXv/EivqlfSLxpKA==", + "dev": true + }, + "node_modules/@types/normalize-package-data": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz", + "integrity": "sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==", + "dev": true + }, + "node_modules/@types/parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==", + "dev": true, + "optional": true + }, + "node_modules/acorn": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.0.1.tgz", + "integrity": "sha512-dmKn4pqZ29iQl2Pvze1zTrps2luvls2PBY//neO2WJ0s10B3AxJXshN+Ph7B4GrhfGhHXrl4dnUwyNNXQcnWGQ==", + "dev": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.0.0.tgz", + "integrity": "sha512-oZRad/3SMOI/pxbbmqyurIx7jHw1wZDcR9G44L8pUVFEomX/0dH89SrM1KaDXuv1NpzAXz6Op/Xu/Qd5XXzdEA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/add-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/add-stream/-/add-stream-1.0.0.tgz", + "integrity": "sha1-anmQQ3ynNtXhKI25K9MmbV9csqo=", + "dev": true + }, + "node_modules/agent-base": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.1.tgz", + "integrity": "sha512-01q25QQDwLSsyfhrKbn8yuur+JNw0H+0Y4JiGIKd3z9aYk/w/2kxD/Upc+t2ZBBSUNff50VjPsSW2YxM8QYKVg==", + "dev": true, + "dependencies": { + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/aggregate-error": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", + "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "dev": true, + "dependencies": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-align": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-3.0.0.tgz", + "integrity": "sha512-ZpClVKqXN3RGBmKibdfWzqCY4lnjEuoNzU5T0oEFpfd/z5qJHVarukridD4juLO2FXMiwUQxr9WqQtaYa8XRYw==", + "dev": true, + "dependencies": { + "string-width": "^3.0.0" + } + }, + "node_modules/ansi-align/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-align/node_modules/emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "node_modules/ansi-align/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/ansi-align/node_modules/string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "dependencies": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-align/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "dependencies": { + "ansi-regex": "^4.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/ansi-escapes": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz", + "integrity": "sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz", + "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==", + "dev": true, + "dependencies": { + "@types/color-name": "^1.1.1", + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/anymatch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz", + "integrity": "sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg==", + "dev": true, + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/append-transform": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/append-transform/-/append-transform-1.0.0.tgz", + "integrity": "sha512-P009oYkeHyU742iSZJzZZywj4QRJdnTWffaKuJQLablCZ1uz6/cW4yaRgcDaoQ+uwOxxnt0gRUcwfsNP2ri0gw==", + "dev": true, + "dependencies": { + "default-require-extensions": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/archy": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/archy/-/archy-1.0.0.tgz", + "integrity": "sha1-+cjBN1fMHde8N5rHeyxipcKGjEA=", + "dev": true + }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/argv": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/argv/-/argv-0.0.2.tgz", + "integrity": "sha1-7L0W+JSbFXGDcRsb2jNPN4QBhas=", + "dev": true, + "engines": { + "node": ">=0.6.10" + } + }, + "node_modules/arr-diff": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/arr-diff/-/arr-diff-4.0.0.tgz", + "integrity": "sha1-1kYQdP6/7HHn4VI1dhoyml3HxSA=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/arr-flatten": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/arr-flatten/-/arr-flatten-1.1.0.tgz", + "integrity": "sha512-L3hKV5R/p5o81R7O02IGnwpDmkp6E982XhtbuwSe3O4qOtMMMtodicASA1Cny2U+aCXcNpml+m4dPsvsJ3jatg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/arr-union": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", + "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/array-find-index": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array-find-index/-/array-find-index-1.0.2.tgz", + "integrity": "sha1-3wEKoSh+Fku9pvlyOwqWoexBh6E=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/array-ify": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/array-ify/-/array-ify-1.0.0.tgz", + "integrity": "sha1-nlKHYrSpBmrRY6aWKjZEGOlibs4=", + "dev": true + }, + "node_modules/array-union": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", + "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/array-uniq": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/array-uniq/-/array-uniq-1.0.3.tgz", + "integrity": "sha1-r2rId6Jcx/dOBYiUdThY39sk/bY=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/array-unique": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/array-unique/-/array-unique-0.3.2.tgz", + "integrity": "sha1-qJS3XUvE9s1nnvMkSp/Y9Gri1Cg=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/arrgv": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/arrgv/-/arrgv-1.0.2.tgz", + "integrity": "sha512-a4eg4yhp7mmruZDQFqVMlxNRFGi/i1r87pt8SDHy0/I8PqSXoUTlWZRdAZo0VXgvEARcujbtTk8kiZRi1uDGRw==", + "dev": true, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/assign-symbols": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/assign-symbols/-/assign-symbols-1.0.0.tgz", + "integrity": "sha1-WWZ/QfrdTyDMvCu5a41Pf3jsA2c=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/astral-regex": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", + "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/async": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", + "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", + "dev": true, + "dependencies": { + "lodash": "^4.17.14" + } + }, + "node_modules/atob": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", + "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + "dev": true, + "bin": { + "atob": "bin/atob.js" + }, + "engines": { + "node": ">= 4.5.0" + } + }, + "node_modules/ava": { + "version": "3.12.1", + "resolved": "https://registry.npmjs.org/ava/-/ava-3.12.1.tgz", + "integrity": "sha512-cS41+X+UfrcPed+CIgne/YV/6eWxaUjHEPH+W8WvNSqWTWku5YitjZGE5cMHFuJxwHELdR541xTBRn8Uwi4PSw==", + "dev": true, + "dependencies": { + "@concordance/react": "^2.0.0", + "acorn": "^8.0.1", + "acorn-walk": "^8.0.0", + "ansi-styles": "^4.2.1", + "arrgv": "^1.0.2", + "arrify": "^2.0.1", + "callsites": "^3.1.0", + "chalk": "^4.1.0", + "chokidar": "^3.4.2", + "chunkd": "^2.0.1", + "ci-info": "^2.0.0", + "ci-parallel-vars": "^1.0.1", + "clean-yaml-object": "^0.1.0", + "cli-cursor": "^3.1.0", + "cli-truncate": "^2.1.0", + "code-excerpt": "^3.0.0", + "common-path-prefix": "^3.0.0", + "concordance": "^5.0.1", + "convert-source-map": "^1.7.0", + "currently-unhandled": "^0.4.1", + "debug": "^4.1.1", + "del": "^5.1.0", + "emittery": "^0.7.1", + "equal-length": "^1.0.0", + "figures": "^3.2.0", + "globby": "^11.0.1", + "ignore-by-default": "^2.0.0", + "import-local": "^3.0.2", + "indent-string": "^4.0.0", + "is-error": "^2.2.2", + "is-plain-object": "^4.1.1", + "is-promise": "^4.0.0", + "lodash": "^4.17.20", + "matcher": "^3.0.0", + "md5-hex": "^3.0.1", + "mem": "^6.1.0", + "ms": "^2.1.2", + "ora": "^5.0.0", + "p-map": "^4.0.0", + "picomatch": "^2.2.2", + "pkg-conf": "^3.1.0", + "plur": "^4.0.0", + "pretty-ms": "^7.0.0", + "read-pkg": "^5.2.0", + "resolve-cwd": "^3.0.0", + "slash": "^3.0.0", + "source-map-support": "^0.5.19", + "stack-utils": "^2.0.2", + "strip-ansi": "^6.0.0", + "supertap": "^1.0.0", + "temp-dir": "^2.0.0", + "trim-off-newlines": "^1.0.1", + "update-notifier": "^4.1.1", + "write-file-atomic": "^3.0.3", + "yargs": "^15.4.1" + }, + "bin": { + "ava": "cli.js" + }, + "engines": { + "node": ">=10.18.0 <11 || >=12.14.0 <12.17.0 || >=12.17.0 <13 || >=14.0.0" + } + }, + "node_modules/ava/node_modules/lodash": { + "version": "4.17.20", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", + "dev": true + }, + "node_modules/balanced-match": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true + }, + "node_modules/base": { + "version": "0.11.2", + "resolved": "https://registry.npmjs.org/base/-/base-0.11.2.tgz", + "integrity": "sha512-5T6P4xPgpp0YDFvSWwEZ4NoE3aM4QBQXDzmVbraCkFj8zHM+mba8SyqB5DbZWyR7mYHo6Y7BdQo3MoA4m0TeQg==", + "dev": true, + "dependencies": { + "cache-base": "^1.0.1", + "class-utils": "^0.3.5", + "component-emitter": "^1.2.1", + "define-property": "^1.0.0", + "isobject": "^3.0.1", + "mixin-deep": "^1.2.0", + "pascalcase": "^0.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/base/node_modules/define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/base/node_modules/is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/base/node_modules/is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/base/node_modules/is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/binary-extensions": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz", + "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/blueimp-md5": { + "version": "2.17.0", + "resolved": "https://registry.npmjs.org/blueimp-md5/-/blueimp-md5-2.17.0.tgz", + "integrity": "sha512-x5PKJHY5rHQYaADj6NwPUR2QRCUVSggPzrUKkeENpj871o9l9IefJbO2jkT5UvYykeOK9dx0VmkIo6dZ+vThYw==", + "dev": true + }, + "node_modules/boxen": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-4.2.0.tgz", + "integrity": "sha512-eB4uT9RGzg2odpER62bBwSLvUeGC+WbRjjyyFhGsKnc8wp/m0+hQsMUvUe3H2V0D5vw0nBdO1hCJoZo5mKeuIQ==", + "dev": true, + "dependencies": { + "ansi-align": "^3.0.0", + "camelcase": "^5.3.1", + "chalk": "^3.0.0", + "cli-boxes": "^2.2.0", + "string-width": "^4.1.0", + "term-size": "^2.1.0", + "type-fest": "^0.8.1", + "widest-line": "^3.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/boxen/node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/boxen/node_modules/chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/boxen/node_modules/type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", + "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "dev": true, + "dependencies": { + "fill-range": "^7.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/buffer-from": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", + "dev": true + }, + "node_modules/builtin-modules": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", + "integrity": "sha1-Jw8HbFpywC9bZaR9+Uxf46J4iS8=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/cache-base": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/cache-base/-/cache-base-1.0.1.tgz", + "integrity": "sha512-AKcdTnFSWATd5/GCPRxr2ChwIJ85CeyrEyjRHlKxQ56d4XJMGym0uAiKn0xbLOGOl3+yRpOTi484dVCEc5AUzQ==", + "dev": true, + "dependencies": { + "collection-visit": "^1.0.0", + "component-emitter": "^1.2.1", + "get-value": "^2.0.6", + "has-value": "^1.0.0", + "isobject": "^3.0.1", + "set-value": "^2.0.0", + "to-object-path": "^0.3.0", + "union-value": "^1.0.0", + "unset-value": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/cacheable-request": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-6.1.0.tgz", + "integrity": "sha512-Oj3cAGPCqOZX7Rz64Uny2GYAZNliQSqfbePrgAQ1wKAihYmCUnraBtJtKcGR4xz7wF+LoJC+ssFZvv5BgF9Igg==", + "dev": true, + "dependencies": { + "clone-response": "^1.0.2", + "get-stream": "^5.1.0", + "http-cache-semantics": "^4.0.0", + "keyv": "^3.0.0", + "lowercase-keys": "^2.0.0", + "normalize-url": "^4.1.0", + "responselike": "^1.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cacheable-request/node_modules/get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "dev": true, + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cacheable-request/node_modules/lowercase-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz", + "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/cachedir": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cachedir/-/cachedir-2.2.0.tgz", + "integrity": "sha512-VvxA0xhNqIIfg0V9AmJkDg91DaJwryutH5rVEZAhcNi4iJFj9f+QxmAjgK1LT9I8OgToX27fypX6/MeCXVbBjQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/caching-transform": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-3.0.2.tgz", + "integrity": "sha512-Mtgcv3lh3U0zRii/6qVgQODdPA4G3zhG+jtbCWj39RXuUFTMzH0vcdMtaJS1jPowd+It2Pqr6y3NJMQqOqCE2w==", + "dev": true, + "dependencies": { + "hasha": "^3.0.0", + "make-dir": "^2.0.0", + "package-hash": "^3.0.0", + "write-file-atomic": "^2.4.2" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/caching-transform/node_modules/make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "dependencies": { + "pify": "^4.0.1", + "semver": "^5.6.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/caching-transform/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/caching-transform/node_modules/write-file-atomic": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-2.4.3.tgz", + "integrity": "sha512-GaETH5wwsX+GcnzhPgKcKjJ6M2Cq3/iZp1WyY/X1CSqrW+jVNM9Y7D8EC2sM4ZG/V8wZlSniJnCKWPmBYAucRQ==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.2" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase-keys": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-6.2.2.tgz", + "integrity": "sha512-YrwaA0vEKazPBkn0ipTiMpSajYDSe+KjQfrjhcBMxJt/znbvlHd8Pw/Vamaz5EB4Wfhs3SUR3Z9mwRu/P3s3Yg==", + "dev": true, + "dependencies": { + "camelcase": "^5.3.1", + "map-obj": "^4.0.0", + "quick-lru": "^4.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/capture-stack-trace": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.1.tgz", + "integrity": "sha512-mYQLZnx5Qt1JgB1WEiMCf2647plpGeQ2NMR/5L0HNZzGQo4fuSPnK+wjfPnKZV0aiJDgzmWqqkV/g7JD+DW0qw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/chalk": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/chardet": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", + "dev": true + }, + "node_modules/chokidar": { + "version": "3.4.2", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.2.tgz", + "integrity": "sha512-IZHaDeBeI+sZJRX7lGcXsdzgvZqKv6sECqsbErJA4mHWfpRrD8B97kSFN4cQz6nGBGiuFia1MKR4d6c1o8Cv7A==", + "dev": true, + "dependencies": { + "anymatch": "~3.1.1", + "braces": "~3.0.2", + "fsevents": "~2.1.2", + "glob-parent": "~5.1.0", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.4.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "optionalDependencies": { + "fsevents": "~2.1.2" + } + }, + "node_modules/chunkd": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/chunkd/-/chunkd-2.0.1.tgz", + "integrity": "sha512-7d58XsFmOq0j6el67Ug9mHf9ELUXsQXYJBkyxhH/k+6Ke0qXRnv0kbemx+Twc6fRJ07C49lcbdgm9FL1Ei/6SQ==", + "dev": true + }, + "node_modules/ci-info": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-2.0.0.tgz", + "integrity": "sha512-5tK7EtrZ0N+OLFMthtqOj4fI2Jeb88C4CAZPu25LDVUgXJ0A3Js4PMGqrn0JU1W0Mh1/Z8wZzYPxqUrXeBboCQ==", + "dev": true + }, + "node_modules/ci-parallel-vars": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ci-parallel-vars/-/ci-parallel-vars-1.0.1.tgz", + "integrity": "sha512-uvzpYrpmidaoxvIQHM+rKSrigjOe9feHYbw4uOI2gdfe1C3xIlxO+kVXq83WQWNniTf8bAxVpy+cQeFQsMERKg==", + "dev": true + }, + "node_modules/class-utils": { + "version": "0.3.6", + "resolved": "https://registry.npmjs.org/class-utils/-/class-utils-0.3.6.tgz", + "integrity": "sha512-qOhPa/Fj7s6TY8H8esGu5QNpMMQxz79h+urzrNYN6mn+9BnxlDGf5QZ+XeCDsxSjPqsSR56XOZOJmpeurnLMeg==", + "dev": true, + "dependencies": { + "arr-union": "^3.1.0", + "define-property": "^0.2.5", + "isobject": "^3.0.0", + "static-extend": "^0.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/class-utils/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/clean-yaml-object": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/clean-yaml-object/-/clean-yaml-object-0.1.0.tgz", + "integrity": "sha1-Y/sRDcLOGoTcIfbZM0h20BCui2g=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/cli-boxes": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.0.tgz", + "integrity": "sha512-gpaBrMAizVEANOpfZp/EEUixTXDyGt7DFzdK5hU+UbWt/J0lB0w20ncZj59Z9a93xHb9u12zF5BS6i9RKbtg4w==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "dev": true, + "dependencies": { + "restore-cursor": "^3.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-spinners": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.4.0.tgz", + "integrity": "sha512-sJAofoarcm76ZGpuooaO0eDy8saEy+YoZBLjC4h8srt4jeBnkYeOgqxgsJQTpyt2LjI5PTfLJHSL+41Yu4fEJA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/cli-truncate": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-2.1.0.tgz", + "integrity": "sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg==", + "dev": true, + "dependencies": { + "slice-ansi": "^3.0.0", + "string-width": "^4.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-width": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz", + "integrity": "sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw==", + "dev": true + }, + "node_modules/cliui": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-6.0.0.tgz", + "integrity": "sha512-t6wbgtoCXvAzst7QgXxJYqPt0usEfbgQdftEPbLL/cvv6HPE5VgvqCuAIDR0NgU52ds6rFwqrgakNLrHEjCbrQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^6.2.0" + } + }, + "node_modules/clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=", + "dev": true, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/clone-response": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz", + "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=", + "dev": true, + "dependencies": { + "mimic-response": "^1.0.0" + } + }, + "node_modules/code-excerpt": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/code-excerpt/-/code-excerpt-3.0.0.tgz", + "integrity": "sha512-VHNTVhd7KsLGOqfX3SyeO8RyYPMp1GJOg194VITk04WMYCv4plV68YWe6TJZxd9MhobjtpMRnVky01gqZsalaw==", + "dev": true, + "dependencies": { + "convert-to-spaces": "^1.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/code-point-at": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", + "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/codecov": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/codecov/-/codecov-3.7.2.tgz", + "integrity": "sha512-fmCjAkTese29DUX3GMIi4EaKGflHa4K51EoMc29g8fBHawdk/+KEq5CWOeXLdd9+AT7o1wO4DIpp/Z1KCqCz1g==", + "dev": true, + "dependencies": { + "argv": "0.0.2", + "ignore-walk": "3.0.3", + "js-yaml": "3.13.1", + "teeny-request": "6.0.1", + "urlgrey": "0.4.4" + }, + "bin": { + "codecov": "bin/codecov" + }, + "engines": { + "node": ">=4.0" + } + }, + "node_modules/codecov/node_modules/js-yaml": { + "version": "3.13.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", + "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/collection-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/collection-visit/-/collection-visit-1.0.0.tgz", + "integrity": "sha1-S8A3PBZLwykbTTaMgpzxqApZ3KA=", + "dev": true, + "dependencies": { + "map-visit": "^1.0.0", + "object-visit": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true + }, + "node_modules/commitizen": { + "version": "4.2.1", + "resolved": "https://registry.npmjs.org/commitizen/-/commitizen-4.2.1.tgz", + "integrity": "sha512-nZsp8IThkDu7C+93BFD/mLShb9Gd6Wsaf90tpKE3x/6u5y/Q52kzanIJpGr0qvIsJ5bCMpgKtr3Lbu3miEJfaA==", + "dev": true, + "dependencies": { + "cachedir": "2.2.0", + "cz-conventional-changelog": "3.2.0", + "dedent": "0.7.0", + "detect-indent": "6.0.0", + "find-node-modules": "2.0.0", + "find-root": "1.1.0", + "fs-extra": "8.1.0", + "glob": "7.1.4", + "inquirer": "6.5.2", + "is-utf8": "^0.2.1", + "lodash": "^4.17.20", + "minimist": "1.2.5", + "strip-bom": "4.0.0", + "strip-json-comments": "3.0.1" + }, + "bin": { + "commitizen": "bin/commitizen", + "cz": "bin/git-cz", + "git-cz": "bin/git-cz" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/commitizen/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/commitizen/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/commitizen/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/commitizen/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "node_modules/commitizen/node_modules/cz-conventional-changelog": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/cz-conventional-changelog/-/cz-conventional-changelog-3.2.0.tgz", + "integrity": "sha512-yAYxeGpVi27hqIilG1nh4A9Bnx4J3Ov+eXy4koL3drrR+IO9GaWPsKjik20ht608Asqi8TQPf0mczhEeyAtMzg==", + "dev": true, + "dependencies": { + "@commitlint/load": ">6.1.1", + "chalk": "^2.4.1", + "commitizen": "^4.0.3", + "conventional-commit-types": "^3.0.0", + "lodash.map": "^4.5.1", + "longest": "^2.0.1", + "word-wrap": "^1.0.3" + }, + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@commitlint/load": ">6.1.1" + } + }, + "node_modules/commitizen/node_modules/glob": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.4.tgz", + "integrity": "sha512-hkLPepehmnKk41pUGm3sYxoFs/umurYfYJCerbXEyFIWcAzvpipAgVkBqqT9RBKMGjnq6kMuyYwha6csxbiM1A==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/commitizen/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/commitizen/node_modules/lodash": { + "version": "4.17.20", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", + "dev": true + }, + "node_modules/commitizen/node_modules/strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/commitizen/node_modules/strip-json-comments": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.0.1.tgz", + "integrity": "sha512-VTyMAUfdm047mwKl+u79WIdrZxtFtn+nBxHeb844XBQ9uMNTuTHdx2hc5RiAJYqwTj3wc/xe5HLSdJSkJ+WfZw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/commitizen/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/common-path-prefix": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/common-path-prefix/-/common-path-prefix-3.0.0.tgz", + "integrity": "sha512-QE33hToZseCH3jS0qN96O/bSh3kaw/h+Tq7ngyY9eWDUnTlTNUyqfqvCXioLe5Na5jFsL78ra/wuBU4iuEgd4w==", + "dev": true + }, + "node_modules/commondir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", + "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=", + "dev": true + }, + "node_modules/compare-func": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/compare-func/-/compare-func-2.0.0.tgz", + "integrity": "sha512-zHig5N+tPWARooBnb0Zx1MFcdfpyJrfTJ3Y5L+IFvUm8rM74hHz66z0gw0x4tijh5CorKkKUCnW82R2vmpeCRA==", + "dev": true, + "dependencies": { + "array-ify": "^1.0.0", + "dot-prop": "^5.1.0" + } + }, + "node_modules/component-emitter": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz", + "integrity": "sha512-Rd3se6QB+sO1TwqZjscQrurpEPIfO0/yYnSin6Q/rD3mOutHvUrCAhJub3r90uNb+SESBuE0QYoB90YdfatsRg==", + "dev": true + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true + }, + "node_modules/concat-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-2.0.0.tgz", + "integrity": "sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==", + "dev": true, + "engines": [ + "node >= 6.0" + ], + "dependencies": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.0.2", + "typedarray": "^0.0.6" + } + }, + "node_modules/concordance": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/concordance/-/concordance-5.0.1.tgz", + "integrity": "sha512-TbNtInKVElgEBnJ1v2Xg+MFX2lvFLbmlv3EuSC5wTfCwpB8kC3w3mffF6cKuUhkn475Ym1f1I4qmuXzx2+uXpw==", + "dev": true, + "dependencies": { + "date-time": "^3.1.0", + "esutils": "^2.0.3", + "fast-diff": "^1.2.0", + "js-string-escape": "^1.0.1", + "lodash": "^4.17.15", + "md5-hex": "^3.0.1", + "semver": "^7.3.2", + "well-known-symbols": "^2.0.0" + }, + "engines": { + "node": ">=10.18.0 <11 || >=12.14.0 <13 || >=14" + } + }, + "node_modules/configstore": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz", + "integrity": "sha512-aMKprgk5YhBNyH25hj8wGt2+D52Sw1DRRIzqBwLp2Ya9mFmY8KPvvtvmna8SxVR9JMZ4kzMD68N22vlaRpkeFA==", + "dev": true, + "dependencies": { + "dot-prop": "^5.2.0", + "graceful-fs": "^4.1.2", + "make-dir": "^3.0.0", + "unique-string": "^2.0.0", + "write-file-atomic": "^3.0.0", + "xdg-basedir": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/conventional-changelog": { + "version": "3.1.23", + "resolved": "https://registry.npmjs.org/conventional-changelog/-/conventional-changelog-3.1.23.tgz", + "integrity": "sha512-sScUu2NHusjRC1dPc5p8/b3kT78OYr95/Bx7Vl8CPB8tF2mG1xei5iylDTRjONV5hTlzt+Cn/tBWrKdd299b7A==", + "dev": true, + "dependencies": { + "conventional-changelog-angular": "^5.0.11", + "conventional-changelog-atom": "^2.0.7", + "conventional-changelog-codemirror": "^2.0.7", + "conventional-changelog-conventionalcommits": "^4.4.0", + "conventional-changelog-core": "^4.2.0", + "conventional-changelog-ember": "^2.0.8", + "conventional-changelog-eslint": "^3.0.8", + "conventional-changelog-express": "^2.0.5", + "conventional-changelog-jquery": "^3.0.10", + "conventional-changelog-jshint": "^2.0.8", + "conventional-changelog-preset-loader": "^2.3.4" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/conventional-changelog-angular": { + "version": "5.0.11", + "resolved": "https://registry.npmjs.org/conventional-changelog-angular/-/conventional-changelog-angular-5.0.11.tgz", + "integrity": "sha512-nSLypht/1yEflhuTogC03i7DX7sOrXGsRn14g131Potqi6cbGbGEE9PSDEHKldabB6N76HiSyw9Ph+kLmC04Qw==", + "dev": true, + "dependencies": { + "compare-func": "^2.0.0", + "q": "^1.5.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/conventional-changelog-atom": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/conventional-changelog-atom/-/conventional-changelog-atom-2.0.7.tgz", + "integrity": "sha512-7dOREZwzB+tCEMjRTDfen0OHwd7vPUdmU0llTy1eloZgtOP4iSLVzYIQqfmdRZEty+3w5Jz+AbhfTJKoKw1JeQ==", + "dev": true, + "dependencies": { + "q": "^1.5.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/conventional-changelog-codemirror": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/conventional-changelog-codemirror/-/conventional-changelog-codemirror-2.0.7.tgz", + "integrity": "sha512-Oralk1kiagn3Gb5cR5BffenWjVu59t/viE6UMD/mQa1hISMPkMYhJIqX+CMeA1zXgVBO+YHQhhokEj99GP5xcg==", + "dev": true, + "dependencies": { + "q": "^1.5.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/conventional-changelog-config-spec": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/conventional-changelog-config-spec/-/conventional-changelog-config-spec-2.1.0.tgz", + "integrity": "sha512-IpVePh16EbbB02V+UA+HQnnPIohgXvJRxHcS5+Uwk4AT5LjzCZJm5sp/yqs5C6KZJ1jMsV4paEV13BN1pvDuxQ==", + "dev": true + }, + "node_modules/conventional-changelog-conventionalcommits": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/conventional-changelog-conventionalcommits/-/conventional-changelog-conventionalcommits-4.4.0.tgz", + "integrity": "sha512-ybvx76jTh08tpaYrYn/yd0uJNLt5yMrb1BphDe4WBredMlvPisvMghfpnJb6RmRNcqXeuhR6LfGZGewbkRm9yA==", + "dev": true, + "dependencies": { + "compare-func": "^2.0.0", + "lodash": "^4.17.15", + "q": "^1.5.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/conventional-changelog-core": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/conventional-changelog-core/-/conventional-changelog-core-4.2.0.tgz", + "integrity": "sha512-8+xMvN6JvdDtPbGBqA7oRNyZD4od1h/SIzrWqHcKZjitbVXrFpozEeyn4iI4af1UwdrabQpiZMaV07fPUTGd4w==", + "dev": true, + "dependencies": { + "add-stream": "^1.0.0", + "conventional-changelog-writer": "^4.0.17", + "conventional-commits-parser": "^3.1.0", + "dateformat": "^3.0.0", + "get-pkg-repo": "^1.0.0", + "git-raw-commits": "2.0.0", + "git-remote-origin-url": "^2.0.0", + "git-semver-tags": "^4.1.0", + "lodash": "^4.17.15", + "normalize-package-data": "^2.3.5", + "q": "^1.5.1", + "read-pkg": "^3.0.0", + "read-pkg-up": "^3.0.0", + "shelljs": "^0.8.3", + "through2": "^3.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/conventional-changelog-core/node_modules/find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dev": true, + "dependencies": { + "locate-path": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/conventional-changelog-core/node_modules/load-json-file": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "parse-json": "^4.0.0", + "pify": "^3.0.0", + "strip-bom": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/conventional-changelog-core/node_modules/locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dev": true, + "dependencies": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/conventional-changelog-core/node_modules/p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "dependencies": { + "p-try": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/conventional-changelog-core/node_modules/p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "dev": true, + "dependencies": { + "p-limit": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/conventional-changelog-core/node_modules/p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/conventional-changelog-core/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/conventional-changelog-core/node_modules/path-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", + "dev": true, + "dependencies": { + "pify": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/conventional-changelog-core/node_modules/pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/conventional-changelog-core/node_modules/read-pkg": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", + "dev": true, + "dependencies": { + "load-json-file": "^4.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/conventional-changelog-core/node_modules/read-pkg-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz", + "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=", + "dev": true, + "dependencies": { + "find-up": "^2.0.0", + "read-pkg": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/conventional-changelog-ember": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/conventional-changelog-ember/-/conventional-changelog-ember-2.0.8.tgz", + "integrity": "sha512-JEMEcUAMg4Q9yxD341OgWlESQ4gLqMWMXIWWUqoQU8yvTJlKnrvcui3wk9JvnZQyONwM2g1MKRZuAjKxr8hAXA==", + "dev": true, + "dependencies": { + "q": "^1.5.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/conventional-changelog-eslint": { + "version": "3.0.8", + "resolved": "https://registry.npmjs.org/conventional-changelog-eslint/-/conventional-changelog-eslint-3.0.8.tgz", + "integrity": "sha512-5rTRltgWG7TpU1PqgKHMA/2ivjhrB+E+S7OCTvj0zM/QGg4vmnVH67Vq/EzvSNYtejhWC+OwzvDrLk3tqPry8A==", + "dev": true, + "dependencies": { + "q": "^1.5.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/conventional-changelog-express": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/conventional-changelog-express/-/conventional-changelog-express-2.0.5.tgz", + "integrity": "sha512-pW2hsjKG+xNx/Qjof8wYlAX/P61hT5gQ/2rZ2NsTpG+PgV7Rc8RCfITvC/zN9K8fj0QmV6dWmUefCteD9baEAw==", + "dev": true, + "dependencies": { + "q": "^1.5.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/conventional-changelog-jquery": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/conventional-changelog-jquery/-/conventional-changelog-jquery-3.0.10.tgz", + "integrity": "sha512-QCW6wF8QgPkq2ruPaxc83jZxoWQxLkt/pNxIDn/oYjMiVgrtqNdd7lWe3vsl0hw5ENHNf/ejXuzDHk6suKsRpg==", + "dev": true, + "dependencies": { + "q": "^1.5.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/conventional-changelog-jshint": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/conventional-changelog-jshint/-/conventional-changelog-jshint-2.0.8.tgz", + "integrity": "sha512-hB/iI0IiZwnZ+seYI+qEQ4b+EMQSEC8jGIvhO2Vpz1E5p8FgLz75OX8oB1xJWl+s4xBMB6f8zJr0tC/BL7YOjw==", + "dev": true, + "dependencies": { + "compare-func": "^2.0.0", + "q": "^1.5.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/conventional-changelog-preset-loader": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/conventional-changelog-preset-loader/-/conventional-changelog-preset-loader-2.3.4.tgz", + "integrity": "sha512-GEKRWkrSAZeTq5+YjUZOYxdHq+ci4dNwHvpaBC3+ENalzFWuCWa9EZXSuZBpkr72sMdKB+1fyDV4takK1Lf58g==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/conventional-changelog-writer": { + "version": "4.0.17", + "resolved": "https://registry.npmjs.org/conventional-changelog-writer/-/conventional-changelog-writer-4.0.17.tgz", + "integrity": "sha512-IKQuK3bib/n032KWaSb8YlBFds+aLmzENtnKtxJy3+HqDq5kohu3g/UdNbIHeJWygfnEbZjnCKFxAW0y7ArZAw==", + "dev": true, + "dependencies": { + "compare-func": "^2.0.0", + "conventional-commits-filter": "^2.0.6", + "dateformat": "^3.0.0", + "handlebars": "^4.7.6", + "json-stringify-safe": "^5.0.1", + "lodash": "^4.17.15", + "meow": "^7.0.0", + "semver": "^6.0.0", + "split": "^1.0.0", + "through2": "^3.0.0" + }, + "bin": { + "conventional-changelog-writer": "cli.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/conventional-changelog-writer/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/conventional-commit-types": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/conventional-commit-types/-/conventional-commit-types-3.0.0.tgz", + "integrity": "sha512-SmmCYnOniSsAa9GqWOeLqc179lfr5TRu5b4QFDkbsrJ5TZjPJx85wtOr3zn+1dbeNiXDKGPbZ72IKbPhLXh/Lg==", + "dev": true + }, + "node_modules/conventional-commits-filter": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/conventional-commits-filter/-/conventional-commits-filter-2.0.6.tgz", + "integrity": "sha512-4g+sw8+KA50/Qwzfr0hL5k5NWxqtrOVw4DDk3/h6L85a9Gz0/Eqp3oP+CWCNfesBvZZZEFHF7OTEbRe+yYSyKw==", + "dev": true, + "dependencies": { + "lodash.ismatch": "^4.4.0", + "modify-values": "^1.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/conventional-commits-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/conventional-commits-parser/-/conventional-commits-parser-3.1.0.tgz", + "integrity": "sha512-RSo5S0WIwXZiRxUGTPuYFbqvrR4vpJ1BDdTlthFgvHt5kEdnd1+pdvwWphWn57/oIl4V72NMmOocFqqJ8mFFhA==", + "dev": true, + "dependencies": { + "is-text-path": "^1.0.1", + "JSONStream": "^1.0.4", + "lodash": "^4.17.15", + "meow": "^7.0.0", + "split2": "^2.0.0", + "through2": "^3.0.0", + "trim-off-newlines": "^1.0.0" + }, + "bin": { + "conventional-commits-parser": "cli.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/conventional-recommended-bump": { + "version": "6.0.10", + "resolved": "https://registry.npmjs.org/conventional-recommended-bump/-/conventional-recommended-bump-6.0.10.tgz", + "integrity": "sha512-2ibrqAFMN3ZA369JgVoSbajdD/BHN6zjY7DZFKTHzyzuQejDUCjQ85S5KHxCRxNwsbDJhTPD5hOKcis/jQhRgg==", + "dev": true, + "dependencies": { + "concat-stream": "^2.0.0", + "conventional-changelog-preset-loader": "^2.3.4", + "conventional-commits-filter": "^2.0.6", + "conventional-commits-parser": "^3.1.0", + "git-raw-commits": "2.0.0", + "git-semver-tags": "^4.1.0", + "meow": "^7.0.0", + "q": "^1.5.1" + }, + "bin": { + "conventional-recommended-bump": "cli.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/convert-source-map": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.7.0.tgz", + "integrity": "sha512-4FJkXzKXEDB1snCFZlLP4gpC3JILicCpGbzG9f9G7tGqGCzETQ2hWPrcinA9oU4wtf2biUaEH5065UnMeR33oA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.1" + } + }, + "node_modules/convert-to-spaces": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/convert-to-spaces/-/convert-to-spaces-1.0.2.tgz", + "integrity": "sha1-fj5Iu+bZl7FBfdyihoIEtNPYVxU=", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/copy-descriptor": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/copy-descriptor/-/copy-descriptor-0.1.1.tgz", + "integrity": "sha1-Z29us8OZl8LuGsOpJP1hJHSPV40=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", + "dev": true + }, + "node_modules/cosmiconfig": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-6.0.0.tgz", + "integrity": "sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg==", + "dev": true, + "optional": true, + "dependencies": { + "@types/parse-json": "^4.0.0", + "import-fresh": "^3.1.0", + "parse-json": "^5.0.0", + "path-type": "^4.0.0", + "yaml": "^1.7.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cosmiconfig/node_modules/parse-json": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.1.0.tgz", + "integrity": "sha512-+mi/lmVVNKFNVyLXV31ERiy2CY5E1/F6QtJFEzoChPRwwngMNXRDQ9GJ5WdE2Z2P4AujsOi0/+2qHID68KwfIQ==", + "dev": true, + "optional": true, + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/cp-file": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/cp-file/-/cp-file-6.2.0.tgz", + "integrity": "sha512-fmvV4caBnofhPe8kOcitBwSn2f39QLjnAnGq3gO9dfd75mUytzKNZB1hde6QHunW2Rt+OwuBOMc3i1tNElbszA==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "make-dir": "^2.0.0", + "nested-error-stacks": "^2.0.0", + "pify": "^4.0.1", + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/cp-file/node_modules/make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "dependencies": { + "pify": "^4.0.1", + "semver": "^5.6.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/cp-file/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/cp-sugar": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cp-sugar/-/cp-sugar-1.0.0.tgz", + "integrity": "sha1-HCb5vcocA89q90SmVSOf+THMq28=", + "dev": true, + "dependencies": { + "cross-spawn-async": "^2.1.6", + "pinkie-promise": "^2.0.0", + "promisify-event": "^1.0.0", + "shell-quote": "^1.4.3" + } + }, + "node_modules/create-error-class": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/create-error-class/-/create-error-class-3.0.2.tgz", + "integrity": "sha1-Br56vvlHo/FKMP1hBnHUAbyot7Y=", + "dev": true, + "dependencies": { + "capture-stack-trace": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/cross-spawn": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.5.tgz", + "integrity": "sha512-eTVLrBSt7fjbDygz805pMnstIs2VTBNkRm0qxZd+M7A5XDdxVRWO5MxGBXZhjY4cqLYLdtrGqRf8mBPmzwSpWQ==", + "dev": true, + "dependencies": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + }, + "engines": { + "node": ">=4.8" + } + }, + "node_modules/cross-spawn-async": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/cross-spawn-async/-/cross-spawn-async-2.2.5.tgz", + "integrity": "sha1-hF/wwINKPe2dFg2sptOQkGuyiMw=", + "dev": true, + "dependencies": { + "lru-cache": "^4.0.0", + "which": "^1.2.8" + } + }, + "node_modules/cross-spawn/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/crypto-random-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/crypto-random-string/-/crypto-random-string-2.0.0.tgz", + "integrity": "sha512-v1plID3y9r/lPhviJ1wrXpLeyUIGAZ2SHNYTEapm7/8A9nLPoyvVp3RK/EPFqn5kEznyWgYZNsRtYYIWbuG8KA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/currently-unhandled": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/currently-unhandled/-/currently-unhandled-0.4.1.tgz", + "integrity": "sha1-mI3zP+qxke95mmE2nddsF635V+o=", + "dev": true, + "dependencies": { + "array-find-index": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/cz-conventional-changelog": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cz-conventional-changelog/-/cz-conventional-changelog-2.1.0.tgz", + "integrity": "sha1-L0vHOQ4yROTfKT5ro1Hkx0Cnx2Q=", + "dev": true, + "dependencies": { + "conventional-commit-types": "^2.0.0", + "lodash.map": "^4.5.1", + "longest": "^1.0.1", + "right-pad": "^1.0.1", + "word-wrap": "^1.0.3" + } + }, + "node_modules/cz-conventional-changelog/node_modules/conventional-commit-types": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/conventional-commit-types/-/conventional-commit-types-2.3.0.tgz", + "integrity": "sha512-6iB39PrcGYdz0n3z31kj6/Km6mK9hm9oMRhwcLnKxE7WNoeRKZbTAobliKrbYZ5jqyCvtcVEfjCiaEzhL3AVmQ==", + "dev": true + }, + "node_modules/cz-conventional-changelog/node_modules/longest": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/longest/-/longest-1.0.1.tgz", + "integrity": "sha1-MKCy2jj3N3DoKUoNIuZiXtd9AJc=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/dargs": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/dargs/-/dargs-4.1.0.tgz", + "integrity": "sha1-A6nbtLXC8Tm/FK5T8LiipqhvThc=", + "dev": true, + "dependencies": { + "number-is-nan": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/date-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/date-time/-/date-time-3.1.0.tgz", + "integrity": "sha512-uqCUKXE5q1PNBXjPqvwhwJf9SwMoAHBgWJ6DcrnS5o+W2JOiIILl0JEdVD8SGujrNS02GGxgwAg2PN2zONgtjg==", + "dev": true, + "dependencies": { + "time-zone": "^1.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/dateformat": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-3.0.3.tgz", + "integrity": "sha512-jyCETtSl3VMZMWeRo7iY1FL19ges1t55hMo5yaam4Jrsm5EPL89UQkoQRyiI+Yf4k8r2ZpdngkV8hr1lIdjb3Q==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "dev": true, + "dependencies": { + "ms": "^2.1.1" + } + }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/decamelize-keys": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/decamelize-keys/-/decamelize-keys-1.1.0.tgz", + "integrity": "sha1-0XGoeTMlKAfrPLYdwcFEXQeN8tk=", + "dev": true, + "dependencies": { + "decamelize": "^1.1.0", + "map-obj": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/decamelize-keys/node_modules/map-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz", + "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/decode-uri-component": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.0.tgz", + "integrity": "sha1-6zkTMzRYd1y4TNGh+uBiEGu4dUU=", + "dev": true, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/decompress-response": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", + "integrity": "sha1-gKTdMjdIOEv6JICDYirt7Jgq3/M=", + "dev": true, + "dependencies": { + "mimic-response": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/dedent": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz", + "integrity": "sha1-JJXduvbrh0q7Dhvp3yLS5aVEMmw=", + "dev": true + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/default-require-extensions": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-2.0.0.tgz", + "integrity": "sha1-9fj7sYp9bVCyH2QfZJ67Uiz+JPc=", + "dev": true, + "dependencies": { + "strip-bom": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/defaults": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz", + "integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=", + "dev": true, + "dependencies": { + "clone": "^1.0.2" + } + }, + "node_modules/defer-to-connect": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-1.1.3.tgz", + "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==", + "dev": true + }, + "node_modules/define-properties": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", + "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", + "dev": true, + "dependencies": { + "object-keys": "^1.0.12" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/define-property": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-2.0.2.tgz", + "integrity": "sha512-jwK2UV4cnPpbcG7+VRARKTZPUWowwXA8bzH5NP6ud0oeAxyYPuGZUAC7hMugpCdz4BeSZl2Dl9k66CHJ/46ZYQ==", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.2", + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/define-property/node_modules/is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/define-property/node_modules/is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/define-property/node_modules/is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/del": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/del/-/del-5.1.0.tgz", + "integrity": "sha512-wH9xOVHnczo9jN2IW68BabcecVPxacIA3g/7z6vhSU/4stOKQzeCRK0yD0A24WiAAUJmmVpWqrERcTxnLo3AnA==", + "dev": true, + "dependencies": { + "globby": "^10.0.1", + "graceful-fs": "^4.2.2", + "is-glob": "^4.0.1", + "is-path-cwd": "^2.2.0", + "is-path-inside": "^3.0.1", + "p-map": "^3.0.0", + "rimraf": "^3.0.0", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/del/node_modules/globby": { + "version": "10.0.2", + "resolved": "https://registry.npmjs.org/globby/-/globby-10.0.2.tgz", + "integrity": "sha512-7dUi7RvCoT/xast/o/dLN53oqND4yk0nsHkhRgn9w65C4PofCLOoJ39iSOg+qVDdWQPIEj+eszMHQ+aLVwwQSg==", + "dev": true, + "dependencies": { + "@types/glob": "^7.1.1", + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.0.3", + "glob": "^7.1.3", + "ignore": "^5.1.1", + "merge2": "^1.2.3", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/del/node_modules/p-map": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-3.0.0.tgz", + "integrity": "sha512-d3qXVTF/s+W+CdJ5A29wywV2n8CQQYahlgz2bFiA+4eVNJbHJodPZ+/gXwPGh0bOqA+j8S+6+ckmvLGPk1QpxQ==", + "dev": true, + "dependencies": { + "aggregate-error": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/detect-file": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/detect-file/-/detect-file-1.0.0.tgz", + "integrity": "sha1-8NZtA2cqglyxtzvbP+YjEMjlUrc=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/detect-indent": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/detect-indent/-/detect-indent-6.0.0.tgz", + "integrity": "sha512-oSyFlqaTHCItVRGK5RmrmjB+CmaMOW7IaNA/kdxqhoa6d17j/5ce9O9eWXmV/KEdRwqpQA+Vqe8a8Bsybu4YnA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/detect-newline": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", + "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/dir-glob": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", + "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", + "dev": true, + "dependencies": { + "path-type": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/dot-prop": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-5.2.0.tgz", + "integrity": "sha512-uEUyaDKoSQ1M4Oq8l45hSE26SnTxL6snNnqvK/VWx5wJhmff5z0FUVJDKDanor/6w3kzE3i7XZOk+7wC0EXr1A==", + "dev": true, + "dependencies": { + "is-obj": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/dotgitignore": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/dotgitignore/-/dotgitignore-2.1.0.tgz", + "integrity": "sha512-sCm11ak2oY6DglEPpCB8TixLjWAxd3kJTs6UIcSasNYxXdFPV+YKlye92c8H4kKFqV5qYMIh7d+cYecEg0dIkA==", + "dev": true, + "dependencies": { + "find-up": "^3.0.0", + "minimatch": "^3.0.4" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/dotgitignore/node_modules/find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "dependencies": { + "locate-path": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/dotgitignore/node_modules/locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/dotgitignore/node_modules/p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "dependencies": { + "p-limit": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/dotgitignore/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/duplexer2": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", + "integrity": "sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=", + "dev": true, + "dependencies": { + "readable-stream": "^2.0.2" + } + }, + "node_modules/duplexer2/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/duplexer2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/duplexer3": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz", + "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=", + "dev": true + }, + "node_modules/elegant-spinner": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/elegant-spinner/-/elegant-spinner-1.0.1.tgz", + "integrity": "sha1-2wQ1IcldfjA/2PNFvtwzSc+wcp4=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/elegant-status": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/elegant-status/-/elegant-status-1.1.0.tgz", + "integrity": "sha1-Qe9KXy0DZCmDurtPZKS1drEVUhU=", + "dev": true, + "dependencies": { + "chalk": "^1.1.1", + "elegant-spinner": "^1.0.1", + "log-update": "^1.0.2", + "os-family": "^1.0.0" + } + }, + "node_modules/elegant-status/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/elegant-status/node_modules/ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/elegant-status/node_modules/chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "dev": true, + "dependencies": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/elegant-status/node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/elegant-status/node_modules/supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/email-addresses": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/email-addresses/-/email-addresses-3.1.0.tgz", + "integrity": "sha512-k0/r7GrWVL32kZlGwfPNgB2Y/mMXVTq/decgLczm/j34whdaspNrZO8CnXPf1laaHxI6ptUlsnAxN+UAPw+fzg==", + "dev": true + }, + "node_modules/emittery": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.7.1.tgz", + "integrity": "sha512-d34LN4L6h18Bzz9xpoku2nPwKxCPlPMr3EEKTkoEBi+1/+b0lcRkRJ1UVyyZaKNeqGR3swcGl6s390DNO4YVgQ==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", + "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "dev": true, + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/equal-length": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/equal-length/-/equal-length-1.0.1.tgz", + "integrity": "sha1-IcoRLUirJLTh5//A5TOdMf38J0w=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "dev": true, + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/es-abstract": { + "version": "1.17.6", + "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", + "integrity": "sha512-Fr89bON3WFyUi5EvAeI48QTWX0AyekGgLA8H+c+7fbfCkJwRWRMLd8CQedNEyJuoYYhmtEqY92pgte1FAhBlhw==", + "dev": true, + "dependencies": { + "es-to-primitive": "^1.2.1", + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1", + "is-callable": "^1.2.0", + "is-regex": "^1.1.0", + "object-inspect": "^1.7.0", + "object-keys": "^1.1.1", + "object.assign": "^4.1.0", + "string.prototype.trimend": "^1.0.1", + "string.prototype.trimstart": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-to-primitive": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", + "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", + "dev": true, + "dependencies": { + "is-callable": "^1.1.4", + "is-date-object": "^1.0.1", + "is-symbol": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "dev": true + }, + "node_modules/escape-goat": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/escape-goat/-/escape-goat-2.1.1.tgz", + "integrity": "sha512-8/uIhbG12Csjy2JEW7D9pHbreaVaS/OpN3ycnyvElTdwM5n6GY6W6e2IPemfvGZeUMqZ9A/3GqIZMgKnBhAw/Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/escape-string-applescript": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-applescript/-/escape-string-applescript-2.0.0.tgz", + "integrity": "sha1-dgvKg4Zo5Aj+XuUs5CyvfLRsUnM=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/execa": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-0.10.0.tgz", + "integrity": "sha512-7XOMnz8Ynx1gGo/3hyV9loYNPWM94jG3+3T3Y8tsfSstFmETmENCMU/A/zj8Lyaj1lkgEepKepvd6240tBRvlw==", + "dev": true, + "dependencies": { + "cross-spawn": "^6.0.0", + "get-stream": "^3.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/execa/node_modules/get-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz", + "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/exit-hook": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/exit-hook/-/exit-hook-1.1.1.tgz", + "integrity": "sha1-8FyiM7SMBdVP/wd2XfhQfpXAL/g=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets": { + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/expand-brackets/-/expand-brackets-2.1.4.tgz", + "integrity": "sha1-t3c14xXOMPa27/D4OwQVGiJEliI=", + "dev": true, + "dependencies": { + "debug": "^2.3.3", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "posix-character-classes": "^0.1.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/expand-brackets/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/expand-brackets/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "node_modules/expand-tilde": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz", + "integrity": "sha1-l+gBqgUt8CRU3kawK/YhZCzchQI=", + "dev": true, + "dependencies": { + "homedir-polyfill": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extend-shallow": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-3.0.2.tgz", + "integrity": "sha1-Jqcarwc7OfshJxcnRhMcJwQCjbg=", + "dev": true, + "dependencies": { + "assign-symbols": "^1.0.0", + "is-extendable": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extend-shallow/node_modules/is-extendable": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", + "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "dev": true, + "dependencies": { + "is-plain-object": "^2.0.4" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extend-shallow/node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/external-editor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", + "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", + "dev": true, + "dependencies": { + "chardet": "^0.7.0", + "iconv-lite": "^0.4.24", + "tmp": "^0.0.33" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/extglob": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/extglob/-/extglob-2.0.4.tgz", + "integrity": "sha512-Nmb6QXkELsuBr24CJSkilo6UHHgbekK5UiZgfE6UHD3Eb27YC6oD+bhcT+tJ6cl8dmsgdQxnWlcry8ksBIBLpw==", + "dev": true, + "dependencies": { + "array-unique": "^0.3.2", + "define-property": "^1.0.0", + "expand-brackets": "^2.1.4", + "extend-shallow": "^2.0.1", + "fragment-cache": "^0.2.1", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/extglob/node_modules/is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fast-diff": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.2.0.tgz", + "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==", + "dev": true + }, + "node_modules/fast-glob": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.4.tgz", + "integrity": "sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ==", + "dev": true, + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.0", + "merge2": "^1.3.0", + "micromatch": "^4.0.2", + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fastq": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.8.0.tgz", + "integrity": "sha512-SMIZoZdLh/fgofivvIkmknUXyPnvxRE3DhtZ5Me3Mrsk5gyPL42F0xr51TdRXskBxHfMp+07bcYzfsYEsSQA9Q==", + "dev": true, + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/figures": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", + "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", + "dev": true, + "dependencies": { + "escape-string-regexp": "^1.0.5" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/file-type": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/file-type/-/file-type-3.9.0.tgz", + "integrity": "sha1-JXoHg4TR24CHvESdEH1SpSZyuek=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/filename-reserved-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/filename-reserved-regex/-/filename-reserved-regex-1.0.0.tgz", + "integrity": "sha1-5hz4BfDeHJhFZ9A4bcXfUO5a9+Q=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/filenamify": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/filenamify/-/filenamify-1.2.1.tgz", + "integrity": "sha1-qfL/0RxQO+0wABUCknI3jx8TZaU=", + "dev": true, + "dependencies": { + "filename-reserved-regex": "^1.0.0", + "strip-outer": "^1.0.0", + "trim-repeated": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/filenamify-url": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/filenamify-url/-/filenamify-url-1.0.0.tgz", + "integrity": "sha1-syvYExnvWGO3MHi+1Q9GpPeXX1A=", + "dev": true, + "dependencies": { + "filenamify": "^1.0.0", + "humanize-url": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fill-range": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", + "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "dev": true, + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/filled-array": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/filled-array/-/filled-array-1.1.0.tgz", + "integrity": "sha1-w8T2xmO5I0WamqKZEtLQMfFQf4Q=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/find-cache-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.1.0.tgz", + "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==", + "dev": true, + "dependencies": { + "commondir": "^1.0.1", + "make-dir": "^2.0.0", + "pkg-dir": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/find-cache-dir/node_modules/find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "dependencies": { + "locate-path": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/find-cache-dir/node_modules/locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/find-cache-dir/node_modules/make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "dependencies": { + "pify": "^4.0.1", + "semver": "^5.6.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/find-cache-dir/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/find-cache-dir/node_modules/p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "dependencies": { + "p-limit": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/find-cache-dir/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/find-cache-dir/node_modules/pkg-dir": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", + "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", + "dev": true, + "dependencies": { + "find-up": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/find-cache-dir/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/find-node-modules": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/find-node-modules/-/find-node-modules-2.0.0.tgz", + "integrity": "sha512-8MWIBRgJi/WpjjfVXumjPKCtmQ10B+fjx6zmSA+770GMJirLhWIzg8l763rhjl9xaeaHbnxPNRQKq2mgMhr+aw==", + "dev": true, + "dependencies": { + "findup-sync": "^3.0.0", + "merge": "^1.2.1" + } + }, + "node_modules/find-root": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/find-root/-/find-root-1.1.0.tgz", + "integrity": "sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==", + "dev": true + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/findup-sync": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/findup-sync/-/findup-sync-3.0.0.tgz", + "integrity": "sha512-YbffarhcicEhOrm4CtrwdKBdCuz576RLdhJDsIfvNtxUuhdRet1qZcsMjqbePtAseKdAnDyM/IyXbu7PRPRLYg==", + "dev": true, + "dependencies": { + "detect-file": "^1.0.0", + "is-glob": "^4.0.0", + "micromatch": "^3.0.4", + "resolve-dir": "^1.0.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/findup-sync/node_modules/braces": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz", + "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==", + "dev": true, + "dependencies": { + "arr-flatten": "^1.1.0", + "array-unique": "^0.3.2", + "extend-shallow": "^2.0.1", + "fill-range": "^4.0.0", + "isobject": "^3.0.1", + "repeat-element": "^1.1.2", + "snapdragon": "^0.8.1", + "snapdragon-node": "^2.0.1", + "split-string": "^3.0.2", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/findup-sync/node_modules/braces/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/findup-sync/node_modules/fill-range": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", + "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=", + "dev": true, + "dependencies": { + "extend-shallow": "^2.0.1", + "is-number": "^3.0.0", + "repeat-string": "^1.6.1", + "to-regex-range": "^2.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/findup-sync/node_modules/fill-range/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/findup-sync/node_modules/is-number": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", + "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/findup-sync/node_modules/is-number/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/findup-sync/node_modules/micromatch": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", + "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==", + "dev": true, + "dependencies": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "braces": "^2.3.1", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "extglob": "^2.0.4", + "fragment-cache": "^0.2.1", + "kind-of": "^6.0.2", + "nanomatch": "^1.2.9", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/findup-sync/node_modules/to-regex-range": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz", + "integrity": "sha1-fIDBe53+vlmeJzZ+DU3VWQFB2zg=", + "dev": true, + "dependencies": { + "is-number": "^3.0.0", + "repeat-string": "^1.6.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/for-in": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", + "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/foreground-child": { + "version": "1.5.6", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-1.5.6.tgz", + "integrity": "sha1-T9ca0t/elnibmApcCilZN8svXOk=", + "dev": true, + "dependencies": { + "cross-spawn": "^4", + "signal-exit": "^3.0.0" + } + }, + "node_modules/foreground-child/node_modules/cross-spawn": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz", + "integrity": "sha1-e5JHYhwjrf3ThWAEqCPL45dCTUE=", + "dev": true, + "dependencies": { + "lru-cache": "^4.0.1", + "which": "^1.2.9" + } + }, + "node_modules/fragment-cache": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/fragment-cache/-/fragment-cache-0.2.1.tgz", + "integrity": "sha1-QpD60n8T6Jvn8zeZxrxaCr//DRk=", + "dev": true, + "dependencies": { + "map-cache": "^0.2.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fs-access": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/fs-access/-/fs-access-1.0.1.tgz", + "integrity": "sha1-1qh/JiJxzv6+wwxVNAf7mV2od3o=", + "dev": true, + "dependencies": { + "null-check": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true + }, + "node_modules/fsevents": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.1.3.tgz", + "integrity": "sha512-Auw9a4AxqWpa9GUfj370BMPzzyncfBABW8Mab7BGWBYDj4Isgq+cDKtx0i6u9jcX9pQDnswsaaOTgTmA5pEjuQ==", + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-pkg-repo": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/get-pkg-repo/-/get-pkg-repo-1.4.0.tgz", + "integrity": "sha1-xztInAbYDMVTbCyFP54FIyBWly0=", + "dev": true, + "dependencies": { + "hosted-git-info": "^2.1.4", + "meow": "^3.3.0", + "normalize-package-data": "^2.3.0", + "parse-github-repo-url": "^1.3.0", + "through2": "^2.0.0" + }, + "bin": { + "get-pkg-repo": "cli.js" + } + }, + "node_modules/get-pkg-repo/node_modules/camelcase": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz", + "integrity": "sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/camelcase-keys": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", + "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=", + "dev": true, + "dependencies": { + "camelcase": "^2.0.0", + "map-obj": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/find-up": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz", + "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=", + "dev": true, + "dependencies": { + "path-exists": "^2.0.0", + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/get-stdin": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz", + "integrity": "sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/indent-string": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-2.1.0.tgz", + "integrity": "sha1-ji1INIdCEhtKghi3oTfppSBJ3IA=", + "dev": true, + "dependencies": { + "repeating": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/load-json-file": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "parse-json": "^2.2.0", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0", + "strip-bom": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/map-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz", + "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/meow": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz", + "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=", + "dev": true, + "dependencies": { + "camelcase-keys": "^2.0.0", + "decamelize": "^1.1.2", + "loud-rejection": "^1.0.0", + "map-obj": "^1.0.1", + "minimist": "^1.1.3", + "normalize-package-data": "^2.3.4", + "object-assign": "^4.0.1", + "read-pkg-up": "^1.0.1", + "redent": "^1.0.0", + "trim-newlines": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", + "dev": true, + "dependencies": { + "error-ex": "^1.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/path-exists": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz", + "integrity": "sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=", + "dev": true, + "dependencies": { + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/path-type": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz", + "integrity": "sha1-WcRPfuSR2nBNpBXaWkBwuk+P5EE=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/read-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz", + "integrity": "sha1-9f+qXs0pyzHAR0vKfXVra7KePyg=", + "dev": true, + "dependencies": { + "load-json-file": "^1.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/read-pkg-up": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz", + "integrity": "sha1-nWPBMnbAZZGNV/ACpX9AobZD+wI=", + "dev": true, + "dependencies": { + "find-up": "^1.0.0", + "read-pkg": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/get-pkg-repo/node_modules/redent": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/redent/-/redent-1.0.0.tgz", + "integrity": "sha1-z5Fqsf1fHxbfsggi3W7H9zDCr94=", + "dev": true, + "dependencies": { + "indent-string": "^2.1.0", + "strip-indent": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/get-pkg-repo/node_modules/strip-bom": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "integrity": "sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4=", + "dev": true, + "dependencies": { + "is-utf8": "^0.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/strip-indent": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-1.0.1.tgz", + "integrity": "sha1-DHlipq3vp7vUrDZkYKY4VSrhoKI=", + "dev": true, + "dependencies": { + "get-stdin": "^4.0.1" + }, + "bin": { + "strip-indent": "cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-pkg-repo/node_modules/through2": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "dev": true, + "dependencies": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + }, + "node_modules/get-pkg-repo/node_modules/trim-newlines": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz", + "integrity": "sha1-WIeWa7WCpFA6QetST301ARgVphM=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/get-stdin": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-5.0.1.tgz", + "integrity": "sha1-Ei4WFZHiH/TFJTAwVpPyDmOTo5g=", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dev": true, + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/get-value": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz", + "integrity": "sha1-3BXKHGcjh8p2vTesCjlbogQqLCg=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/gh-pages": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/gh-pages/-/gh-pages-2.2.0.tgz", + "integrity": "sha512-c+yPkNOPMFGNisYg9r4qvsMIjVYikJv7ImFOhPIVPt0+AcRUamZ7zkGRLHz7FKB0xrlZ+ddSOJsZv9XAFVXLmA==", + "dev": true, + "dependencies": { + "async": "^2.6.1", + "commander": "^2.18.0", + "email-addresses": "^3.0.1", + "filenamify-url": "^1.0.0", + "fs-extra": "^8.1.0", + "globby": "^6.1.0" + }, + "bin": { + "gh-pages": "bin/gh-pages.js", + "gh-pages-clean": "bin/gh-pages-clean.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/gh-pages/node_modules/array-union": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", + "integrity": "sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=", + "dev": true, + "dependencies": { + "array-uniq": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/gh-pages/node_modules/globby": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/globby/-/globby-6.1.0.tgz", + "integrity": "sha1-9abXDoOV4hyFj7BInWTfAkJNUGw=", + "dev": true, + "dependencies": { + "array-union": "^1.0.1", + "glob": "^7.0.3", + "object-assign": "^4.0.1", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/gh-pages/node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/git-raw-commits": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/git-raw-commits/-/git-raw-commits-2.0.0.tgz", + "integrity": "sha512-w4jFEJFgKXMQJ0H0ikBk2S+4KP2VEjhCvLCNqbNRQC8BgGWgLKNCO7a9K9LI+TVT7Gfoloje502sEnctibffgg==", + "dev": true, + "dependencies": { + "dargs": "^4.0.1", + "lodash.template": "^4.0.2", + "meow": "^4.0.0", + "split2": "^2.0.0", + "through2": "^2.0.0" + }, + "bin": { + "git-raw-commits": "cli.js" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/git-raw-commits/node_modules/arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/git-raw-commits/node_modules/camelcase": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-4.1.0.tgz", + "integrity": "sha1-1UVjW+HjPFQmScaRc+Xeas+uNN0=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/camelcase-keys": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-4.2.0.tgz", + "integrity": "sha1-oqpfsa9oh1glnDLBQUJteJI7m3c=", + "dev": true, + "dependencies": { + "camelcase": "^4.1.0", + "map-obj": "^2.0.0", + "quick-lru": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/find-up": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-2.1.0.tgz", + "integrity": "sha1-RdG35QbHF93UgndaK3eSCjwMV6c=", + "dev": true, + "dependencies": { + "locate-path": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/indent-string": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz", + "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/load-json-file": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "parse-json": "^4.0.0", + "pify": "^3.0.0", + "strip-bom": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/locate-path": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-2.0.0.tgz", + "integrity": "sha1-K1aLJl7slExtnA3pw9u7ygNUzY4=", + "dev": true, + "dependencies": { + "p-locate": "^2.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/map-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-2.0.0.tgz", + "integrity": "sha1-plzSkIepJZi4eRJXpSPgISIqwfk=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/meow": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/meow/-/meow-4.0.1.tgz", + "integrity": "sha512-xcSBHD5Z86zaOc+781KrupuHAzeGXSLtiAOmBsiLDiPSaYSB6hdew2ng9EBAnZ62jagG9MHAOdxpDi/lWBFJ/A==", + "dev": true, + "dependencies": { + "camelcase-keys": "^4.0.0", + "decamelize-keys": "^1.0.0", + "loud-rejection": "^1.0.0", + "minimist": "^1.1.3", + "minimist-options": "^3.0.1", + "normalize-package-data": "^2.3.4", + "read-pkg-up": "^3.0.0", + "redent": "^2.0.0", + "trim-newlines": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/minimist-options": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/minimist-options/-/minimist-options-3.0.2.tgz", + "integrity": "sha512-FyBrT/d0d4+uiZRbqznPXqw3IpZZG3gl3wKWiX784FycUKVwBt0uLBFkQrtE4tZOrgo78nZp2jnKz3L65T5LdQ==", + "dev": true, + "dependencies": { + "arrify": "^1.0.1", + "is-plain-obj": "^1.1.0" + }, + "engines": { + "node": ">= 4" + } + }, + "node_modules/git-raw-commits/node_modules/p-limit": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz", + "integrity": "sha512-vvcXsLAJ9Dr5rQOPk7toZQZJApBl2K4J6dANSsEuh6QI41JYcsS/qhTGa9ErIUUgK3WNQoJYvylxvjqmiqEA9Q==", + "dev": true, + "dependencies": { + "p-try": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/p-locate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-2.0.0.tgz", + "integrity": "sha1-IKAQOyIqcMj9OcwuWAaA893l7EM=", + "dev": true, + "dependencies": { + "p-limit": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/path-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", + "dev": true, + "dependencies": { + "pify": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/quick-lru": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-1.1.0.tgz", + "integrity": "sha1-Q2CxfGETatOAeDl/8RQW4Ybc+7g=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/read-pkg": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", + "dev": true, + "dependencies": { + "load-json-file": "^4.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/read-pkg-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-3.0.0.tgz", + "integrity": "sha1-PtSWaF26D4/hGNBpHcUfSh/5bwc=", + "dev": true, + "dependencies": { + "find-up": "^2.0.0", + "read-pkg": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/git-raw-commits/node_modules/redent": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/redent/-/redent-2.0.0.tgz", + "integrity": "sha1-wbIAe0LVfrE4kHmzyDM2OdXhzKo=", + "dev": true, + "dependencies": { + "indent-string": "^3.0.0", + "strip-indent": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/git-raw-commits/node_modules/strip-indent": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-2.0.0.tgz", + "integrity": "sha1-XvjbKV0B5u1sv3qrlpmNeCJSe2g=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-raw-commits/node_modules/through2": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "dev": true, + "dependencies": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + }, + "node_modules/git-raw-commits/node_modules/trim-newlines": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-2.0.0.tgz", + "integrity": "sha1-tAPQuRvlDDMd/EuC7s6yLD3hbSA=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-remote-origin-url": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/git-remote-origin-url/-/git-remote-origin-url-2.0.0.tgz", + "integrity": "sha1-UoJlna4hBxRaERJhEq0yFuxfpl8=", + "dev": true, + "dependencies": { + "gitconfiglocal": "^1.0.0", + "pify": "^2.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/git-remote-origin-url/node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/git-semver-tags": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/git-semver-tags/-/git-semver-tags-4.1.0.tgz", + "integrity": "sha512-TcxAGeo03HdErzKzi4fDD+xEL7gi8r2Y5YSxH6N2XYdVSV5UkBwfrt7Gqo1b+uSHCjy/sa9Y6BBBxxFLxfbhTg==", + "dev": true, + "dependencies": { + "meow": "^7.0.0", + "semver": "^6.0.0" + }, + "bin": { + "git-semver-tags": "cli.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/git-semver-tags/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/gitconfiglocal": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/gitconfiglocal/-/gitconfiglocal-1.0.0.tgz", + "integrity": "sha1-QdBF84UaXqiPA/JMocYXgRRGS5s=", + "dev": true, + "dependencies": { + "ini": "^1.3.2" + } + }, + "node_modules/glob": { + "version": "7.1.6", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", + "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/glob-parent": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz", + "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/global-dirs": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-2.0.1.tgz", + "integrity": "sha512-5HqUqdhkEovj2Of/ms3IeS/EekcO54ytHRLV4PEY2rhRwrHXLQjeVEES0Lhka0xwNDtGYn58wyC4s5+MHsOO6A==", + "dev": true, + "dependencies": { + "ini": "^1.3.5" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/global-modules": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/global-modules/-/global-modules-1.0.0.tgz", + "integrity": "sha512-sKzpEkf11GpOFuw0Zzjzmt4B4UZwjOcG757PPvrfhxcLFbq0wpsgpOqxpxtxFiCG4DtG93M6XRVbF2oGdev7bg==", + "dev": true, + "dependencies": { + "global-prefix": "^1.0.1", + "is-windows": "^1.0.1", + "resolve-dir": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/global-prefix": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-1.0.2.tgz", + "integrity": "sha1-2/dDxsFJklk8ZVVoy2btMsASLr4=", + "dev": true, + "dependencies": { + "expand-tilde": "^2.0.2", + "homedir-polyfill": "^1.0.1", + "ini": "^1.3.4", + "is-windows": "^1.0.1", + "which": "^1.2.14" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/globby": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/globby/-/globby-11.0.1.tgz", + "integrity": "sha512-iH9RmgwCmUJHi2z5o2l3eTtGBtXek1OYlHrbcxOYugyHLmAsZrPj43OtHThd62Buh/Vv6VyCBD2bdyWcGNQqoQ==", + "dev": true, + "dependencies": { + "array-union": "^2.1.0", + "dir-glob": "^3.0.1", + "fast-glob": "^3.1.1", + "ignore": "^5.1.4", + "merge2": "^1.3.0", + "slash": "^3.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/got": { + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", + "integrity": "sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q==", + "dev": true, + "dependencies": { + "@sindresorhus/is": "^0.14.0", + "@szmarczak/http-timer": "^1.1.2", + "cacheable-request": "^6.0.0", + "decompress-response": "^3.3.0", + "duplexer3": "^0.1.4", + "get-stream": "^4.1.0", + "lowercase-keys": "^1.0.1", + "mimic-response": "^1.0.1", + "p-cancelable": "^1.0.0", + "to-readable-stream": "^1.0.0", + "url-parse-lax": "^3.0.0" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.4.tgz", + "integrity": "sha512-WjKPNJF79dtJAVniUlGGWHYGz2jWxT6VhN/4m1NdkbZ2nOsEF+cI1Edgql5zCRhs/VsQYRvrXctxktVXZUkixw==", + "dev": true + }, + "node_modules/handlebars": { + "version": "4.7.6", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.6.tgz", + "integrity": "sha512-1f2BACcBfiwAfStCKZNrUCgqNZkGsAT7UM3kkYtXuLo0KnaVfjKOyf7PRzB6++aK9STyT1Pd2ZCPe3EGOXleXA==", + "dev": true, + "dependencies": { + "minimist": "^1.2.5", + "neo-async": "^2.6.0", + "source-map": "^0.6.1", + "uglify-js": "^3.1.4", + "wordwrap": "^1.0.0" + }, + "bin": { + "handlebars": "bin/handlebars" + }, + "engines": { + "node": ">=0.4.7" + }, + "optionalDependencies": { + "uglify-js": "^3.1.4" + } + }, + "node_modules/hard-rejection": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/hard-rejection/-/hard-rejection-2.1.0.tgz", + "integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/has-ansi": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "integrity": "sha1-NPUEnOHs3ysGSa8+8k5F7TVBbZE=", + "dev": true, + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-ansi/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", + "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", + "dev": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/has-value": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-value/-/has-value-1.0.0.tgz", + "integrity": "sha1-GLKB2lhbHFxR3vJMkw7SmgvmsXc=", + "dev": true, + "dependencies": { + "get-value": "^2.0.6", + "has-values": "^1.0.0", + "isobject": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-values": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/has-values/-/has-values-1.0.0.tgz", + "integrity": "sha1-lbC2P+whRmGab+V/51Yo1aOe/k8=", + "dev": true, + "dependencies": { + "is-number": "^3.0.0", + "kind-of": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-values/node_modules/is-number": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", + "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-values/node_modules/is-number/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-values/node_modules/kind-of": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-4.0.0.tgz", + "integrity": "sha1-IIE989cSkosgc3hpGkUGb65y3Vc=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/has-yarn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/has-yarn/-/has-yarn-2.1.0.tgz", + "integrity": "sha512-UqBRqi4ju7T+TqGNdqAO0PaSVGsDGJUBQvk9eUWNGRY1CFGDzYhLWoM7JQEemnlvVcv/YEmc2wNW8BC24EnUsw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/hasha": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hasha/-/hasha-3.0.0.tgz", + "integrity": "sha1-UqMvq4Vp1BymmmH/GiFPjrfIvTk=", + "dev": true, + "dependencies": { + "is-stream": "^1.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/highlight.js": { + "version": "10.1.1", + "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-10.1.1.tgz", + "integrity": "sha512-b4L09127uVa+9vkMgPpdUQP78ickGbHEQTWeBrQFTJZ4/n2aihWOGS0ZoUqAwjVmfjhq/C76HRzkqwZhK4sBbg==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/homedir-polyfill": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz", + "integrity": "sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA==", + "dev": true, + "dependencies": { + "parse-passwd": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/hosted-git-info": { + "version": "2.8.8", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.8.tgz", + "integrity": "sha512-f/wzC2QaWBs7t9IYqB4T3sR1xviIViXJRJTWBlx2Gf3g0Xi5vI7Yy4koXQ1c9OYDGHN9sBy1DQ2AB8fqZBWhUg==", + "dev": true + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true + }, + "node_modules/http-cache-semantics": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz", + "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==", + "dev": true + }, + "node_modules/http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "dev": true, + "dependencies": { + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/https-proxy-agent": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", + "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==", + "dev": true, + "dependencies": { + "agent-base": "5", + "debug": "4" + }, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/https-proxy-agent/node_modules/agent-base": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", + "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==", + "dev": true, + "engines": { + "node": ">= 6.0.0" + } + }, + "node_modules/humanize-url": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/humanize-url/-/humanize-url-1.0.1.tgz", + "integrity": "sha1-9KuZ4NKIF0yk4eUEB8VfuuRk7/8=", + "dev": true, + "dependencies": { + "normalize-url": "^1.0.0", + "strip-url-auth": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/humanize-url/node_modules/normalize-url": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-1.9.1.tgz", + "integrity": "sha1-LMDWazHqIwNkWENuNiDYWVTGbDw=", + "dev": true, + "dependencies": { + "object-assign": "^4.0.1", + "prepend-http": "^1.0.0", + "query-string": "^4.1.0", + "sort-keys": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/humanize-url/node_modules/prepend-http": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", + "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ignore": { + "version": "5.1.8", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz", + "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, + "node_modules/ignore-by-default": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-2.0.0.tgz", + "integrity": "sha512-+mQSgMRiFD3L3AOxLYOCxjIq4OnAmo5CIuC+lj5ehCJcPtV++QacEV7FdpzvYxH6DaOySWzQU6RR0lPLy37ckA==", + "dev": true, + "engines": { + "node": ">=10 <11 || >=12 <13 || >=14" + } + }, + "node_modules/ignore-walk": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz", + "integrity": "sha512-m7o6xuOaT1aqheYHKf8W6J5pYH85ZI9w077erOzLje3JsB1gkafkAhHHY19dqjulgIZHFm32Cp5uNZgcQqdJKw==", + "dev": true, + "dependencies": { + "minimatch": "^3.0.4" + } + }, + "node_modules/import-fresh": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", + "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", + "dev": true, + "optional": true, + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/import-fresh/node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "optional": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/import-lazy": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/import-lazy/-/import-lazy-2.1.0.tgz", + "integrity": "sha1-BWmOPUXIjo1+nZLLBYTnfwlvPkM=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/import-local": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.0.2.tgz", + "integrity": "sha512-vjL3+w0oulAVZ0hBHnxa/Nm5TAurf9YLQJDhqRZyqb+VKGOB6LU8t9H1Nr5CIo16vh9XfJTOoHwU0B71S557gA==", + "dev": true, + "dependencies": { + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" + }, + "bin": { + "import-local-fixture": "fixtures/cli.js" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha1-khi5srkoojixPcT7a21XbyMUU+o=", + "dev": true, + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true + }, + "node_modules/ini": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", + "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/inquirer": { + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz", + "integrity": "sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ==", + "dev": true, + "dependencies": { + "ansi-escapes": "^3.2.0", + "chalk": "^2.4.2", + "cli-cursor": "^2.1.0", + "cli-width": "^2.0.0", + "external-editor": "^3.0.3", + "figures": "^2.0.0", + "lodash": "^4.17.12", + "mute-stream": "0.0.7", + "run-async": "^2.2.0", + "rxjs": "^6.4.0", + "string-width": "^2.1.0", + "strip-ansi": "^5.1.0", + "through": "^2.3.6" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/inquirer/node_modules/ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/inquirer/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/inquirer/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/inquirer/node_modules/cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", + "dev": true, + "dependencies": { + "restore-cursor": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/inquirer/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/inquirer/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "node_modules/inquirer/node_modules/figures": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", + "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", + "dev": true, + "dependencies": { + "escape-string-regexp": "^1.0.5" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/inquirer/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/inquirer/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/inquirer/node_modules/mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/inquirer/node_modules/mute-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", + "dev": true + }, + "node_modules/inquirer/node_modules/onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", + "dev": true, + "dependencies": { + "mimic-fn": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/inquirer/node_modules/restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", + "dev": true, + "dependencies": { + "onetime": "^2.0.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/inquirer/node_modules/string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "dependencies": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/inquirer/node_modules/string-width/node_modules/strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "dependencies": { + "ansi-regex": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/inquirer/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "dependencies": { + "ansi-regex": "^4.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/inquirer/node_modules/strip-ansi/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/inquirer/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/interpret": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/interpret/-/interpret-1.4.0.tgz", + "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", + "dev": true, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/irregular-plurals": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-3.2.0.tgz", + "integrity": "sha512-YqTdPLfwP7YFN0SsD3QUVCkm9ZG2VzOXv3DOrw5G5mkMbVwptTwVcFv7/C0vOpBmgTxAeTG19XpUs1E522LW9Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-accessor-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz", + "integrity": "sha1-qeEss66Nh2cn7u84Q/igiXtcmNY=", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-accessor-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha1-d8mYQFJ6qOyxqLppe4BkWnqSap0=", + "dev": true + }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-buffer": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "dev": true + }, + "node_modules/is-callable": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.0.tgz", + "integrity": "sha512-pyVD9AaGLxtg6srb2Ng6ynWJqkHU9bEM087AKck0w8QwDarTfNcpIYoU8x8Hv2Icm8u6kFJM18Dag8lyqGkviw==", + "dev": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-ci": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-2.0.0.tgz", + "integrity": "sha512-YfJT7rkpQB0updsdHLGWrvhBJfcfzNNawYDNIyQXJz0IViGf75O8EBPKSdvw2rF+LGCsX4FZ8tcr3b19LcZq4w==", + "dev": true, + "dependencies": { + "ci-info": "^2.0.0" + }, + "bin": { + "is-ci": "bin.js" + } + }, + "node_modules/is-data-descriptor": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz", + "integrity": "sha1-C17mSDiOLIYCgueT8YVv7D8wG1Y=", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-data-descriptor/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-date-object": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.2.tgz", + "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==", + "dev": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-descriptor": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-0.1.6.tgz", + "integrity": "sha512-avDYr0SB3DwO9zsMov0gKCESFYqCnE4hq/4z3TdUlukEy5t9C0YRq7HLrsN52NAcqXKaepeCD0n+B0arnVG3Hg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^0.1.6", + "is-data-descriptor": "^0.1.4", + "kind-of": "^5.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-descriptor/node_modules/kind-of": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-5.1.0.tgz", + "integrity": "sha512-NGEErnH6F2vUuXDh+OlbcKW7/wOcfdRHaZ7VWtqCztfHri/++YKmP51OdWeGPuqCOba6kk2OTe5d02VmTB80Pw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-error": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/is-error/-/is-error-2.2.2.tgz", + "integrity": "sha512-IOQqts/aHWbiisY5DuPJQ0gcbvaLFCa7fBa9xoLfxBZvQ+ZI/Zh9xoI7Gk+G64N0FdK4AbibytHht2tWgpJWLg==", + "dev": true + }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha1-YrEQ4omkcUGOPsNqYX1HLjAd/Ik=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha1-qIwCU1eR8C7TfHahueqXc8gz+MI=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-finite": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-finite/-/is-finite-1.1.0.tgz", + "integrity": "sha512-cdyMtqX/BOqqNBBiKlIVkytNHm49MtMlYyn1zxzvJKWmFMlGzm+ry5BBfYyeY9YmNKbRSo/o7OX9w9ale0wg3w==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.1.tgz", + "integrity": "sha512-5G0tKtBTFImOqDnLB2hG6Bp2qcKEFduo4tZu9MT/H6NQv/ghhy30o55ufafxJ/LdH79LLs2Kfrn85TLKyA7BUg==", + "dev": true, + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-installed-globally": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-installed-globally/-/is-installed-globally-0.3.2.tgz", + "integrity": "sha512-wZ8x1js7Ia0kecP/CHM/3ABkAmujX7WPvQk6uu3Fly/Mk44pySulQpnHG46OMjHGXApINnV4QhY3SWnECO2z5g==", + "dev": true, + "dependencies": { + "global-dirs": "^2.0.1", + "is-path-inside": "^3.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-interactive": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", + "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-npm": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-4.0.0.tgz", + "integrity": "sha512-96ECIfh9xtDDlPylNPXhzjsykHsMJZ18ASpaWzQyBr4YRTcVjUvzaHayDAES2oU/3KpljhHUjtSRNiDwi0F0ig==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-obj": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz", + "integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-path-cwd": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz", + "integrity": "sha512-w942bTcih8fdJPJmQHFzkS76NEP8Kzzvmw92cXsazb8intwLqPibPPdXf4ANdKV3rYMuuQYGIWtvz9JilB3NFQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/is-path-inside": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.2.tgz", + "integrity": "sha512-/2UGPSgmtqwo1ktx8NDHjuPwZWmHhO+gj0f93EkhLB5RgW9RZevWYYlIkS6zePc6U2WpOdQYIwHe9YC4DWEBVg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/is-plain-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", + "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-plain-object": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-4.1.1.tgz", + "integrity": "sha512-5Aw8LLVsDlZsETVMhoMXzqsXwQqr/0vlnBYzIXJbYo2F4yYlhLHs+Ez7Bod7IIQKWkJbJfxrWD7pA1Dw1TKrwA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-promise": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", + "integrity": "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==", + "dev": true + }, + "node_modules/is-redirect": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", + "integrity": "sha1-HQPd7VO9jbDzDCbk+V02/HyH3CQ=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.0.tgz", + "integrity": "sha512-iI97M8KTWID2la5uYXlkbSDQIg4F6o1sYboZKKTDpnDQMLtUL86zxhgDet3Q2SriaYsyGqZ6Mn2SjbRKeLHdqw==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-retry-allowed": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz", + "integrity": "sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha1-EtSj3U5o4Lec6428hBc66A2RykQ=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-symbol": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", + "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/is-text-path": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-text-path/-/is-text-path-1.0.1.tgz", + "integrity": "sha1-Thqg+1G/vLPpJogAE5cgLBd1tm4=", + "dev": true, + "dependencies": { + "text-extensions": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=", + "dev": true + }, + "node_modules/is-utf8": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", + "integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI=", + "dev": true + }, + "node_modules/is-windows": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", + "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-yarn-global": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.3.0.tgz", + "integrity": "sha512-VjSeb/lHmkoyd8ryPVIKvOCn4D1koMqY+vqyjjUfc3xyKtP4dYOxM44sZrnqQSzSds3xyOrUTLTC9LVCVgLngw==", + "dev": true + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", + "dev": true + }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/istanbul-lib-coverage": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.5.tgz", + "integrity": "sha512-8aXznuEPCJvGnMSRft4udDRDtb1V3pkQkMMI5LI+6HuQz5oQ4J2UFn1H82raA3qJtyOLkkwVqICBQkjnGtn5mA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/istanbul-lib-hook": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/istanbul-lib-hook/-/istanbul-lib-hook-2.0.7.tgz", + "integrity": "sha512-vrRztU9VRRFDyC+aklfLoeXyNdTfga2EI3udDGn4cZ6fpSXpHLV9X6CHvfoMCPtggg8zvDDmC4b9xfu0z6/llA==", + "dev": true, + "dependencies": { + "append-transform": "^1.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/istanbul-lib-instrument": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-3.3.0.tgz", + "integrity": "sha512-5nnIN4vo5xQZHdXno/YDXJ0G+I3dAm4XgzfSVTPLQpj/zAV2dV6Juy0yaf10/zrJOJeHoN3fraFe+XRq2bFVZA==", + "dev": true, + "dependencies": { + "@babel/generator": "^7.4.0", + "@babel/parser": "^7.4.3", + "@babel/template": "^7.4.0", + "@babel/traverse": "^7.4.3", + "@babel/types": "^7.4.0", + "istanbul-lib-coverage": "^2.0.5", + "semver": "^6.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/istanbul-lib-instrument/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/istanbul-lib-report": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-2.0.8.tgz", + "integrity": "sha512-fHBeG573EIihhAblwgxrSenp0Dby6tJMFR/HvlerBsrCTD5bkUuoNtn3gVh29ZCS824cGGBPn7Sg7cNk+2xUsQ==", + "dev": true, + "dependencies": { + "istanbul-lib-coverage": "^2.0.5", + "make-dir": "^2.1.0", + "supports-color": "^6.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/istanbul-lib-report/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/istanbul-lib-report/node_modules/make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "dependencies": { + "pify": "^4.0.1", + "semver": "^5.6.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/istanbul-lib-report/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/istanbul-lib-report/node_modules/supports-color": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-6.1.0.tgz", + "integrity": "sha512-qe1jfm1Mg7Nq/NSh6XE24gPXROEVsWHxC1LIx//XNlD9iw7YZQGjZNjYN7xGaEG6iKdA8EtNFW6R0gjnVXp+wQ==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/istanbul-lib-source-maps": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-3.0.6.tgz", + "integrity": "sha512-R47KzMtDJH6X4/YW9XTx+jrLnZnscW4VpNN+1PViSYTejLVPWv7oov+Duf8YQSPyVRUvueQqz1TcsC6mooZTXw==", + "dev": true, + "dependencies": { + "debug": "^4.1.1", + "istanbul-lib-coverage": "^2.0.5", + "make-dir": "^2.1.0", + "rimraf": "^2.6.3", + "source-map": "^0.6.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/istanbul-lib-source-maps/node_modules/make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "dependencies": { + "pify": "^4.0.1", + "semver": "^5.6.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/istanbul-lib-source-maps/node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/istanbul-lib-source-maps/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/istanbul-reports": { + "version": "2.2.7", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-2.2.7.tgz", + "integrity": "sha512-uu1F/L1o5Y6LzPVSVZXNOoD/KXpJue9aeLRd0sM9uMXfZvzomB0WxVamWb5ue8kA2vVWEmW7EG+A5n3f1kqHKg==", + "dev": true, + "dependencies": { + "html-escaper": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/js-string-escape": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/js-string-escape/-/js-string-escape-1.0.1.tgz", + "integrity": "sha1-4mJbrbwNZ8dTPp7cEGjFh65BN+8=", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true + }, + "node_modules/js-yaml": { + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.0.tgz", + "integrity": "sha512-/4IbIeHcD9VMHFqDR/gQ7EdZdLimOvW2DdcxFjdyyZ9NsbS+ccrXqVWDtab/lRl5AlUqmpBx8EhPaWR+OtY17A==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsesc": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", + "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", + "dev": true, + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/json-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.0.tgz", + "integrity": "sha1-Wx85evx11ne96Lz8Dkfh+aPZqJg=", + "dev": true + }, + "node_modules/json-parse-better-errors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", + "integrity": "sha512-mrqyZKfX5EhL7hvqcV6WG1yYjnjeuYDzDhhcAAUrq8Po85NBQBJP+ZDUT75qZQ98IkUoBqdkExkukOU7Ts2wrw==", + "dev": true + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.0.tgz", + "integrity": "sha512-o3aP+RsWDJZayj1SbHNQAI8x0v3T3SKiGoZlNYfbUP1S3omJQ6i9CnqADqkSPaOAxwua4/1YWx5CM7oiChJt2Q==", + "dev": true + }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", + "dev": true + }, + "node_modules/jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/jsonparse": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", + "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=", + "dev": true, + "engines": [ + "node >= 0.2.0" + ] + }, + "node_modules/JSONStream": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", + "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", + "dev": true, + "dependencies": { + "jsonparse": "^1.2.0", + "through": ">=2.2.7 <3" + }, + "bin": { + "JSONStream": "bin.js" + }, + "engines": { + "node": "*" + } + }, + "node_modules/keyv": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz", + "integrity": "sha512-9ykJ/46SN/9KPM/sichzQ7OvXyGDYKGTaDlKMGCAlg2UK8KRy4jb0d8sFc+0Tt0YYnThq8X2RZgCg74RPxgcVA==", + "dev": true, + "dependencies": { + "json-buffer": "3.0.0" + } + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/klaw": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/klaw/-/klaw-1.3.1.tgz", + "integrity": "sha1-QIhDO0azsbolnXh4XY6W9zugJDk=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.9" + } + }, + "node_modules/latest-version": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-5.1.0.tgz", + "integrity": "sha512-weT+r0kTkRQdCdYCNtkMwWXQTMEswKrFBkm4ckQOMVhhqhIMI1UT2hMj+1iigIhgSZm5gTmrRXBNoGUgaTY1xA==", + "dev": true, + "dependencies": { + "package-json": "^6.3.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lazy-req": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/lazy-req/-/lazy-req-1.1.0.tgz", + "integrity": "sha1-va6+rTD42CQDnODOFJ1Nqge6H6w=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/lines-and-columns": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.1.6.tgz", + "integrity": "sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA=", + "dev": true + }, + "node_modules/load-json-file": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-5.3.0.tgz", + "integrity": "sha512-cJGP40Jc/VXUsp8/OrnyKyTZ1y6v/dphm3bioS+RrKXjK2BB6wHUd6JptZEFDGgGahMT+InnZO5i1Ei9mpC8Bw==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.15", + "parse-json": "^4.0.0", + "pify": "^4.0.1", + "strip-bom": "^3.0.0", + "type-fest": "^0.3.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lodash": { + "version": "4.17.19", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz", + "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ==", + "dev": true + }, + "node_modules/lodash._reinterpolate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/lodash._reinterpolate/-/lodash._reinterpolate-3.0.0.tgz", + "integrity": "sha1-DM8tiRZq8Ds2Y8eWU4t1rG4RTZ0=", + "dev": true + }, + "node_modules/lodash.flattendeep": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.flattendeep/-/lodash.flattendeep-4.4.0.tgz", + "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", + "dev": true + }, + "node_modules/lodash.ismatch": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.ismatch/-/lodash.ismatch-4.4.0.tgz", + "integrity": "sha1-dWy1FQyjum8RCFp4hJZF8Yj4Xzc=", + "dev": true + }, + "node_modules/lodash.map": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.map/-/lodash.map-4.6.0.tgz", + "integrity": "sha1-dx7Hg540c9nEzeKLGTlMNWL09tM=", + "dev": true + }, + "node_modules/lodash.template": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.template/-/lodash.template-4.5.0.tgz", + "integrity": "sha512-84vYFxIkmidUiFxidA/KjjH9pAycqW+h980j7Fuz5qxRtO9pgB7MDFTdys1N7A5mcucRiDyEq4fusljItR1T/A==", + "dev": true, + "dependencies": { + "lodash._reinterpolate": "^3.0.0", + "lodash.templatesettings": "^4.0.0" + } + }, + "node_modules/lodash.templatesettings": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/lodash.templatesettings/-/lodash.templatesettings-4.2.0.tgz", + "integrity": "sha512-stgLz+i3Aa9mZgnjr/O+v9ruKZsPsndy7qPZOchbqk2cnTU1ZaldKK+v7m54WoKIyxiuMZTKT2H81F8BeAc3ZQ==", + "dev": true, + "dependencies": { + "lodash._reinterpolate": "^3.0.0" + } + }, + "node_modules/lodash.toarray": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/lodash.toarray/-/lodash.toarray-4.4.0.tgz", + "integrity": "sha1-JMS/zWsvuji/0FlNsRedjptlZWE=", + "dev": true + }, + "node_modules/log-symbols": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", + "integrity": "sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==", + "dev": true, + "dependencies": { + "chalk": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/log-update": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/log-update/-/log-update-1.0.2.tgz", + "integrity": "sha1-GZKfZMQJPS0ucHWh2tivWcKWuNE=", + "dev": true, + "dependencies": { + "ansi-escapes": "^1.0.0", + "cli-cursor": "^1.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/log-update/node_modules/ansi-escapes": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-1.4.0.tgz", + "integrity": "sha1-06ioOzGapneTZisT52HHkRQiMG4=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/log-update/node_modules/cli-cursor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-1.0.2.tgz", + "integrity": "sha1-ZNo/fValRBLll5S9Ytw1KV6PKYc=", + "dev": true, + "dependencies": { + "restore-cursor": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/log-update/node_modules/onetime": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-1.1.0.tgz", + "integrity": "sha1-ofeDj4MUxRbwXs78vEzP4EtO14k=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/log-update/node_modules/restore-cursor": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-1.0.1.tgz", + "integrity": "sha1-NGYfRohjJ/7SmRR5FSJS35LapUE=", + "dev": true, + "dependencies": { + "exit-hook": "^1.0.0", + "onetime": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/longest": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/longest/-/longest-2.0.1.tgz", + "integrity": "sha1-eB4YMpaqlPbU2RbcM10NF676I/g=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/loud-rejection": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/loud-rejection/-/loud-rejection-1.6.0.tgz", + "integrity": "sha1-W0b4AUft7leIcPCG0Eghz5mOVR8=", + "dev": true, + "dependencies": { + "currently-unhandled": "^0.4.1", + "signal-exit": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/lowercase-keys": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-1.0.1.tgz", + "integrity": "sha512-G2Lj61tXDnVFFOi8VZds+SoQjtQC3dgokKdDG2mTm1tx4m50NUHBOZSBwQQHyy0V12A0JTG4icfZQH+xPyh8VA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/lru-cache": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz", + "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==", + "dev": true, + "dependencies": { + "pseudomap": "^1.0.2", + "yallist": "^2.1.2" + } + }, + "node_modules/lunr": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/lunr/-/lunr-2.3.8.tgz", + "integrity": "sha512-oxMeX/Y35PNFuZoHp+jUj5OSEmLCaIH4KTFJh7a93cHBoFmpw2IoPs22VIz7vyO2YUnx2Tn9dzIwO2P/4quIRg==", + "dev": true + }, + "node_modules/make-dir": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-3.1.0.tgz", + "integrity": "sha512-g3FeP20LNwhALb/6Cz6Dd4F2ngze0jz7tbzrD2wAV+o9FeNHe4rL+yK2md0J/fiSf1sa1ADhXqi5+oVwOM/eGw==", + "dev": true, + "dependencies": { + "semver": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/map-age-cleaner": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz", + "integrity": "sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w==", + "dev": true, + "dependencies": { + "p-defer": "^1.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/map-cache": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz", + "integrity": "sha1-wyq9C9ZSXZsFFkW7TyasXcmKDb8=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/map-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-4.1.0.tgz", + "integrity": "sha512-glc9y00wgtwcDmp7GaE/0b0OnxpNJsVf3ael/An6Fe2Q51LLwN1er6sdomLRzz5h0+yMpiYLhWYF5R7HeqVd4g==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/map-visit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/map-visit/-/map-visit-1.0.0.tgz", + "integrity": "sha1-7Nyo8TFE5mDxtb1B8S80edmN+48=", + "dev": true, + "dependencies": { + "object-visit": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/marked": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/marked/-/marked-1.0.0.tgz", + "integrity": "sha512-Wo+L1pWTVibfrSr+TTtMuiMfNzmZWiOPeO7rZsQUY5bgsxpHesBEcIWJloWVTFnrMXnf/TL30eTFSGJddmQAng==", + "dev": true, + "bin": { + "marked": "bin/marked" + }, + "engines": { + "node": ">= 8.16.2" + } + }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "dev": true, + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/matcher/node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/md5-hex": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/md5-hex/-/md5-hex-3.0.1.tgz", + "integrity": "sha512-BUiRtTtV39LIJwinWBjqVsU9xhdnz7/i889V859IBFpuqGAj6LuOvHv5XLbgZ2R7ptJoJaEcxkv88/h25T7Ciw==", + "dev": true, + "dependencies": { + "blueimp-md5": "^2.10.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/mem": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/mem/-/mem-6.1.0.tgz", + "integrity": "sha512-RlbnLQgRHk5lwqTtpEkBTQ2ll/CG/iB+J4Hy2Wh97PjgZgXgWJWrFF+XXujh3UUVLvR4OOTgZzcWMMwnehlEUg==", + "dev": true, + "dependencies": { + "map-age-cleaner": "^0.1.3", + "mimic-fn": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/mem/node_modules/mimic-fn": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-3.1.0.tgz", + "integrity": "sha512-Ysbi9uYW9hFyfrThdDEQuykN4Ey6BuwPD2kpI5ES/nFTDn/98yxYNLZJcgUAKPT/mcrLLKaGzJR9YVxJrIdASQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/memorystream": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/memorystream/-/memorystream-0.3.1.tgz", + "integrity": "sha1-htcJCzDORV1j+64S3aUaR93K+bI=", + "dev": true, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/meow": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/meow/-/meow-7.1.1.tgz", + "integrity": "sha512-GWHvA5QOcS412WCo8vwKDlTelGLsCGBVevQB5Kva961rmNfun0PCbv5+xta2kUMFJyR8/oWnn7ddeKdosbAPbA==", + "dev": true, + "dependencies": { + "@types/minimist": "^1.2.0", + "camelcase-keys": "^6.2.2", + "decamelize-keys": "^1.1.0", + "hard-rejection": "^2.1.0", + "minimist-options": "4.1.0", + "normalize-package-data": "^2.5.0", + "read-pkg-up": "^7.0.1", + "redent": "^3.0.0", + "trim-newlines": "^3.0.0", + "type-fest": "^0.13.1", + "yargs-parser": "^18.1.3" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/meow/node_modules/read-pkg-up": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-7.0.1.tgz", + "integrity": "sha512-zK0TB7Xd6JpCLmlLmufqykGE+/TlOePD6qKClNW7hHDKFh/J7/7gCWGR7joEQEW1bKq3a3yUZSObOoWLFQ4ohg==", + "dev": true, + "dependencies": { + "find-up": "^4.1.0", + "read-pkg": "^5.2.0", + "type-fest": "^0.8.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/meow/node_modules/read-pkg-up/node_modules/type-fest": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", + "integrity": "sha512-4dbzIzqvjtgiM5rw1k5rEHtBANKmdudhGyBEajN01fEyhaAIhsoKNy6y7+IN93IfpFtwY9iqi7kD+xwKhQsNJA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/meow/node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "dev": true, + "engines": { + "node": ">=10" + } + }, + "node_modules/merge": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/merge/-/merge-1.2.1.tgz", + "integrity": "sha512-VjFo4P5Whtj4vsLzsYBu5ayHhoHJ0UqNm7ibvShmbmoz7tGi0vXaoJbGdB+GmDMLUdg8DpQXEIeVDAe8MaABvQ==", + "dev": true + }, + "node_modules/merge-source-map": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/merge-source-map/-/merge-source-map-1.1.0.tgz", + "integrity": "sha512-Qkcp7P2ygktpMPh2mCQZaf3jhN6D3Z/qVZHSdWvQ+2Ef5HgRAPBO57A77+ENm0CPx2+1Ce/MYKi3ymqdfuqibw==", + "dev": true, + "dependencies": { + "source-map": "^0.6.1" + } + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "dev": true, + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.2.tgz", + "integrity": "sha512-y7FpHSbMUMoyPbYUSzO6PaZ6FyRnQOpHuKwbo1G+Knck95XVU4QAiKdGEnj5wwoS7PlOgthX/09u5iFJ+aYf5Q==", + "dev": true, + "dependencies": { + "braces": "^3.0.1", + "picomatch": "^2.0.5" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/mimic-response": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", + "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/min-indent": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", + "integrity": "sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/minimatch": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", + "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", + "dev": true + }, + "node_modules/minimist-options": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/minimist-options/-/minimist-options-4.1.0.tgz", + "integrity": "sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==", + "dev": true, + "dependencies": { + "arrify": "^1.0.1", + "is-plain-obj": "^1.1.0", + "kind-of": "^6.0.3" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/minimist-options/node_modules/arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mixin-deep": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz", + "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==", + "dev": true, + "dependencies": { + "for-in": "^1.0.2", + "is-extendable": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mixin-deep/node_modules/is-extendable": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", + "integrity": "sha512-arnXMxT1hhoKo9k1LZdmlNyJdDDfy2v0fXjFlmok4+i8ul/6WlbVge9bhM74OpNPQPMGUToDtz+KXa1PneJxOA==", + "dev": true, + "dependencies": { + "is-plain-object": "^2.0.4" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mixin-deep/node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "dependencies": { + "minimist": "^1.2.5" + }, + "bin": { + "mkdirp": "bin/cmd.js" + } + }, + "node_modules/modify-values": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/modify-values/-/modify-values-1.0.1.tgz", + "integrity": "sha512-xV2bxeN6F7oYjZWTe/YPAy6MN2M+sL4u/Rlm2AHCIVGfo2p1yGmBHQ6vHehl4bRTZBdHu3TSkWdYgkwpYzAGSw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mount-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mount-point/-/mount-point-3.0.0.tgz", + "integrity": "sha1-Zly57evoDREOZY21bDHQrvUaj5c=", + "dev": true, + "dependencies": { + "@sindresorhus/df": "^1.0.1", + "pify": "^2.3.0", + "pinkie-promise": "^2.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mount-point/node_modules/@sindresorhus/df": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@sindresorhus/df/-/df-1.0.1.tgz", + "integrity": "sha1-xptm9S9vzdKHyAffIQMF2694UA0=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/mount-point/node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", + "dev": true + }, + "node_modules/mute-stream": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", + "dev": true + }, + "node_modules/nanomatch": { + "version": "1.2.13", + "resolved": "https://registry.npmjs.org/nanomatch/-/nanomatch-1.2.13.tgz", + "integrity": "sha512-fpoe2T0RbHwBTBUOftAfBPaDEi06ufaUai0mE6Yn1kacc3SnTErfb/h+X94VXzI64rKFHYImXSvdwGGCmwOqCA==", + "dev": true, + "dependencies": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "fragment-cache": "^0.2.1", + "is-windows": "^1.0.2", + "kind-of": "^6.0.2", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/neo-async": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.1.tgz", + "integrity": "sha512-iyam8fBuCUpWeKPGpaNMetEocMt364qkCsfL9JuhjXX6dRnguRVOfk2GZaDpPjcOKiiXCPINZC1GczQ7iTq3Zw==", + "dev": true + }, + "node_modules/nested-error-stacks": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.1.0.tgz", + "integrity": "sha512-AO81vsIO1k1sM4Zrd6Hu7regmJN1NSiAja10gc4bX3F0wd+9rQmcuHQaHVQCYIEC8iFXnE+mavh23GOt7wBgug==", + "dev": true + }, + "node_modules/nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", + "dev": true + }, + "node_modules/node-emoji": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/node-emoji/-/node-emoji-1.8.1.tgz", + "integrity": "sha512-+ktMAh1Jwas+TnGodfCfjUbJKoANqPaJFN0z0iqh41eqD8dvguNzcitVSBSVK1pidz0AqGbLKcoVuVLRVZ/aVg==", + "dev": true, + "dependencies": { + "lodash.toarray": "^4.4.0" + } + }, + "node_modules/node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "dev": true, + "engines": { + "node": "4.x || >=6.0.0" + } + }, + "node_modules/node-status-codes": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-status-codes/-/node-status-codes-1.0.0.tgz", + "integrity": "sha1-WuVUHQJGRdMqWPzdyc7s6nrjrC8=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/normalize-package-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", + "integrity": "sha512-/5CMN3T0R4XTj4DcGaexo+roZSdSFW/0AOOTROrjxzCG1wrWXEsGbRKevjlIL+ZDE4sZlJr5ED4YW0yqmkK+eA==", + "dev": true, + "dependencies": { + "hosted-git-info": "^2.1.4", + "resolve": "^1.10.0", + "semver": "2 || 3 || 4 || 5", + "validate-npm-package-license": "^3.0.1" + } + }, + "node_modules/normalize-package-data/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/normalize-url": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz", + "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/npm-run-all": { + "version": "4.1.5", + "resolved": "https://registry.npmjs.org/npm-run-all/-/npm-run-all-4.1.5.tgz", + "integrity": "sha512-Oo82gJDAVcaMdi3nuoKFavkIHBRVqQ1qvMb+9LHk/cF4P6B2m8aP04hGf7oL6wZ9BuGwX1onlLhpuoofSyoQDQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "chalk": "^2.4.1", + "cross-spawn": "^6.0.5", + "memorystream": "^0.3.1", + "minimatch": "^3.0.4", + "pidtree": "^0.3.0", + "read-pkg": "^3.0.0", + "shell-quote": "^1.6.1", + "string.prototype.padend": "^3.0.0" + }, + "bin": { + "npm-run-all": "bin/npm-run-all/index.js", + "run-p": "bin/run-p/index.js", + "run-s": "bin/run-s/index.js" + }, + "engines": { + "node": ">= 4" + } + }, + "node_modules/npm-run-all/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/npm-run-all/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "node_modules/npm-run-all/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/load-json-file": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "parse-json": "^4.0.0", + "pify": "^3.0.0", + "strip-bom": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/path-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", + "dev": true, + "dependencies": { + "pify": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/read-pkg": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", + "dev": true, + "dependencies": { + "load-json-file": "^4.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-all/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha1-NakjLfo11wZ7TLLd8jV7GHFTbF8=", + "dev": true, + "dependencies": { + "path-key": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/null-check": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/null-check/-/null-check-1.0.0.tgz", + "integrity": "sha1-l33/1xdgErnsMNKjnbXPcqBDnt0=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/number-is-nan": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", + "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/nyc": { + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/nyc/-/nyc-14.1.1.tgz", + "integrity": "sha512-OI0vm6ZGUnoGZv/tLdZ2esSVzDwUC88SNs+6JoSOMVxA+gKMB8Tk7jBwgemLx4O40lhhvZCVw1C+OYLOBOPXWw==", + "dev": true, + "dependencies": { + "archy": "^1.0.0", + "caching-transform": "^3.0.2", + "convert-source-map": "^1.6.0", + "cp-file": "^6.2.0", + "find-cache-dir": "^2.1.0", + "find-up": "^3.0.0", + "foreground-child": "^1.5.6", + "glob": "^7.1.3", + "istanbul-lib-coverage": "^2.0.5", + "istanbul-lib-hook": "^2.0.7", + "istanbul-lib-instrument": "^3.3.0", + "istanbul-lib-report": "^2.0.8", + "istanbul-lib-source-maps": "^3.0.6", + "istanbul-reports": "^2.2.4", + "js-yaml": "^3.13.1", + "make-dir": "^2.1.0", + "merge-source-map": "^1.1.0", + "resolve-from": "^4.0.0", + "rimraf": "^2.6.3", + "signal-exit": "^3.0.2", + "spawn-wrap": "^1.4.2", + "test-exclude": "^5.2.3", + "uuid": "^3.3.2", + "yargs": "^13.2.2", + "yargs-parser": "^13.0.0" + }, + "bin": { + "nyc": "bin/nyc.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/nyc/node_modules/ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/nyc/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/nyc/node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/nyc/node_modules/cliui": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz", + "integrity": "sha512-PYeGSEmmHM6zvoef2w8TPzlrnNpXIjTipYK780YswmIP9vjxmd6Y2a3CB2Ks6/AU8NHjZugXvo8w3oWM2qnwXA==", + "dev": true, + "dependencies": { + "string-width": "^3.1.0", + "strip-ansi": "^5.2.0", + "wrap-ansi": "^5.1.0" + } + }, + "node_modules/nyc/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/nyc/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "node_modules/nyc/node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/nyc/node_modules/emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "node_modules/nyc/node_modules/find-up": { + "version": "3.0.0", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "dependencies": { + "locate-path": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/nyc/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/nyc/node_modules/locate-path": { + "version": "3.0.0", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/nyc/node_modules/make-dir": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", + "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", + "dev": true, + "dependencies": { + "pify": "^4.0.1", + "semver": "^5.6.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/nyc/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/nyc/node_modules/p-locate": { + "version": "3.0.0", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "dependencies": { + "p-limit": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/nyc/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/nyc/node_modules/resolve-from": { + "version": "4.0.0", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/nyc/node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/nyc/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/nyc/node_modules/string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "dependencies": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/nyc/node_modules/strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "dependencies": { + "ansi-regex": "^4.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/nyc/node_modules/wrap-ansi": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-5.1.0.tgz", + "integrity": "sha512-QC1/iN/2/RPVJ5jYK8BGttj5z83LmSKmvbvrXPNCLZSEb32KKVDJDl/MOt2N01qU2H/FkzEa9PKto1BqDjtd7Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.0", + "string-width": "^3.0.0", + "strip-ansi": "^5.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/nyc/node_modules/yargs": { + "version": "13.3.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-13.3.2.tgz", + "integrity": "sha512-AX3Zw5iPruN5ie6xGRIDgqkT+ZhnRlZMLMHAs8tg7nRruy2Nb+i5o9bwghAogtM08q1dpr2LVoS8KSTMYpWXUw==", + "dev": true, + "dependencies": { + "cliui": "^5.0.0", + "find-up": "^3.0.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^3.0.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^13.1.2" + } + }, + "node_modules/nyc/node_modules/yargs-parser": { + "version": "13.1.2", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz", + "integrity": "sha512-3lbsNRf/j+A4QuSZfDRA7HRSfWrzO0YjqTJd5kjAq37Zep1CEgaYmrH9Q3GwPiB9cHyd1Y1UwggGhJGoxipbzg==", + "dev": true, + "dependencies": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + } + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/object-copy/-/object-copy-0.1.0.tgz", + "integrity": "sha1-fn2Fi3gb18mRpBupde04EnVOmYw=", + "dev": true, + "dependencies": { + "copy-descriptor": "^0.1.0", + "define-property": "^0.2.5", + "kind-of": "^3.0.3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-copy/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object-inspect": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.8.0.tgz", + "integrity": "sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA==", + "dev": true + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object-visit": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/object-visit/-/object-visit-1.0.1.tgz", + "integrity": "sha1-95xEk68MU3e1n+OdOV5BBC3QRbs=", + "dev": true, + "dependencies": { + "isobject": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/object.assign": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", + "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", + "dev": true, + "dependencies": { + "define-properties": "^1.1.2", + "function-bind": "^1.1.1", + "has-symbols": "^1.0.0", + "object-keys": "^1.0.11" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/object.pick": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/object.pick/-/object.pick-1.3.0.tgz", + "integrity": "sha1-h6EKxMFpS9Lhy/U1kaZhQftd10c=", + "dev": true, + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/opn": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/opn/-/opn-4.0.2.tgz", + "integrity": "sha1-erwi5kTf9jsKltWrfyeQwPAavJU=", + "dev": true, + "dependencies": { + "object-assign": "^4.0.1", + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/opn-cli/-/opn-cli-3.1.0.tgz", + "integrity": "sha1-+BmubK4LQRvQFJuFYP5siK2tIPg=", + "dev": true, + "dependencies": { + "file-type": "^3.6.0", + "get-stdin": "^5.0.1", + "meow": "^3.7.0", + "opn": "^4.0.0", + "temp-write": "^2.1.0" + }, + "bin": { + "opn": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/opn-cli/node_modules/camelcase": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz", + "integrity": "sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/camelcase-keys": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", + "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=", + "dev": true, + "dependencies": { + "camelcase": "^2.0.0", + "map-obj": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/find-up": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz", + "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=", + "dev": true, + "dependencies": { + "path-exists": "^2.0.0", + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/indent-string": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-2.1.0.tgz", + "integrity": "sha1-ji1INIdCEhtKghi3oTfppSBJ3IA=", + "dev": true, + "dependencies": { + "repeating": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/load-json-file": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "parse-json": "^2.2.0", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0", + "strip-bom": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/map-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz", + "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/meow": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz", + "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=", + "dev": true, + "dependencies": { + "camelcase-keys": "^2.0.0", + "decamelize": "^1.1.2", + "loud-rejection": "^1.0.0", + "map-obj": "^1.0.1", + "minimist": "^1.1.3", + "normalize-package-data": "^2.3.4", + "object-assign": "^4.0.1", + "read-pkg-up": "^1.0.1", + "redent": "^1.0.0", + "trim-newlines": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", + "dev": true, + "dependencies": { + "error-ex": "^1.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/path-exists": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz", + "integrity": "sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=", + "dev": true, + "dependencies": { + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/path-type": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz", + "integrity": "sha1-WcRPfuSR2nBNpBXaWkBwuk+P5EE=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/read-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz", + "integrity": "sha1-9f+qXs0pyzHAR0vKfXVra7KePyg=", + "dev": true, + "dependencies": { + "load-json-file": "^1.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/read-pkg-up": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz", + "integrity": "sha1-nWPBMnbAZZGNV/ACpX9AobZD+wI=", + "dev": true, + "dependencies": { + "find-up": "^1.0.0", + "read-pkg": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/redent": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/redent/-/redent-1.0.0.tgz", + "integrity": "sha1-z5Fqsf1fHxbfsggi3W7H9zDCr94=", + "dev": true, + "dependencies": { + "indent-string": "^2.1.0", + "strip-indent": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/strip-bom": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "integrity": "sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4=", + "dev": true, + "dependencies": { + "is-utf8": "^0.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/strip-indent": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-1.0.1.tgz", + "integrity": "sha1-DHlipq3vp7vUrDZkYKY4VSrhoKI=", + "dev": true, + "dependencies": { + "get-stdin": "^4.0.1" + }, + "bin": { + "strip-indent": "cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/strip-indent/node_modules/get-stdin": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz", + "integrity": "sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/opn-cli/node_modules/trim-newlines": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz", + "integrity": "sha1-WIeWa7WCpFA6QetST301ARgVphM=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/ora": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ora/-/ora-5.0.0.tgz", + "integrity": "sha512-s26qdWqke2kjN/wC4dy+IQPBIMWBJlSU/0JZhk30ZDBLelW25rv66yutUWARMigpGPzcXHb+Nac5pNhN/WsARw==", + "dev": true, + "dependencies": { + "chalk": "^4.1.0", + "cli-cursor": "^3.1.0", + "cli-spinners": "^2.4.0", + "is-interactive": "^1.0.0", + "log-symbols": "^4.0.0", + "mute-stream": "0.0.8", + "strip-ansi": "^6.0.0", + "wcwidth": "^1.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/os-family": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/os-family/-/os-family-1.1.0.tgz", + "integrity": "sha512-E3Orl5pvDJXnVmpaAA2TeNNpNhTMl4o5HghuWhOivBjEiTnJSrMYSa5uZMek1lBEvu8kKEsa2YgVcGFVDqX/9w==", + "dev": true + }, + "node_modules/os-homedir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", + "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/os-tmpdir": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", + "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/osenv": { + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/osenv/-/osenv-0.1.5.tgz", + "integrity": "sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==", + "dev": true, + "dependencies": { + "os-homedir": "^1.0.0", + "os-tmpdir": "^1.0.0" + } + }, + "node_modules/p-cancelable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-1.1.0.tgz", + "integrity": "sha512-s73XxOZ4zpt1edZYZzvhqFa6uvQc1vwUa0K0BdtIZgQMAJj9IbebH+JkgKZc9h+B05PKHLOTl4ajG1BmNrVZlw==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/p-defer": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz", + "integrity": "sha1-n26xgvbJqozXQwBKfU+WsZaw+ww=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "dev": true, + "dependencies": { + "aggregate-error": "^3.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/package-hash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/package-hash/-/package-hash-3.0.0.tgz", + "integrity": "sha512-lOtmukMDVvtkL84rJHI7dpTYq+0rli8N2wlnqUcBuDWCfVhRUfOmnR9SsoHFMLpACvEV60dX7rd0rFaYDZI+FA==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.15", + "hasha": "^3.0.0", + "lodash.flattendeep": "^4.4.0", + "release-zalgo": "^1.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/package-json": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-6.5.0.tgz", + "integrity": "sha512-k3bdm2n25tkyxcjSKzB5x8kfVxlMdgsbPr0GkZcwHsLpba6cBjqCt1KlcChKEvxHIcTB1FVMuwoijZ26xex5MQ==", + "dev": true, + "dependencies": { + "got": "^9.6.0", + "registry-auth-token": "^4.0.0", + "registry-url": "^5.0.0", + "semver": "^6.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/package-json/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "optional": true, + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-github-repo-url": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/parse-github-repo-url/-/parse-github-repo-url-1.4.1.tgz", + "integrity": "sha1-nn2LslKmy2ukJZUGC3v23z28H1A=", + "dev": true + }, + "node_modules/parse-json": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-4.0.0.tgz", + "integrity": "sha1-vjX1Qlvh9/bHRxhPmKeIy5lHfuA=", + "dev": true, + "dependencies": { + "error-ex": "^1.3.1", + "json-parse-better-errors": "^1.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/parse-ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/parse-ms/-/parse-ms-2.1.0.tgz", + "integrity": "sha512-kHt7kzLoS9VBZfUsiKjv43mr91ea+U05EyKkEtqp7vNbHxmaVuEqN7XxeEVnGrMtYOAxGrDElSi96K7EgO1zCA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-passwd": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/parse-passwd/-/parse-passwd-1.0.0.tgz", + "integrity": "sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pascalcase": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz", + "integrity": "sha1-s2PlXoAGym/iF4TS2yK9FdeRfxQ=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha1-QRyttXTFoUDTpLGRDUDYDMn0C0A=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/path-parse": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.6.tgz", + "integrity": "sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw==", + "dev": true + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/picomatch": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", + "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==", + "dev": true, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/pidtree": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/pidtree/-/pidtree-0.3.1.tgz", + "integrity": "sha512-qQbW94hLHEqCg7nhby4yRC7G2+jYHY4Rguc2bjw7Uug4GIJuu1tvf2uHaZv5Q8zdt+WKJ6qK1FOI6amaWUo5FA==", + "dev": true, + "bin": { + "pidtree": "bin/pidtree.js" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/pinkie": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz", + "integrity": "sha1-clVrgM+g1IqXToDnckjoDtT3+HA=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pinkie-promise": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "integrity": "sha1-ITXW36ejWMBprJsXh3YogihFD/o=", + "dev": true, + "dependencies": { + "pinkie": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pkg-conf": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/pkg-conf/-/pkg-conf-3.1.0.tgz", + "integrity": "sha512-m0OTbR/5VPNPqO1ph6Fqbj7Hv6QU7gR/tQW40ZqrL1rjgCU85W6C1bJn0BItuJqnR98PWzw7Z8hHeChD1WrgdQ==", + "dev": true, + "dependencies": { + "find-up": "^3.0.0", + "load-json-file": "^5.2.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-conf/node_modules/find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "dependencies": { + "locate-path": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-conf/node_modules/locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-conf/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-conf/node_modules/p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "dependencies": { + "p-limit": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-conf/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pkg-dir/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/pkg-dir/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/plur": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/plur/-/plur-4.0.0.tgz", + "integrity": "sha512-4UGewrYgqDFw9vV6zNV+ADmPAUAfJPKtGvb/VdpQAx25X5f3xXdGdyOEVFwkl8Hl/tl7+xbeHqSEM+D5/TirUg==", + "dev": true, + "dependencies": { + "irregular-plurals": "^3.2.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/posix-character-classes": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/posix-character-classes/-/posix-character-classes-0.1.1.tgz", + "integrity": "sha1-AerA/jta9xoqbAL+q7jB/vfgDqs=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/prepend-http": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-2.0.0.tgz", + "integrity": "sha1-6SQ0v6XqjBn0HN/UAddBo8gZ2Jc=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/prettier": { + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-1.19.1.tgz", + "integrity": "sha512-s7PoyDv/II1ObgQunCbB9PdLmUcBZcnWOcxDh7O0N/UwDEsHyqkW+Qh28jW+mVuCdx7gLB0BotYI1Y6uI9iyew==", + "dev": true, + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pretty-ms": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pretty-ms/-/pretty-ms-7.0.0.tgz", + "integrity": "sha512-J3aPWiC5e9ZeZFuSeBraGxSkGMOvulSWsxDByOcbD1Pr75YL3LSNIKIb52WXbCLE1sS5s4inBBbryjF4Y05Ceg==", + "dev": true, + "dependencies": { + "parse-ms": "^2.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", + "dev": true + }, + "node_modules/progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/promisify-event": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/promisify-event/-/promisify-event-1.0.0.tgz", + "integrity": "sha1-vXUj6ga3AWLzcJeQFrU6aGxg6Q8=", + "dev": true, + "dependencies": { + "pinkie-promise": "^2.0.0" + } + }, + "node_modules/pseudomap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", + "integrity": "sha1-8FKijacOYYkX7wqKw0wa5aaChrM=", + "dev": true + }, + "node_modules/publish-please": { + "version": "5.5.1", + "resolved": "https://registry.npmjs.org/publish-please/-/publish-please-5.5.1.tgz", + "integrity": "sha512-KFX+79e3QhNswHp1wLfwXN7rLf6tO3+7jcY5ELjOVR8rtjqDiXu9jrDbHqSapI1xwiPMdersn2O4c8PwUuOZtA==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "chalk": "2.4.1", + "cp-sugar": "1.0.0", + "elegant-status": "1.1.0", + "inquirer": "6.2.0", + "is-ci": "1.2.1", + "lodash": "4.17.12", + "micromatch": "3.1.10", + "node-emoji": "1.8.1", + "osenv": "0.1.5", + "semver": "5.6.0" + }, + "bin": { + "publish-please": "bin/publish-please.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/publish-please/node_modules/ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/publish-please/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/publish-please/node_modules/braces": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/braces/-/braces-2.3.2.tgz", + "integrity": "sha512-aNdbnj9P8PjdXU4ybaWLK2IF3jc/EoDYbC7AazW6to3TRsfXxscC9UXOB5iDiEQrkyIbWp2SLQda4+QAa7nc3w==", + "dev": true, + "dependencies": { + "arr-flatten": "^1.1.0", + "array-unique": "^0.3.2", + "extend-shallow": "^2.0.1", + "fill-range": "^4.0.0", + "isobject": "^3.0.1", + "repeat-element": "^1.1.2", + "snapdragon": "^0.8.1", + "snapdragon-node": "^2.0.1", + "split-string": "^3.0.2", + "to-regex": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/publish-please/node_modules/braces/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/publish-please/node_modules/chalk": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.1.tgz", + "integrity": "sha512-ObN6h1v2fTJSmUXoS3nMQ92LbDK9be4TV+6G+omQlGJFdcUX5heKi1LZ1YnRMIgwTLEj3E24bT6tYni50rlCfQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/publish-please/node_modules/ci-info": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-1.6.0.tgz", + "integrity": "sha512-vsGdkwSCDpWmP80ncATX7iea5DWQemg1UgCW5J8tqjU3lYw4FBYuj89J0CTVomA7BEfvSZd84GmHko+MxFQU2A==", + "dev": true + }, + "node_modules/publish-please/node_modules/cli-cursor": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-2.1.0.tgz", + "integrity": "sha1-s12sN2R5+sw+lHR9QdDQ9SOP/LU=", + "dev": true, + "dependencies": { + "restore-cursor": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/publish-please/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/publish-please/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "node_modules/publish-please/node_modules/figures": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/figures/-/figures-2.0.0.tgz", + "integrity": "sha1-OrGi0qYsi/tDGgyUy3l6L84nyWI=", + "dev": true, + "dependencies": { + "escape-string-regexp": "^1.0.5" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/publish-please/node_modules/fill-range": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-4.0.0.tgz", + "integrity": "sha1-1USBHUKPmOsGpj3EAtJAPDKMOPc=", + "dev": true, + "dependencies": { + "extend-shallow": "^2.0.1", + "is-number": "^3.0.0", + "repeat-string": "^1.6.1", + "to-regex-range": "^2.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/publish-please/node_modules/fill-range/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/publish-please/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/publish-please/node_modules/inquirer": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-6.2.0.tgz", + "integrity": "sha512-QIEQG4YyQ2UYZGDC4srMZ7BjHOmNk1lR2JQj5UknBapklm6WHA+VVH7N+sUdX3A7NeCfGF8o4X1S3Ao7nAcIeg==", + "dev": true, + "dependencies": { + "ansi-escapes": "^3.0.0", + "chalk": "^2.0.0", + "cli-cursor": "^2.1.0", + "cli-width": "^2.0.0", + "external-editor": "^3.0.0", + "figures": "^2.0.0", + "lodash": "^4.17.10", + "mute-stream": "0.0.7", + "run-async": "^2.2.0", + "rxjs": "^6.1.0", + "string-width": "^2.1.0", + "strip-ansi": "^4.0.0", + "through": "^2.3.6" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/publish-please/node_modules/is-ci": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/is-ci/-/is-ci-1.2.1.tgz", + "integrity": "sha512-s6tfsaQaQi3JNciBH6shVqEDvhGut0SUXr31ag8Pd8BBbVVlcGfWhpPmEOoM6RJ5TFhbypvf5yyRw/VXW1IiWg==", + "dev": true, + "dependencies": { + "ci-info": "^1.5.0" + }, + "bin": { + "is-ci": "bin.js" + } + }, + "node_modules/publish-please/node_modules/is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/publish-please/node_modules/is-number": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-3.0.0.tgz", + "integrity": "sha1-JP1iAaR4LPUFYcgQJ2r8fRLXEZU=", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/publish-please/node_modules/is-number/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/publish-please/node_modules/lodash": { + "version": "4.17.12", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.12.tgz", + "integrity": "sha512-+CiwtLnsJhX03p20mwXuvhoebatoh5B3tt+VvYlrPgZC1g36y+RRbkufX95Xa+X4I59aWEacDFYwnJZiyBh9gA==", + "dev": true + }, + "node_modules/publish-please/node_modules/micromatch": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz", + "integrity": "sha512-MWikgl9n9M3w+bpsY3He8L+w9eF9338xRl8IAO5viDizwSzziFEyUzo2xrrloB64ADbTf8uA8vRqqttDTOmccg==", + "dev": true, + "dependencies": { + "arr-diff": "^4.0.0", + "array-unique": "^0.3.2", + "braces": "^2.3.1", + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "extglob": "^2.0.4", + "fragment-cache": "^0.2.1", + "kind-of": "^6.0.2", + "nanomatch": "^1.2.9", + "object.pick": "^1.3.0", + "regex-not": "^1.0.0", + "snapdragon": "^0.8.1", + "to-regex": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/publish-please/node_modules/mimic-fn": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz", + "integrity": "sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/publish-please/node_modules/mute-stream": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.7.tgz", + "integrity": "sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s=", + "dev": true + }, + "node_modules/publish-please/node_modules/onetime": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-2.0.1.tgz", + "integrity": "sha1-BnQoIw/WdEOyeUsiu6UotoZ5YtQ=", + "dev": true, + "dependencies": { + "mimic-fn": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/publish-please/node_modules/restore-cursor": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-2.0.0.tgz", + "integrity": "sha1-n37ih/gv0ybU/RYpI9YhKe7g368=", + "dev": true, + "dependencies": { + "onetime": "^2.0.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/publish-please/node_modules/semver": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.6.0.tgz", + "integrity": "sha512-RS9R6R35NYgQn++fkDWaOmqGoj4Ek9gGs+DPxNUZKuwE183xjJroKvyo1IzVFeXvUrvmALy6FWD5xrdJT25gMg==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/publish-please/node_modules/string-width": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", + "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "dev": true, + "dependencies": { + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^4.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/publish-please/node_modules/strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "dependencies": { + "ansi-regex": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/publish-please/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/publish-please/node_modules/to-regex-range": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-2.1.1.tgz", + "integrity": "sha1-fIDBe53+vlmeJzZ+DU3VWQFB2zg=", + "dev": true, + "dependencies": { + "is-number": "^3.0.0", + "repeat-string": "^1.6.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dev": true, + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/pupa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/pupa/-/pupa-2.0.1.tgz", + "integrity": "sha512-hEJH0s8PXLY/cdXh66tNEQGndDrIKNqNC5xmrysZy3i5C3oEoLna7YAOad+7u125+zH1HNXUmGEkrhb3c2VriA==", + "dev": true, + "dependencies": { + "escape-goat": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/q": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", + "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=", + "dev": true, + "engines": { + "node": ">=0.6.0", + "teleport": ">=0.2.0" + } + }, + "node_modules/query-string": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/query-string/-/query-string-4.3.4.tgz", + "integrity": "sha1-u7aTucqRXCMlFbIosaArYJBD2+s=", + "dev": true, + "dependencies": { + "object-assign": "^4.1.0", + "strict-uri-encode": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/quick-lru": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-4.0.1.tgz", + "integrity": "sha512-ARhCpm70fzdcvNQfPoy49IaanKkTlRWF2JMzqhcJbhSFRZv7nPTvZJdcY7301IPmvW+/p0RgIWnQDLJxifsQ7g==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/read-all-stream": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/read-all-stream/-/read-all-stream-3.1.0.tgz", + "integrity": "sha1-NcPhd/IHjveJ7kv6+kNzB06u9Po=", + "dev": true, + "dependencies": { + "pinkie-promise": "^2.0.0", + "readable-stream": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/read-all-stream/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/read-all-stream/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/read-pkg": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-5.2.0.tgz", + "integrity": "sha512-Ug69mNOpfvKDAc2Q8DRpMjjzdtrnv9HcSMX+4VsZxD1aZ6ZzrIE7rlzXBtWTyhULSMKg076AW6WR5iZpD0JiOg==", + "dev": true, + "dependencies": { + "@types/normalize-package-data": "^2.4.0", + "normalize-package-data": "^2.5.0", + "parse-json": "^5.0.0", + "type-fest": "^0.6.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/read-pkg-up": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-4.0.0.tgz", + "integrity": "sha512-6etQSH7nJGsK0RbG/2TeDzZFa8shjQ1um+SwQQ5cwKy0dhSXdOncEhb1CPpvQG4h7FyOV6EB6YlV0yJvZQNAkA==", + "dev": true, + "dependencies": { + "find-up": "^3.0.0", + "read-pkg": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/read-pkg-up/node_modules/find-up": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", + "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", + "dev": true, + "dependencies": { + "locate-path": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/read-pkg-up/node_modules/load-json-file": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-4.0.0.tgz", + "integrity": "sha1-L19Fq5HjMhYjT9U62rZo607AmTs=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "parse-json": "^4.0.0", + "pify": "^3.0.0", + "strip-bom": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/read-pkg-up/node_modules/locate-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", + "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", + "dev": true, + "dependencies": { + "p-locate": "^3.0.0", + "path-exists": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/read-pkg-up/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/read-pkg-up/node_modules/p-locate": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", + "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", + "dev": true, + "dependencies": { + "p-limit": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/read-pkg-up/node_modules/path-exists": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", + "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/read-pkg-up/node_modules/path-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", + "dev": true, + "dependencies": { + "pify": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/read-pkg-up/node_modules/pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/read-pkg-up/node_modules/read-pkg": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-3.0.0.tgz", + "integrity": "sha1-nLxoaXj+5l0WwA4rGcI3/Pbjg4k=", + "dev": true, + "dependencies": { + "load-json-file": "^4.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/read-pkg/node_modules/parse-json": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.1.0.tgz", + "integrity": "sha512-+mi/lmVVNKFNVyLXV31ERiy2CY5E1/F6QtJFEzoChPRwwngMNXRDQ9GJ5WdE2Z2P4AujsOi0/+2qHID68KwfIQ==", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/read-pkg/node_modules/type-fest": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.6.0.tgz", + "integrity": "sha512-q+MB8nYR1KDLrgr4G5yemftpMC7/QLqVndBmEEdqzmNj5dcFOO4Oo8qlwZE3ULT3+Zim1F8Kq4cBnikNhlCMlg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/readdirp": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.4.0.tgz", + "integrity": "sha512-0xe001vZBnJEK+uKcj8qOhyAKPzIT+gStxWr3LCB0DwcXR5NZJ3IaC+yGnHCYzB/S7ov3m3EEbZI2zeNvX+hGQ==", + "dev": true, + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, + "node_modules/rechoir": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.6.2.tgz", + "integrity": "sha1-hSBLVNuoLVdC4oyWdW70OvUOM4Q=", + "dev": true, + "dependencies": { + "resolve": "^1.1.6" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/redent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", + "integrity": "sha512-6tDA8g98We0zd0GvVeMT9arEOnTw9qM03L9cJXaCjrip1OO764RDBLBfrB4cwzNGDj5OA5ioymC9GkizgWJDUg==", + "dev": true, + "dependencies": { + "indent-string": "^4.0.0", + "strip-indent": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/regex-not": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/regex-not/-/regex-not-1.0.2.tgz", + "integrity": "sha512-J6SDjUgDxQj5NusnOtdFxDwN/+HWykR8GELwctJ7mdqhcyy1xEc4SRFHUXvxTp661YaVKAjfRLZ9cCqS6tn32A==", + "dev": true, + "dependencies": { + "extend-shallow": "^3.0.2", + "safe-regex": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/registry-auth-token": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-4.2.0.tgz", + "integrity": "sha512-P+lWzPrsgfN+UEpDS3U8AQKg/UjZX6mQSJueZj3EK+vNESoqBSpBUD3gmu4sF9lOsjXWjF11dQKUqemf3veq1w==", + "dev": true, + "dependencies": { + "rc": "^1.2.8" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/registry-url": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-5.1.0.tgz", + "integrity": "sha512-8acYXXTI0AkQv6RAOjE3vOaIXZkT9wo4LOFbBKYQEEnnMNBpKqdUrI6S4NT0KPIo/WVvJ5tE/X5LF/TQUf0ekw==", + "dev": true, + "dependencies": { + "rc": "^1.2.8" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/release-zalgo": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/release-zalgo/-/release-zalgo-1.0.0.tgz", + "integrity": "sha1-CXALflB0Mpc5Mw5TXFqQ+2eFFzA=", + "dev": true, + "dependencies": { + "es6-error": "^4.0.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/repeat-element": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/repeat-element/-/repeat-element-1.1.3.tgz", + "integrity": "sha512-ahGq0ZnV5m5XtZLMb+vP76kcAM5nkLqk0lpqAuojSKGgQtn4eRi4ZZGm2olo2zKFH+sMsWaqOCW1dqAnOru72g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha1-jcrkcOHIirwtYA//Sndihtp15jc=", + "dev": true, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/repeating": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/repeating/-/repeating-2.0.1.tgz", + "integrity": "sha1-UhTFOpJtNVJwdSf7q0FdvAjQbdo=", + "dev": true, + "dependencies": { + "is-finite": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/require-main-filename": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", + "integrity": "sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==", + "dev": true + }, + "node_modules/resolve": { + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.17.0.tgz", + "integrity": "sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w==", + "dev": true, + "dependencies": { + "path-parse": "^1.0.6" + } + }, + "node_modules/resolve-cwd": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", + "dev": true, + "dependencies": { + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-dir": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/resolve-dir/-/resolve-dir-1.0.1.tgz", + "integrity": "sha1-eaQGRMNivoLybv/nOcm7U4IEb0M=", + "dev": true, + "dependencies": { + "expand-tilde": "^2.0.0", + "global-modules": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-global": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-global/-/resolve-global-1.0.0.tgz", + "integrity": "sha512-zFa12V4OLtT5XUX/Q4VLvTfBf+Ok0SPc1FNGM/z9ctUdiU618qwKpWnd0CHs3+RqROfyEg/DhuHbMWYqcgljEw==", + "dev": true, + "optional": true, + "dependencies": { + "global-dirs": "^0.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-global/node_modules/global-dirs": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/global-dirs/-/global-dirs-0.1.1.tgz", + "integrity": "sha1-sxnA3UYH81PzvpzKTHL8FIxJ9EU=", + "dev": true, + "optional": true, + "dependencies": { + "ini": "^1.3.4" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/resolve-url": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz", + "integrity": "sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=", + "dev": true + }, + "node_modules/responselike": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/responselike/-/responselike-1.0.2.tgz", + "integrity": "sha1-kYcg7ztjHFZCvgaPFa3lpG9Loec=", + "dev": true, + "dependencies": { + "lowercase-keys": "^1.0.0" + } + }, + "node_modules/restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "dev": true, + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ret": { + "version": "0.1.15", + "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz", + "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==", + "dev": true, + "engines": { + "node": ">=0.12" + } + }, + "node_modules/reusify": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", + "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", + "dev": true, + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/right-pad": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/right-pad/-/right-pad-1.0.1.tgz", + "integrity": "sha1-jKCMLLtbVedNr6lr9/0aJ9VoyNA=", + "dev": true, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/run-applescript": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-3.2.0.tgz", + "integrity": "sha512-Ep0RsvAjnRcBX1p5vogbaBdAGu/8j/ewpvGqnQYunnLd9SM0vWcPJewPKNnWFggf0hF0pwIgwV5XK7qQ7UZ8Qg==", + "dev": true, + "dependencies": { + "execa": "^0.10.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/run-async": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", + "dev": true, + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/run-parallel": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz", + "integrity": "sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q==", + "dev": true + }, + "node_modules/rxjs": { + "version": "6.6.0", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.6.0.tgz", + "integrity": "sha512-3HMA8z/Oz61DUHe+SdOiQyzIf4tOx5oQHmMir7IZEu6TMqCLHT4LRcmNaUS0NwOz8VLvmmBduMsoaUvMaIiqzg==", + "dev": true, + "dependencies": { + "tslib": "^1.9.0" + }, + "engines": { + "npm": ">=2.0.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "dev": true + }, + "node_modules/safe-regex": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz", + "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=", + "dev": true, + "dependencies": { + "ret": "~0.1.10" + } + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, + "node_modules/semver": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver-diff": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-3.1.1.tgz", + "integrity": "sha512-GX0Ix/CJcHyB8c4ykpHGIAvLyOwOobtM/8d+TQkAd81/bEjgPHrfba41Vpesr7jX/t8Uh+R3EX9eAS5be+jQYg==", + "dev": true, + "dependencies": { + "semver": "^6.3.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/semver-diff/node_modules/semver": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", + "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "dev": true, + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/serialize-error": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-2.1.0.tgz", + "integrity": "sha1-ULZ51WNc34Rme9yOWa9OW4HV9go=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/set-blocking": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", + "integrity": "sha1-BF+XgtARrppoA93TgrJDkrPYkPc=", + "dev": true + }, + "node_modules/set-value": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz", + "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==", + "dev": true, + "dependencies": { + "extend-shallow": "^2.0.1", + "is-extendable": "^0.1.1", + "is-plain-object": "^2.0.3", + "split-string": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/set-value/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/set-value/node_modules/is-plain-object": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz", + "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==", + "dev": true, + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha1-RKrGW2lbAzmJaMOfNj/uXer98eo=", + "dev": true, + "dependencies": { + "shebang-regex": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha1-2kL0l0DAtC2yypcoVxyxkMmO/qM=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/shell-quote": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/shell-quote/-/shell-quote-1.7.2.tgz", + "integrity": "sha512-mRz/m/JVscCrkMyPqHc/bczi3OQHkLTqXHEFu0zDhK/qfv3UcOA4SVmRCLmos4bhjr9ekVQubj/R7waKapmiQg==", + "dev": true + }, + "node_modules/shelljs": { + "version": "0.8.4", + "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.8.4.tgz", + "integrity": "sha512-7gk3UZ9kOfPLIAbslLzyWeGiEqx9e3rxwZM0KE6EL8GlGwjym9Mrlx5/p33bWTu9YG6vcS4MBxYZDHYr5lr8BQ==", + "dev": true, + "dependencies": { + "glob": "^7.0.0", + "interpret": "^1.0.0", + "rechoir": "^0.6.2" + }, + "bin": { + "shjs": "bin/shjs" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/signal-exit": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.3.tgz", + "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", + "dev": true + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/slice-ansi": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-3.0.0.tgz", + "integrity": "sha512-pSyv7bSTC7ig9Dcgbw9AuRNUb5k5V6oDudjZoMBSr13qpLBG7tB+zgCkARjq7xIUgdz5P1Qe8u+rSGdouOOIyQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "astral-regex": "^2.0.0", + "is-fullwidth-code-point": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/slide": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/slide/-/slide-1.1.6.tgz", + "integrity": "sha1-VusCfWW00tzmyy4tMsTUr8nh1wc=", + "dev": true, + "engines": { + "node": "*" + } + }, + "node_modules/snapdragon": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", + "integrity": "sha512-FtyOnWN/wCHTVXOMwvSv26d+ko5vWlIDD6zoUJ7LW8vh+ZBC8QdljveRP+crNrtBwioEUWy/4dMtbBjA4ioNlg==", + "dev": true, + "dependencies": { + "base": "^0.11.1", + "debug": "^2.2.0", + "define-property": "^0.2.5", + "extend-shallow": "^2.0.1", + "map-cache": "^0.2.2", + "source-map": "^0.5.6", + "source-map-resolve": "^0.5.0", + "use": "^3.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-node": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/snapdragon-node/-/snapdragon-node-2.1.1.tgz", + "integrity": "sha512-O27l4xaMYt/RSQ5TR3vpWCAB5Kb/czIcqUFOM/C4fYcLnbZUc1PkjTAMjof2pBWaSTwOUd6qUHcFGVGj7aIwnw==", + "dev": true, + "dependencies": { + "define-property": "^1.0.0", + "isobject": "^3.0.0", + "snapdragon-util": "^3.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-node/node_modules/define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-1.0.0.tgz", + "integrity": "sha1-dp66rz9KY6rTr56NMEybvnm/sOY=", + "dev": true, + "dependencies": { + "is-descriptor": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-node/node_modules/is-accessor-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-accessor-descriptor/-/is-accessor-descriptor-1.0.0.tgz", + "integrity": "sha512-m5hnHTkcVsPfqx3AKlyttIPb7J+XykHvJP2B9bZDjlhLIoEq4XoK64Vg7boZlVWYK6LUY94dYPEE7Lh0ZkZKcQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-node/node_modules/is-data-descriptor": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-data-descriptor/-/is-data-descriptor-1.0.0.tgz", + "integrity": "sha512-jbRXy1FmtAoCjQkVmIVYwuuqDFUbaOeDjmed1tOGPrsMhtJA4rD9tkgA0F1qJ3gRFRXcHYVkdeaP50Q5rE/jLQ==", + "dev": true, + "dependencies": { + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-node/node_modules/is-descriptor": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-descriptor/-/is-descriptor-1.0.2.tgz", + "integrity": "sha512-2eis5WqQGV7peooDyLmNEPUrps9+SXX5c9pL3xEB+4e9HnGuDa7mB7kHxHw4CbqS9k1T2hOH3miL8n8WtiYVtg==", + "dev": true, + "dependencies": { + "is-accessor-descriptor": "^1.0.0", + "is-data-descriptor": "^1.0.0", + "kind-of": "^6.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-util": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/snapdragon-util/-/snapdragon-util-3.0.1.tgz", + "integrity": "sha512-mbKkMdQKsjX4BAL4bRYTj21edOf8cN7XHdYUJEe+Zn99hVEYcMvKPct1IqNe7+AZPirn8BCDOQBHQZknqmKlZQ==", + "dev": true, + "dependencies": { + "kind-of": "^3.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon-util/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/snapdragon/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/snapdragon/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", + "dev": true + }, + "node_modules/snapdragon/node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/sort-keys": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/sort-keys/-/sort-keys-1.1.2.tgz", + "integrity": "sha1-RBttTTRnmPG05J6JIK37oOVD+a0=", + "dev": true, + "dependencies": { + "is-plain-obj": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-resolve": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/source-map-resolve/-/source-map-resolve-0.5.3.tgz", + "integrity": "sha512-Htz+RnsXWk5+P2slx5Jh3Q66vhQj1Cllm0zvnaY98+NFx+Dv2CF/f5O/t8x+KaNdrdIAsruNzoh/KpialbqAnw==", + "dev": true, + "dependencies": { + "atob": "^2.1.2", + "decode-uri-component": "^0.2.0", + "resolve-url": "^0.2.1", + "source-map-url": "^0.4.0", + "urix": "^0.1.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.19", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.19.tgz", + "integrity": "sha512-Wonm7zOCIJzBGQdB+thsPar0kYuCIzYvxZwlBa87yi/Mdjv7Tip2cyVbLj5o0cFPN4EVkuTwb3GDDyUx2DGnGw==", + "dev": true, + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/source-map-url": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/source-map-url/-/source-map-url-0.4.0.tgz", + "integrity": "sha1-PpNdfd1zYxuXZZlW1VEo6HtQhKM=", + "dev": true + }, + "node_modules/spawn-wrap": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/spawn-wrap/-/spawn-wrap-1.4.3.tgz", + "integrity": "sha512-IgB8md0QW/+tWqcavuFgKYR/qIRvJkRLPJDFaoXtLLUaVcCDK0+HeFTkmQHj3eprcYhc+gOl0aEA1w7qZlYezw==", + "dev": true, + "dependencies": { + "foreground-child": "^1.5.6", + "mkdirp": "^0.5.0", + "os-homedir": "^1.0.1", + "rimraf": "^2.6.2", + "signal-exit": "^3.0.2", + "which": "^1.3.0" + } + }, + "node_modules/spawn-wrap/node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/spdx-correct": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.1.tgz", + "integrity": "sha512-cOYcUWwhCuHCXi49RhFRCyJEK3iPj1Ziz9DpViV3tbZOwXD49QzIN3MpOLJNxh2qwq2lJJZaKMVw9qNi4jTC0w==", + "dev": true, + "dependencies": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-exceptions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.3.0.tgz", + "integrity": "sha512-/tTrYOC7PPI1nUAgx34hUpqXuyJG+DTHJTnIULG4rDygi4xu/tfgmq1e1cIRwRzwZgo4NLySi+ricLkZkw4i5A==", + "dev": true + }, + "node_modules/spdx-expression-parse": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", + "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", + "dev": true, + "dependencies": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-license-ids": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.5.tgz", + "integrity": "sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q==", + "dev": true + }, + "node_modules/split": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/split/-/split-1.0.1.tgz", + "integrity": "sha512-mTyOoPbrivtXnwnIxZRFYRrPNtEFKlpB2fvjSnCQUiAA6qAZzqwna5envK4uk6OIeP17CsdF3rSBGYVBsU0Tkg==", + "dev": true, + "dependencies": { + "through": "2" + }, + "engines": { + "node": "*" + } + }, + "node_modules/split-string": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz", + "integrity": "sha512-NzNVhJDYpwceVVii8/Hu6DKfD2G+NrQHlS/V/qgv763EYudVwEcMQNxd2lh+0VrUByXN/oJkl5grOhYWvQUYiw==", + "dev": true, + "dependencies": { + "extend-shallow": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/split2": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-2.2.0.tgz", + "integrity": "sha512-RAb22TG39LhI31MbreBgIuKiIKhVsawfTgEGqKHTK87aG+ul/PB8Sqoi3I7kVdRWiCfrKxK3uo4/YUkpNvhPbw==", + "dev": true, + "dependencies": { + "through2": "^2.0.2" + } + }, + "node_modules/split2/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/split2/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/split2/node_modules/through2": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", + "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", + "dev": true, + "dependencies": { + "readable-stream": "~2.3.6", + "xtend": "~4.0.1" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", + "dev": true + }, + "node_modules/stack-utils": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.2.tgz", + "integrity": "sha512-0H7QK2ECz3fyZMzQ8rH0j2ykpfbnd20BFtfg/SqVC2+sCTtcw0aDTGB7dk+de4U4uUeuz6nOtJcrkFFLG1B0Rg==", + "dev": true, + "dependencies": { + "escape-string-regexp": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/stack-utils/node_modules/escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/standard-version": { + "resolved": "git+ssh://git@github.com/conventional-changelog/standard-version.git#a1c2053509cde8b79254d69b7d6cbec035dc17f7", + "dev": true, + "dependencies": { + "chalk": "^2.4.2", + "conventional-changelog": "3.1.23", + "conventional-changelog-config-spec": "2.1.0", + "conventional-changelog-conventionalcommits": "4.4.0", + "conventional-recommended-bump": "6.0.10", + "detect-indent": "^6.0.0", + "detect-newline": "^3.1.0", + "dotgitignore": "^2.1.0", + "figures": "^3.1.0", + "find-up": "^4.1.0", + "fs-access": "^1.0.1", + "git-semver-tags": "^4.0.0", + "semver": "^7.1.1", + "stringify-package": "^1.0.1", + "yargs": "^15.3.1" + }, + "bin": { + "standard-version": "bin/cli.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/standard-version/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/standard-version/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/standard-version/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/standard-version/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "node_modules/standard-version/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/standard-version/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/standard-version/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/static-extend": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/static-extend/-/static-extend-0.1.2.tgz", + "integrity": "sha1-YICcOcv/VTNyJv1eC1IPNB8ftcY=", + "dev": true, + "dependencies": { + "define-property": "^0.2.5", + "object-copy": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/static-extend/node_modules/define-property": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/define-property/-/define-property-0.2.5.tgz", + "integrity": "sha1-w1se+RjsPJkPmlvFe+BKrOxcgRY=", + "dev": true, + "dependencies": { + "is-descriptor": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stream-events": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", + "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", + "dev": true, + "dependencies": { + "stubs": "^3.0.0" + } + }, + "node_modules/strict-uri-encode": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", + "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string_decoder/node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + }, + "node_modules/string-width": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", + "integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string.prototype.padend": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string.prototype.padend/-/string.prototype.padend-3.1.0.tgz", + "integrity": "sha512-3aIv8Ffdp8EZj8iLwREGpQaUZiPyrWrpzMBHvkiSW/bK/EGve9np07Vwy7IJ5waydpGXzQZu/F8Oze2/IWkBaA==", + "dev": true, + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.0-next.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/string.prototype.trimend": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz", + "integrity": "sha512-LRPxFUaTtpqYsTeNKaFOw3R4bxIzWOnbQ837QfBylo8jIxtcbK/A/sMV7Q+OAV/vWo+7s25pOE10KYSjaSO06g==", + "dev": true, + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" + } + }, + "node_modules/string.prototype.trimstart": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz", + "integrity": "sha512-XxZn+QpvrBI1FOcg6dIpxUPgWCPuNXvMD72aaRaUQv1eD4e/Qy8i/hFTe0BUmD60p/QA6bh1avmuPTfNjqVWRw==", + "dev": true, + "dependencies": { + "define-properties": "^1.1.3", + "es-abstract": "^1.17.5" + } + }, + "node_modules/stringify-package": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/stringify-package/-/stringify-package-1.0.1.tgz", + "integrity": "sha512-sa4DUQsYciMP1xhKWGuFM04fB0LG/9DlluZoSVywUMRNvzid6XucHK0/90xGxRoHrAaROrcHK1aPKaijCtSrhg==", + "dev": true + }, + "node_modules/strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-bom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", + "integrity": "sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-indent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-3.0.0.tgz", + "integrity": "sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==", + "dev": true, + "dependencies": { + "min-indent": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-outer": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/strip-outer/-/strip-outer-1.0.1.tgz", + "integrity": "sha512-k55yxKHwaXnpYGsOzg4Vl8+tDrWylxDEpknGjhTiZB8dFRU5rTo9CAzeycivxV3s+zlTKwrs6WxMxR95n26kwg==", + "dev": true, + "dependencies": { + "escape-string-regexp": "^1.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/strip-url-auth": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/strip-url-auth/-/strip-url-auth-1.0.1.tgz", + "integrity": "sha1-IrD6OkE4WzO+PzMVUbu4N/oM164=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stubs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", + "integrity": "sha1-6NK6H6nJBXAwPAMLaQD31fiavls=", + "dev": true + }, + "node_modules/supertap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supertap/-/supertap-1.0.0.tgz", + "integrity": "sha512-HZJ3geIMPgVwKk2VsmO5YHqnnJYl6bV5A9JW2uzqV43WmpgliNEYbuvukfor7URpaqpxuw3CfZ3ONdVbZjCgIA==", + "dev": true, + "dependencies": { + "arrify": "^1.0.1", + "indent-string": "^3.2.0", + "js-yaml": "^3.10.0", + "serialize-error": "^2.1.0", + "strip-ansi": "^4.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/supertap/node_modules/ansi-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", + "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/supertap/node_modules/arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha1-iYUI2iIm84DfkEcoRWhJwVAaSw0=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/supertap/node_modules/indent-string": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-3.2.0.tgz", + "integrity": "sha1-Sl/W0nzDMvN+VBmlBNu4NxBckok=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/supertap/node_modules/strip-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", + "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=", + "dev": true, + "dependencies": { + "ansi-regex": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/supports-color": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", + "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", + "dev": true, + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/teeny-request": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-6.0.1.tgz", + "integrity": "sha512-TAK0c9a00ELOqLrZ49cFxvPVogMUFaWY8dUsQc/0CuQPGF+BOxOQzXfE413BAk2kLomwNplvdtMpeaeGWmoc2g==", + "dev": true, + "dependencies": { + "http-proxy-agent": "^4.0.0", + "https-proxy-agent": "^4.0.0", + "node-fetch": "^2.2.0", + "stream-events": "^1.0.5", + "uuid": "^3.3.2" + } + }, + "node_modules/temp-dir": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-2.0.0.tgz", + "integrity": "sha512-aoBAniQmmwtcKp/7BzsH8Cxzv8OL736p7v1ihGb5e9DJ9kTwGWHrQrVB5+lfVDzfGrdRzXch+ig7LHaY1JTOrg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/temp-write": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/temp-write/-/temp-write-2.1.0.tgz", + "integrity": "sha1-WYkJGODvCdVIqqNC9L00CdhATpY=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "mkdirp": "^0.5.0", + "os-tmpdir": "^1.0.0", + "pify": "^2.2.0", + "pinkie-promise": "^2.0.0", + "uuid": "^2.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/temp-write/node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/temp-write/node_modules/uuid": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", + "integrity": "sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho=", + "dev": true + }, + "node_modules/term-size": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/term-size/-/term-size-2.2.0.tgz", + "integrity": "sha512-a6sumDlzyHVJWb8+YofY4TW112G6p2FCPEAFk+59gIYHv3XHRhm9ltVQ9kli4hNWeQBwSpe8cRN25x0ROunMOw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/test-exclude": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-5.2.3.tgz", + "integrity": "sha512-M+oxtseCFO3EDtAaGH7iiej3CBkzXqFMbzqYAACdzKui4eZA+pq3tZEwChvOdNfa7xxy8BfbmgJSIr43cC/+2g==", + "dev": true, + "dependencies": { + "glob": "^7.1.3", + "minimatch": "^3.0.4", + "read-pkg-up": "^4.0.0", + "require-main-filename": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/text-extensions": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-1.9.0.tgz", + "integrity": "sha512-wiBrwC1EhBelW12Zy26JeOUkQ5mRu+5o8rpsJk5+2t+Y5vE7e842qtZDQ2g1NpX/29HdyFeJ4nSIhI47ENSxlQ==", + "dev": true, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/through": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", + "dev": true + }, + "node_modules/through2": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-3.0.2.tgz", + "integrity": "sha512-enaDQ4MUyP2W6ZyT6EsMzqBPZaM/avg8iuo+l2d3QCs0J+6RaqkHV/2/lOwDTueBHeJ/2LG9lrLW3d5rWPucuQ==", + "dev": true, + "dependencies": { + "inherits": "^2.0.4", + "readable-stream": "2 || 3" + } + }, + "node_modules/time-zone": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/time-zone/-/time-zone-1.0.0.tgz", + "integrity": "sha1-mcW/VZWJZq9tBtg73zgA3IL67F0=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/timed-out": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/timed-out/-/timed-out-3.1.3.tgz", + "integrity": "sha1-lYYL/MXHbCd/j4Mm/Q9bLiDrohc=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tmp": { + "version": "0.0.33", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", + "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", + "dev": true, + "dependencies": { + "os-tmpdir": "~1.0.2" + }, + "engines": { + "node": ">=0.6.0" + } + }, + "node_modules/to-fast-properties": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", + "integrity": "sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/to-object-path": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz", + "integrity": "sha1-KXWIt7Dn4KwI4E5nL4XB9JmeF68=", + "dev": true, + "dependencies": { + "kind-of": "^3.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/to-object-path/node_modules/kind-of": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz", + "integrity": "sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=", + "dev": true, + "dependencies": { + "is-buffer": "^1.1.5" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/to-readable-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-1.0.0.tgz", + "integrity": "sha512-Iq25XBt6zD5npPhlLVXGFN3/gyR2/qODcKNNyTMd4vbm39HUaOiAM4PMq0eMVC/Tkxz+Zjdsc55g9yyz+Yq00Q==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/to-regex": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/to-regex/-/to-regex-3.0.2.tgz", + "integrity": "sha512-FWtleNAtZ/Ki2qtqej2CXTOayOH9bHDQF+Q48VpWyDXjbYxA4Yz8iDB31zXOBUlOHHKidDbqGVrTUvQMPmBGBw==", + "dev": true, + "dependencies": { + "define-property": "^2.0.2", + "extend-shallow": "^3.0.2", + "regex-not": "^1.0.2", + "safe-regex": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/trash": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/trash/-/trash-4.3.0.tgz", + "integrity": "sha512-f36TKwIaBiXm63xSrn8OTNghg5CYHBsFVJvcObMo76LRpgariuRi2CqXQHw1VzfeximD0igdGaonOG6N760BtQ==", + "dev": true, + "dependencies": { + "escape-string-applescript": "^2.0.0", + "fs-extra": "^0.30.0", + "globby": "^7.1.1", + "p-map": "^1.2.0", + "p-try": "^1.0.0", + "pify": "^3.0.0", + "run-applescript": "^3.0.0", + "uuid": "^3.1.0", + "xdg-trashdir": "^2.1.1" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/trash-cli": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/trash-cli/-/trash-cli-1.4.0.tgz", + "integrity": "sha1-MojYkMgkpcyXimxEip8ymwa+Bp0=", + "dev": true, + "dependencies": { + "meow": "^3.7.0", + "trash": "^4.0.0", + "update-notifier": "^1.0.2" + }, + "bin": { + "trash": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/trash-cli/node_modules/ansi-align": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/ansi-align/-/ansi-align-1.1.0.tgz", + "integrity": "sha1-LwwWWIKXOa3V67FeawxuNCPwFro=", + "dev": true, + "dependencies": { + "string-width": "^1.0.1" + } + }, + "node_modules/trash-cli/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/boxen": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/boxen/-/boxen-0.6.0.tgz", + "integrity": "sha1-g2TUJIrDT/DvGy8r9JpsYM4NgbY=", + "dev": true, + "dependencies": { + "ansi-align": "^1.1.0", + "camelcase": "^2.1.0", + "chalk": "^1.1.1", + "cli-boxes": "^1.0.0", + "filled-array": "^1.0.0", + "object-assign": "^4.0.1", + "repeating": "^2.0.0", + "string-width": "^1.0.1", + "widest-line": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/camelcase": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz", + "integrity": "sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/camelcase-keys": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz", + "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=", + "dev": true, + "dependencies": { + "camelcase": "^2.0.0", + "map-obj": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "dev": true, + "dependencies": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/cli-boxes": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-1.0.0.tgz", + "integrity": "sha1-T6kXw+WclKAEzWH47lCdplFocUM=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/configstore": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/configstore/-/configstore-2.1.0.tgz", + "integrity": "sha1-c3o6cDbpiGECqmCZ5HuzOrGroaE=", + "dev": true, + "dependencies": { + "dot-prop": "^3.0.0", + "graceful-fs": "^4.1.2", + "mkdirp": "^0.5.0", + "object-assign": "^4.0.1", + "os-tmpdir": "^1.0.0", + "osenv": "^0.1.0", + "uuid": "^2.0.1", + "write-file-atomic": "^1.1.2", + "xdg-basedir": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/dot-prop": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/dot-prop/-/dot-prop-3.0.0.tgz", + "integrity": "sha1-G3CK8JSknJoOfbyteQq6U52sEXc=", + "dev": true, + "dependencies": { + "is-obj": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/find-up": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz", + "integrity": "sha1-ay6YIrGizgpgq2TWEOzK1TyyTQ8=", + "dev": true, + "dependencies": { + "path-exists": "^2.0.0", + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/get-stdin": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz", + "integrity": "sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/got": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/got/-/got-5.7.1.tgz", + "integrity": "sha1-X4FjWmHkplifGAVp6k44FoClHzU=", + "dev": true, + "dependencies": { + "create-error-class": "^3.0.1", + "duplexer2": "^0.1.4", + "is-redirect": "^1.0.0", + "is-retry-allowed": "^1.0.0", + "is-stream": "^1.0.0", + "lowercase-keys": "^1.0.0", + "node-status-codes": "^1.0.0", + "object-assign": "^4.0.1", + "parse-json": "^2.1.0", + "pinkie-promise": "^2.0.0", + "read-all-stream": "^3.0.0", + "readable-stream": "^2.0.5", + "timed-out": "^3.0.0", + "unzip-response": "^1.0.2", + "url-parse-lax": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0 <7" + } + }, + "node_modules/trash-cli/node_modules/indent-string": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-2.1.0.tgz", + "integrity": "sha1-ji1INIdCEhtKghi3oTfppSBJ3IA=", + "dev": true, + "dependencies": { + "repeating": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/is-fullwidth-code-point": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", + "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "dev": true, + "dependencies": { + "number-is-nan": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/is-npm": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz", + "integrity": "sha1-8vtjpl5JBbQGyGBydloaTceTufQ=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/is-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/is-obj/-/is-obj-1.0.1.tgz", + "integrity": "sha1-PkcprB9f3gJc19g6iW2rn09n2w8=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/latest-version": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/latest-version/-/latest-version-2.0.0.tgz", + "integrity": "sha1-VvjWE5YghHuAF/jx9NeOIRMkFos=", + "dev": true, + "dependencies": { + "package-json": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/load-json-file": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz", + "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "parse-json": "^2.2.0", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0", + "strip-bom": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/map-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz", + "integrity": "sha1-2TPOuSBdgr3PSIb2dCvcK03qFG0=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/meow": { + "version": "3.7.0", + "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz", + "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=", + "dev": true, + "dependencies": { + "camelcase-keys": "^2.0.0", + "decamelize": "^1.1.2", + "loud-rejection": "^1.0.0", + "map-obj": "^1.0.1", + "minimist": "^1.1.3", + "normalize-package-data": "^2.3.4", + "object-assign": "^4.0.1", + "read-pkg-up": "^1.0.1", + "redent": "^1.0.0", + "trim-newlines": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/package-json": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/package-json/-/package-json-2.4.0.tgz", + "integrity": "sha1-DRW9Z9HLvduyyiIv8u24a8sxqLs=", + "dev": true, + "dependencies": { + "got": "^5.0.0", + "registry-auth-token": "^3.0.1", + "registry-url": "^3.0.3", + "semver": "^5.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/parse-json": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz", + "integrity": "sha1-9ID0BDTvgHQfhGkJn43qGPVaTck=", + "dev": true, + "dependencies": { + "error-ex": "^1.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/path-exists": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz", + "integrity": "sha1-D+tsZPD8UY2adU3V77YscCJ2H0s=", + "dev": true, + "dependencies": { + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/path-type": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz", + "integrity": "sha1-WcRPfuSR2nBNpBXaWkBwuk+P5EE=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "pify": "^2.0.0", + "pinkie-promise": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/prepend-http": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/prepend-http/-/prepend-http-1.0.4.tgz", + "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/read-pkg": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz", + "integrity": "sha1-9f+qXs0pyzHAR0vKfXVra7KePyg=", + "dev": true, + "dependencies": { + "load-json-file": "^1.0.0", + "normalize-package-data": "^2.3.2", + "path-type": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/read-pkg-up": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz", + "integrity": "sha1-nWPBMnbAZZGNV/ACpX9AobZD+wI=", + "dev": true, + "dependencies": { + "find-up": "^1.0.0", + "read-pkg": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dev": true, + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/trash-cli/node_modules/redent": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/redent/-/redent-1.0.0.tgz", + "integrity": "sha1-z5Fqsf1fHxbfsggi3W7H9zDCr94=", + "dev": true, + "dependencies": { + "indent-string": "^2.1.0", + "strip-indent": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/registry-auth-token": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/registry-auth-token/-/registry-auth-token-3.4.0.tgz", + "integrity": "sha512-4LM6Fw8eBQdwMYcES4yTnn2TqIasbXuwDx3um+QRs7S55aMKCBKBxvPXl2RiUjHwuJLTyYfxSpmfSAjQpcuP+A==", + "dev": true, + "dependencies": { + "rc": "^1.1.6", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/trash-cli/node_modules/registry-url": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/registry-url/-/registry-url-3.1.0.tgz", + "integrity": "sha1-PU74cPc93h138M+aOBQyRE4XSUI=", + "dev": true, + "dependencies": { + "rc": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/trash-cli/node_modules/semver-diff": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-2.1.0.tgz", + "integrity": "sha1-S7uEN8jTfksM8aaP1ybsbWRdbTY=", + "dev": true, + "dependencies": { + "semver": "^5.0.3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/trash-cli/node_modules/string-width": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", + "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "dev": true, + "dependencies": { + "code-point-at": "^1.0.0", + "is-fullwidth-code-point": "^1.0.0", + "strip-ansi": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/strip-bom": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", + "integrity": "sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4=", + "dev": true, + "dependencies": { + "is-utf8": "^0.2.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/strip-indent": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-1.0.1.tgz", + "integrity": "sha1-DHlipq3vp7vUrDZkYKY4VSrhoKI=", + "dev": true, + "dependencies": { + "get-stdin": "^4.0.1" + }, + "bin": { + "strip-indent": "cli.js" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", + "dev": true, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/trash-cli/node_modules/trim-newlines": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz", + "integrity": "sha1-WIeWa7WCpFA6QetST301ARgVphM=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/update-notifier": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-1.0.3.tgz", + "integrity": "sha1-j5LFFUgr1oMbfJMBPnD4dVLHz1o=", + "dev": true, + "dependencies": { + "boxen": "^0.6.0", + "chalk": "^1.0.0", + "configstore": "^2.0.0", + "is-npm": "^1.0.0", + "latest-version": "^2.0.0", + "lazy-req": "^1.1.0", + "semver-diff": "^2.0.0", + "xdg-basedir": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/url-parse-lax": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-1.0.0.tgz", + "integrity": "sha1-evjzA2Rem9eaJy56FKxovAYJ2nM=", + "dev": true, + "dependencies": { + "prepend-http": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/uuid": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-2.0.3.tgz", + "integrity": "sha1-Z+LoY3lyFVMN/zGOW/nc6/1Hsho=", + "dev": true + }, + "node_modules/trash-cli/node_modules/widest-line": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-1.0.0.tgz", + "integrity": "sha1-DAnIXCqUaD0Nfq+O4JfVZL8OEFw=", + "dev": true, + "dependencies": { + "string-width": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash-cli/node_modules/write-file-atomic": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-1.3.4.tgz", + "integrity": "sha1-+Aek8LHZ6ROuekgRLmzDrxmRtF8=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.11", + "imurmurhash": "^0.1.4", + "slide": "^1.1.5" + } + }, + "node_modules/trash-cli/node_modules/xdg-basedir": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-2.0.0.tgz", + "integrity": "sha1-7byQPMOF/ARSPZZqM1UEtVBNG9I=", + "dev": true, + "dependencies": { + "os-homedir": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash/node_modules/array-union": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/array-union/-/array-union-1.0.2.tgz", + "integrity": "sha1-mjRBDk9OPaI96jdb5b5w8kd47Dk=", + "dev": true, + "dependencies": { + "array-uniq": "^1.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trash/node_modules/dir-glob": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-2.2.2.tgz", + "integrity": "sha512-f9LBi5QWzIW3I6e//uxZoLBlUt9kcp66qo0sSCxL6YZKc75R1c4MFCoe/LaZiBGmgujvQdxc5Bn3QhfyvK5Hsw==", + "dev": true, + "dependencies": { + "path-type": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/trash/node_modules/fs-extra": { + "version": "0.30.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-0.30.0.tgz", + "integrity": "sha1-8jP/zAjU2n1DLapEl3aYnbHfk/A=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.2", + "jsonfile": "^2.1.0", + "klaw": "^1.0.0", + "path-is-absolute": "^1.0.0", + "rimraf": "^2.2.8" + } + }, + "node_modules/trash/node_modules/globby": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/globby/-/globby-7.1.1.tgz", + "integrity": "sha1-+yzP+UAfhgCUXfral0QMypcrhoA=", + "dev": true, + "dependencies": { + "array-union": "^1.0.1", + "dir-glob": "^2.0.0", + "glob": "^7.1.2", + "ignore": "^3.3.5", + "pify": "^3.0.0", + "slash": "^1.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/trash/node_modules/ignore": { + "version": "3.3.10", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-3.3.10.tgz", + "integrity": "sha512-Pgs951kaMm5GXP7MOvxERINe3gsaVjUWFm+UZPSq9xYriQAksyhg0csnS0KXSNRD5NmNdapXEpjxG49+AKh/ug==", + "dev": true + }, + "node_modules/trash/node_modules/jsonfile": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz", + "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=", + "dev": true, + "dependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/trash/node_modules/p-map": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-1.2.0.tgz", + "integrity": "sha512-r6zKACMNhjPJMTl8KcFH4li//gkrXWfbD6feV8l6doRHlzljFWGJ2AP6iKaCJXyZmAUMOPtvbW7EXkbWO/pLEA==", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/trash/node_modules/p-try": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", + "integrity": "sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/trash/node_modules/path-type": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-3.0.0.tgz", + "integrity": "sha512-T2ZUsdZFHgA3u4e5PfPbjd7HDDpxPnQb5jN0SrDsjNSuVXHJqtwTnWqG0B1jZrgmJ/7lj1EmVIByWt1gxGkWvg==", + "dev": true, + "dependencies": { + "pify": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/trash/node_modules/pify": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-3.0.0.tgz", + "integrity": "sha1-5aSs0sEB/fPZpNB/DbxNtJ3SgXY=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/trash/node_modules/rimraf": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", + "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", + "dev": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + } + }, + "node_modules/trash/node_modules/slash": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-1.0.0.tgz", + "integrity": "sha1-xB8vbDn8FtHNF61LXYlhFK5HDVU=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trim-newlines": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-3.0.0.tgz", + "integrity": "sha512-C4+gOpvmxaSMKuEf9Qc134F1ZuOHVXKRbtEflf4NTtuuJDEIJ9p5PXsalL8SkeRw+qit1Mo+yuvMPAKwWg/1hA==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/trim-off-newlines": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz", + "integrity": "sha1-n5up2e+odkw4dpi8v+sshI8RrbM=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/trim-repeated": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/trim-repeated/-/trim-repeated-1.0.0.tgz", + "integrity": "sha1-42RqLqTokTEr9+rObPsFOAvAHCE=", + "dev": true, + "dependencies": { + "escape-string-regexp": "^1.0.2" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tslib": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.13.0.tgz", + "integrity": "sha512-i/6DQjL8Xf3be4K/E6Wgpekn5Qasl1usyw++dAA35Ue5orEn65VIxOA+YvNNl9HV3qv70T7CNwjODHZrLwvd1Q==", + "dev": true + }, + "node_modules/tslint": { + "version": "5.20.1", + "resolved": "https://registry.npmjs.org/tslint/-/tslint-5.20.1.tgz", + "integrity": "sha1-5AHortoBUrxE3QfmFANPP4DGe30=", + "dev": true, + "dependencies": { + "@babel/code-frame": "^7.0.0", + "builtin-modules": "^1.1.1", + "chalk": "^2.3.0", + "commander": "^2.12.1", + "diff": "^4.0.1", + "glob": "^7.1.1", + "js-yaml": "^3.13.1", + "minimatch": "^3.0.4", + "mkdirp": "^0.5.1", + "resolve": "^1.3.2", + "semver": "^5.3.0", + "tslib": "^1.8.0", + "tsutils": "^2.29.0" + }, + "bin": { + "tslint": "bin/tslint" + }, + "engines": { + "node": ">=4.8.0" + } + }, + "node_modules/tslint-config-prettier": { + "version": "1.18.0", + "resolved": "https://registry.npmjs.org/tslint-config-prettier/-/tslint-config-prettier-1.18.0.tgz", + "integrity": "sha512-xPw9PgNPLG3iKRxmK7DWr+Ea/SzrvfHtjFt5LBl61gk2UBG/DB9kCXRjv+xyIU1rUtnayLeMUVJBcMX8Z17nDg==", + "dev": true, + "bin": { + "tslint-config-prettier-check": "bin/check.js" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/tslint-immutable": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/tslint-immutable/-/tslint-immutable-4.9.1.tgz", + "integrity": "sha512-iIFCq08H4YyNIX0bV5N6fGQtAmjc4OQZKQCgBP5WHgQaITyGAHPVmAw+Yf7qe0zbRCvCDZdrdEC/191fLGFiww==", + "dev": true + }, + "node_modules/tslint/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/tslint/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/tslint/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/tslint/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=", + "dev": true + }, + "node_modules/tslint/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", + "dev": true, + "engines": { + "node": ">=4" + } + }, + "node_modules/tslint/node_modules/semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "dev": true, + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/tslint/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/tsutils": { + "version": "2.29.0", + "resolved": "https://registry.npmjs.org/tsutils/-/tsutils-2.29.0.tgz", + "integrity": "sha1-MrSIUBRnrL7dS4VJhnOggSrKC5k=", + "dev": true, + "dependencies": { + "tslib": "^1.8.1" + } + }, + "node_modules/type-fest": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.3.1.tgz", + "integrity": "sha512-cUGJnCdr4STbePCgqNFbpVNCepa+kAVohJs1sLhxzdH+gnEoOd8VhbYa7pD3zZYGiURWM2xzEII3fQcRizDkYQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=", + "dev": true + }, + "node_modules/typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "dev": true, + "dependencies": { + "is-typedarray": "^1.0.0" + } + }, + "node_modules/typedoc": { + "version": "0.17.8", + "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.17.8.tgz", + "integrity": "sha512-/OyrHCJ8jtzu+QZ+771YaxQ9s4g5Z3XsQE3Ma7q+BL392xxBn4UMvvCdVnqKC2T/dz03/VXSLVKOP3lHmDdc/w==", + "dev": true, + "dependencies": { + "fs-extra": "^8.1.0", + "handlebars": "^4.7.6", + "highlight.js": "^10.0.0", + "lodash": "^4.17.15", + "lunr": "^2.3.8", + "marked": "1.0.0", + "minimatch": "^3.0.0", + "progress": "^2.0.3", + "shelljs": "^0.8.4", + "typedoc-default-themes": "^0.10.2" + }, + "bin": { + "typedoc": "bin/typedoc" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/typedoc-default-themes": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/typedoc-default-themes/-/typedoc-default-themes-0.10.2.tgz", + "integrity": "sha512-zo09yRj+xwLFE3hyhJeVHWRSPuKEIAsFK5r2u47KL/HBKqpwdUSanoaz5L34IKiSATFrjG5ywmIu98hPVMfxZg==", + "dev": true, + "dependencies": { + "lunr": "^2.3.8" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/typescript": { + "version": "3.9.7", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.7.tgz", + "integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, + "node_modules/uglify-js": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.10.0.tgz", + "integrity": "sha512-Esj5HG5WAyrLIdYU74Z3JdG2PxdIusvj6IWHMtlyESxc7kcDz7zYlYjpnSokn1UbpV0d/QX9fan7gkCNd/9BQA==", + "dev": true, + "optional": true, + "bin": { + "uglifyjs": "bin/uglifyjs" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/union-value": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz", + "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==", + "dev": true, + "dependencies": { + "arr-union": "^3.1.0", + "get-value": "^2.0.6", + "is-extendable": "^0.1.1", + "set-value": "^2.0.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unique-string": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unique-string/-/unique-string-2.0.0.tgz", + "integrity": "sha512-uNaeirEPvpZWSgzwsPGtU2zVSTrn/8L5q/IexZmH0eH6SA73CmAA5U4GwORTxQAZs95TAXLNqeLoPPNO5gZfWg==", + "dev": true, + "dependencies": { + "crypto-random-string": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "dev": true, + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/unset-value": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unset-value/-/unset-value-1.0.0.tgz", + "integrity": "sha1-g3aHP30jNRef+x5vw6jtDfyKtVk=", + "dev": true, + "dependencies": { + "has-value": "^0.3.1", + "isobject": "^3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unset-value/node_modules/has-value": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/has-value/-/has-value-0.3.1.tgz", + "integrity": "sha1-ex9YutpiyoJ+wKIHgCVlSEWZXh8=", + "dev": true, + "dependencies": { + "get-value": "^2.0.3", + "has-values": "^0.1.4", + "isobject": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unset-value/node_modules/has-value/node_modules/isobject": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-2.1.0.tgz", + "integrity": "sha1-8GVWEJaj8dou9GJy+BXIQNh+DIk=", + "dev": true, + "dependencies": { + "isarray": "1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unset-value/node_modules/has-values": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/has-values/-/has-values-0.1.4.tgz", + "integrity": "sha1-bWHeldkd/Km5oCCJrThL/49it3E=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/unzip-response": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/unzip-response/-/unzip-response-1.0.2.tgz", + "integrity": "sha1-uYTwh3/AqJwsdzzB73tbIytbBv4=", + "dev": true, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/update-notifier": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/update-notifier/-/update-notifier-4.1.1.tgz", + "integrity": "sha512-9y+Kds0+LoLG6yN802wVXoIfxYEwh3FlZwzMwpCZp62S2i1/Jzeqb9Eeeju3NSHccGGasfGlK5/vEHbAifYRDg==", + "dev": true, + "dependencies": { + "boxen": "^4.2.0", + "chalk": "^3.0.0", + "configstore": "^5.0.1", + "has-yarn": "^2.1.0", + "import-lazy": "^2.1.0", + "is-ci": "^2.0.0", + "is-installed-globally": "^0.3.1", + "is-npm": "^4.0.0", + "is-yarn-global": "^0.3.0", + "latest-version": "^5.0.0", + "pupa": "^2.0.1", + "semver-diff": "^3.1.1", + "xdg-basedir": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/update-notifier/node_modules/chalk": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-3.0.0.tgz", + "integrity": "sha512-4D3B6Wf41KOYRFdszmDqMCGq5VV/uMAB273JILmO+3jAlh8X4qDtdtgCR3fxtbLEMzSx22QdhnDcJvu2u1fVwg==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/urix": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz", + "integrity": "sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=", + "dev": true + }, + "node_modules/url-parse-lax": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/url-parse-lax/-/url-parse-lax-3.0.0.tgz", + "integrity": "sha1-FrXK/Afb42dsGxmZF3gj1lA6yww=", + "dev": true, + "dependencies": { + "prepend-http": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/urlgrey": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/urlgrey/-/urlgrey-0.4.4.tgz", + "integrity": "sha1-iS/pWWCAXoVRnxzUOJ8stMu3ZS8=", + "dev": true + }, + "node_modules/use": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/use/-/use-3.1.1.tgz", + "integrity": "sha512-cwESVXlO3url9YWlFW/TA9cshCEhtu7IKJ/p5soJ/gGpj7vbvFrAY/eIioQ6Dw23KjZhYgiIo8HOs1nQ2vr/oQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/user-home": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/user-home/-/user-home-2.0.0.tgz", + "integrity": "sha1-nHC/2Babwdy/SGBODwS4tJzenp8=", + "dev": true, + "dependencies": { + "os-homedir": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", + "dev": true + }, + "node_modules/uuid": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", + "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", + "dev": true, + "bin": { + "uuid": "bin/uuid" + } + }, + "node_modules/validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "dev": true, + "dependencies": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "node_modules/wcwidth": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", + "integrity": "sha1-8LDc+RW8X/FSivrbLA4XtTLaL+g=", + "dev": true, + "dependencies": { + "defaults": "^1.0.3" + } + }, + "node_modules/well-known-symbols": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz", + "integrity": "sha512-ZMjC3ho+KXo0BfJb7JgtQ5IBuvnShdlACNkKkdsqBmYw3bPAaJfPeYUo6tLUaT5tG/Gkh7xkpBhKRQ9e7pyg9Q==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dev": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "which": "bin/which" + } + }, + "node_modules/which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha1-2e8H3Od7mQK4o6j6SzHD4/fm6Ho=", + "dev": true + }, + "node_modules/widest-line": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-3.1.0.tgz", + "integrity": "sha512-NsmoXalsWVDMGupxZ5R08ka9flZjjiLvHVAWYOKtiKM8ujtZWr9cRffak+uSE48+Ob8ObalXpwyeUiyDD6QFgg==", + "dev": true, + "dependencies": { + "string-width": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha1-J1hIEIkUVqQXHI0CJkQa3pDLyus=", + "dev": true + }, + "node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + }, + "node_modules/write-file-atomic": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", + "integrity": "sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q==", + "dev": true, + "dependencies": { + "imurmurhash": "^0.1.4", + "is-typedarray": "^1.0.0", + "signal-exit": "^3.0.2", + "typedarray-to-buffer": "^3.1.5" + } + }, + "node_modules/xdg-basedir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz", + "integrity": "sha512-PSNhEJDejZYV7h50BohL09Er9VaIefr2LMAf3OEmpCkjOi34eYyQYAXUTjEQtZJTKcF0E2UKTh+osDLsgNim9Q==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/xdg-trashdir": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/xdg-trashdir/-/xdg-trashdir-2.1.1.tgz", + "integrity": "sha512-KcVhPaOu2ZurYNHSRTf1+ZHORkTZGCQ+u0JHN17QixRISJq4pXOnjt/lQcehvtHL5QAKhSzKgyjrcNnPdkPBHA==", + "dev": true, + "dependencies": { + "@sindresorhus/df": "^2.1.0", + "mount-point": "^3.0.0", + "pify": "^2.2.0", + "user-home": "^2.0.0", + "xdg-basedir": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/xdg-trashdir/node_modules/pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/xdg-trashdir/node_modules/xdg-basedir": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-2.0.0.tgz", + "integrity": "sha1-7byQPMOF/ARSPZZqM1UEtVBNG9I=", + "dev": true, + "dependencies": { + "os-homedir": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "dev": true, + "engines": { + "node": ">=0.4" + } + }, + "node_modules/y18n": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", + "integrity": "sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==", + "dev": true + }, + "node_modules/yallist": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz", + "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=", + "dev": true + }, + "node_modules/yaml": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz", + "integrity": "sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg==", + "dev": true, + "optional": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/yargs": { + "version": "15.4.1", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.4.1.tgz", + "integrity": "sha512-aePbxDmcYW++PaqBsJ+HYUFwCdv4LVvdnhBy78E57PIor8/OVvhMrADFFEDh8DHDFRv/O9i3lPhsENjO7QX0+A==", + "dev": true, + "dependencies": { + "cliui": "^6.0.0", + "decamelize": "^1.2.0", + "find-up": "^4.1.0", + "get-caller-file": "^2.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^2.0.0", + "set-blocking": "^2.0.0", + "string-width": "^4.2.0", + "which-module": "^2.0.0", + "y18n": "^4.0.0", + "yargs-parser": "^18.1.2" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs-parser": { + "version": "18.1.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", + "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", + "dev": true, + "dependencies": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/yargs/node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "engines": { + "node": ">=6" + } + }, + "node_modules/yargs/node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/yargs/node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/yargs/node_modules/yargs-parser": { + "version": "18.1.3", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-18.1.3.tgz", + "integrity": "sha512-o50j0JeToy/4K6OZcaQmW6lyXXKhq7csREXcDwk2omFPJEwUNOVtJKvmDr9EI1fAJZUyZcRF7kxGBWmRXudrCQ==", + "dev": true, + "dependencies": { + "camelcase": "^5.0.0", + "decamelize": "^1.2.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/yargs/node_modules/yargs-parser/node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha1-9lNNFRSCabIDUue+4m9QH5oZEpA=", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + } + }, "dependencies": { "@babel/code-frame": { "version": "7.10.4", @@ -374,16 +11860,6 @@ "dev": true, "optional": true }, - "JSONStream": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", - "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", - "dev": true, - "requires": { - "jsonparse": "^1.2.0", - "through": ">=2.2.7 <3" - } - }, "acorn": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.0.1.tgz", @@ -1661,8 +13137,8 @@ "integrity": "sha512-RSo5S0WIwXZiRxUGTPuYFbqvrR4vpJ1BDdTlthFgvHt5kEdnd1+pdvwWphWn57/oIl4V72NMmOocFqqJ8mFFhA==", "dev": true, "requires": { - "JSONStream": "^1.0.4", "is-text-path": "^1.0.1", + "JSONStream": "^1.0.4", "lodash": "^4.17.15", "meow": "^7.0.0", "split2": "^2.0.0", @@ -4604,6 +16080,16 @@ "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=", "dev": true }, + "JSONStream": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", + "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", + "dev": true, + "requires": { + "jsonparse": "^1.2.0", + "through": ">=2.2.7 <3" + } + }, "keyv": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/keyv/-/keyv-3.1.0.tgz", @@ -5442,7 +16928,6 @@ }, "find-up": { "version": "3.0.0", - "resolved": false, "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", "dev": true, "requires": { @@ -5457,7 +16942,6 @@ }, "locate-path": { "version": "3.0.0", - "resolved": false, "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", "dev": true, "requires": { @@ -5486,7 +16970,6 @@ }, "p-locate": { "version": "3.0.0", - "resolved": false, "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", "dev": true, "requires": { @@ -5501,7 +16984,6 @@ }, "resolve-from": { "version": "4.0.0", - "resolved": false, "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", "dev": true }, @@ -7506,9 +18988,9 @@ } }, "standard-version": { - "version": "github:conventional-changelog/standard-version#a1c2053509cde8b79254d69b7d6cbec035dc17f7", - "from": "github:conventional-changelog/standard-version#master", + "version": "git+ssh://git@github.com/conventional-changelog/standard-version.git#a1c2053509cde8b79254d69b7d6cbec035dc17f7", "dev": true, + "from": "standard-version@github:conventional-changelog/standard-version#master", "requires": { "chalk": "^2.4.2", "conventional-changelog": "3.1.23", @@ -7625,6 +19107,23 @@ "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=", "dev": true }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "requires": { + "safe-buffer": "~5.2.0" + }, + "dependencies": { + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + } + } + }, "string-width": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz", @@ -7666,23 +19165,6 @@ "es-abstract": "^1.17.5" } }, - "string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", - "dev": true, - "requires": { - "safe-buffer": "~5.2.0" - }, - "dependencies": { - "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true - } - } - }, "stringify-package": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/stringify-package/-/stringify-package-1.0.1.tgz", @@ -8443,6 +19925,15 @@ "semver": "^5.0.3" } }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dev": true, + "requires": { + "safe-buffer": "~5.1.0" + } + }, "string-width": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", @@ -8454,15 +19945,6 @@ "strip-ansi": "^3.0.0" } }, - "string_decoder": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", - "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", - "dev": true, - "requires": { - "safe-buffer": "~5.1.0" - } - }, "strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", @@ -8653,11 +20135,6 @@ "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", "dev": true }, - "js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha1-GSA/tZmR35jjoocFDUZHzerzJJk=" - }, "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", diff --git a/openwpm/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py index a6a4a57a3..8be156ba2 100644 --- a/openwpm/storage/arrow_storage.py +++ b/openwpm/storage/arrow_storage.py @@ -2,18 +2,19 @@ import logging import random from abc import abstractmethod +from collections import defaultdict from typing import Any, DefaultDict, Dict, List import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -from automation.types import VisitId from pyarrow import Table +from openwpm.types import VisitId + from .parquet_schema import PQ_SCHEMAS from .storage_providers import StructuredStorageProvider, TableName -SITE_VISITS_INDEX = TableName("_site_visits_index") INCOMPLETE_VISITS = TableName("incomplete_visits") CACHE_SIZE = 500 @@ -28,19 +29,19 @@ def __init__(self) -> None: self.logger = logging.getLogger("openwpm") def factory_function() -> DefaultDict[TableName, List[Dict[str, Any]]]: - return DefaultDict(list) + return defaultdict(list) # Raw records per VisitId and Table self._records: DefaultDict[ VisitId, DefaultDict[TableName, List[Dict[str, Any]]] - ] = DefaultDict(factory_function) + ] = defaultdict(factory_function) # Record batches by TableName - self._batches: DefaultDict[TableName, List[pa.RecordBatch]] = DefaultDict(list) + self._batches: DefaultDict[TableName, List[pa.RecordBatch]] = defaultdict(list) # Used to synchronize the finalizing and the flushing self.storing_condition = asyncio.Condition() - self.is_flushing = False + self.flushing = False self._instance_id = random.getrandbits(32) @@ -82,11 +83,6 @@ def _create_batch(self, visit_id: VisitId) -> None: exc_info=True, ) pass - # We construct a special index file from the site_visits data - # to make it easier to query the dataset - if table_name == "site_visits": - for item in data: - self._batches[SITE_VISITS_INDEX].append(item) del self._records[visit_id] @@ -107,6 +103,7 @@ async def finalize_visit_id( # 2. No new batches should be created while saving out all the batches async with self.storing_condition: if self.flushing: + # This way we wait if there is an on going flush await self.storing_condition.wait() self._create_batch(visit_id) @@ -117,8 +114,6 @@ async def finalize_visit_id( else: await self.storing_condition.wait() - raise NotImplementedError() - @abstractmethod async def write_table(self, table_name: TableName, table: Table) -> None: """Write out the table to persistent storage @@ -134,7 +129,7 @@ async def flush_cache(self, cond: asyncio.Condition = None) -> None: got_cond = not not cond if not got_cond: cond = self.storing_condition - cond.acquire() + await cond.acquire() assert cond.locked() for table_name, batches in self._batches.items(): table = pa.Table.from_batches(batches) diff --git a/test/conftest.py b/test/conftest.py index be555ccfd..131254d90 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -49,8 +49,8 @@ def default_params(num_browsers, tmpdir) -> Tuple[ManagerParams, List[BrowserPar manager_params["data_directory"] = data_dir manager_params["log_directory"] = data_dir for i in range(num_browsers): - browser_params[i]["display_mode"] = display_mode - manager_params["db"] = join( - manager_params["data_directory"], manager_params["database_name"] + browser_params[i]["display_mode"] = "headless" + manager_params["db"] = os.sep.join( + [manager_params["data_directory"], manager_params["database_name"]] ) return manager_params, browser_params diff --git a/test/openwpm_jstest.py b/test/openwpm_jstest.py index 235633123..d7365c154 100644 --- a/test/openwpm_jstest.py +++ b/test/openwpm_jstest.py @@ -1,6 +1,7 @@ import re -from ..openwpm.utilities import db_utils +from openwpm.utilities import db_utils + from .openwpmtest import OpenWPMTest diff --git a/test/storage/test_memory_storage_provider.py b/test/storage/test_memory_storage_provider.py index 47632cc40..865a84a17 100644 --- a/test/storage/test_memory_storage_provider.py +++ b/test/storage/test_memory_storage_provider.py @@ -23,7 +23,6 @@ memory_unstructured = "memory_unstructured" leveldb = "leveldb" - structured_scenarios: List[Tuple[str, Dict[str, Any]]] = [ (memory_structured, {"structured_provider": memory_structured}), (sqllite, {"structured_provider": sqllite}), @@ -74,8 +73,10 @@ async def test_basic_access( await structured_provider.store_record( TableName("site_visits"), VisitId(2), data ) - await structured_provider.finalize_visit_id(VisitId(2)) - await structured_provider.flush_cache() + await asyncio.gather( + structured_provider.finalize_visit_id(VisitId(2)), + structured_provider.flush_cache(), + ) @pytest.mark.asyncio @@ -86,4 +87,4 @@ async def test_basic_unstructured_storing(self) -> None: test_string = "This is my test string" blob = test_string.encode() prov = MemoryUnstructuredProvider() - prov.store_blob("test", blob, compressed=False) + await prov.store_blob("test", blob, compressed=False) From 12a60a08a00c8ce52c4eda530a214f33d22d343c Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 25 Nov 2020 18:09:05 +0100 Subject: [PATCH 034/139] Starting to rewrite tests --- demo.py | 9 +++--- openwpm/storage/parquet_schema.py | 1 + openwpm/storage/schema.sql | 5 ++-- openwpm/storage/storage_controller.py | 2 +- openwpm/task_manager.py | 4 +-- test/conftest.py | 41 +++++++++++++++++++++++---- test/openwpmtest.py | 14 +-------- test/test_timer.py | 32 +++++++++------------ 8 files changed, 62 insertions(+), 46 deletions(-) diff --git a/demo.py b/demo.py index b17a17a02..9d57ab2fe 100644 --- a/demo.py +++ b/demo.py @@ -1,9 +1,10 @@ import logging import os -from openwpm import command_sequence, task_manager +from openwpm.command_sequence import CommandSequence from openwpm.storage.in_memory_storage import MemoryUnstructuredProvider from openwpm.storage.sql_provider import SqlLiteStorageProvider +from openwpm.task_manager import TaskManager, load_default_params # The list of sites that we wish to crawl NUM_BROWSERS = 1 @@ -15,7 +16,7 @@ # Loads the default manager params # and NUM_BROWSERS copies of the default browser params -manager_params, browser_params = task_manager.load_default_params(NUM_BROWSERS) +manager_params, browser_params = load_default_params(NUM_BROWSERS) # Update browser configuration (use this for per-browser settings) for i in range(NUM_BROWSERS): @@ -44,7 +45,7 @@ # Instantiates the measurement platform # Commands time out by default after 60 seconds -manager = task_manager.TaskManager( +manager = TaskManager( manager_params, browser_params, SqlLiteStorageProvider( @@ -57,7 +58,7 @@ for site in sites: # Parallelize sites over all number of browsers set above. - command_sequence = command_sequence.CommandSequence( + command_sequence = CommandSequence( site, reset=True, callback=lambda success, val=site: print("CommandSequence {} done".format(val)), diff --git a/openwpm/storage/parquet_schema.py b/openwpm/storage/parquet_schema.py index 1d6155068..dbd6dab1d 100644 --- a/openwpm/storage/parquet_schema.py +++ b/openwpm/storage/parquet_schema.py @@ -23,6 +23,7 @@ pa.field("command_status", pa.string()), pa.field("error", pa.string()), pa.field("traceback", pa.string()), + pa.field("duration", pa.int64()), ] PQ_SCHEMAS["crawl_history"] = pa.schema(fields) diff --git a/openwpm/storage/schema.sql b/openwpm/storage/schema.sql index 742aa8316..35b1f9434 100644 --- a/openwpm/storage/schema.sql +++ b/openwpm/storage/schema.sql @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS site_visits ( browser_id INTEGER NOT NULL, site_url VARCHAR(500) NOT NULL, site_rank INTEGER, - FOREIGN KEY(browser_id) REFERENCES crawl(id)); + FOREIGN KEY(browser_id) REFERENCES crawl(browser_id)); /* # crawl_history @@ -39,8 +39,9 @@ CREATE TABLE IF NOT EXISTS crawl_history ( command_status TEXT, error TEXT, traceback TEXT, + duration INTEGER, dtg DATETIME DEFAULT (CURRENT_TIMESTAMP), - FOREIGN KEY(browser_id) REFERENCES crawl(id)); + FOREIGN KEY(browser_id) REFERENCES crawl(browser_id)); /* # http_requests diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 794e59da0..542af7205 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -293,7 +293,7 @@ class StorageControllerHandle: def __init__( self, structured_storage: StructuredStorageProvider, - unstructured_storage: UnstructuredStorageProvider, + unstructured_storage: Optional[UnstructuredStorageProvider], ) -> None: self.listener_address: Optional[Tuple[str, int]] = None diff --git a/openwpm/task_manager.py b/openwpm/task_manager.py index 5386693f2..77aa90297 100644 --- a/openwpm/task_manager.py +++ b/openwpm/task_manager.py @@ -79,7 +79,7 @@ def __init__( manager_params: ManagerParams, browser_params: List[BrowserParams], structured_storage_provider: StructuredStorageProvider, - unstructured_storage_provider: UnstructuredStorageProvider, + unstructured_storage_provider: Optional[UnstructuredStorageProvider], logger_kwargs: Dict[Any, Any] = {}, ) -> None: """Initialize the TaskManager with browser and manager config params @@ -282,7 +282,7 @@ def _manager_watchdog(self) -> None: def _launch_aggregators( self, structured_storage_provider: StructuredStorageProvider, - unstructured_storage_provider: UnstructuredStorageProvider, + unstructured_storage_provider: Optional[UnstructuredStorageProvider], ) -> None: """Launch the necessary data aggregators""" self.data_aggregator_handle = StorageControllerHandle( diff --git a/test/conftest.py b/test/conftest.py index 1fccf79f7..61ca9a491 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,10 +1,15 @@ +import logging import os import subprocess -from typing import List, Tuple +from pathlib import Path +from typing import Callable, List, Tuple import pytest from openwpm import task_manager +from openwpm.commands import browser_commands +from openwpm.storage.sql_provider import SqlLiteStorageProvider +from openwpm.task_manager import TaskManager from openwpm.types import BrowserParams, ManagerParams from . import utilities @@ -42,15 +47,39 @@ def prepare_test_setup(request): @pytest.fixture() -def default_params(num_browsers, tmpdir) -> Tuple[ManagerParams, List[BrowserParams]]: +def default_params( + tmp_path: Path, num_browsers: int = 2 +) -> Tuple[ManagerParams, List[BrowserParams]]: """Just a simple wrapper around task_manager.load_default_params""" - data_dir = tmpdir + data_dir = str(tmp_path) manager_params, browser_params = task_manager.load_default_params(num_browsers) manager_params["data_directory"] = data_dir manager_params["log_directory"] = data_dir for i in range(num_browsers): browser_params[i]["display_mode"] = "headless" - manager_params["db"] = os.sep.join( - [manager_params["data_directory"], manager_params["database_name"]] - ) + manager_params["db"] = str(tmp_path / manager_params["database_name"]) return manager_params, browser_params + + +@pytest.fixture() +def task_manager_creator() -> Callable[ + [Tuple[ManagerParams, List[BrowserParams]]], TaskManager +]: + """We create a callable that returns a TaskManager that has + been configured with the Manager and BrowserParams""" + + def _create_task_manager( + params: Tuple[ManagerParams, List[BrowserParams]] + ) -> TaskManager: + manager_params, browser_params = params + structured_provider = SqlLiteStorageProvider(manager_params["db"]) + manager = task_manager.TaskManager( + manager_params, + browser_params, + structured_provider, + None, + # logger_kwargs={"log_level_console": logging.DEBUG}, + ) + return manager + + return _create_task_manager diff --git a/test/openwpmtest.py b/test/openwpmtest.py index 89758c3e8..6e931342e 100644 --- a/test/openwpmtest.py +++ b/test/openwpmtest.py @@ -13,7 +13,7 @@ class OpenWPMTest: - NUM_BROWSERS = 1 + NUM_BROWSERS = 2 @pytest.fixture(autouse=True) def set_tmpdir(self, tmpdir): @@ -58,15 +58,3 @@ def get_test_config( manager_params["data_directory"], manager_params["database_name"] ) return manager_params, browser_params - - def is_installed(self, cmd): - """Check if a program is available via the standard PATH lookup.""" - path = os.environ["PATH"].split(os.pathsep) - for d in path: - candidate = join(d, cmd) - if isfile(candidate) and os.access(candidate, os.X_OK): - return True - return False - - def assert_is_installed(self, cmd): - assert self.is_installed(cmd), "Cannot find %s in your system" % cmd diff --git a/test/test_timer.py b/test/test_timer.py index 7edda62b2..f3203b85c 100644 --- a/test/test_timer.py +++ b/test/test_timer.py @@ -8,23 +8,19 @@ TEST_URL = BASE_TEST_URL + "/" + TEST_FILE -class TestCommandDuration(OpenWPMTest): - def get_config(self, data_dir=""): - return self.get_test_config(data_dir) +def test_command_duration(default_params, task_manager_creator): + manager_params = default_params[0] + manager = task_manager_creator(default_params) + manager.get(url=TEST_URL, sleep=5) + manager.close() - def test_command_duration(self): - manager_params, browser_params = self.get_config() - manager = task_manager.TaskManager(manager_params, browser_params) - manager.get(url=TEST_URL, sleep=5) - manager.close() + get_command = db_utils.query_db( + manager_params["db"], + "SELECT duration FROM crawl_history WHERE command = 'GetCommand'", + as_tuple=True, + )[0] - get_command = db_utils.query_db( - manager_params["db"], - "SELECT duration FROM crawl_history WHERE command = \"\"", - as_tuple=True, - )[0] - - assert get_command[0] > (5 * 1000) # milliseconds conversion for sleep time - assert get_command[0] <= ( - (5 * 1000) + 2 * 1000 - ) # milliseconds conversion for sleep time + time duration a command took (milliseconds) + assert get_command[0] > (5 * 1000) # milliseconds conversion for sleep time + assert get_command[0] <= ( + (5 * 1000) + 2 * 1000 + ) # milliseconds conversion for sleep time + time duration a command took (milliseconds) From 84bff66065495745dc2e399ac4404535e26129e7 Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 25 Nov 2020 18:52:01 +0100 Subject: [PATCH 035/139] Setting up fixtures --- test/conftest.py | 21 +++---- test/manual_test.py | 2 +- test/openwpmtest.py | 4 ++ test/test_callback.py | 32 +++++------ test/test_callstack_instrument.py | 94 +++++++++++++++---------------- 5 files changed, 72 insertions(+), 81 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 61ca9a491..46361bc08 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -23,21 +23,16 @@ ) -def create_xpi(): +@pytest.fixture(scope="session") +def xpi(): # Creates a new xpi using npm run build. print("Building new xpi") subprocess.check_call(["npm", "run", "build"], cwd=EXTENSION_DIR) -@pytest.fixture(scope="session", autouse=True) -def prepare_test_setup(request): +@pytest.fixture(scope="session") +def server(request): """Run an HTTP server during the tests.""" - - if "pyonly" in request.config.invocation_params.args: - return - - create_xpi() - print("Starting local_http_server") server, server_thread = utilities.start_server() yield @@ -48,7 +43,7 @@ def prepare_test_setup(request): @pytest.fixture() def default_params( - tmp_path: Path, num_browsers: int = 2 + tmp_path: Path, num_browsers: int = 1 ) -> Tuple[ManagerParams, List[BrowserParams]]: """Just a simple wrapper around task_manager.load_default_params""" data_dir = str(tmp_path) @@ -62,9 +57,9 @@ def default_params( @pytest.fixture() -def task_manager_creator() -> Callable[ - [Tuple[ManagerParams, List[BrowserParams]]], TaskManager -]: +def task_manager_creator( + server, xpi +) -> Callable[[Tuple[ManagerParams, List[BrowserParams]]], TaskManager]: """We create a callable that returns a TaskManager that has been configured with the Manager and BrowserParams""" diff --git a/test/manual_test.py b/test/manual_test.py index 1627b6f23..53b624f2e 100644 --- a/test/manual_test.py +++ b/test/manual_test.py @@ -132,7 +132,7 @@ def cleanup_server(): with open(browser_params_file, "r") as f: browser_params.update(json.loads(f.read())) js_request = browser_params["js_instrument_settings"] - js_request_as_string = jsi.convert_browser_params_to_js_string(js_request) + js_request_as_string = jsi.clean_js_instrumentation_settings(js_request) browser_params["js_instrument_settings"] = js_request_as_string profile_dir = driver.capabilities["moz:profile"] diff --git a/test/openwpmtest.py b/test/openwpmtest.py index 6e931342e..ed66fea07 100644 --- a/test/openwpmtest.py +++ b/test/openwpmtest.py @@ -1,5 +1,6 @@ import logging import os +from abc import ABCMeta, abstractmethod from os.path import isfile, join from typing import List, Tuple @@ -24,6 +25,9 @@ def set_tmpdir(self, tmpdir): """ self.tmpdir = str(tmpdir) + def get_config(self, data_dir) -> Tuple[ManagerParams, List[BrowserParams]]: + pass + def visit(self, page_url, data_dir="", sleep_after=0): """Visit a test page with the given parameters.""" manager_params, browser_params = self.get_config(data_dir) diff --git a/test/test_callback.py b/test/test_callback.py index 51ad396c7..70953befc 100644 --- a/test/test_callback.py +++ b/test/test_callback.py @@ -8,27 +8,21 @@ from .utilities import BASE_TEST_URL -class TestCallbackCommand(OpenWPMTest): +def test_local_callbacks(default_params, task_manager_creator) -> None: """Test test the Aggregators as well as the entire callback machinery to see if all callbacks get correctly called""" + manager = task_manager_creator(default_params) + TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html" - def get_config(self, data_dir=""): - return self.get_test_config(data_dir) + def callback(argument: List[int], success: bool) -> None: + argument.extend([1, 2, 3]) - def test_local_callbacks(self) -> None: - manager_params, browser_params = self.get_config() - TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html" - manager = TaskManager(manager_params, browser_params) + my_list: List[int] = [] + sequence = CommandSequence( + TEST_SITE, reset=True, blocking=True, callback=partial(callback, my_list) + ) + sequence.get() - def callback(argument: List[int], success: bool) -> None: - argument.extend([1, 2, 3]) - - my_list: List[int] = [] - sequence = CommandSequence( - TEST_SITE, reset=True, blocking=True, callback=partial(callback, my_list) - ) - sequence.get() - - manager.execute_command_sequence(sequence) - manager.close() - assert my_list == [1, 2, 3] + manager.execute_command_sequence(sequence) + manager.close() + assert my_list == [1, 2, 3] diff --git a/test/test_callstack_instrument.py b/test/test_callstack_instrument.py index f394808d7..f8c88f1dd 100644 --- a/test/test_callstack_instrument.py +++ b/test/test_callstack_instrument.py @@ -27,9 +27,11 @@ "onload@" + HTTP_STACKTRACE_TEST_URL + ":1:1;null" ) -HTTP_STACKTRACES = set( - (STACK_TRACE_INJECT_IMAGE, STACK_TRACE_INJECT_PIXEL, STACK_TRACE_INJECT_JS) -) +HTTP_STACKTRACES = { + STACK_TRACE_INJECT_IMAGE, + STACK_TRACE_INJECT_PIXEL, + STACK_TRACE_INJECT_JS, +} # parsed HTTP call stack dict CALL_STACK_INJECT_IMAGE = [ { @@ -56,53 +58,49 @@ ] -class TestCallstackInstrument(OpenWPMTest): - def get_config(self, data_dir=""): - manager_params, browser_params = self.get_test_config(data_dir) - # Record HTTP Requests and Responses - browser_params[0]["http_instrument"] = True - # Record JS Web API calls - browser_params[0]["js_instrument"] = True - # Record the callstack of all WebRequests made - browser_params[0]["callstack_instrument"] = True - return manager_params, browser_params - - def test_http_stacktrace(self): - test_url = utilities.BASE_TEST_URL + "/http_stacktrace.html" - manager_params, browser_params = self.get_config() - manager = task_manager.TaskManager(manager_params, browser_params) - manager.get(test_url, sleep=10) - db = manager_params["db"] - manager.close() - rows = db_utils.query_db( - db, - ( - "SELECT hr.url, c.call_stack" - " FROM callstacks c" - " JOIN http_requests hr" - " ON c.request_id=hr.request_id" - " AND c.visit_id= hr.visit_id" - " AND c.browser_id = hr.browser_id;" - ), +def test_http_stacktrace(default_params, task_manager_creator): + manager_params, browser_params = default_params + # Record HTTP Requests and Responses + browser_params[0]["http_instrument"] = True + # Record JS Web API calls + browser_params[0]["js_instrument"] = True + # Record the callstack of all WebRequests made + browser_params[0]["callstack_instrument"] = True + test_url = utilities.BASE_TEST_URL + "/http_stacktrace.html" + manager = task_manager_creator(manager_params, browser_params) + manager.get(test_url, sleep=10) + db = manager_params["db"] + manager.close() + rows = db_utils.query_db( + db, + ( + "SELECT hr.url, c.call_stack" + " FROM callstacks c" + " JOIN http_requests hr" + " ON c.request_id=hr.request_id" + " AND c.visit_id= hr.visit_id" + " AND c.browser_id = hr.browser_id;" + ), + ) + print("Printing callstacks contents") + observed_records = set() + for row in rows: + print(row["call_stack"]) + url, call_stack = row + test_urls = ( + "inject_pixel.js", + "test_image.png", + "Blank.gif", ) - print("Printing callstacks contents") - observed_records = set() - for row in rows: - print(row["call_stack"]) - url, call_stack = row - test_urls = ( - "inject_pixel.js", - "test_image.png", - "Blank.gif", - ) - if url.endswith(test_urls): - observed_records.add(call_stack) - assert HTTP_STACKTRACES == observed_records + if url.endswith(test_urls): + observed_records.add(call_stack) + assert HTTP_STACKTRACES == observed_records + - def test_parse_http_stack_trace_str(self): - stacktrace = STACK_TRACE_INJECT_IMAGE - stack_frames = parse_http_stack_trace_str(stacktrace) - assert stack_frames == CALL_STACK_INJECT_IMAGE +def test_parse_http_stack_trace_str(): + stacktrace = STACK_TRACE_INJECT_IMAGE + stack_frames = parse_http_stack_trace_str(stacktrace) + assert stack_frames == CALL_STACK_INJECT_IMAGE # TODO: webext instrumentation doesn't support req_call_stack yet. # def test_http_stacktrace_nonjs_loads(self): From 4eb5c23f2061023d590aaaf67da5caa0fb46c693 Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 25 Nov 2020 21:17:04 +0100 Subject: [PATCH 036/139] Attempting to fix all the tests --- demo.py | 1 - test/conftest.py | 5 +- test/openwpmtest.py | 10 +- test/storage/test_memory_storage_provider.py | 4 +- test/test_custom_function_command.py | 100 ++- test/test_dns_instrument.py | 29 +- test/test_env.py | 11 - test/test_extension.py | 2 +- test/test_js_instrument_py.py | 10 +- test/test_mp_logger.py | 208 ++--- test/test_profile.py | 74 +- test/test_simple_commands.py | 818 ++++++++++--------- test/test_storage_vectors.py | 67 +- test/test_webdriver_utils.py | 32 +- 14 files changed, 687 insertions(+), 684 deletions(-) delete mode 100644 test/test_env.py diff --git a/demo.py b/demo.py index 9d57ab2fe..3a8266822 100644 --- a/demo.py +++ b/demo.py @@ -2,7 +2,6 @@ import os from openwpm.command_sequence import CommandSequence -from openwpm.storage.in_memory_storage import MemoryUnstructuredProvider from openwpm.storage.sql_provider import SqlLiteStorageProvider from openwpm.task_manager import TaskManager, load_default_params diff --git a/test/conftest.py b/test/conftest.py index 46361bc08..b027070e3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -7,12 +7,12 @@ import pytest from openwpm import task_manager -from openwpm.commands import browser_commands from openwpm.storage.sql_provider import SqlLiteStorageProvider from openwpm.task_manager import TaskManager from openwpm.types import BrowserParams, ManagerParams from . import utilities +from .openwpmtest import NUM_BROWSERS EXTENSION_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), @@ -43,7 +43,7 @@ def server(request): @pytest.fixture() def default_params( - tmp_path: Path, num_browsers: int = 1 + tmp_path: Path, num_browsers: int = NUM_BROWSERS ) -> Tuple[ManagerParams, List[BrowserParams]]: """Just a simple wrapper around task_manager.load_default_params""" data_dir = str(tmp_path) @@ -73,7 +73,6 @@ def _create_task_manager( browser_params, structured_provider, None, - # logger_kwargs={"log_level_console": logging.DEBUG}, ) return manager diff --git a/test/openwpmtest.py b/test/openwpmtest.py index ed66fea07..c3f0b0f15 100644 --- a/test/openwpmtest.py +++ b/test/openwpmtest.py @@ -12,10 +12,11 @@ from . import utilities +NUM_BROWSERS = 2 -class OpenWPMTest: - NUM_BROWSERS = 2 +@pytest.mark.usefixtures("xpi", "server") +class OpenWPMTest: @pytest.fixture(autouse=True) def set_tmpdir(self, tmpdir): """Create a tmpdir fixture to be used in `get_test_config`. @@ -33,10 +34,7 @@ def visit(self, page_url, data_dir="", sleep_after=0): manager_params, browser_params = self.get_config(data_dir) structured_provider = SqlLiteStorageProvider(manager_params["db"]) manager = task_manager.TaskManager( - manager_params, - browser_params, - structured_provider, - logger_kwargs={"log_level_console": logging.DEBUG}, + manager_params, browser_params, structured_provider, None ) if not page_url.startswith("http"): page_url = utilities.BASE_TEST_URL + page_url diff --git a/test/storage/test_memory_storage_provider.py b/test/storage/test_memory_storage_provider.py index 865a84a17..97fcb128b 100644 --- a/test/storage/test_memory_storage_provider.py +++ b/test/storage/test_memory_storage_provider.py @@ -57,7 +57,7 @@ def pytest_generate_tests(metafunc: Any) -> Any: @pytest.mark.asyncio -class TestStructuredStorageProvider(OpenWPMTest): +class TestStructuredStorageProvider: scenarios = structured_scenarios async def test_basic_access( @@ -80,7 +80,7 @@ async def test_basic_access( @pytest.mark.asyncio -class TestUnstructuredStorageProvide(OpenWPMTest): +class TestUnstructuredStorageProvide: scenarios: List[Tuple[str, Dict[str, Any]]] = [(memory_unstructured, {})] async def test_basic_unstructured_storing(self) -> None: diff --git a/test/test_custom_function_command.py b/test/test_custom_function_command.py index 9c7cab40e..c1e509904 100644 --- a/test/test_custom_function_command.py +++ b/test/test_custom_function_command.py @@ -22,64 +22,58 @@ } -class TestCustomFunctionCommand(OpenWPMTest): - """Test `custom_function` command's ability to handle inline functions""" +def test_custom_function(default_params, task_manager_creator): + """ Test `custom_function` with an inline func that collects links """ - def get_config(self, data_dir=""): - return self.get_test_config(data_dir) + from openwpm.socket_interface import ClientSocket - def test_custom_function(self): - """ Test `custom_function` with an inline func that collects links """ - - from openwpm.socket_interface import ClientSocket + def collect_links(table_name, scheme, **kwargs): + """ Collect links with `scheme` and save in table `table_name` """ + driver = kwargs["driver"] + manager_params = kwargs["manager_params"] + browser_id = kwargs["command"].browser_id + visit_id = kwargs["command"].visit_id + link_urls = [ + x + for x in ( + element.get_attribute("href") + for element in driver.find_elements_by_tag_name("a") + ) + if x.startswith(scheme + "://") + ] + current_url = driver.current_url - def collect_links(table_name, scheme, **kwargs): - """ Collect links with `scheme` and save in table `table_name` """ - driver = kwargs["driver"] - manager_params = kwargs["manager_params"] - browser_id = kwargs["command"].browser_id - visit_id = kwargs["command"].visit_id - link_urls = [ - x - for x in ( - element.get_attribute("href") - for element in driver.find_elements_by_tag_name("a") - ) - if x.startswith(scheme + "://") - ] - current_url = driver.current_url + sock = ClientSocket() + sock.connect(*manager_params["aggregator_address"]) - sock = ClientSocket() - sock.connect(*manager_params["aggregator_address"]) + query = ( + "CREATE TABLE IF NOT EXISTS %s (" + "top_url TEXT, link TEXT, " + "visit_id INTEGER, browser_id INTEGER);" % table_name + ) + sock.send(("create_table", query)) + for link in link_urls: query = ( - "CREATE TABLE IF NOT EXISTS %s (" - "top_url TEXT, link TEXT, " - "visit_id INTEGER, browser_id INTEGER);" % table_name + table_name, + { + "top_url": current_url, + "link": link, + "visit_id": visit_id, + "browser_id": browser_id, + }, ) - sock.send(("create_table", query)) + sock.send(query) + sock.close() - for link in link_urls: - query = ( - table_name, - { - "top_url": current_url, - "link": link, - "visit_id": visit_id, - "browser_id": browser_id, - }, - ) - sock.send(query) - sock.close() - - manager_params, browser_params = self.get_config() - manager = task_manager.TaskManager(manager_params, browser_params) - cs = command_sequence.CommandSequence(url_a) - cs.get(sleep=0, timeout=60) - cs.run_custom_function(collect_links, ("page_links", "http")) - manager.execute_command_sequence(cs) - manager.close() - query_result = db_utils.query_db( - manager_params["db"], "SELECT top_url, link FROM page_links;", as_tuple=True - ) - assert PAGE_LINKS == set(query_result) + manager_params, browser_params = default_params + manager = task_manager_creator(default_params) + cs = command_sequence.CommandSequence(url_a) + cs.get(sleep=0, timeout=60) + cs.run_custom_function(collect_links, ("page_links", "http")) + manager.execute_command_sequence(cs) + manager.close() + query_result = db_utils.query_db( + manager_params["db"], "SELECT top_url, link FROM page_links;", as_tuple=True + ) + assert PAGE_LINKS == set(query_result) diff --git a/test/test_dns_instrument.py b/test/test_dns_instrument.py index 222f1a4de..4399c1ed1 100644 --- a/test/test_dns_instrument.py +++ b/test/test_dns_instrument.py @@ -1,21 +1,18 @@ from openwpm.utilities import db_utils -from .openwpmtest import OpenWPMTest +def test_name_resolution(default_params, task_manager_creator): + manager_params, browser_params = default_params + for browser_param in browser_params: + browser_param["dns_instrument"] = True -class TestDNSInstrument(OpenWPMTest): - def get_config(self, data_dir=""): - manager_params, browser_params = self.get_test_config(data_dir) - for browser_param in browser_params: - browser_param["dns_instrument"] = True - return manager_params, browser_params + manager = task_manager_creator((manager_params, browser_params)) + manager.get("http://localtest.me:8000") + manager.close() - def test_name_resolution(self): - db = self.visit("http://localtest.me:8000") - result = db_utils.query_db(db, "SELECT * FROM dns_responses") - result = result[0] - print(result.keys()) - assert result["used_address"] == "127.0.0.1" - assert result["addresses"] == "127.0.0.1" - assert result["hostname"] == "localtest.me" - assert result["canonical_name"] == "localtest.me" + result = db_utils.query_db(manager_params["db"], "SELECT * FROM dns_responses") + result = result[0] + assert result["used_address"] == "127.0.0.1" + assert result["addresses"] == "127.0.0.1" + assert result["hostname"] == "localtest.me" + assert result["canonical_name"] == "localtest.me" diff --git a/test/test_env.py b/test/test_env.py deleted file mode 100644 index d9189133a..000000000 --- a/test/test_env.py +++ /dev/null @@ -1,11 +0,0 @@ -from os.path import isfile - -from openwpm.utilities.platform_utils import get_firefox_binary_path - -from .openwpmtest import OpenWPMTest - - -class TestDependencies(OpenWPMTest): - def test_dependencies(self): - firefox_binary_path = get_firefox_binary_path() - assert isfile(firefox_binary_path) diff --git a/test/test_extension.py b/test/test_extension.py index 64f82ef30..c69229f32 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -244,7 +244,7 @@ u"test_cookie=Test-0123456789; " "expires=Tue, 31 Dec 2030 00:00:00 UTC; path=/", ) -DOCUMENT_COOKIE_READ_WRITE = set([DOCUMENT_COOKIE_READ, DOCUMENT_COOKIE_WRITE]) +DOCUMENT_COOKIE_READ_WRITE = {DOCUMENT_COOKIE_READ, DOCUMENT_COOKIE_WRITE} class TestExtension(OpenWPMTest): diff --git a/test/test_js_instrument_py.py b/test/test_js_instrument_py.py index 33be8c4af..eafc6e3f3 100644 --- a/test/test_js_instrument_py.py +++ b/test/test_js_instrument_py.py @@ -1,19 +1,17 @@ +""" +Test function that converts our python +objects to our JS string +""" import pytest from jsonschema.exceptions import ValidationError from openwpm import js_instrumentation as jsi -pytestmark = pytest.mark.pyonly - def _no_whitespace(x): return "".join(x.split()) -# Test function that converts our python -# objects to our JS string - - def test_python_to_js_lower_true_false(): inpy = [ { diff --git a/test/test_mp_logger.py b/test/test_mp_logger.py index 43e595c67..e37bf53dd 100644 --- a/test/test_mp_logger.py +++ b/test/test_mp_logger.py @@ -78,105 +78,109 @@ def child_proc_logging_exception(): ) -class TestMPLogger(OpenWPMTest): - def get_logfile_path(self, directory): - return os.path.join(directory, "mplogger.log") - - def get_logfile_contents(self, logfile): - with open(logfile, "r") as f: - content = f.read().strip() - return content - - def test_multiprocess(self, tmpdir): - # Set up loggingserver - log_file = self.get_logfile_path(str(tmpdir)) - openwpm_logger = mp_logger.MPLogger(log_file) - - child_process_1 = Process(target=child_proc, args=(0,)) - child_process_1.daemon = True - child_process_1.start() - child_process_2 = Process(target=child_proc, args=(1,)) - child_process_2.daemon = True - child_process_2.start() - - # Send some sample logs - logger.info(PARENT_INFO_STR_1) - logger.error(PARENT_ERROR_STR) - logger.critical(PARENT_CRITICAL_STR) - logger.debug(PARENT_DEBUG_STR) - logger.warning(PARENT_WARNING_STR) - - logger1 = logging.getLogger("test1") - logger2 = logging.getLogger("test2") - logger1.info(NAMED_LOGGER_INFO_1) - logger2.info(NAMED_LOGGER_INFO_2) - - # Close the logging server - time.sleep(2) # give some time for logs to be sent - openwpm_logger.close() - child_process_1.join() - child_process_2.join() - print("Child processes joined...") - - log_content = self.get_logfile_contents(log_file) - for child in range(2): - assert log_content.count(CHILD_INFO_STR_1 % child) == 1 - assert log_content.count(CHILD_INFO_STR_2 % child) == 1 - assert log_content.count(CHILD_ERROR_STR % child) == 1 - assert log_content.count(CHILD_CRITICAL_STR % child) == 1 - assert log_content.count(CHILD_DEBUG_STR % child) == 1 - assert log_content.count(CHILD_WARNING_STR % child) == 1 - assert log_content.count(PARENT_INFO_STR_1) == 1 - assert log_content.count(PARENT_ERROR_STR) == 1 - assert log_content.count(PARENT_CRITICAL_STR) == 1 - assert log_content.count(PARENT_DEBUG_STR) == 1 - assert log_content.count(PARENT_WARNING_STR) == 1 - - def test_multiple_instances(self, tmpdir): - os.makedirs(str(tmpdir) + "-1") - self.test_multiprocess(str(tmpdir) + "-1") - os.makedirs(str(tmpdir) + "-2") - self.test_multiprocess(str(tmpdir) + "-2") - - @pytest.mark.skipif( - "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", - reason="Flaky on Travis CI", - ) - def test_child_process_with_exception(self, tmpdir): - log_file = self.get_logfile_path(str(tmpdir)) - openwpm_logger = mp_logger.MPLogger(log_file) - - child_process_1 = Process(target=child_proc_with_exception, args=(0,)) - child_process_1.daemon = True - child_process_1.start() - child_process_2 = Process(target=child_proc_with_exception, args=(1,)) - child_process_2.daemon = True - child_process_2.start() - - # Close the logging server - time.sleep(2) # give some time for logs to be sent - child_process_1.join() - child_process_2.join() - print("Child processes joined...") - openwpm_logger.close() - - log_content = self.get_logfile_contents(log_file) - for child in range(2): - assert log_content.count(CHILD_INFO_STR_1 % child) == 1 - assert log_content.count(CHILD_INFO_STR_2 % child) == 1 - assert log_content.count(CHILD_EXCEPTION_STR % child) == 1 - - @pytest.mark.skipif( - "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", - reason="Flaky on Travis CI", - ) - def test_child_process_logging(self, tmpdir): - log_file = self.get_logfile_path(str(tmpdir)) - openwpm_logger = mp_logger.MPLogger(log_file) - child_process = Process(target=child_proc_logging_exception()) - child_process.daemon = True - child_process.start() - openwpm_logger.close() - child_process.join() - log_content = self.get_logfile_contents(log_file) - assert "I'm logging an exception" in log_content +def get_logfile_path(directory): + return os.path.join(directory, "mplogger.log") + + +def get_logfile_contents(logfile): + with open(logfile, "r") as f: + content = f.read().strip() + return content + + +def test_multiprocess(tmpdir): + # Set up loggingserver + log_file = get_logfile_path(str(tmpdir)) + openwpm_logger = mp_logger.MPLogger(log_file) + + child_process_1 = Process(target=child_proc, args=(0,)) + child_process_1.daemon = True + child_process_1.start() + child_process_2 = Process(target=child_proc, args=(1,)) + child_process_2.daemon = True + child_process_2.start() + + # Send some sample logs + logger.info(PARENT_INFO_STR_1) + logger.error(PARENT_ERROR_STR) + logger.critical(PARENT_CRITICAL_STR) + logger.debug(PARENT_DEBUG_STR) + logger.warning(PARENT_WARNING_STR) + + logger1 = logging.getLogger("test1") + logger2 = logging.getLogger("test2") + logger1.info(NAMED_LOGGER_INFO_1) + logger2.info(NAMED_LOGGER_INFO_2) + + # Close the logging server + time.sleep(2) # give some time for logs to be sent + openwpm_logger.close() + child_process_1.join() + child_process_2.join() + print("Child processes joined...") + + log_content = get_logfile_contents(log_file) + for child in range(2): + assert log_content.count(CHILD_INFO_STR_1 % child) == 1 + assert log_content.count(CHILD_INFO_STR_2 % child) == 1 + assert log_content.count(CHILD_ERROR_STR % child) == 1 + assert log_content.count(CHILD_CRITICAL_STR % child) == 1 + assert log_content.count(CHILD_DEBUG_STR % child) == 1 + assert log_content.count(CHILD_WARNING_STR % child) == 1 + assert log_content.count(PARENT_INFO_STR_1) == 1 + assert log_content.count(PARENT_ERROR_STR) == 1 + assert log_content.count(PARENT_CRITICAL_STR) == 1 + assert log_content.count(PARENT_DEBUG_STR) == 1 + assert log_content.count(PARENT_WARNING_STR) == 1 + + +def test_multiple_instances(tmpdir): + os.makedirs(str(tmpdir) + "-1") + test_multiprocess(str(tmpdir) + "-1") + os.makedirs(str(tmpdir) + "-2") + test_multiprocess(str(tmpdir) + "-2") + + +@pytest.mark.skipif( + "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", + reason="Flaky on Travis CI", +) +def test_child_process_with_exception(tmpdir): + log_file = get_logfile_path(str(tmpdir)) + openwpm_logger = mp_logger.MPLogger(log_file) + + child_process_1 = Process(target=child_proc_with_exception, args=(0,)) + child_process_1.daemon = True + child_process_1.start() + child_process_2 = Process(target=child_proc_with_exception, args=(1,)) + child_process_2.daemon = True + child_process_2.start() + + # Close the logging server + time.sleep(2) # give some time for logs to be sent + child_process_1.join() + child_process_2.join() + print("Child processes joined...") + openwpm_logger.close() + + log_content = get_logfile_contents(log_file) + for child in range(2): + assert log_content.count(CHILD_INFO_STR_1 % child) == 1 + assert log_content.count(CHILD_INFO_STR_2 % child) == 1 + assert log_content.count(CHILD_EXCEPTION_STR % child) == 1 + + +@pytest.mark.skipif( + "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", + reason="Flaky on Travis CI", +) +def test_child_process_logging(tmpdir): + log_file = get_logfile_path(str(tmpdir)) + openwpm_logger = mp_logger.MPLogger(log_file) + child_process = Process(target=child_proc_logging_exception()) + child_process.daemon = True + child_process.start() + openwpm_logger.close() + child_process.join() + log_content = get_logfile_contents(log_file) + assert "I'm logging an exception" in log_content diff --git a/test/test_profile.py b/test/test_profile.py index 93495f890..847771348 100644 --- a/test/test_profile.py +++ b/test/test_profile.py @@ -83,40 +83,44 @@ def test_profile_saved_when_launch_crashes(self): manager.close() assert isfile(join(browser_params[0]["profile_archive_dir"], "profile.tar.gz")) - def test_seed_persistance(self): - def test_config_is_set(*args, **kwargs): - driver = kwargs["driver"] - driver.get("about:config") - result = driver.execute_script( - """ - var prefs = Components - .classes["@mozilla.org/preferences-service;1"] - .getService(Components.interfaces.nsIPrefBranch); - try { - return prefs.getBoolPref("test_pref") - } catch (e) { - return false; - } + +def test_seed_persistance(default_params, task_manager_creator): + def test_config_is_set(*args, **kwargs): + driver = kwargs["driver"] + driver.get("about:config") + result = driver.execute_script( """ - ) - assert result - - manager_params, browser_params = self.get_test_config(num_browsers=1) - browser_params[0]["seed_tar"] = "." - command_sequences = [] - for _ in range(2): - cs = CommandSequence(url="https://example.com", reset=True) - cs.get() - cs.run_custom_function(test_config_is_set) - command_sequences.append(cs) - manager = task_manager.TaskManager(manager_params, browser_params) - for cs in command_sequences: - manager.execute_command_sequence(cs) - manager.close() - query_result = db_utils.query_db( - manager_params["db"], - "SELECT * FROM crawl_history;", + var prefs = Components + .classes["@mozilla.org/preferences-service;1"] + .getService(Components.interfaces.nsIPrefBranch); + try { + return prefs.getBoolPref("test_pref") + } catch (e) { + return false; + } + """ ) - assert len(query_result) > 0 - for row in query_result: - assert row["command_status"] == "ok", f"Command {tuple(row)} was not ok" + assert result + + manager_params, browser_params = default_params + for browser_param in browser_params: + browser_param["seed_tar"] = "." + manager = task_manager_creator(default_params) + + command_sequences = [] + for _ in range(2): + cs = CommandSequence(url="https://example.com", reset=True) + cs.get() + cs.run_custom_function(test_config_is_set) + command_sequences.append(cs) + + for cs in command_sequences: + manager.execute_command_sequence(cs) + manager.close() + query_result = db_utils.query_db( + manager_params["db"], + "SELECT * FROM crawl_history;", + ) + assert len(query_result) > 0 + for row in query_result: + assert row["command_status"] == "ok", f"Command {tuple(row)} was not ok" diff --git a/test/test_simple_commands.py b/test/test_simple_commands.py index 95c073e13..3f1e23ae8 100644 --- a/test/test_simple_commands.py +++ b/test/test_simple_commands.py @@ -1,13 +1,23 @@ +"""Test correctness of simple commands and check +that resulting data is properly keyed. + +This entire test class is parametrized to run against +both headless and xvfb modes to ensure both are exercised +during the test suite and there are no obvious problems. +""" import glob import gzip import json import os import re +from typing import Callable, List, Tuple from urllib.parse import urlparse +import pytest from PIL import Image from openwpm import command_sequence, task_manager +from openwpm.types import BrowserParams, ManagerParams from openwpm.utilities import db_utils from . import utilities @@ -78,403 +88,421 @@ } -def pytest_generate_tests(metafunc): - # Source: https://docs.pytest.org/en/latest/example/parametrize.html#a-quick-port-of-testscenarios # noqa - idlist = [] - argvalues = [] - for scenario in metafunc.cls.scenarios: - idlist.append(scenario[0]) - items = scenario[1].items() - argnames = [x[0] for x in items] - argvalues.append([x[1] for x in items]) - metafunc.parametrize(argnames, argvalues, ids=idlist, scope="class") - - -class TestSimpleCommands(OpenWPMTest): - """Test correctness of simple commands and check - that resulting data is properly keyed. +scenarios = [ + pytest.param("headless", id="headless"), + pytest.param("xvfb", id="xvfb"), +] - This entire test class is parametrized to run against - both headless and xvfb modes to ensure both are exercized - during the test suite and there are no obvious problems. - """ - scenarios = [ - ("headless", {"display_mode": "headless"}), - ("xvfb", {"display_mode": "xvfb"}), - ] +@pytest.fixture() +def http_params( + default_params, +) -> Callable[[str], Tuple[ManagerParams, List[BrowserParams]]]: + manager_params, browser_params = default_params + for browser_param in browser_params: + browser_param["http_instrument"] = True - def get_config(self, display_mode): - manager_params, browser_params = self.get_test_config(display_mode=display_mode) - browser_params[0]["http_instrument"] = True + def parameterize(display_mode: str) -> Tuple[ManagerParams, List[BrowserParams]]: + for browser_param in browser_params: + browser_param["display_mode"] = display_mode return manager_params, browser_params - def test_get_site_visits_table_valid(self, display_mode): - """Check that get works and populates db correctly.""" - # Run the test crawl - manager_params, browser_params = self.get_config(display_mode) - manager = task_manager.TaskManager(manager_params, browser_params) - - # Set up two sequential get commands to two URLS - cs_a = command_sequence.CommandSequence(url_a) - cs_a.get(sleep=1) - cs_b = command_sequence.CommandSequence(url_b) - cs_b.get(sleep=1) - - # Perform the get commands - manager.execute_command_sequence(cs_a) - manager.execute_command_sequence(cs_b) - manager.close() - - qry_res = db_utils.query_db( - manager_params["db"], "SELECT site_url FROM site_visits" - ) - - # We had two separate page visits - assert len(qry_res) == 2 - - assert qry_res[0][0] == url_a - assert qry_res[1][0] == url_b - - def test_get_http_tables_valid(self, display_mode): - """Check that get works and populates http tables correctly.""" - # Run the test crawl - manager_params, browser_params = self.get_config(display_mode) - manager = task_manager.TaskManager(manager_params, browser_params) - - # Set up two sequential get commands to two URLS - cs_a = command_sequence.CommandSequence(url_a) - cs_a.get(sleep=1) - cs_b = command_sequence.CommandSequence(url_b) - cs_b.get(sleep=1) - - manager.execute_command_sequence(cs_a) - manager.execute_command_sequence(cs_b) - manager.close() - - qry_res = db_utils.query_db( - manager_params["db"], "SELECT visit_id, site_url FROM site_visits" - ) - - # Construct dict mapping site_url to visit_id - visit_ids = dict() - for row in qry_res: - visit_ids[row[1]] = row[0] - - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_requests" " WHERE url = ?", - (url_a,), - ) - assert qry_res[0][0] == visit_ids[url_a] - - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_requests" " WHERE url = ?", - (url_b,), - ) - assert qry_res[0][0] == visit_ids[url_b] - - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_responses" " WHERE url = ?", - (url_a,), - ) - assert qry_res[0][0] == visit_ids[url_a] - - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_responses" " WHERE url = ?", - (url_b,), - ) - assert qry_res[0][0] == visit_ids[url_b] - - def test_browse_site_visits_table_valid(self, display_mode): - """Check that CommandSequence.browse() populates db correctly.""" - # Run the test crawl - manager_params, browser_params = self.get_config(display_mode) - manager = task_manager.TaskManager(manager_params, browser_params) - - # Set up two sequential browse commands to two URLS - cs_a = command_sequence.CommandSequence(url_a, site_rank=0) - cs_a.browse(num_links=1, sleep=1) - cs_b = command_sequence.CommandSequence(url_b, site_rank=1) - cs_b.browse(num_links=1, sleep=1) - - manager.execute_command_sequence(cs_a) - manager.execute_command_sequence(cs_b) - manager.close() - - qry_res = db_utils.query_db( - manager_params["db"], "SELECT site_url, site_rank" " FROM site_visits" - ) - - # We had two separate page visits - assert len(qry_res) == 2 - - assert qry_res[0][0] == url_a - assert qry_res[0][1] == 0 - assert qry_res[1][0] == url_b - assert qry_res[1][1] == 1 - - def test_browse_http_table_valid(self, display_mode): - """Check CommandSequence.browse() works and populates http tables correctly. - - NOTE: Since the browse command is choosing links randomly, there is a - (very small -- 2*0.5^20) chance this test will fail with valid - code. - """ - # Run the test crawl - manager_params, browser_params = self.get_config(display_mode) - manager = task_manager.TaskManager(manager_params, browser_params) - - # Set up two sequential browse commands to two URLS - cs_a = command_sequence.CommandSequence(url_a) - cs_a.browse(num_links=20, sleep=1) - cs_b = command_sequence.CommandSequence(url_b) - cs_b.browse(num_links=1, sleep=1) - - manager.execute_command_sequence(cs_a) - manager.execute_command_sequence(cs_b) - manager.close() - - qry_res = db_utils.query_db( - manager_params["db"], "SELECT visit_id, site_url FROM site_visits" - ) - - # Construct dict mapping site_url to visit_id - visit_ids = dict() - for row in qry_res: - visit_ids[row[1]] = row[0] - - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_requests" " WHERE url = ?", - (url_a,), - ) - assert qry_res[0][0] == visit_ids[url_a] - - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_requests" " WHERE url = ?", - (url_b,), - ) - assert qry_res[0][0] == visit_ids[url_b] - - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_responses" " WHERE url = ?", - (url_a,), - ) - assert qry_res[0][0] == visit_ids[url_a] - - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_responses" " WHERE url = ?", - (url_b,), - ) - assert qry_res[0][0] == visit_ids[url_b] - - # Page simple_a.html has three links: - # 1) An absolute link to simple_c.html - # 2) A relative link to simple_d.html - # 3) A javascript: link - # 4) A link to www.google.com - # 5) A link to example.com?localtest.me - # We should see page visits for 1 and 2, but not 3-5. - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_responses" " WHERE url = ?", - (url_c,), - ) - assert qry_res[0][0] == visit_ids[url_a] - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_responses" " WHERE url = ?", - (url_d,), - ) - assert qry_res[0][0] == visit_ids[url_a] - - # We expect 4 urls: a,c,d and a favicon request - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT COUNT(DISTINCT url) FROM http_responses" " WHERE visit_id = ?", - (visit_ids[url_a],), - ) - assert qry_res[0][0] == 4 - - def test_browse_wrapper_http_table_valid(self, display_mode): - """Check that TaskManager.browse() wrapper works and populates - http tables correctly. - - NOTE: Since the browse command is choosing links randomly, there is a - (very small -- 2*0.5^20) chance this test will fail with valid - code. - """ - # Run the test crawl - manager_params, browser_params = self.get_config(display_mode) - manager = task_manager.TaskManager(manager_params, browser_params) - - # Set up two sequential browse commands to two URLS - manager.browse(url_a, num_links=20, sleep=1) - manager.browse(url_b, num_links=1, sleep=1) - manager.close() - - qry_res = db_utils.query_db( - manager_params["db"], "SELECT visit_id, site_url FROM site_visits" - ) - - # Construct dict mapping site_url to visit_id - visit_ids = dict() - for row in qry_res: - visit_ids[row[1]] = row[0] - - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_requests" " WHERE url = ?", - (url_a,), - ) - assert qry_res[0][0] == visit_ids[url_a] - - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_requests" " WHERE url = ?", - (url_b,), - ) - assert qry_res[0][0] == visit_ids[url_b] - - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_responses" " WHERE url = ?", - (url_a,), - ) - assert qry_res[0][0] == visit_ids[url_a] - - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_responses" " WHERE url = ?", - (url_b,), - ) - assert qry_res[0][0] == visit_ids[url_b] - - # Page simple_a.html has three links: - # 1) An absolute link to simple_c.html - # 2) A relative link to simple_d.html - # 3) A javascript: link - # 4) A link to www.google.com - # 5) A link to example.com?localtest.me - # We should see page visits for 1 and 2, but not 3-5. - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_responses" " WHERE url = ?", - (url_c,), - ) - assert qry_res[0][0] == visit_ids[url_a] - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM http_responses" " WHERE url = ?", - (url_d,), - ) - assert qry_res[0][0] == visit_ids[url_a] - - # We expect 4 urls: a,c,d and a favicon request - qry_res = db_utils.query_db( - manager_params["db"], - "SELECT COUNT(DISTINCT url) FROM http_responses" " WHERE visit_id = ?", - (visit_ids[url_a],), - ) - assert qry_res[0][0] == 4 - - def test_save_screenshot_valid(self, display_mode): - """Check that 'save_screenshot' works""" - # Run the test crawl - manager_params, browser_params = self.get_config(display_mode) - manager = task_manager.TaskManager(manager_params, browser_params) - cs = command_sequence.CommandSequence(url_a) - cs.get(sleep=1) - cs.save_screenshot("test") - cs.screenshot_full_page("test_full") - manager.execute_command_sequence(cs) - manager.close() - - # Check that viewport image is not blank - pattern = os.path.join(str(self.tmpdir), "screenshots", "1-*-test.png") - screenshot = glob.glob(pattern)[0] - im = Image.open(screenshot) - bands = im.split() - is_blank = all(band.getextrema() == (255, 255) for band in bands) - assert not is_blank - - # Check that full page screenshot is not blank - pattern = os.path.join(str(self.tmpdir), "screenshots", "1-*-test_full.png") - screenshot = glob.glob(pattern)[0] - im = Image.open(screenshot) - bands = im.split() - is_blank = all(band.getextrema() == (255, 255) for band in bands) - assert not is_blank - - def test_dump_page_source_valid(self, display_mode): - """Check that 'dump_page_source' works and source is saved properly.""" - # Run the test crawl - manager_params, browser_params = self.get_config(display_mode) - manager = task_manager.TaskManager(manager_params, browser_params) - cs = command_sequence.CommandSequence(url_a) - cs.get(sleep=1) - cs.dump_page_source(suffix="test") - manager.execute_command_sequence(cs) - manager.close() - - # Source filename is of the follow structure: - # `sources/-(-suffix).html` - # thus for this test we expect `sources/1--test.html`. - outfile = os.path.join(str(self.tmpdir), "sources", "1-*-test.html") - source_file = glob.glob(outfile)[0] - with open(source_file, "rb") as f: - actual_source = f.read() - with open("./test_pages/expected_source.html", "rb") as f: - expected_source = f.read() - - assert actual_source == expected_source - - def test_recursive_dump_page_source_valid(self, display_mode): - """Check that 'recursive_dump_page_source' works""" - # Run the test crawl - manager_params, browser_params = self.get_config(display_mode) - manager = task_manager.TaskManager(manager_params, browser_params) - cs = command_sequence.CommandSequence(NESTED_FRAMES_URL) - cs.get(sleep=1) - cs.recursive_dump_page_source() - manager.execute_command_sequence(cs) - manager.close() - - outfile = os.path.join(str(self.tmpdir), "sources", "1-*.json.gz") - src_file = glob.glob(outfile)[0] - with gzip.GzipFile(src_file, "rb") as f: - visit_source = json.loads(f.read().decode("utf-8")) - - observed_parents = dict() - - def verify_frame(frame, parent_frames=[]): - # Verify structure - observed_parents[frame["doc_url"]] = list(parent_frames) # copy - - # Verify source - path = urlparse(frame["doc_url"]).path - expected_source = "" - with open("." + path, "r") as f: - expected_source = re.sub(r"\s", "", f.read().lower()) - if expected_source.startswith(""): - expected_source = expected_source[14:] - observed_source = re.sub(r"\s", "", frame["source"].lower()) - if observed_source.startswith(""): - observed_source = observed_source[14:] - assert observed_source == expected_source - - # Verify children - parent_frames.append(frame["doc_url"]) - for key, child_frame in frame["iframes"].items(): - verify_frame(child_frame, parent_frames) - parent_frames.pop() - - verify_frame(visit_source) - assert EXPECTED_PARENTS == observed_parents + return parameterize + + +@pytest.mark.parametrize("display_mode", scenarios) +def test_get_site_visits_table_valid(http_params, task_manager_creator, display_mode): + """Check that get works and populates db correctly.""" + # Run the test crawl + manager_params, browser_params = http_params(display_mode) + manager = task_manager_creator((manager_params, browser_params)) + + # Set up two sequential get commands to two URLS + cs_a = command_sequence.CommandSequence(url_a) + cs_a.get(sleep=1) + cs_b = command_sequence.CommandSequence(url_b) + cs_b.get(sleep=1) + + # Perform the get commands + manager.execute_command_sequence(cs_a) + manager.execute_command_sequence(cs_b) + manager.close() + + qry_res = db_utils.query_db( + manager_params["db"], "SELECT site_url FROM site_visits ORDER BY site_url" + ) + + # We had two separate page visits + assert len(qry_res) == 2 + + assert qry_res[0][0] == url_a + assert qry_res[1][0] == url_b + + +@pytest.mark.parametrize("display_mode", scenarios) +def test_get_http_tables_valid(http_params, task_manager_creator, display_mode): + """Check that get works and populates http tables correctly.""" + # Run the test crawl + manager_params, browser_params = http_params(display_mode) + manager = task_manager_creator((manager_params, browser_params)) + + # Set up two sequential get commands to two URLS + cs_a = command_sequence.CommandSequence(url_a) + cs_a.get(sleep=1) + cs_b = command_sequence.CommandSequence(url_b) + cs_b.get(sleep=1) + + manager.execute_command_sequence(cs_a) + manager.execute_command_sequence(cs_b) + manager.close() + + qry_res = db_utils.query_db( + manager_params["db"], "SELECT visit_id, site_url FROM site_visits" + ) + + # Construct dict mapping site_url to visit_id + visit_ids = dict() + for row in qry_res: + visit_ids[row[1]] = row[0] + + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_requests WHERE url = ?", + (url_a,), + ) + assert qry_res[0][0] == visit_ids[url_a] + + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_requests WHERE url = ?", + (url_b,), + ) + assert qry_res[0][0] == visit_ids[url_b] + + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_responses WHERE url = ?", + (url_a,), + ) + assert qry_res[0][0] == visit_ids[url_a] + + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_responses WHERE url = ?", + (url_b,), + ) + assert qry_res[0][0] == visit_ids[url_b] + + +@pytest.mark.parametrize("display_mode", scenarios) +def test_browse_site_visits_table_valid( + http_params, task_manager_creator, display_mode +): + """Check that CommandSequence.browse() populates db correctly.""" + # Run the test crawl + manager_params, browser_params = http_params(display_mode) + manager = task_manager_creator((manager_params, browser_params)) + + # Set up two sequential browse commands to two URLS + cs_a = command_sequence.CommandSequence(url_a, site_rank=0) + cs_a.browse(num_links=1, sleep=1) + cs_b = command_sequence.CommandSequence(url_b, site_rank=1) + cs_b.browse(num_links=1, sleep=1) + + manager.execute_command_sequence(cs_a) + manager.execute_command_sequence(cs_b) + manager.close() + + qry_res = db_utils.query_db( + manager_params["db"], "SELECT site_url, site_rank FROM site_visits" + ) + + # We had two separate page visits + assert len(qry_res) == 2 + + assert qry_res[0][0] == url_a + assert qry_res[0][1] == 0 + assert qry_res[1][0] == url_b + assert qry_res[1][1] == 1 + + +@pytest.mark.parametrize("display_mode", scenarios) +def test_browse_http_table_valid(http_params, task_manager_creator, display_mode): + """Check CommandSequence.browse() works and populates http tables correctly. + + NOTE: Since the browse command is choosing links randomly, there is a + (very small -- 2*0.5^20) chance this test will fail with valid + code. + """ + # Run the test crawl + manager_params, browser_params = http_params(display_mode) + manager = task_manager_creator((manager_params, browser_params)) + + # Set up two sequential browse commands to two URLS + cs_a = command_sequence.CommandSequence(url_a) + cs_a.browse(num_links=20, sleep=1) + cs_b = command_sequence.CommandSequence(url_b) + cs_b.browse(num_links=1, sleep=1) + + manager.execute_command_sequence(cs_a) + manager.execute_command_sequence(cs_b) + manager.close() + + qry_res = db_utils.query_db( + manager_params["db"], "SELECT visit_id, site_url FROM site_visits" + ) + + # Construct dict mapping site_url to visit_id + visit_ids = dict() + for row in qry_res: + visit_ids[row[1]] = row[0] + + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_requests WHERE url = ?", + (url_a,), + ) + assert qry_res[0][0] == visit_ids[url_a] + + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_requests WHERE url = ?", + (url_b,), + ) + assert qry_res[0][0] == visit_ids[url_b] + + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_responses WHERE url = ?", + (url_a,), + ) + assert qry_res[0][0] == visit_ids[url_a] + + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_responses WHERE url = ?", + (url_b,), + ) + assert qry_res[0][0] == visit_ids[url_b] + + # Page simple_a.html has three links: + # 1) An absolute link to simple_c.html + # 2) A relative link to simple_d.html + # 3) A javascript: link + # 4) A link to www.google.com + # 5) A link to example.com?localtest.me + # We should see page visits for 1 and 2, but not 3-5. + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_responses WHERE url = ?", + (url_c,), + ) + assert qry_res[0][0] == visit_ids[url_a] + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_responses WHERE url = ?", + (url_d,), + ) + assert qry_res[0][0] == visit_ids[url_a] + + # We expect 4 urls: a,c,d and a favicon request + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT COUNT(DISTINCT url) FROM http_responses WHERE visit_id = ?", + (visit_ids[url_a],), + ) + assert qry_res[0][0] == 4 + + +@pytest.mark.parametrize("display_mode", scenarios) +def test_browse_wrapper_http_table_valid( + http_params, task_manager_creator, display_mode +): + """Check that TaskManager.browse() wrapper works and populates + http tables correctly. + + NOTE: Since the browse command is choosing links randomly, there is a + (very small -- 2*0.5^20) chance this test will fail with valid + code. + """ + # Run the test crawl + manager_params, browser_params = http_params(display_mode) + manager = task_manager_creator((manager_params, browser_params)) + + # Set up two sequential browse commands to two URLS + manager.browse(url_a, num_links=20, sleep=1) + manager.browse(url_b, num_links=1, sleep=1) + manager.close() + + qry_res = db_utils.query_db( + manager_params["db"], "SELECT visit_id, site_url FROM site_visits" + ) + + # Construct dict mapping site_url to visit_id + visit_ids = dict() + for row in qry_res: + visit_ids[row[1]] = row[0] + + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_requests WHERE url = ?", + (url_a,), + ) + assert qry_res[0][0] == visit_ids[url_a] + + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_requests WHERE url = ?", + (url_b,), + ) + assert qry_res[0][0] == visit_ids[url_b] + + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_responses WHERE url = ?", + (url_a,), + ) + assert qry_res[0][0] == visit_ids[url_a] + + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_responses WHERE url = ?", + (url_b,), + ) + assert qry_res[0][0] == visit_ids[url_b] + + # Page simple_a.html has three links: + # 1) An absolute link to simple_c.html + # 2) A relative link to simple_d.html + # 3) A javascript: link + # 4) A link to www.google.com + # 5) A link to example.com?localtest.me + # We should see page visits for 1 and 2, but not 3-5. + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_responses WHERE url = ?", + (url_c,), + ) + assert qry_res[0][0] == visit_ids[url_a] + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT visit_id FROM http_responses WHERE url = ?", + (url_d,), + ) + assert qry_res[0][0] == visit_ids[url_a] + + # We expect 4 urls: a,c,d and a favicon request + qry_res = db_utils.query_db( + manager_params["db"], + "SELECT COUNT(DISTINCT url) FROM http_responses WHERE visit_id = ?", + (visit_ids[url_a],), + ) + assert qry_res[0][0] == 4 + + +@pytest.mark.parametrize("display_mode", scenarios) +def test_save_screenshot_valid(http_params, task_manager_creator, display_mode): + """Check that 'save_screenshot' works""" + # Run the test crawl + manager_params, browser_params = http_params(display_mode) + manager = task_manager_creator((manager_params, browser_params)) + + cs = command_sequence.CommandSequence(url_a) + cs.get(sleep=1) + cs.save_screenshot("test") + cs.screenshot_full_page("test_full") + manager.execute_command_sequence(cs) + manager.close() + + # Check that viewport image is not blank + pattern = os.path.join( + manager_params["data_directory"], "screenshots", "1-*-test.png" + ) + screenshot = glob.glob(pattern)[0] + im = Image.open(screenshot) + bands = im.split() + is_blank = all(band.getextrema() == (255, 255) for band in bands) + assert not is_blank + + # Check that full page screenshot is not blank + pattern = os.path.join( + manager_params["data_directory"], "screenshots", "1-*-test_full.png" + ) + screenshot = glob.glob(pattern)[0] + im = Image.open(screenshot) + bands = im.split() + is_blank = all(band.getextrema() == (255, 255) for band in bands) + assert not is_blank + + +@pytest.mark.parametrize("display_mode", scenarios) +def test_dump_page_source_valid(http_params, task_manager_creator, display_mode): + """Check that 'dump_page_source' works and source is saved properly.""" + # Run the test crawl + manager_params, browser_params = http_params(display_mode) + manager = task_manager_creator((manager_params, browser_params)) + + cs = command_sequence.CommandSequence(url_a) + cs.get(sleep=1) + cs.dump_page_source(suffix="test") + manager.execute_command_sequence(cs) + manager.close() + + # Source filename is of the follow structure: + # `sources/-(-suffix).html` + # thus for this test we expect `sources/1--test.html`. + outfile = os.path.join(manager_params["data_directory"], "sources", "1-*-test.html") + source_file = glob.glob(outfile)[0] + with open(source_file, "rb") as f: + actual_source = f.read() + with open("./test_pages/expected_source.html", "rb") as f: + expected_source = f.read() + + assert actual_source == expected_source + + +@pytest.mark.parametrize("display_mode", scenarios) +def test_recursive_dump_page_source_valid( + http_params, task_manager_creator, display_mode +): + """Check that 'recursive_dump_page_source' works""" + # Run the test crawl + manager_params, browser_params = http_params(display_mode) + manager = task_manager_creator((manager_params, browser_params)) + cs = command_sequence.CommandSequence(NESTED_FRAMES_URL) + cs.get(sleep=1) + cs.recursive_dump_page_source() + manager.execute_command_sequence(cs) + manager.close() + + outfile = os.path.join(manager_params["data_directory"], "sources", "1-*.json.gz") + src_file = glob.glob(outfile)[0] + with gzip.GzipFile(src_file, "rb") as f: + visit_source = json.loads(f.read().decode("utf-8")) + + observed_parents = dict() + + def verify_frame(frame, parent_frames=[]): + # Verify structure + observed_parents[frame["doc_url"]] = list(parent_frames) # copy + + # Verify source + path = urlparse(frame["doc_url"]).path + expected_source = "" + with open("." + path, "r") as f: + expected_source = re.sub(r"\s", "", f.read().lower()) + if expected_source.startswith(""): + expected_source = expected_source[14:] + observed_source = re.sub(r"\s", "", frame["source"].lower()) + if observed_source.startswith(""): + observed_source = observed_source[14:] + assert observed_source == expected_source + + # Verify children + parent_frames.append(frame["doc_url"]) + for key, child_frame in frame["iframes"].items(): + verify_frame(child_frame, parent_frames) + parent_frames.pop() + + verify_frame(visit_source) + assert EXPECTED_PARENTS == observed_parents diff --git a/test/test_storage_vectors.py b/test/test_storage_vectors.py index 0a3535dd1..3c39abfa6 100644 --- a/test/test_storage_vectors.py +++ b/test/test_storage_vectors.py @@ -1,3 +1,10 @@ +"""Runs some basic tests to check that the saving of +storage vectors (i.e. profile cookies) works. + +NOTE: These tests are very basic and should be expanded +on to check for completeness and correctness. +""" + from typing import List, Tuple from openwpm import command_sequence, task_manager @@ -23,38 +30,28 @@ ) -class TestStorageVectors(OpenWPMTest): - """Runs some basic tests to check that the saving of - storage vectors (i.e. profile cookies) works. - - NOTE: These tests are very basic and should be expanded - on to check for completeness and correctness. - """ - - def test_js_profile_cookies( - self, default_params: Tuple[ManagerParams, List[BrowserParams]] - ): - """ Check that profile cookies set by JS are saved """ - # Run the test crawl - manager_params, browser_params = default_params - browser_params[0]["cookie_instrument"] = True - manager = task_manager.TaskManager(*default_params) - url = utilities.BASE_TEST_URL + "/js_cookie.html" - cs = command_sequence.CommandSequence(url) - cs.get(sleep=3, timeout=120) - manager.execute_command_sequence(cs) - manager.close() - # Check that the JS cookie we stored is recorded - qry_res = db_utils.query_db( - manager_params["db"], - ( - "SELECT visit_id, record_type, change_cause, is_http_only, " - "is_host_only, is_session, host, is_secure, name, path, " - "value, same_site FROM javascript_cookies" - ), - as_tuple=True, - ) - assert len(qry_res) == 1 # we store only one cookie - cookies = qry_res[0] # take the first cookie - # compare URL, domain, name, value, origin, path - assert cookies == expected_js_cookie +def test_js_profile_cookies(default_params, task_manager_creator): + """ Check that profile cookies set by JS are saved """ + # Run the test crawl + manager_params, browser_params = default_params + browser_params[0]["cookie_instrument"] = True + manager = task_manager_creator(default_params) + url = utilities.BASE_TEST_URL + "/js_cookie.html" + cs = command_sequence.CommandSequence(url) + cs.get(sleep=3, timeout=120) + manager.execute_command_sequence(cs) + manager.close() + # Check that the JS cookie we stored is recorded + qry_res = db_utils.query_db( + manager_params["db"], + ( + "SELECT visit_id, record_type, change_cause, is_http_only, " + "is_host_only, is_session, host, is_secure, name, path, " + "value, same_site FROM javascript_cookies" + ), + as_tuple=True, + ) + assert len(qry_res) == 1 # we store only one cookie + cookies = qry_res[0] # take the first cookie + # compare URL, domain, name, value, origin, path + assert cookies == expected_js_cookie diff --git a/test/test_webdriver_utils.py b/test/test_webdriver_utils.py index 6be31b003..ddb913d54 100644 --- a/test/test_webdriver_utils.py +++ b/test/test_webdriver_utils.py @@ -16,21 +16,17 @@ def test_parse_neterror(): assert parse_neterror(text) == "dnsNotFound" -class TestCustomFunctionCommand(OpenWPMTest): - def get_config(self, data_dir=""): - return self.get_test_config(data_dir) - - def test_parse_neterror_integration(self): - manager_params, browser_params = self.get_config() - manager = task_manager.TaskManager(manager_params, browser_params) - manager.get("http://website.invalid") - manager.close() - - get_command = db_utils.query_db( - manager_params["db"], - "SELECT command_status, error FROM crawl_history WHERE command = \"\"", - as_tuple=True, - )[0] - - assert get_command[0] == "neterror" - assert get_command[1] == "dnsNotFound" +def test_parse_neterror_integration(default_params, task_manager_creator): + manager_params = default_params[0] + manager = task_manager_creator(default_params) + manager.get("http://website.invalid") + manager.close() + + get_command = db_utils.query_db( + manager_params["db"], + "SELECT command_status, error FROM crawl_history WHERE command ='GetCommand'", + as_tuple=True, + )[0] + + assert get_command[0] == "neterror" + assert get_command[1] == "dnsNotFound" From 9b03e300d6232c32dec211befb71f0e1f4119278 Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 25 Nov 2020 22:06:12 +0100 Subject: [PATCH 037/139] Still fixing tests --- openwpm/storage/sql_provider.py | 3 ++- openwpm/storage/storage_controller.py | 4 ++-- test/conftest.py | 18 ++++++++++++++++++ test/test_callstack_instrument.py | 2 +- test/test_extension.py | 20 +++++++------------- test/test_simple_commands.py | 16 ---------------- test/test_storage_vectors.py | 8 ++++---- 7 files changed, 34 insertions(+), 37 deletions(-) diff --git a/openwpm/storage/sql_provider.py b/openwpm/storage/sql_provider.py index 43e3fb805..65cdfc3b7 100644 --- a/openwpm/storage/sql_provider.py +++ b/openwpm/storage/sql_provider.py @@ -60,8 +60,9 @@ async def store_record( % (type(e), e, statement, repr(args)) ) + @staticmethod def _generate_insert( - self, table: TableName, data: Dict[str, Any] + table: TableName, data: Dict[str, Any] ) -> Tuple[str, List[Any]]: """Generate a SQL query from `record`""" statement = "INSERT INTO %s (" % table diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 542af7205..d862a618b 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -106,7 +106,7 @@ async def handler( self._last_record_received = time.time() record_type, data = record - self.logger.info("Received record for record_type %s", record_type) + self.logger.debug("Received record for record_type %s", record_type) if record_type == RECORD_TYPE_CREATE: raise RuntimeError( @@ -260,7 +260,7 @@ async def _run(self) -> None: ) sockets = server.sockets assert sockets is not None - assert len(sockets) == 1 + # assert len(sockets) == 1 socketname = sockets[0].getsockname() self.status_queue.put(socketname) status_queue_update = asyncio.create_task( diff --git a/test/conftest.py b/test/conftest.py index b027070e3..38b4cd0bd 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -77,3 +77,21 @@ def _create_task_manager( return manager return _create_task_manager + + +@pytest.fixture() +def http_params( + default_params, +) -> Callable[[str], Tuple[ManagerParams, List[BrowserParams]]]: + manager_params, browser_params = default_params + for browser_param in browser_params: + browser_param["http_instrument"] = True + + def parameterize( + display_mode: str = "headless", + ) -> Tuple[ManagerParams, List[BrowserParams]]: + for browser_param in browser_params: + browser_param["display_mode"] = display_mode + return manager_params, browser_params + + return parameterize diff --git a/test/test_callstack_instrument.py b/test/test_callstack_instrument.py index f8c88f1dd..23add8bb7 100644 --- a/test/test_callstack_instrument.py +++ b/test/test_callstack_instrument.py @@ -67,7 +67,7 @@ def test_http_stacktrace(default_params, task_manager_creator): # Record the callstack of all WebRequests made browser_params[0]["callstack_instrument"] = True test_url = utilities.BASE_TEST_URL + "/http_stacktrace.html" - manager = task_manager_creator(manager_params, browser_params) + manager = task_manager_creator((manager_params, browser_params)) manager.get(test_url, sleep=10) db = manager_params["db"] manager.close() diff --git a/test/test_extension.py b/test/test_extension.py index c69229f32..4a9af5038 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -280,18 +280,12 @@ def test_canvas_fingerprinting(self): assert CANVAS_CALLS == observed_rows def test_extension_gets_correct_visit_id(self): - manager_params, browser_params = self.get_config() - manager = task_manager.TaskManager(manager_params, browser_params) - url_a = utilities.BASE_TEST_URL + "/simple_a.html" url_b = utilities.BASE_TEST_URL + "/simple_b.html" + self.visit(url_a) + db = self.visit(url_b) - manager.get(url_a) - manager.get(url_b) - manager.close() - qry_res = db_utils.query_db( - manager_params["db"], "SELECT visit_id, site_url FROM site_visits" - ) + qry_res = db_utils.query_db(db, "SELECT visit_id, site_url FROM site_visits") # Construct dict mapping site_url to visit_id visit_ids = dict() @@ -299,14 +293,14 @@ def test_extension_gets_correct_visit_id(self): visit_ids[row[1]] = row[0] simple_a_visit_id = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM javascript WHERE " "symbol=?", + db, + "SELECT visit_id FROM javascript WHERE symbol=?", ("window.navigator.userAgent",), ) simple_b_visit_id = db_utils.query_db( - manager_params["db"], - "SELECT visit_id FROM javascript WHERE " "symbol=?", + db, + "SELECT visit_id FROM javascript WHERE symbol=?", ("window.navigator.platform",), ) diff --git a/test/test_simple_commands.py b/test/test_simple_commands.py index 3f1e23ae8..3a050f2d0 100644 --- a/test/test_simple_commands.py +++ b/test/test_simple_commands.py @@ -94,22 +94,6 @@ ] -@pytest.fixture() -def http_params( - default_params, -) -> Callable[[str], Tuple[ManagerParams, List[BrowserParams]]]: - manager_params, browser_params = default_params - for browser_param in browser_params: - browser_param["http_instrument"] = True - - def parameterize(display_mode: str) -> Tuple[ManagerParams, List[BrowserParams]]: - for browser_param in browser_params: - browser_param["display_mode"] = display_mode - return manager_params, browser_params - - return parameterize - - @pytest.mark.parametrize("display_mode", scenarios) def test_get_site_visits_table_valid(http_params, task_manager_creator, display_mode): """Check that get works and populates db correctly.""" diff --git a/test/test_storage_vectors.py b/test/test_storage_vectors.py index 3c39abfa6..ff0555b41 100644 --- a/test/test_storage_vectors.py +++ b/test/test_storage_vectors.py @@ -15,7 +15,6 @@ from .openwpmtest import OpenWPMTest expected_js_cookie = ( - 1, # visit_id u"added-or-changed", # record_type u"explicit", # change_cause 0, # is_http_only @@ -34,8 +33,9 @@ def test_js_profile_cookies(default_params, task_manager_creator): """ Check that profile cookies set by JS are saved """ # Run the test crawl manager_params, browser_params = default_params - browser_params[0]["cookie_instrument"] = True - manager = task_manager_creator(default_params) + for browser_param in browser_params: + browser_param["cookie_instrument"] = True + manager = task_manager_creator((manager_params, browser_params)) url = utilities.BASE_TEST_URL + "/js_cookie.html" cs = command_sequence.CommandSequence(url) cs.get(sleep=3, timeout=120) @@ -45,7 +45,7 @@ def test_js_profile_cookies(default_params, task_manager_creator): qry_res = db_utils.query_db( manager_params["db"], ( - "SELECT visit_id, record_type, change_cause, is_http_only, " + "SELECT record_type, change_cause, is_http_only, " "is_host_only, is_session, host, is_secure, name, path, " "value, same_site FROM javascript_cookies" ), From 95bfcd50de6ba46bbb891d9c58e4237b26473fa8 Mon Sep 17 00:00:00 2001 From: vringar Date: Thu, 26 Nov 2020 00:04:18 +0100 Subject: [PATCH 038/139] Broken content saving --- openwpm/storage/leveldb.py | 42 ++++++++++++++++++++- openwpm/storage/sql_provider.py | 4 ++ openwpm/storage/storage_controller.py | 3 +- openwpm/utilities/db_utils.py | 15 ++++---- test/test_custom_function_command.py | 27 +++++++------- test/test_http_instrumentation.py | 53 ++++++++++++++++----------- 6 files changed, 99 insertions(+), 45 deletions(-) diff --git a/openwpm/storage/leveldb.py b/openwpm/storage/leveldb.py index 5850cc546..a626155c3 100644 --- a/openwpm/storage/leveldb.py +++ b/openwpm/storage/leveldb.py @@ -1,5 +1,45 @@ +from pathlib import Path + +import plyvel + from .storage_providers import UnstructuredStorageProvider +LDB_BATCH_SIZE = 100 + class LevelDbProvider(UnstructuredStorageProvider): - ... + def __init__(self, db_path: Path): + self.ldb = plyvel.DB( + str(db_path), + create_if_missing=True, + write_buffer_size=128 * 10 ** 6, + compression="snappy", + ) + self.content_batch = self.ldb.write_batch() + self._ldb_counter = 0 + self._ldb_commit_time = 0 + + async def flush_cache(self) -> None: + """Write out content batch to LevelDB database""" + self.content_batch.write() + self.content_batch = self.ldb.write_batch() + + async def shutdown(self) -> None: + self.ldb.close() + print("Ldb is closed:", self.ldb.closed) + + async def store_blob( + self, + filename: str, + blob: bytes, + compressed: bool = True, + overwrite: bool = False, + ) -> None: + content_hash = str(filename).encode("ascii") + if self.ldb.get(content_hash) is not None: + return + self.content_batch.put(content_hash, blob) + self._ldb_counter += 1 + + if self._ldb_counter >= LDB_BATCH_SIZE: + await self.flush_cache() diff --git a/openwpm/storage/sql_provider.py b/openwpm/storage/sql_provider.py index 65cdfc3b7..39ad70e09 100644 --- a/openwpm/storage/sql_provider.py +++ b/openwpm/storage/sql_provider.py @@ -78,6 +78,10 @@ def _generate_insert( statement = statement + ") " + value_str + ")" return statement, values + def execute_statement(self, statement: str): + self.cur.execute(statement) + self.db.commit() + async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False ) -> None: diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index d862a618b..42e1c9337 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -118,7 +118,6 @@ async def handler( ) if record_type == RECORD_TYPE_CONTENT: - assert isinstance(data, tuple) assert len(data) == 2 if self.unstructured_storage is None: self.logger.error( @@ -133,7 +132,7 @@ async def handler( ) continue - if not "visit_id" in data: + if "visit_id" not in data: self.logger.error( "Skipping record: No visit_id contained in record %r", record ) diff --git a/openwpm/utilities/db_utils.py b/openwpm/utilities/db_utils.py index d54bbb894..4a1c7eef6 100644 --- a/openwpm/utilities/db_utils.py +++ b/openwpm/utilities/db_utils.py @@ -1,10 +1,10 @@ import os import sqlite3 +from pathlib import Path +from typing import AnyStr, Iterator, Tuple import plyvel -CONTENT_DB_NAME = "content.ldb" - def query_db(db, query, params=None, as_tuple=False): """Run a query against the given db. @@ -22,16 +22,15 @@ def query_db(db, query, params=None, as_tuple=False): return rows -def get_content(data_directory): +def get_content(db_name: Path) -> Iterator[Tuple[AnyStr, AnyStr]]: """Yield key, value pairs from the deduplicated leveldb content database Parameters ---------- - data_directory : string - root directory of the crawl files containing the content database + db_name : Path + The full path to the current db """ - db_path = os.path.join(data_directory, CONTENT_DB_NAME) - db = plyvel.DB(db_path, create_if_missing=False, compression="snappy") + db = plyvel.DB(str(db_name), create_if_missing=False, compression="snappy") for content_hash, content in db.iterator(): yield content_hash, content db.close() @@ -43,7 +42,7 @@ def get_javascript_entries(db, all_columns=False, as_tuple=False): else: select_columns = "script_url, symbol, operation, value, arguments" - return query_db(db, "SELECT %s FROM javascript" % select_columns, as_tuple=as_tuple) + return query_db(db, f"SELECT {select_columns} FROM javascript", as_tuple=as_tuple) def any_command_failed(db): diff --git a/test/test_custom_function_command.py b/test/test_custom_function_command.py index c1e509904..6452155af 100644 --- a/test/test_custom_function_command.py +++ b/test/test_custom_function_command.py @@ -1,8 +1,10 @@ from openwpm import command_sequence, task_manager +from openwpm.socket_interface import ClientSocket +from openwpm.storage.sql_provider import SqlLiteStorageProvider +from openwpm.task_manager import TaskManager from openwpm.utilities import db_utils from . import utilities -from .openwpmtest import OpenWPMTest url_a = utilities.BASE_TEST_URL + "/simple_a.html" @@ -22,10 +24,9 @@ } -def test_custom_function(default_params, task_manager_creator): +def test_custom_function(default_params, xpi, server): """ Test `custom_function` with an inline func that collects links """ - - from openwpm.socket_interface import ClientSocket + table_name = "page_links" def collect_links(table_name, scheme, **kwargs): """ Collect links with `scheme` and save in table `table_name` """ @@ -46,13 +47,6 @@ def collect_links(table_name, scheme, **kwargs): sock = ClientSocket() sock.connect(*manager_params["aggregator_address"]) - query = ( - "CREATE TABLE IF NOT EXISTS %s (" - "top_url TEXT, link TEXT, " - "visit_id INTEGER, browser_id INTEGER);" % table_name - ) - sock.send(("create_table", query)) - for link in link_urls: query = ( table_name, @@ -67,10 +61,17 @@ def collect_links(table_name, scheme, **kwargs): sock.close() manager_params, browser_params = default_params - manager = task_manager_creator(default_params) + storage_provider = SqlLiteStorageProvider(manager_params["db"]) + storage_provider.execute_statement( + """CREATE TABLE IF NOT EXISTS %s ( + top_url TEXT, link TEXT, + visit_id INTEGER, browser_id INTEGER);""" + % table_name + ) + manager = TaskManager(manager_params, browser_params, storage_provider, None) cs = command_sequence.CommandSequence(url_a) cs.get(sleep=0, timeout=60) - cs.run_custom_function(collect_links, ("page_links", "http")) + cs.run_custom_function(collect_links, (table_name, "http")) manager.execute_command_sequence(cs) manager.close() query_result = db_utils.query_db( diff --git a/test/test_http_instrumentation.py b/test/test_http_instrumentation.py index 7ef899b1e..c566d40cf 100644 --- a/test/test_http_instrumentation.py +++ b/test/test_http_instrumentation.py @@ -5,6 +5,7 @@ import json import os from hashlib import sha256 +from pathlib import Path from time import sleep from typing import Set, Tuple from urllib.parse import urlparse @@ -12,6 +13,8 @@ import pytest from openwpm import command_sequence, task_manager +from openwpm.storage.leveldb import LevelDbProvider +from openwpm.storage.sql_provider import SqlLiteStorageProvider from openwpm.utilities import db_utils from . import utilities @@ -733,27 +736,6 @@ def test_cache_hits_recorded(self): observed_records.add((src, dst)) assert HTTP_CACHED_REDIRECTS == observed_records - def test_javascript_saving(self, tmpdir): - """ check that javascript content is saved and hashed correctly """ - test_url = utilities.BASE_TEST_URL + "/http_test_page.html" - manager_params, browser_params = self.get_test_config(str(tmpdir)) - browser_params[0]["http_instrument"] = True - browser_params[0]["save_content"] = "script" - manager = task_manager.TaskManager(manager_params, browser_params) - manager.get(url=test_url, sleep=1) - manager.close() - expected_hashes = { - "0110c0521088c74f179615cd7c404816816126fa657550032f75ede67a66c7cc", - "b34744034cd61e139f85f6c4c92464927bed8343a7ac08acf9fb3c6796f80f08", - } - for chash, content in db_utils.get_content(str(tmpdir)): - chash = chash.decode("ascii").lower() - pyhash = sha256(content).hexdigest().lower() - assert pyhash == chash # Verify expected key (sha256 of content) - assert chash in expected_hashes - expected_hashes.remove(chash) - assert len(expected_hashes) == 0 # All expected hashes have been seen - def test_document_saving(self, tmpdir): """ check that document content is saved and hashed correctly """ test_url = utilities.BASE_TEST_URL + "/http_test_page.html" @@ -1030,3 +1012,32 @@ def type_filenames_into_form(**kwargs): u"upload-img": img_file_content, } assert expected_body == post_body_decoded + + +def test_javascript_saving(http_params, xpi, server): + """ check that javascript content is saved and hashed correctly """ + test_url = utilities.BASE_TEST_URL + "/http_test_page.html" + manager_params, browser_params = http_params() + + for browser_param in browser_params: + browser_param["http_instrument"] = True + browser_param["save_content"] = "script" + structured_storage = SqlLiteStorageProvider(db_path=manager_params["db"]) + ldb_path = Path(manager_params["data_directory"]) / "content.ldb" + unstructured_storage = LevelDbProvider(db_path=ldb_path) + manager = task_manager.TaskManager( + manager_params, browser_params, structured_storage, unstructured_storage + ) + manager.get(url=test_url, sleep=1) + manager.close() + expected_hashes = { + "0110c0521088c74f179615cd7c404816816126fa657550032f75ede67a66c7cc", + "b34744034cd61e139f85f6c4c92464927bed8343a7ac08acf9fb3c6796f80f08", + } + for chash, content in db_utils.get_content(ldb_path): + chash = chash.decode("ascii").lower() + pyhash = sha256(content).hexdigest().lower() + assert pyhash == chash # Verify expected key (sha256 of content) + assert chash in expected_hashes + expected_hashes.remove(chash) + assert len(expected_hashes) == 0 # All expected hashes have been seen From 1b2f16272122e246567485ac03d76cbdc0a637b6 Mon Sep 17 00:00:00 2001 From: vringar Date: Thu, 26 Nov 2020 00:48:44 +0100 Subject: [PATCH 039/139] Added node --- environment.yaml | 1 + scripts/environment-unpinned.yaml | 2 +- scripts/travis.sh | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/environment.yaml b/environment.yaml index a17f5b302..ad304762b 100644 --- a/environment.yaml +++ b/environment.yaml @@ -13,6 +13,7 @@ dependencies: - localstack=0.11.1.1 - multiprocess=0.70.11.1 - mypy=0.790 +- nodejs=14.15.1 - pandas=1.1.4 - pillow=8.0.1 - pip=20.2.4 diff --git a/scripts/environment-unpinned.yaml b/scripts/environment-unpinned.yaml index e30cb16b4..d23a6c666 100644 --- a/scripts/environment-unpinned.yaml +++ b/scripts/environment-unpinned.yaml @@ -10,7 +10,7 @@ dependencies: - geckodriver - leveldb - multiprocess - - nodejs<15.0.0 + - nodejs==14.*.* - pandas - pip - pillow diff --git a/scripts/travis.sh b/scripts/travis.sh index 6ef3b8055..11de2a1ea 100755 --- a/scripts/travis.sh +++ b/scripts/travis.sh @@ -3,7 +3,7 @@ if [[ "$TESTS" == "webextension" ]]; then cd openwpm/Extension/webext-instrumentation; npm test; else - python -m pytest --cov=../openwpm --cov-report=xml $TESTS -s -v --durations=10; + python -m pytest --cov=openwpm --cov-report=xml $TESTS -s -v --durations=10; exit_code=$?; if [[ "$exit_code" -ne 0 ]]; then exit $exit_code; From f01756b93f5051ae79705942e992d2975b4431c6 Mon Sep 17 00:00:00 2001 From: vringar Date: Thu, 26 Nov 2020 15:51:07 +0100 Subject: [PATCH 040/139] Fixed screenshot tests --- test/test_simple_commands.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_simple_commands.py b/test/test_simple_commands.py index 3a050f2d0..57bad506f 100644 --- a/test/test_simple_commands.py +++ b/test/test_simple_commands.py @@ -400,7 +400,7 @@ def test_save_screenshot_valid(http_params, task_manager_creator, display_mode): # Check that viewport image is not blank pattern = os.path.join( - manager_params["data_directory"], "screenshots", "1-*-test.png" + manager_params["data_directory"], "screenshots", "*-*-test.png" ) screenshot = glob.glob(pattern)[0] im = Image.open(screenshot) @@ -410,7 +410,7 @@ def test_save_screenshot_valid(http_params, task_manager_creator, display_mode): # Check that full page screenshot is not blank pattern = os.path.join( - manager_params["data_directory"], "screenshots", "1-*-test_full.png" + manager_params["data_directory"], "screenshots", "*-*-test_full.png" ) screenshot = glob.glob(pattern)[0] im = Image.open(screenshot) From c5dfcd685f103cdca1544b58f4ac34248db7e4a6 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 27 Nov 2020 11:32:43 +0100 Subject: [PATCH 041/139] Fixing more tests --- demo.py | 3 +- test/test_http_instrumentation.py | 181 +++++++++++++++++------------- test/test_simple_commands.py | 4 +- 3 files changed, 108 insertions(+), 80 deletions(-) diff --git a/demo.py b/demo.py index 3a8266822..bfadcf6e8 100644 --- a/demo.py +++ b/demo.py @@ -54,11 +54,12 @@ ) # Visits the sites -for site in sites: +for index, site in enumerate(sites): # Parallelize sites over all number of browsers set above. command_sequence = CommandSequence( site, + site_rank=index, reset=True, callback=lambda success, val=site: print("CommandSequence {} done".format(val)), ) diff --git a/test/test_http_instrumentation.py b/test/test_http_instrumentation.py index c566d40cf..73602e1f7 100644 --- a/test/test_http_instrumentation.py +++ b/test/test_http_instrumentation.py @@ -13,6 +13,7 @@ import pytest from openwpm import command_sequence, task_manager +from openwpm.command_sequence import CommandSequence from openwpm.storage.leveldb import LevelDbProvider from openwpm.storage.sql_provider import SqlLiteStorageProvider from openwpm.utilities import db_utils @@ -659,83 +660,6 @@ def test_page_visit(self): observed_records.add((src, dst, location)) assert HTTP_REDIRECTS == observed_records - def test_cache_hits_recorded(self): - """Verify all http responses are recorded, including cached responses - - Note that we expect to see all of the same requests and responses - during the second vist (even if cached) except for images. Cached - images do not trigger Observer Notification events. - See Bug 634073: https://bugzilla.mozilla.org/show_bug.cgi?id=634073 - - The test page includes an image which does several permanent redirects - before returning a 404. We expect to see new requests and responses - for this image when the page is reloaded. Additionally, the redirects - should be cached. - """ - test_url = utilities.BASE_TEST_URL + "/http_test_page.html" - manager_params, browser_params = self.get_config() - manager = task_manager.TaskManager(manager_params, browser_params) - manager.get(test_url, sleep=5) - manager.get(test_url, sleep=5) - manager.close() - db = manager_params["db"] - - request_id_to_url = dict() - - # HTTP Requests - rows = db_utils.query_db(db, "SELECT * FROM http_requests WHERE visit_id = 2") - observed_records = set() - for row in rows: - # HACK: favicon caching is unpredictable, don't bother checking it - if row["url"].split("?")[0].endswith("favicon.ico"): - continue - observed_records.add( - ( - row["url"].split("?")[0], - row["top_level_url"], - row["triggering_origin"], - row["loading_origin"], - row["loading_href"], - row["is_XHR"], - row["is_third_party_channel"], - row["is_third_party_to_top_window"], - row["resource_type"], - ) - ) - request_id_to_url[row["request_id"]] = row["url"] - assert HTTP_CACHED_REQUESTS == observed_records - - # HTTP Responses - rows = db_utils.query_db(db, "SELECT * FROM http_responses WHERE visit_id = 2") - observed_records = set() - for row in rows: - # HACK: favicon caching is unpredictable, don't bother checking it - if row["url"].split("?")[0].endswith("favicon.ico"): - continue - observed_records.add( - ( - row["url"].split("?")[0], - # TODO: referrer isn't available yet in the - # webext instrumentation | row['referrer'], - row["is_cached"], - ) - ) - assert row["request_id"] in request_id_to_url - assert request_id_to_url[row["request_id"]] == row["url"] - assert HTTP_CACHED_RESPONSES == observed_records - - # HTTP Redirects - rows = db_utils.query_db(db, "SELECT * FROM http_redirects WHERE visit_id = 2") - observed_records = set() - for row in rows: - # TODO: new_request_id isn't supported yet - # src = request_id_to_url[row['old_request_id']].split('?')[0] - # dst = request_id_to_url[row['new_request_id']].split('?')[0] - src = row["old_request_url"].split("?")[0] - dst = row["new_request_url"].split("?")[0] - observed_records.add((src, dst)) - assert HTTP_CACHED_REDIRECTS == observed_records - def test_document_saving(self, tmpdir): """ check that document content is saved and hashed correctly """ test_url = utilities.BASE_TEST_URL + "/http_test_page.html" @@ -1041,3 +965,106 @@ def test_javascript_saving(http_params, xpi, server): assert chash in expected_hashes expected_hashes.remove(chash) assert len(expected_hashes) == 0 # All expected hashes have been seen + + +def test_cache_hits_recorded(http_params, task_manager_creator): + """Verify all http responses are recorded, including cached responses + + Note that we expect to see all of the same requests and responses + during the second vist (even if cached) except for images. Cached + images do not trigger Observer Notification events. + See Bug 634073: https://bugzilla.mozilla.org/show_bug.cgi?id=634073 + + The test page includes an image which does several permanent redirects + before returning a 404. We expect to see new requests and responses + for this image when the page is reloaded. Additionally, the redirects + should be cached. + """ + test_url = utilities.BASE_TEST_URL + "/http_test_page.html" + manager_params, browser_params = http_params() + # ensuring that we only spawn one browser + manager = task_manager_creator((manager_params, [browser_params[0]])) + for i in range(2): + cs = CommandSequence(test_url, site_rank=i) + cs.get(sleep=5) + manager.execute_command_sequence(cs) + + manager.close() + db = manager_params["db"] + + request_id_to_url = dict() + + # HTTP Requests + rows = db_utils.query_db( + db, + """ + SELECT hr.* + FROM http_requests as hr + JOIN site_visits sv ON sv.visit_id = hr.visit_id and sv.browser_id = hr.browser_id + WHERE sv.site_rank = 1""", + ) + observed_records = set() + for row in rows: + # HACK: favicon caching is unpredictable, don't bother checking it + if row["url"].split("?")[0].endswith("favicon.ico"): + continue + observed_records.add( + ( + row["url"].split("?")[0], + row["top_level_url"], + row["triggering_origin"], + row["loading_origin"], + row["loading_href"], + row["is_XHR"], + row["is_third_party_channel"], + row["is_third_party_to_top_window"], + row["resource_type"], + ) + ) + request_id_to_url[row["request_id"]] = row["url"] + assert observed_records == HTTP_CACHED_REQUESTS + + # HTTP Responses + rows = db_utils.query_db( + db, + """ + SELECT hp.* + FROM http_responses as hp + JOIN site_visits sv ON sv.visit_id = hp.visit_id and sv.browser_id = hp.browser_id + WHERE sv.site_rank = 1""", + ) + observed_records = set() + for row in rows: + # HACK: favicon caching is unpredictable, don't bother checking it + if row["url"].split("?")[0].endswith("favicon.ico"): + continue + observed_records.add( + ( + row["url"].split("?")[0], + # TODO: referrer isn't available yet in the + # webext instrumentation | row['referrer'], + row["is_cached"], + ) + ) + assert row["request_id"] in request_id_to_url + assert request_id_to_url[row["request_id"]] == row["url"] + assert HTTP_CACHED_RESPONSES == observed_records + + # HTTP Redirects + rows = db_utils.query_db( + db, + """ + SELECT hr.* + FROM http_redirects as hr + JOIN site_visits sv ON sv.visit_id = hr.visit_id and sv.browser_id = hr.browser_id + WHERE sv.site_rank = 1""", + ) + observed_records = set() + for row in rows: + # TODO: new_request_id isn't supported yet + # src = request_id_to_url[row['old_request_id']].split('?')[0] + # dst = request_id_to_url[row['new_request_id']].split('?')[0] + src = row["old_request_url"].split("?")[0] + dst = row["new_request_url"].split("?")[0] + observed_records.add((src, dst)) + assert HTTP_CACHED_REDIRECTS == observed_records diff --git a/test/test_simple_commands.py b/test/test_simple_commands.py index 57bad506f..d755ed6be 100644 --- a/test/test_simple_commands.py +++ b/test/test_simple_commands.py @@ -435,7 +435,7 @@ def test_dump_page_source_valid(http_params, task_manager_creator, display_mode) # Source filename is of the follow structure: # `sources/-(-suffix).html` # thus for this test we expect `sources/1--test.html`. - outfile = os.path.join(manager_params["data_directory"], "sources", "1-*-test.html") + outfile = os.path.join(manager_params["data_directory"], "sources", "*-*-test.html") source_file = glob.glob(outfile)[0] with open(source_file, "rb") as f: actual_source = f.read() @@ -459,7 +459,7 @@ def test_recursive_dump_page_source_valid( manager.execute_command_sequence(cs) manager.close() - outfile = os.path.join(manager_params["data_directory"], "sources", "1-*.json.gz") + outfile = os.path.join(manager_params["data_directory"], "sources", "*-*.json.gz") src_file = glob.glob(outfile)[0] with gzip.GzipFile(src_file, "rb") as f: visit_source = json.loads(f.read().decode("utf-8")) From 9d635d32a1f3e257168b8432aed7206873225875 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 27 Nov 2020 11:46:57 +0100 Subject: [PATCH 042/139] Fixed tests --- test/test_http_instrumentation.py | 120 +++++++++++++++++------------- 1 file changed, 69 insertions(+), 51 deletions(-) diff --git a/test/test_http_instrumentation.py b/test/test_http_instrumentation.py index 73602e1f7..4030a823f 100644 --- a/test/test_http_instrumentation.py +++ b/test/test_http_instrumentation.py @@ -660,57 +660,6 @@ def test_page_visit(self): observed_records.add((src, dst, location)) assert HTTP_REDIRECTS == observed_records - def test_document_saving(self, tmpdir): - """ check that document content is saved and hashed correctly """ - test_url = utilities.BASE_TEST_URL + "/http_test_page.html" - expected_hashes = { - "2390eceab422db15bc45940b7e042e83e6cbd5f279f57e714bc4ad6cded7f966", - "25343f42d9ffa5c082745f775b172db87d6e14dfbc3160b48669e06d727bfc8d", - } - manager_params, browser_params = self.get_test_config(str(tmpdir)) - browser_params[0]["http_instrument"] = True - browser_params[0]["save_content"] = "main_frame,sub_frame" - manager = task_manager.TaskManager(manager_params, browser_params) - manager.get(url=test_url, sleep=1) - manager.close() - for chash, content in db_utils.get_content(str(tmpdir)): - chash = chash.decode("ascii").lower() - pyhash = sha256(content).hexdigest().lower() - assert pyhash == chash # Verify expected key (sha256 of content) - assert chash in expected_hashes - expected_hashes.remove(chash) - assert len(expected_hashes) == 0 # All expected hashes have been seen - - def test_content_saving(self, tmpdir): - """ check that content is saved and hashed correctly """ - test_url = utilities.BASE_TEST_URL + "/http_test_page.html" - manager_params, browser_params = self.get_test_config(str(tmpdir)) - browser_params[0]["http_instrument"] = True - browser_params[0]["save_content"] = True - manager = task_manager.TaskManager(manager_params, browser_params) - manager.get(url=test_url, sleep=1) - manager.close() - db = manager_params["db"] - rows = db_utils.query_db(db, "SELECT * FROM http_responses;") - disk_content = dict() - for row in rows: - if "MAGIC_REDIRECT" in row["url"] or "404" in row["url"]: - continue - path = urlparse(row["url"]).path - with open(os.path.join(BASE_PATH, path[1:]), "rb") as f: - content = f.read() - chash = sha256(content).hexdigest() - assert chash == row["content_hash"] - disk_content[chash] = content - - ldb_content = dict() - for chash, content in db_utils.get_content(str(tmpdir)): - chash = chash.decode("ascii") - ldb_content[chash] = content - - for k, v in disk_content.items(): - assert v == ldb_content[k] - def test_worker_script_requests(self): """Check correct URL attribution for requests made by worker script""" test_url = utilities.BASE_TEST_URL + "/http_worker_page.html" @@ -967,6 +916,75 @@ def test_javascript_saving(http_params, xpi, server): assert len(expected_hashes) == 0 # All expected hashes have been seen +def test_document_saving(http_params, xpi, server): + """ check that document content is saved and hashed correctly """ + test_url = utilities.BASE_TEST_URL + "/http_test_page.html" + expected_hashes = { + "2390eceab422db15bc45940b7e042e83e6cbd5f279f57e714bc4ad6cded7f966", + "25343f42d9ffa5c082745f775b172db87d6e14dfbc3160b48669e06d727bfc8d", + } + manager_params, browser_params = http_params() + for browser_param in browser_params: + browser_param["http_instrument"] = True + browser_param["save_content"] = "main_frame,sub_frame" + + structured_storage = SqlLiteStorageProvider(db_path=manager_params["db"]) + ldb_path = Path(manager_params["data_directory"]) / "content.ldb" + unstructured_storage = LevelDbProvider(db_path=ldb_path) + manager = task_manager.TaskManager( + manager_params, browser_params, structured_storage, unstructured_storage + ) + + manager.get(url=test_url, sleep=1) + manager.close() + for chash, content in db_utils.get_content(ldb_path): + chash = chash.decode("ascii").lower() + pyhash = sha256(content).hexdigest().lower() + assert pyhash == chash # Verify expected key (sha256 of content) + assert chash in expected_hashes + expected_hashes.remove(chash) + assert len(expected_hashes) == 0 # All expected hashes have been seen + + +def test_content_saving(http_params, xpi, server): + """ check that content is saved and hashed correctly """ + test_url = utilities.BASE_TEST_URL + "/http_test_page.html" + manager_params, browser_params = http_params() + for browser_param in browser_params: + browser_param["http_instrument"] = True + browser_param["save_content"] = True + + structured_storage = SqlLiteStorageProvider(db_path=manager_params["db"]) + ldb_path = Path(manager_params["data_directory"]) / "content.ldb" + unstructured_storage = LevelDbProvider(db_path=ldb_path) + manager = task_manager.TaskManager( + manager_params, browser_params, structured_storage, unstructured_storage + ) + manager.get(url=test_url, sleep=1) + manager.close() + + db = manager_params["db"] + rows = db_utils.query_db(db, "SELECT * FROM http_responses;") + disk_content = dict() + for row in rows: + if "MAGIC_REDIRECT" in row["url"] or "404" in row["url"]: + continue + path = urlparse(row["url"]).path + with open(os.path.join(BASE_PATH, path[1:]), "rb") as f: + content = f.read() + chash = sha256(content).hexdigest() + assert chash == row["content_hash"] + disk_content[chash] = content + + ldb_content = dict() + for chash, content in db_utils.get_content(ldb_path): + chash = chash.decode("ascii") + ldb_content[chash] = content + + for k, v in disk_content.items(): + assert v == ldb_content[k] + + def test_cache_hits_recorded(http_params, task_manager_creator): """Verify all http responses are recorded, including cached responses From ceb1d9869dda53ef793632154e267d1eb3cf556a Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 27 Nov 2020 12:56:08 +0100 Subject: [PATCH 043/139] Implemented local_storage.py --- openwpm/storage/local_storage.py | 47 ++++++++++++++++++++++++++-- openwpm/storage/storage_providers.py | 4 +-- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/openwpm/storage/local_storage.py b/openwpm/storage/local_storage.py index 978299d8e..57f766fda 100644 --- a/openwpm/storage/local_storage.py +++ b/openwpm/storage/local_storage.py @@ -1,10 +1,51 @@ +import logging +from pathlib import Path + +import pyarrow.parquet as pq +from pyarrow.lib import Table + from .arrow_storage import ArrowProvider -from .storage_providers import UnstructuredStorageProvider +from .storage_providers import TableName, UnstructuredStorageProvider class LocalArrowProvider(ArrowProvider): - ... + """ Stores Parquet files under storage_path/table_name/n.parquet""" + + def __init__(self, storage_path: Path) -> None: + super().__init__() + self.storage_path = storage_path + self.logger = logging.getLogger("openwpm") + + async def write_table(self, table_name: TableName, table: Table) -> None: + pq.write_to_dataset(table, self.storage_path / table_name) + + async def shutdown(self) -> None: + pass class LocalGzipProvider(UnstructuredStorageProvider): - ... + """ Stores files as storage_path/hash.zip """ + + def __init__(self, storage_path: Path) -> None: + super().__init__() + self.storage_path = storage_path + self.logger = logging.getLogger("openwpm") + + async def store_blob( + self, filename: str, blob: bytes, overwrite: bool = False + ) -> None: + path = self.storage_path / (filename + "zip") + if path.exists() and not overwrite: + self.logger.debug( + "File %s already exists on disk. Not overwriting", filename + ) + return + compressed = self._compress(blob) + with path.open(mode="wb") as f: + f.write(compressed.read()) + + async def flush_cache(self) -> None: + pass + + async def shutdown(self) -> None: + pass diff --git a/openwpm/storage/storage_providers.py b/openwpm/storage/storage_providers.py index 642814449..1872fe61e 100644 --- a/openwpm/storage/storage_providers.py +++ b/openwpm/storage/storage_providers.py @@ -64,13 +64,13 @@ async def store_blob( self, filename: str, blob: bytes, - compressed: bool = True, overwrite: bool = False, ) -> None: """Stores the given bytes under the provided filename""" pass - def _compress(self, blob: bytes) -> io.BytesIO: + @staticmethod + def _compress(blob: bytes) -> io.BytesIO: """Takes a byte blob and compresses it with gzip The returned BytesIO object is at stream position 0. This means it can be treated like a zip file on disk. From 11fb99fa55ff91010e602e8e4f78a15c1e5d4d69 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 27 Nov 2020 12:58:10 +0100 Subject: [PATCH 044/139] Cleaned up flush_cache --- openwpm/storage/arrow_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwpm/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py index 8be156ba2..c38b085ce 100644 --- a/openwpm/storage/arrow_storage.py +++ b/openwpm/storage/arrow_storage.py @@ -126,7 +126,7 @@ async def flush_cache(self, cond: asyncio.Condition = None) -> None: So we either grab the storing condition ourselves or the caller needs to pass us the locked storing_condition """ - got_cond = not not cond + got_cond = cond is not None if not got_cond: cond = self.storing_condition await cond.acquire() From 17835b4f5cd0a6cb1464ab9f254ee6c81a5895fa Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 27 Nov 2020 13:01:41 +0100 Subject: [PATCH 045/139] Fixing more tests --- test/test_simple_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_simple_commands.py b/test/test_simple_commands.py index d755ed6be..f9af0e1a0 100644 --- a/test/test_simple_commands.py +++ b/test/test_simple_commands.py @@ -198,7 +198,8 @@ def test_browse_site_visits_table_valid( manager.close() qry_res = db_utils.query_db( - manager_params["db"], "SELECT site_url, site_rank FROM site_visits" + manager_params["db"], + "SELECT site_url, site_rank FROM site_visits ORDER BY site_rank", ) # We had two separate page visits From cc9ed52522edaf16b05e63a330a0af1d9a1286d8 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 27 Nov 2020 14:37:59 +0100 Subject: [PATCH 046/139] Wrote test for LocalArrowProvider --- openwpm/storage/local_storage.py | 2 +- test/conftest.py | 1 + test/storage/test_local_storage_provider.py | 33 ++++++++++++++++++++ test/storage/test_memory_storage_provider.py | 3 +- 4 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 test/storage/test_local_storage_provider.py diff --git a/openwpm/storage/local_storage.py b/openwpm/storage/local_storage.py index 57f766fda..1b2b28ede 100644 --- a/openwpm/storage/local_storage.py +++ b/openwpm/storage/local_storage.py @@ -17,7 +17,7 @@ def __init__(self, storage_path: Path) -> None: self.logger = logging.getLogger("openwpm") async def write_table(self, table_name: TableName, table: Table) -> None: - pq.write_to_dataset(table, self.storage_path / table_name) + pq.write_to_dataset(table, str(self.storage_path / table_name)) async def shutdown(self) -> None: pass diff --git a/test/conftest.py b/test/conftest.py index 38b4cd0bd..ec50c9335 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -46,6 +46,7 @@ def default_params( tmp_path: Path, num_browsers: int = NUM_BROWSERS ) -> Tuple[ManagerParams, List[BrowserParams]]: """Just a simple wrapper around task_manager.load_default_params""" + assert len(tmp_path.iterdir()) == 0 data_dir = str(tmp_path) manager_params, browser_params = task_manager.load_default_params(num_browsers) manager_params["data_directory"] = data_dir diff --git a/test/storage/test_local_storage_provider.py b/test/storage/test_local_storage_provider.py new file mode 100644 index 000000000..886b1dc73 --- /dev/null +++ b/test/storage/test_local_storage_provider.py @@ -0,0 +1,33 @@ +import asyncio + +import pytest +from pandas import DataFrame +from pyarrow.parquet import ParquetDataset + +from openwpm.storage.local_storage import LocalArrowProvider +from openwpm.storage.storage_providers import TableName +from openwpm.types import VisitId + + +@pytest.mark.asyncio +async def test_local_arrow_storage_provider(tmp_path): + structured_provider = LocalArrowProvider(tmp_path) + data = { + "visit_id": 2, + "browser_id": 3, + "site_url": "https://example.com", + "site_rank": 4, + } + await structured_provider.store_record(TableName("site_visits"), VisitId(2), data) + await asyncio.gather( + structured_provider.finalize_visit_id(VisitId(2)), + structured_provider.flush_cache(), + ) + dataset = ParquetDataset(tmp_path / "site_visits") + df: DataFrame = dataset.read().to_pandas() + assert df.shape[0] == 1 + for row in df.itertuples(index=False): + assert row.visit_id == 2 + assert row.browser_id == 3 + assert row.site_rank == 4 + assert row.site_url == "https://example.com" diff --git a/test/storage/test_memory_storage_provider.py b/test/storage/test_memory_storage_provider.py index 97fcb128b..ba1300075 100644 --- a/test/storage/test_memory_storage_provider.py +++ b/test/storage/test_memory_storage_provider.py @@ -41,7 +41,7 @@ def structured_provider( return SqlLiteStorageProvider(tmp_path / "test_db.sqllite") elif request.param == memory_arrow: return MemoryArrowProvider() - raise ValueError("invalid internal test config") + request.raiseerror("invalid internal test config") def pytest_generate_tests(metafunc: Any) -> Any: @@ -66,7 +66,6 @@ async def test_basic_access( data = { "visit_id": 2, "browser_id": 3, - "instance_id": 4, "site_url": "https://example.com", } From 0098181b0db060d4d0adf454d6009a7982637835 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 27 Nov 2020 16:08:11 +0100 Subject: [PATCH 047/139] Introduced tests for local_storage_provider.py --- openwpm/storage/parquet_schema.py | 8 + openwpm/storage/schema.sql | 2 + openwpm/storage/storage_providers.py | 4 + test/conftest.py | 8 + test/storage/test_local_storage_provider.py | 39 ++-- test/storage/test_values.py | 225 ++++++++++++++++++++ 6 files changed, 266 insertions(+), 20 deletions(-) create mode 100644 test/storage/test_values.py diff --git a/openwpm/storage/parquet_schema.py b/openwpm/storage/parquet_schema.py index dbd6dab1d..d33d61af1 100644 --- a/openwpm/storage/parquet_schema.py +++ b/openwpm/storage/parquet_schema.py @@ -1,3 +1,11 @@ +""" +Arrow schema for our ArrowProvider.py + +IF YOU CHANGE THIS FILE ALSO CHANGE schema.sql and test_values.py +AND Schema-Documentation.md + +""" + import pyarrow as pa PQ_SCHEMAS = dict() diff --git a/openwpm/storage/schema.sql b/openwpm/storage/schema.sql index 35b1f9434..aab712670 100644 --- a/openwpm/storage/schema.sql +++ b/openwpm/storage/schema.sql @@ -1,6 +1,8 @@ /* This file is sourced during the initialization * of the crawler. Make sure everything is CREATE * IF NOT EXISTS, otherwise there will be errors + * IF YOU CHANGE THIS FILE ALSO CHANGE test_values.py and parquet_schema.py + * AND Schema-Documentation.md */ CREATE TABLE IF NOT EXISTS task ( diff --git a/openwpm/storage/storage_providers.py b/openwpm/storage/storage_providers.py index 1872fe61e..abc3a7cb7 100644 --- a/openwpm/storage/storage_providers.py +++ b/openwpm/storage/storage_providers.py @@ -37,6 +37,10 @@ class StructuredStorageProvider(StorageProvider): def __init__(self) -> None: super().__init__() + # TODO: Discuss if we want visit_id here + # It will always be part of the record and make the interface bigger + # than it needs to be + # But not having to access the data is also convenient @abstractmethod async def store_record( self, table: TableName, visit_id: VisitId, record: Dict[str, Any] diff --git a/test/conftest.py b/test/conftest.py index ec50c9335..8ac9f747a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -7,6 +7,7 @@ import pytest from openwpm import task_manager +from openwpm.mp_logger import MPLogger from openwpm.storage.sql_provider import SqlLiteStorageProvider from openwpm.task_manager import TaskManager from openwpm.types import BrowserParams, ManagerParams @@ -96,3 +97,10 @@ def parameterize( return manager_params, browser_params return parameterize + + +@pytest.fixture() +def mp_logger(tmp_path): + logger = MPLogger(tmp_path / "openwpm.log") + yield logger + logger.close() diff --git a/test/storage/test_local_storage_provider.py b/test/storage/test_local_storage_provider.py index 886b1dc73..09442809f 100644 --- a/test/storage/test_local_storage_provider.py +++ b/test/storage/test_local_storage_provider.py @@ -8,26 +8,25 @@ from openwpm.storage.storage_providers import TableName from openwpm.types import VisitId +from .test_values import TEST_VALUES + @pytest.mark.asyncio -async def test_local_arrow_storage_provider(tmp_path): +async def test_local_arrow_storage_provider(tmp_path, mp_logger): structured_provider = LocalArrowProvider(tmp_path) - data = { - "visit_id": 2, - "browser_id": 3, - "site_url": "https://example.com", - "site_rank": 4, - } - await structured_provider.store_record(TableName("site_visits"), VisitId(2), data) - await asyncio.gather( - structured_provider.finalize_visit_id(VisitId(2)), - structured_provider.flush_cache(), - ) - dataset = ParquetDataset(tmp_path / "site_visits") - df: DataFrame = dataset.read().to_pandas() - assert df.shape[0] == 1 - for row in df.itertuples(index=False): - assert row.visit_id == 2 - assert row.browser_id == 3 - assert row.site_rank == 4 - assert row.site_url == "https://example.com" + visit_ids = set() + for table_name, test_data in TEST_VALUES.items(): + visit_id = VisitId(test_data["visit_id"]) + visit_ids.add(visit_id) + await structured_provider.store_record( + TableName(table_name), visit_id, test_data + ) + task_list = list(structured_provider.finalize_visit_id(i) for i in visit_ids) + task_list.append(structured_provider.flush_cache()) + ret_vals = await asyncio.gather(*task_list) + for table_name, test_data in TEST_VALUES.items(): + dataset = ParquetDataset(tmp_path / table_name) + df: DataFrame = dataset.read().to_pandas() + assert df.shape[0] == 1 + for row in df.itertuples(index=False): + assert row._asdict() == test_data diff --git a/test/storage/test_values.py b/test/storage/test_values.py new file mode 100644 index 000000000..2c79c20c3 --- /dev/null +++ b/test/storage/test_values.py @@ -0,0 +1,225 @@ +""" This file should contain one entry for every table +so that we can test storing and loading for every single entry +for every structured storage provider. + +IF YOU CHANGE THIS FILE ALSO CHANGE schema.sql and parquet_schema.py +AND Schema-Documentation.md +""" + +import random +import string + +TEST_VALUES = dict() + + +def random_word(length): + letters = string.ascii_lowercase + return "".join(random.choice(letters) for _ in range(length)) + + +# site_visits +fields = { + "visit_id": random.randint(0, 2 ** 63 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "site_url": random_word(12), + "site_rank": random.randint(0, 2 ** 31 - 1), +} +TEST_VALUES["site_visits"] = fields + +# crawl_history +fields = { + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "command": random_word(12), + "arguments": random_word(12), + "retry_number": random.randint(0, 2 ** 7 - 1), + "command_status": random_word(12), + "error": random_word(12), + "traceback": random_word(12), + "duration": random.randint(0, 2 ** 63 - 1), +} +TEST_VALUES["crawl_history"] = fields + +# http_requests +fields = { + "incognito": random.randint(0, 2 ** 31 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "extension_session_uuid": random_word(12), + "event_ordinal": random.randint(0, 2 ** 63 - 1), + "window_id": random.randint(0, 2 ** 63 - 1), + "tab_id": random.randint(0, 2 ** 63 - 1), + "frame_id": random.randint(0, 2 ** 63 - 1), + "url": random_word(12), + "top_level_url": random_word(12), + "parent_frame_id": random.randint(0, 2 ** 63 - 1), + "frame_ancestors": random_word(12), + "method": random_word(12), + "referrer": random_word(12), + "headers": random_word(12), + "request_id": random.randint(0, 2 ** 63 - 1), + "is_XHR": random.choice([True, False]), + "is_third_party_channel": random.choice([True, False]), + "is_third_party_to_top_window": random.choice([True, False]), + "triggering_origin": random_word(12), + "loading_origin": random_word(12), + "loading_href": random_word(12), + "req_call_stack": random_word(12), + "resource_type": random_word(12), + "post_body": random_word(12), + "post_body_raw": random_word(12), + "time_stamp": random_word(12), +} +TEST_VALUES["http_requests"] = fields + +# http_responses +fields = { + "incognito": random.randint(0, 2 ** 31 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "extension_session_uuid": random_word(12), + "event_ordinal": random.randint(0, 2 ** 63 - 1), + "window_id": random.randint(0, 2 ** 63 - 1), + "tab_id": random.randint(0, 2 ** 63 - 1), + "frame_id": random.randint(0, 2 ** 63 - 1), + "url": random_word(12), + "method": random_word(12), + "response_status": random.randint(0, 2 ** 63 - 1), + "response_status_text": random_word(12), + "is_cached": random.choice([True, False]), + "headers": random_word(12), + "request_id": random.randint(0, 2 ** 63 - 1), + "location": random_word(12), + "time_stamp": random_word(12), + "content_hash": random_word(12), +} +TEST_VALUES["http_responses"] = fields + +# http_redirects +fields = { + "incognito": random.randint(0, 2 ** 31 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "old_request_url": random_word(12), + "old_request_id": random_word(12), + "new_request_url": random_word(12), + "new_request_id": random_word(12), + "extension_session_uuid": random_word(12), + "event_ordinal": random.randint(0, 2 ** 63 - 1), + "window_id": random.randint(0, 2 ** 63 - 1), + "tab_id": random.randint(0, 2 ** 63 - 1), + "frame_id": random.randint(0, 2 ** 63 - 1), + "response_status": random.randint(0, 2 ** 63 - 1), + "response_status_text": random_word(12), + "headers": random_word(12), + "time_stamp": random_word(12), +} +TEST_VALUES["http_redirects"] = fields + +# javascript +fields = { + "incognito": random.randint(0, 2 ** 31 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "extension_session_uuid": random_word(12), + "event_ordinal": random.randint(0, 2 ** 63 - 1), + "page_scoped_event_ordinal": random.randint(0, 2 ** 63 - 1), + "window_id": random.randint(0, 2 ** 63 - 1), + "tab_id": random.randint(0, 2 ** 63 - 1), + "frame_id": random.randint(0, 2 ** 63 - 1), + "script_url": random_word(12), + "script_line": random_word(12), + "script_col": random_word(12), + "func_name": random_word(12), + "script_loc_eval": random_word(12), + "document_url": random_word(12), + "top_level_url": random_word(12), + "call_stack": random_word(12), + "symbol": random_word(12), + "operation": random_word(12), + "value": random_word(12), + "arguments": random_word(12), + "time_stamp": random_word(12), +} +TEST_VALUES["javascript"] = fields + +# javascript_cookies +fields = { + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "extension_session_uuid": random_word(12), + "event_ordinal": random.randint(0, 2 ** 63 - 1), + "record_type": random_word(12), + "change_cause": random_word(12), + "expiry": random_word(12), + "is_http_only": random.choice([True, False]), + "is_host_only": random.choice([True, False]), + "is_session": random.choice([True, False]), + "host": random_word(12), + "is_secure": random.choice([True, False]), + "name": random_word(12), + "path": random_word(12), + "value": random_word(12), + "same_site": random_word(12), + "first_party_domain": random_word(12), + "store_id": random_word(12), + "time_stamp": random_word(12), +} +TEST_VALUES["javascript_cookies"] = fields + +# navigations +fields = { + "incognito": random.randint(0, 2 ** 31 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "extension_session_uuid": random_word(12), + "process_id": random.randint(0, 2 ** 63 - 1), + "window_id": random.randint(0, 2 ** 63 - 1), + "tab_id": random.randint(0, 2 ** 63 - 1), + "tab_opener_tab_id": random.randint(0, 2 ** 63 - 1), + "frame_id": random.randint(0, 2 ** 63 - 1), + "parent_frame_id": random.randint(0, 2 ** 63 - 1), + "window_width": random.randint(0, 2 ** 63 - 1), + "window_height": random.randint(0, 2 ** 63 - 1), + "window_type": random_word(12), + "tab_width": random.randint(0, 2 ** 63 - 1), + "tab_height": random.randint(0, 2 ** 63 - 1), + "tab_cookie_store_id": random_word(12), + "uuid": random_word(12), + "url": random_word(12), + "transition_qualifiers": random_word(12), + "transition_type": random_word(12), + "before_navigate_event_ordinal": random.randint(0, 2 ** 63 - 1), + "before_navigate_time_stamp": random_word(12), + "committed_event_ordinal": random.randint(0, 2 ** 63 - 1), + "time_stamp": random_word(12), +} +TEST_VALUES["navigations"] = fields + +# callstacks +fields = { + "visit_id": random.randint(0, 2 ** 63 - 1), + "request_id": random.randint(0, 2 ** 63 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "call_stack": random_word(12), +} +TEST_VALUES["callstacks"] = fields + +# incomplete_visits +fields = { + "visit_id": random.randint(0, 2 ** 63 - 1), +} +TEST_VALUES["incomplete_visits"] = fields + +# dns_responses +fields = { + "request_id": random.randint(0, 2 ** 63 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "hostname": random_word(12), + "addresses": random_word(12), + "canonical_name": random_word(12), + "is_TRR": random.choice([True, False]), + "time_stamp": random_word(12), +} +TEST_VALUES["dns_responses"] = fields From bf4f92cfb9c4d1cf112c6d9dc3261026173b0c94 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 30 Nov 2020 21:24:07 +0100 Subject: [PATCH 048/139] Asserting test dir is empty --- test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index 8ac9f747a..9315b0082 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -47,7 +47,7 @@ def default_params( tmp_path: Path, num_browsers: int = NUM_BROWSERS ) -> Tuple[ManagerParams, List[BrowserParams]]: """Just a simple wrapper around task_manager.load_default_params""" - assert len(tmp_path.iterdir()) == 0 + assert len(list(tmp_path.iterdir())) == 0 data_dir = str(tmp_path) manager_params, browser_params = task_manager.load_default_params(num_browsers) manager_params["data_directory"] = data_dir From 5c0a1e11e06e873a0cec29247573d4837b2221c3 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 4 Dec 2020 11:58:09 +0100 Subject: [PATCH 049/139] Creating subfolder for different aggregators --- openwpm/storage/cloud_storage/__init__.py | 0 openwpm/storage/cloud_storage/gcp_storage.py | 21 ++++++++++++++++++++ openwpm/storage/cloud_storage/s3_storage.py | 6 ++++++ openwpm/storage/s3_storage.py | 2 -- scripts/environment-unpinned-dev.yaml | 9 +-------- scripts/environment-unpinned.yaml | 2 +- 6 files changed, 29 insertions(+), 11 deletions(-) create mode 100644 openwpm/storage/cloud_storage/__init__.py create mode 100644 openwpm/storage/cloud_storage/gcp_storage.py create mode 100644 openwpm/storage/cloud_storage/s3_storage.py delete mode 100644 openwpm/storage/s3_storage.py diff --git a/openwpm/storage/cloud_storage/__init__.py b/openwpm/storage/cloud_storage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openwpm/storage/cloud_storage/gcp_storage.py b/openwpm/storage/cloud_storage/gcp_storage.py new file mode 100644 index 000000000..c2ccdfad6 --- /dev/null +++ b/openwpm/storage/cloud_storage/gcp_storage.py @@ -0,0 +1,21 @@ +from pyarrow.lib import Table + +from ..arrow_storage import ArrowProvider +from ..storage_providers import TableName, UnstructuredStorageProvider + + +class GcsStructuredProvider(ArrowProvider): + """This class allows you to upload Parquet files to GCS. + This might not actually be the thing that we want to do + long term but seeing as GCS is the S3 equivalent of GCP + it is the easiest way forward. + """ + + def __init__(self): + super().__init__() + + async def write_table(self, table_name: TableName, table: Table) -> None: + pass + + async def shutdown(self) -> None: + pass diff --git a/openwpm/storage/cloud_storage/s3_storage.py b/openwpm/storage/cloud_storage/s3_storage.py new file mode 100644 index 000000000..68fa8d845 --- /dev/null +++ b/openwpm/storage/cloud_storage/s3_storage.py @@ -0,0 +1,6 @@ +from ..arrow_storage import ArrowProvider +from ..storage_providers import UnstructuredStorageProvider + + +class S3StorageProvider(ArrowProvider): + pass diff --git a/openwpm/storage/s3_storage.py b/openwpm/storage/s3_storage.py deleted file mode 100644 index a1f78c0b9..000000000 --- a/openwpm/storage/s3_storage.py +++ /dev/null @@ -1,2 +0,0 @@ -from .arrow_storage import ArrowProvider -from .storage_providers import UnstructuredStorageProvider diff --git a/scripts/environment-unpinned-dev.yaml b/scripts/environment-unpinned-dev.yaml index 3b3fc7b06..455b85557 100644 --- a/scripts/environment-unpinned-dev.yaml +++ b/scripts/environment-unpinned-dev.yaml @@ -5,16 +5,9 @@ dependencies: - codecov - pytest-cov - ipython - - localstack==0.11.1.1 # See https://github.com/mozilla/OpenWPM/pull/682 - pip - pre-commit - pytest - mypy - pytest-asyncio - - pip: - # Select depenedencies from localstack[full] that we need - - amazon-kclpy - - crontab - - flask-cors - - moto-ext==1.3.15.15 # See https://github.com/mozilla/OpenWPM/pull/682 - - subprocess32 + diff --git a/scripts/environment-unpinned.yaml b/scripts/environment-unpinned.yaml index d23a6c666..50d956ce9 100644 --- a/scripts/environment-unpinned.yaml +++ b/scripts/environment-unpinned.yaml @@ -19,7 +19,7 @@ dependencies: - python - pyvirtualdisplay==0.2.5 # https://github.com/mozilla/OpenWPM/issues/694 - redis-py - - s3fs==0.4.0 # https://github.com/mozilla/OpenWPM/issues/614 + - s3fs - selenium - sentry-sdk - tabulate From 598146336748eccd2210eb80b91c9d30e4eb6762 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 4 Dec 2020 15:44:16 +0100 Subject: [PATCH 050/139] New depencies and init() --- environment.yaml | 14 +- openwpm/storage/cloud_storage/gcp_storage.py | 4 +- openwpm/storage/leveldb.py | 11 +- openwpm/storage/sql_provider.py | 9 +- openwpm/storage/storage_controller.py | 5 +- openwpm/storage/storage_providers.py | 9 ++ scripts/environment-unpinned.yaml | 1 + scripts/prune-environment.py | 15 +- test/test_s3_aggregator.py | 144 ------------------- test/utilities.py | 101 ------------- 10 files changed, 49 insertions(+), 264 deletions(-) delete mode 100644 test/test_s3_aggregator.py diff --git a/environment.yaml b/environment.yaml index ad304762b..656185705 100644 --- a/environment.yaml +++ b/environment.yaml @@ -7,38 +7,32 @@ dependencies: - click=7.1.2 - codecov=2.1.10 - dill=0.3.3 +- gcsfs=0.7.1 - geckodriver=0.28.0 - ipython=7.19.0 - leveldb=1.22 -- localstack=0.11.1.1 - multiprocess=0.70.11.1 - mypy=0.790 - nodejs=14.15.1 - pandas=1.1.4 - pillow=8.0.1 -- pip=20.2.4 -- pre-commit=2.9.0 +- pip=20.3.1 +- pre-commit=2.9.2 - psutil=5.7.3 - pyarrow=2.0.0 -- pytest-asyncio=0.14.0 - pytest-cov=2.10.1 - pytest=6.1.2 - python=3.8.6 - pyvirtualdisplay=0.2.5 - redis-py=3.5.3 -- s3fs=0.4.0 +- s3fs=0.5.1 - selenium=3.141.0 - sentry-sdk=0.19.4 - tabulate=0.8.7 - tblib=1.6.0 - wget=1.20.1 - pip: - - amazon-kclpy==2.0.1 - - crontab==0.22.9 - domain-utils==0.7.1 - - flask-cors==3.0.9 - jsonschema==3.2.0 - - moto-ext==1.3.15.15 - plyvel==1.3.0 - - subprocess32==3.5.4 name: openwpm diff --git a/openwpm/storage/cloud_storage/gcp_storage.py b/openwpm/storage/cloud_storage/gcp_storage.py index c2ccdfad6..0c6a37bc3 100644 --- a/openwpm/storage/cloud_storage/gcp_storage.py +++ b/openwpm/storage/cloud_storage/gcp_storage.py @@ -1,3 +1,4 @@ +from fs_gcsfs import GCSFS from pyarrow.lib import Table from ..arrow_storage import ArrowProvider @@ -11,8 +12,9 @@ class GcsStructuredProvider(ArrowProvider): it is the easiest way forward. """ - def __init__(self): + def __init__(self, bucket_name: str, path: str) -> None: super().__init__() + self.file_system = GCSFS(bucket_name) async def write_table(self, table_name: TableName, table: Table) -> None: pass diff --git a/openwpm/storage/leveldb.py b/openwpm/storage/leveldb.py index a626155c3..53d995dae 100644 --- a/openwpm/storage/leveldb.py +++ b/openwpm/storage/leveldb.py @@ -9,15 +9,20 @@ class LevelDbProvider(UnstructuredStorageProvider): def __init__(self, db_path: Path): + self.db_path = db_path + self._ldb_counter = 0 + self._ldb_commit_time = 0 + self.ldb = None + self.content_batch = None + + async def init(self) -> None: self.ldb = plyvel.DB( - str(db_path), + str(self.db_path), create_if_missing=True, write_buffer_size=128 * 10 ** 6, compression="snappy", ) self.content_batch = self.ldb.write_batch() - self._ldb_counter = 0 - self._ldb_commit_time = 0 async def flush_cache(self) -> None: """Write out content batch to LevelDB database""" diff --git a/openwpm/storage/sql_provider.py b/openwpm/storage/sql_provider.py index 39ad70e09..f5295a2b8 100644 --- a/openwpm/storage/sql_provider.py +++ b/openwpm/storage/sql_provider.py @@ -16,11 +16,16 @@ class SqlLiteStorageProvider(StructuredStorageProvider): def __init__(self, db_path: PathLike) -> None: super().__init__() - self.db = sqlite3.connect(db_path, check_same_thread=False) - self.cur = self.db.cursor() + self.db_path = db_path self._sql_counter = 0 self._sql_commit_time = 0 self.logger = logging.getLogger("openwpm") + self.db = None + self.cur = None + + async def init(self) -> None: + self.db = sqlite3.connect(self.db_path) + self.cur = self.db.cursor() self._create_tables() def _create_tables(self) -> None: diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 42e1c9337..9e3f979e4 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -254,12 +254,15 @@ async def finish_tasks(self) -> None: await task async def _run(self) -> None: + + await self.structured_storage.init() + if self.unstructured_storage: + await self.unstructured_storage.init() server: asyncio.AbstractServer = await asyncio.start_server( self._handler, "localhost", 0, family=socket.AF_INET ) sockets = server.sockets assert sockets is not None - # assert len(sockets) == 1 socketname = sockets[0].getsockname() self.status_queue.put(socketname) status_queue_update = asyncio.create_task( diff --git a/openwpm/storage/storage_providers.py b/openwpm/storage/storage_providers.py index abc3a7cb7..118f02573 100644 --- a/openwpm/storage/storage_providers.py +++ b/openwpm/storage/storage_providers.py @@ -20,6 +20,15 @@ class StorageProvider(ABC): Inherit from StructuredStorageProvider or UnstructuredStorageProvider instead """ + @abstractmethod + async def init(self) -> None: + """Initializes the StorageProvider for use + + Guaranteed to be called in the process the + StorageController runs in. + """ + pass + @abstractmethod async def flush_cache(self) -> None: """ Blockingly write out any cached data to the respective storage """ diff --git a/scripts/environment-unpinned.yaml b/scripts/environment-unpinned.yaml index 50d956ce9..5e174b8a5 100644 --- a/scripts/environment-unpinned.yaml +++ b/scripts/environment-unpinned.yaml @@ -7,6 +7,7 @@ dependencies: - click - dill # - firefox-unbranded - when it's available + - gcsfs - geckodriver - leveldb - multiprocess diff --git a/scripts/prune-environment.py b/scripts/prune-environment.py index 171bb4347..ab2c0abca 100644 --- a/scripts/prune-environment.py +++ b/scripts/prune-environment.py @@ -28,15 +28,26 @@ def iterate_deps(xs, ys, accumulator): deps_not_pip = [] deps_pip = [] + iterate_deps( env_pinned["dependencies"][:-1], env_unpinned["dependencies"][:-1] + env_unpinned_dev["dependencies"][:-1], deps_not_pip, ) + +# Checking if there are any pip dependencies +try: + deps_pip_unpinned = env_unpinned["dependencies"][-1]["pip"] +except: + deps_pip_unpinned = [] +try: + deps_pip_unpinned_dev = env_unpinned_dev["dependencies"][-1]["pip"] +except: + deps_pip_unpinned_dev = [] + iterate_deps( env_pinned["dependencies"][-1]["pip"], - env_unpinned["dependencies"][-1]["pip"] - + env_unpinned_dev["dependencies"][-1]["pip"], + deps_pip_unpinned_dev + deps_pip_unpinned, deps_pip, ) pruned_dependencies = [ diff --git a/test/test_s3_aggregator.py b/test/test_s3_aggregator.py deleted file mode 100644 index 4e4ebce03..000000000 --- a/test/test_s3_aggregator.py +++ /dev/null @@ -1,144 +0,0 @@ -import json -import os -import time -from collections import defaultdict - -import boto3 -import pytest -from localstack.services import infra -from multiprocess import Queue - -from openwpm import task_manager -from openwpm.command_sequence import CommandSequence -from openwpm.storage.parquet_schema import PQ_SCHEMAS - -from .openwpmtest import OpenWPMTest -from .utilities import BASE_TEST_URL, LocalS3Dataset, LocalS3Session, local_s3_bucket - - -class TestS3Aggregator(OpenWPMTest): - @classmethod - def setup_class(cls): - infra.start_infra(asynchronous=True, apis=["s3"]) - boto3.DEFAULT_SESSION = LocalS3Session() - cls.s3_client = boto3.client("s3") - cls.s3_resource = boto3.resource("s3") - - @classmethod - def teardown_class(cls): - infra.stop_infra() - infra.check_infra(retries=2, expect_shutdown=True, apis=["s3"]) - - def get_config(self, num_browsers=1, data_dir=""): - manager_params, browser_params = self.get_test_config( - data_dir, num_browsers=num_browsers - ) - manager_params["output_format"] = "s3" - manager_params["s3_bucket"] = local_s3_bucket(self.s3_resource) - manager_params["s3_directory"] = "s3-aggregator-tests" - for i in range(num_browsers): - browser_params[i]["http_instrument"] = True - browser_params[i]["js_instrument"] = True - browser_params[i]["cookie_instrument"] = True - browser_params[i]["navigation_instrument"] = True - browser_params[i]["callstack_instrument"] = True - browser_params[i]["dns_instrument"] = True - return manager_params, browser_params - - @pytest.mark.skipif( - "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", - reason="Localstack fails to start on Travis", - ) - def test_basic_properties(self): - TEST_SITE = "%s/s3_aggregator.html" % BASE_TEST_URL - NUM_VISITS = 2 - NUM_BROWSERS = 4 - manager_params, browser_params = self.get_config(num_browsers=NUM_BROWSERS) - manager = task_manager.TaskManager(manager_params, browser_params) - for _ in range(NUM_VISITS * NUM_BROWSERS): - manager.get(TEST_SITE, sleep=1) - manager.close() - - dataset = LocalS3Dataset( - manager_params["s3_bucket"], manager_params["s3_directory"] - ) - - # Test visit_id consistency - visit_ids = defaultdict(set) - expected_tables = dict(PQ_SCHEMAS) - # We don't expect incomplete visits to exist - # since the visit shouldn't be interrupted - expected_tables.pop("incomplete_visits") - for table_name in expected_tables: - table = dataset.load_table(table_name) - visit_ids[table_name] = table.visit_id.unique() - actual = len(visit_ids[table_name]) - expected = NUM_VISITS * NUM_BROWSERS - assert actual == expected, ( - f"Table {table_name} had {actual} " f"visit_ids, we expected {expected}" - ) - for vid in visit_ids[table_name]: - assert (vid >= 0) and (vid < (1 << 53)) - for table_name, ids in visit_ids.items(): - assert set(ids) == set(visit_ids["site_visits"]) - - # Ensure http table is created - assert TEST_SITE in dataset.load_table("http_requests").top_level_url.unique() - - # Ensure config directory is created and contains the correct number - # of configuration files - config_file = dataset.list_files("config", prepend_root=True) - assert len(config_file) == 1 # only one instance started in test - config = json.loads(str(dataset.get_file(config_file[0]), "utf-8")) - assert len(config["browser_params"]) == NUM_BROWSERS - - @pytest.mark.skipif( - "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", - reason="Localstack fails to start on Travis", - ) - def test_commit_on_timeout(self): - TEST_SITE = "%s/s3_aggregator.html" % BASE_TEST_URL - manager_params, browser_params = self.get_config(num_browsers=1) - manager_params["s3_directory"] = "s3-aggregator-tests-2" - manager = task_manager.TaskManager(manager_params, browser_params) - manager.get(TEST_SITE, sleep=1) - dataset = LocalS3Dataset( - manager_params["s3_bucket"], manager_params["s3_directory"] - ) - with pytest.raises((FileNotFoundError, OSError)): - requests = dataset.load_table("http_requests") - time.sleep(45) # Current timeout - dataset2 = LocalS3Dataset( - manager_params["s3_bucket"], manager_params["s3_directory"] - ) - requests = dataset2.load_table("http_requests") - assert TEST_SITE in requests.top_level_url.unique() - manager.close() - - @pytest.mark.skipif( - "TRAVIS" in os.environ and os.environ["TRAVIS"] == "true", - reason="Localstack fails to start on Travis", - ) - def test_s3_callbacks(self): - TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html" - manager_params, browser_params = self.get_config() - dataset = LocalS3Dataset( - manager_params["s3_bucket"], manager_params["s3_directory"] - ) - manager = task_manager.TaskManager(manager_params, browser_params) - queue = Queue() - - def ensure_site_in_s3(success: bool): - # Ensure http table is created - queue.put( - TEST_SITE in dataset.load_table("http_requests").top_level_url.unique() - ) - - sequence = CommandSequence( - TEST_SITE, reset=True, blocking=True, callback=ensure_site_in_s3 - ) - sequence.get() - manager.execute_command_sequence(sequence) - manager.close() - - assert queue.get() diff --git a/test/utilities.py b/test/utilities.py index eb7668645..476b916e4 100644 --- a/test/utilities.py +++ b/test/utilities.py @@ -5,7 +5,6 @@ from os.path import dirname, realpath from urllib.parse import parse_qs, urlparse -import boto3 import pyarrow.parquet as pq import s3fs from botocore.credentials import Credentials @@ -110,103 +109,3 @@ def start_server(): thread.start() print("...serving at port", LOCAL_WEBSERVER_PORT) return server, thread - - -class LocalS3Session(object): - """ - Ensures that the local s3 service is used when - setup as the default boto3 Session - Based on localstack_client/session.py - """ - - def __init__( - self, - aws_access_key_id="accesskey", - aws_secret_access_key="secretkey", - aws_session_token="token", - region_name="us-east-1", - endpoint_url="http://localhost:4572", - botocore_session=None, - profile_name=None, - localstack_host=None, - ): - self.env = "local" - self.session = boto3.session.Session() - self.aws_access_key_id = aws_access_key_id - self.aws_secret_access_key = aws_secret_access_key - self.aws_session_token = aws_session_token - self.region_name = region_name - self.endpoint_url = endpoint_url - - def resource(self, service_name, **kwargs): - return self.session.resource( - service_name, - endpoint_url=self.endpoint_url, - aws_access_key_id=self.aws_access_key_id, - aws_secret_access_key=self.aws_secret_access_key, - region_name=self.region_name, - verify=False, - ) - - def get_credentials(self): - return Credentials( - access_key=self.aws_access_key_id, - secret_key=self.aws_secret_access_key, - token=self.aws_session_token, - ) - - def client(self, service_name, **kwargs): - return self.session.client( - service_name, - endpoint_url=self.endpoint_url, - aws_access_key_id=self.aws_access_key_id, - aws_secret_access_key=self.aws_secret_access_key, - region_name=self.region_name, - verify=False, - ) - - -def local_s3_bucket(resource, name="localstack-foo"): - bucket = resource.Bucket(name) - bucket.create() - return name - - -class LocalS3Dataset(object): - def __init__(self, bucket, directory): - self.bucket = bucket - self.root_directory = directory - self.visits_uri = "%s/%s/visits/%%s" % (self.bucket, self.root_directory) - self.s3_fs = s3fs.S3FileSystem(session=LocalS3Session()) - boto3.DEFAULT_SESSION = LocalS3Session() - self.s3_client = boto3.client("s3") - self.s3_resource = boto3.resource("s3") - - def load_table(self, table_name): - return ( - pq.ParquetDataset(self.visits_uri % table_name, filesystem=self.s3_fs) - .read_pandas() - .to_pandas() - ) - - def list_files(self, directory, prepend_root=False): - bucket = self.s3_resource.Bucket(self.bucket) - files = list() - if prepend_root: - prefix = "%s/%s/" % (self.root_directory, directory) - else: - prefix = directory - for summary in bucket.objects.filter(Prefix=prefix): - files.append(summary.key) - return files - - def get_file(self, filename, prepend_root=False): - if prepend_root: - key = "%s/%s" % (self.root_directory, filename) - else: - key = filename - obj = self.s3_client.get_object(Bucket=self.bucket, Key=key) - body = obj["Body"] - content = body.read() - body.close() - return content From ba56b34ee13f5d27e71df76db93f4f48d22b7b40 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 4 Dec 2020 18:18:47 +0100 Subject: [PATCH 051/139] Everything is terribly broken --- demo.py | 10 ++- openwpm/storage/cloud_storage/gcp_storage.py | 87 +++++++++++++++++++- test/storage/test_gcp.py | 29 +++++++ test/storage/test_local_storage_provider.py | 2 +- 4 files changed, 121 insertions(+), 7 deletions(-) create mode 100644 test/storage/test_gcp.py diff --git a/demo.py b/demo.py index bfadcf6e8..52a32c811 100644 --- a/demo.py +++ b/demo.py @@ -2,6 +2,10 @@ import os from openwpm.command_sequence import CommandSequence +from openwpm.storage.cloud_storage.gcp_storage import ( + GcsStructuredProvider, + GcsUnstructuredProvider, +) from openwpm.storage.sql_provider import SqlLiteStorageProvider from openwpm.task_manager import TaskManager, load_default_params @@ -43,13 +47,13 @@ logging_params = {"log_level_console": logging.DEBUG} # Instantiates the measurement platform +project = "senglehardt-openwpm-test-1" +bucket_name = "openwpm-test-bucket" # Commands time out by default after 60 seconds manager = TaskManager( manager_params, browser_params, - SqlLiteStorageProvider( - os.path.expanduser(manager_params["data_directory"] + "crawl-data.sqlite") - ), + GcsStructuredProvider(project=project, bucket_name=bucket_name, base_path="visits"), None, ) diff --git a/openwpm/storage/cloud_storage/gcp_storage.py b/openwpm/storage/cloud_storage/gcp_storage.py index 0c6a37bc3..9f6f04034 100644 --- a/openwpm/storage/cloud_storage/gcp_storage.py +++ b/openwpm/storage/cloud_storage/gcp_storage.py @@ -1,4 +1,10 @@ -from fs_gcsfs import GCSFS +import asyncio +import logging +from functools import partial +from typing import Any, Callable + +import pyarrow.parquet as pq +from gcsfs import GCSFileSystem from pyarrow.lib import Table from ..arrow_storage import ArrowProvider @@ -12,11 +18,86 @@ class GcsStructuredProvider(ArrowProvider): it is the easiest way forward. """ - def __init__(self, bucket_name: str, path: str) -> None: + def __init__( + self, + project: str, + bucket_name: str, + base_path: str, + token: str = None, + ) -> None: super().__init__() - self.file_system = GCSFS(bucket_name) + self.project = project + self.bucket_name = bucket_name + self.base_path = base_path + self.token = token + self.file_system = None + self.base_path = f"{self.bucket_name}/{base_path}/{{table_name}}" + + async def init(self) -> None: + self.file_system = GCSFileSystem( + project=self.project, + token=self.token, + access="read_write", + check_connection=True, + ) async def write_table(self, table_name: TableName, table: Table) -> None: + self.file_system.start_transaction() + pq.write_to_dataset( + table, + self.base_path.format(table_name=table_name), + filesystem=self.file_system, + ) + self.file_system.end_transaction() + + async def shutdown(self) -> None: + pass + + +class GcsUnstructuredProvider(UnstructuredStorageProvider): + """This class allows you to upload Parquet files to GCS. + This might not actually be the thing that we want to do + long term but seeing as GCS is the S3 equivalent of GCP + it is the easiest way forward. + """ + + def __init__( + self, + project: str, + bucket_name: str, + base_path: str, + token: str = None, + ) -> None: + super().__init__() + self.project = project + self.bucket_name = bucket_name + self.base_path = base_path + self.token = token + self.file_system = None + self.base_path = f"{bucket_name}/{base_path}/{{filename}}" + + self.file_name_cache = set() + """The set of all filenames ever uploaded, checked before uploading""" + self.logger = logging.getLogger("openwpm") + + async def init(self) -> None: + pass + + async def store_blob( + self, filename: str, blob: bytes, overwrite: bool = False + ) -> None: + target_path = self.base_path.format(filename=filename) + if not overwrite and ( + filename in self.file_name_cache or self.file_system.exists(target_path) + ): + self.logger.info("Not saving out file %s as it already exists", filename) + return + + with self.file_system.open(target_path, mode="wb") as f: + f.write(blob) + self.file_name_cache.add(filename) + + async def flush_cache(self) -> None: pass async def shutdown(self) -> None: diff --git a/test/storage/test_gcp.py b/test/storage/test_gcp.py new file mode 100644 index 000000000..4e5a7d419 --- /dev/null +++ b/test/storage/test_gcp.py @@ -0,0 +1,29 @@ +import asyncio + +import pytest + +from openwpm.storage.cloud_storage.gcp_storage import GcsStructuredProvider +from openwpm.storage.storage_providers import TableName +from openwpm.types import VisitId +from test.storage.test_values import TEST_VALUES + + +@pytest.mark.skip +@pytest.mark.asyncio +async def test_gcp_structured(mp_logger): + project = "senglehardt-openwpm-test-1" + bucket_name = "openwpm-test-bucket" + structured_provider = GcsStructuredProvider( + project=project, bucket_name=bucket_name, base_path="visits" + ) + await structured_provider.init() + visit_ids = set() + for table_name, test_data in TEST_VALUES.items(): + visit_id = VisitId(test_data["visit_id"]) + visit_ids.add(visit_id) + await structured_provider.store_record( + TableName(table_name), visit_id, test_data + ) + task_list = list(structured_provider.finalize_visit_id(i) for i in visit_ids) + task_list.append(structured_provider.flush_cache()) + await asyncio.gather(*task_list) diff --git a/test/storage/test_local_storage_provider.py b/test/storage/test_local_storage_provider.py index 09442809f..1afa01435 100644 --- a/test/storage/test_local_storage_provider.py +++ b/test/storage/test_local_storage_provider.py @@ -23,7 +23,7 @@ async def test_local_arrow_storage_provider(tmp_path, mp_logger): ) task_list = list(structured_provider.finalize_visit_id(i) for i in visit_ids) task_list.append(structured_provider.flush_cache()) - ret_vals = await asyncio.gather(*task_list) + await asyncio.gather(*task_list) for table_name, test_data in TEST_VALUES.items(): dataset = ParquetDataset(tmp_path / table_name) df: DataFrame = dataset.read().to_pandas() From 74ae07c11669679a890b5b08b02eeb01fc87ef53 Mon Sep 17 00:00:00 2001 From: vringar Date: Tue, 8 Dec 2020 00:15:52 +0100 Subject: [PATCH 052/139] Figured out finalize_visit_id --- demo.py | 7 +- openwpm/storage/arrow_storage.py | 33 ++++----- openwpm/storage/cloud_storage/gcp_storage.py | 8 ++- openwpm/storage/in_memory_storage.py | 51 ++++++++++++-- openwpm/storage/leveldb.py | 13 +++- openwpm/storage/local_storage.py | 7 ++ openwpm/storage/sql_provider.py | 32 +++++++-- openwpm/storage/storage_controller.py | 70 +++++++++++++++---- openwpm/storage/storage_providers.py | 11 +-- test/conftest.py | 7 +- test/storage/__init__.py | 2 + test/storage/test_gcp.py | 5 +- test/storage/test_storage_controller.py | 72 ++++++++++++++++++-- test/storage/test_values.py | 2 + 14 files changed, 255 insertions(+), 65 deletions(-) diff --git a/demo.py b/demo.py index 52a32c811..3a54be979 100644 --- a/demo.py +++ b/demo.py @@ -53,7 +53,12 @@ manager = TaskManager( manager_params, browser_params, - GcsStructuredProvider(project=project, bucket_name=bucket_name, base_path="visits"), + GcsStructuredProvider( + project=project, + bucket_name=bucket_name, + base_path="visits", + token="/home/stefan/.config/gcloud/legacy_credentials/szabka@mozilla.com/adc.json", + ), None, ) diff --git a/openwpm/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py index c38b085ce..421eed51a 100644 --- a/openwpm/storage/arrow_storage.py +++ b/openwpm/storage/arrow_storage.py @@ -3,7 +3,7 @@ import random from abc import abstractmethod from collections import defaultdict -from typing import Any, DefaultDict, Dict, List +from typing import Any, Awaitable, DefaultDict, Dict, List import pandas as pd import pyarrow as pa @@ -40,9 +40,8 @@ def factory_function() -> DefaultDict[TableName, List[Dict[str, Any]]]: self._batches: DefaultDict[TableName, List[pa.RecordBatch]] = defaultdict(list) # Used to synchronize the finalizing and the flushing - self.storing_condition = asyncio.Condition() - self.flushing = False - + self.storing_lock = asyncio.Lock() + self.flush_event = asyncio.Event() self._instance_id = random.getrandbits(32) async def store_record( @@ -94,25 +93,22 @@ def _is_cache_full(self) -> bool: async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False - ) -> None: + ) -> Awaitable[None]: if interrupted: await self.store_record(INCOMPLETE_VISITS, visit_id, {"visit_id": visit_id}) # This code is pretty tricky as there are a number of things going on # 1. No finalize_visit_id shouldn't return unless the visit has been saved to storage # 2. No new batches should be created while saving out all the batches - async with self.storing_condition: - if self.flushing: - # This way we wait if there is an on going flush - await self.storing_condition.wait() + async with self.storing_lock: self._create_batch(visit_id) - if self._is_cache_full(): - self.flushing = True - await self.flush_cache(self.storing_condition) - self.flushing = False - else: - await self.storing_condition.wait() + await self.flush_cache(self.storing_lock) + + async def wait_on_condition(event: asyncio.Event) -> None: + await event.wait() + + return wait_on_condition(self.flush_event) @abstractmethod async def write_table(self, table_name: TableName, table: Table) -> None: @@ -120,7 +116,7 @@ async def write_table(self, table_name: TableName, table: Table) -> None: This should only return once it's actually saved out """ - async def flush_cache(self, cond: asyncio.Condition = None) -> None: + async def flush_cache(self, cond: asyncio.Lock = None) -> None: """We need to hack around the fact that asyncio has no reentrant lock and which prevents us from creating a reentrant condition So we either grab the storing condition ourselves or the caller needs @@ -128,13 +124,12 @@ async def flush_cache(self, cond: asyncio.Condition = None) -> None: """ got_cond = cond is not None if not got_cond: - cond = self.storing_condition + cond = self.storing_lock await cond.acquire() assert cond.locked() for table_name, batches in self._batches.items(): table = pa.Table.from_batches(batches) await self.write_table(table_name, table) - cond.notify_all() - + self.flush_event.set() if not got_cond: cond.release() diff --git a/openwpm/storage/cloud_storage/gcp_storage.py b/openwpm/storage/cloud_storage/gcp_storage.py index 9f6f04034..ca6bb7eb6 100644 --- a/openwpm/storage/cloud_storage/gcp_storage.py +++ b/openwpm/storage/cloud_storage/gcp_storage.py @@ -1,7 +1,7 @@ import asyncio import logging from functools import partial -from typing import Any, Callable +from typing import Any, Callable, Set import pyarrow.parquet as pq from gcsfs import GCSFileSystem @@ -34,11 +34,13 @@ def __init__( self.base_path = f"{self.bucket_name}/{base_path}/{{table_name}}" async def init(self) -> None: + event_loop = asyncio.get_event_loop() self.file_system = GCSFileSystem( project=self.project, token=self.token, access="read_write", - check_connection=True, + loop=event_loop, + asynchronous=True, ) async def write_table(self, table_name: TableName, table: Table) -> None: @@ -76,7 +78,7 @@ def __init__( self.file_system = None self.base_path = f"{bucket_name}/{base_path}/{{filename}}" - self.file_name_cache = set() + self.file_name_cache: Set[str] = set() """The set of all filenames ever uploaded, checked before uploading""" self.logger = logging.getLogger("openwpm") diff --git a/openwpm/storage/in_memory_storage.py b/openwpm/storage/in_memory_storage.py index f960fddc3..b18e7cba0 100644 --- a/openwpm/storage/in_memory_storage.py +++ b/openwpm/storage/in_memory_storage.py @@ -1,6 +1,7 @@ +import asyncio import logging from collections import defaultdict -from typing import Any, DefaultDict, Dict, List, Tuple +from typing import Any, Awaitable, DefaultDict, Dict, List, Tuple from multiprocess import Queue from pyarrow import Table @@ -27,33 +28,63 @@ class MemoryStructuredProvider(StructuredStorageProvider): """ This storage provider passes all it's data to the MemoryStructuredProviderHandle in process safe way. + This makes it ideal for testing and for small crawls where no persistence is required + + It also aims to only save out data as late as possible to ensure that storage_controller + only relies on the guarantees given in the interface. """ + async def init(self) -> None: + pass + def __init__(self) -> None: super().__init__() self.queue = Queue() self.handle = MemoryProviderHandle(self.queue) self.logger = logging.getLogger("openwpm") + self.cache1: DefaultDict[ + VisitId, DefaultDict[TableName, List[Dict[str, Any]]] + ] = defaultdict(lambda: defaultdict(list)) + """The cache for entries before they are finalized""" + self.cache2: DefaultDict[TableName, List[Dict[str, Any]]] = defaultdict(list) + """For all entries that have been finalized but not yet flushed out to the queue""" async def flush_cache(self) -> None: - pass + self.logger.info("Flushing cache") + + for table, record_list in self.cache2.items(): + self.logger.info(f"Saving out {len(record_list)} entries for {table}") + for record in record_list: + self.queue.put((table, record)) + self.cache2.clear() async def store_record( self, table: TableName, visit_id: VisitId, record: Dict[str, Any] ) -> None: - self.logger.debug( + self.logger.info( "Saving into table %s for visit_id %d record %r", table, visit_id, record ) - self.queue.put((table, record)) + self.cache1[visit_id][table].append(record) async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False - ) -> None: - pass + ) -> Awaitable[None]: + self.logger.info( + f"Finalizing visit_id {visit_id} which was {'' if interrupted else 'not'} interrupted" + ) + for table, record_list in self.cache1[visit_id].items(): + self.cache2[table].extend(record_list) + + del self.cache1[visit_id] + + fut = asyncio.get_event_loop().create_future() + fut.set_result(None) + return fut async def shutdown(self) -> None: - pass + if self.cache1 != {} or self.cache2 != {}: + self.logger.error("Shutting down with unsaved records") class MemoryProviderHandle: @@ -78,6 +109,9 @@ class MemoryUnstructuredProvider(UnstructuredStorageProvider): Use this provider for writing tests and for small crawls where no persistence is required """ + async def init(self) -> None: + pass + def __init__(self) -> None: self.storage: Dict[str, bytes] = {} @@ -103,6 +137,9 @@ async def shutdown(self) -> None: class MemoryArrowProvider(ArrowProvider): + async def init(self) -> None: + pass + def __init__(self) -> None: super().__init__() self.queue = Queue() diff --git a/openwpm/storage/leveldb.py b/openwpm/storage/leveldb.py index 53d995dae..96d42ef7c 100644 --- a/openwpm/storage/leveldb.py +++ b/openwpm/storage/leveldb.py @@ -1,6 +1,8 @@ from pathlib import Path +from typing import Optional import plyvel +from plyvel._plyvel import WriteBatch from .storage_providers import UnstructuredStorageProvider @@ -12,8 +14,8 @@ def __init__(self, db_path: Path): self.db_path = db_path self._ldb_counter = 0 self._ldb_commit_time = 0 - self.ldb = None - self.content_batch = None + self.ldb: Optional[plyvel.DB] = None + self.content_batch: Optional[WriteBatch] = None async def init(self) -> None: self.ldb = plyvel.DB( @@ -26,10 +28,13 @@ async def init(self) -> None: async def flush_cache(self) -> None: """Write out content batch to LevelDB database""" + assert self.content_batch is not None + assert self.ldb is not None self.content_batch.write() self.content_batch = self.ldb.write_batch() async def shutdown(self) -> None: + assert self.ldb is not None self.ldb.close() print("Ldb is closed:", self.ldb.closed) @@ -37,9 +42,11 @@ async def store_blob( self, filename: str, blob: bytes, - compressed: bool = True, overwrite: bool = False, ) -> None: + assert self.ldb is not None + assert self.content_batch is not None + content_hash = str(filename).encode("ascii") if self.ldb.get(content_hash) is not None: return diff --git a/openwpm/storage/local_storage.py b/openwpm/storage/local_storage.py index 1b2b28ede..1ecfffe60 100644 --- a/openwpm/storage/local_storage.py +++ b/openwpm/storage/local_storage.py @@ -1,4 +1,5 @@ import logging +from abc import ABC from pathlib import Path import pyarrow.parquet as pq @@ -11,6 +12,9 @@ class LocalArrowProvider(ArrowProvider): """ Stores Parquet files under storage_path/table_name/n.parquet""" + async def init(self) -> None: + pass + def __init__(self, storage_path: Path) -> None: super().__init__() self.storage_path = storage_path @@ -26,6 +30,9 @@ async def shutdown(self) -> None: class LocalGzipProvider(UnstructuredStorageProvider): """ Stores files as storage_path/hash.zip """ + async def init(self) -> None: + pass + def __init__(self, storage_path: Path) -> None: super().__init__() self.storage_path = storage_path diff --git a/openwpm/storage/sql_provider.py b/openwpm/storage/sql_provider.py index f5295a2b8..9d28e8b8b 100644 --- a/openwpm/storage/sql_provider.py +++ b/openwpm/storage/sql_provider.py @@ -3,8 +3,15 @@ import os import sqlite3 from os import PathLike -from sqlite3 import IntegrityError, InterfaceError, OperationalError, ProgrammingError -from typing import Any, Dict, List, Tuple +from sqlite3 import ( + Connection, + Cursor, + IntegrityError, + InterfaceError, + OperationalError, + ProgrammingError, +) +from typing import Any, AsyncGenerator, Awaitable, Dict, List, Optional, Tuple from openwpm.types import VisitId @@ -20,8 +27,8 @@ def __init__(self, db_path: PathLike) -> None: self._sql_counter = 0 self._sql_commit_time = 0 self.logger = logging.getLogger("openwpm") - self.db = None - self.cur = None + self.db: Optional[Connection] = None + self.cur: Optional[Cursor] = None async def init(self) -> None: self.db = sqlite3.connect(self.db_path) @@ -30,11 +37,13 @@ async def init(self) -> None: def _create_tables(self) -> None: """Create tables (if this is a new database)""" + assert self.db is not None with open(SCHEMA_FILE, "r") as f: self.db.executescript(f.read()) self.db.commit() async def flush_cache(self) -> None: + assert self.db is not None self.db.commit() async def store_record( @@ -43,6 +52,7 @@ async def store_record( """Submit a record to be stored The storing might not happen immediately """ + assert self.cur is not None statement, args = self._generate_insert(table=table, data=record) for i in range(len(args)): if isinstance(args[i], bytes): @@ -83,17 +93,27 @@ def _generate_insert( statement = statement + ") " + value_str + ")" return statement, values - def execute_statement(self, statement: str): + def execute_statement(self, statement: str) -> None: + assert self.cur is not None + assert self.db is not None self.cur.execute(statement) self.db.commit() async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False - ) -> None: + ) -> Awaitable[None]: + assert self.cur is not None + if interrupted: self.logger.warning("Visit with visit_id %d got interrupted", visit_id) self.cur.execute("INSERT INTO incomplete_visits VALUES (?)", (visit_id,)) + async def done(): + return + + return done() + async def shutdown(self) -> None: + assert self.db is not None self.db.commit() self.db.close() diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 9e3f979e4..72bc4edac 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -7,7 +7,17 @@ import threading import time from collections import defaultdict -from typing import Any, DefaultDict, Dict, List, Literal, NoReturn, Optional, Tuple +from typing import ( + Any, + Awaitable, + DefaultDict, + Dict, + List, + Literal, + NoReturn, + Optional, + Tuple, +) from multiprocess import Queue @@ -63,7 +73,6 @@ def __init__( self.shutdown_queue = shutdown_queue self._shutdown_flag = False self._relaxed = False - self.record_queue: Queue = None # Initialized on `startup` self.logger = logging.getLogger("openwpm") self.current_tasks: DefaultDict[VisitId, List[asyncio.Task]] = defaultdict(list) self.structured_storage = structured_storage @@ -81,7 +90,7 @@ async def _handler( await self.handler(reader, writer) except Exception as e: self.logger.error( - "An exception occured while listening for data", exc_info=e + "An exception occurred while processing for records", exc_info=e ) async def handler( @@ -145,7 +154,7 @@ async def handler( continue table_name = TableName(record_type) - + # Turning these into task to be able to verify self.current_tasks[visit_id].append( asyncio.create_task( self.structured_storage.store_record( @@ -161,29 +170,47 @@ async def _handle_meta(self, visit_id: VisitId, data: Dict[str, Any]) -> None: Supported message types: - finalize: A message sent by the extension to signal that a visit_id is complete. + - initialize: TODO: Start complaining if we receive data for a visit_id + before the initialize event happened. (This might not be easy + because of `site_visits` """ action: str = data["action"] if action == ACTION_TYPE_INITIALIZE: return elif action == ACTION_TYPE_FINALIZE: - self.logger.info("Awaiting all tasks for visit_id %d", visit_id) success = data["success"] - for task in self.current_tasks[visit_id]: - await task - self.logger.debug( - "Awaited all tasks for visit_id %d while finalizing", visit_id - ) - - await self.structured_storage.finalize_visit_id( - visit_id, interrupted=not success - ) + completion_token = await self.finalize_visit_id(visit_id, success) + await completion_token self.completion_queue.put((visit_id, success)) del self.current_tasks[visit_id] else: raise ValueError("Unexpected action: %s", action) + async def finalize_visit_id( + self, visit_id: VisitId, success: bool + ) -> Awaitable[None]: + """Makes sure all records for a given visit_id + have been processed before we invoke finalize_visit_id + on the structured_storage + """ + self.logger.info("Awaiting all tasks for visit_id %d", visit_id) + + for task in self.current_tasks[visit_id]: + await task + self.logger.debug( + "Awaited all tasks for visit_id %d while finalizing", visit_id + ) + completion_token = await self.structured_storage.finalize_visit_id( + visit_id, interrupted=not success + ) + return completion_token + async def update_status_queue(self) -> NoReturn: - """Send manager process a status update.""" + """Send manager process a status update. + + This coroutine will get cancelled with an exception + so there is no need for an orderly return + """ while True: await asyncio.sleep(STATUS_UPDATE_INTERVAL) visit_id_count = len(self.current_tasks.keys()) @@ -280,6 +307,19 @@ async def _run(self) -> None: await server.wait_closed() await self.finish_tasks() + + finalization_tokens = {} + visit_ids = list(self.current_tasks.keys()) + for visit_id in visit_ids: + finalization_tokens[visit_id] = await self.finalize_visit_id( + visit_id, success=False + ) + await self.structured_storage.flush_cache() + for visit_id, token in finalization_tokens.items(): + await token + self.completion_queue.put((visit_id, False)) + del self.current_tasks[visit_id] + await self.shutdown() def run(self) -> None: diff --git a/openwpm/storage/storage_providers.py b/openwpm/storage/storage_providers.py index 118f02573..0a835177c 100644 --- a/openwpm/storage/storage_providers.py +++ b/openwpm/storage/storage_providers.py @@ -1,7 +1,7 @@ import gzip import io from abc import ABC, abstractmethod -from typing import Any, Dict, List, NewType, Tuple +from typing import Any, AsyncGenerator, Awaitable, Dict, List, NewType, Tuple from openwpm.types import VisitId @@ -62,11 +62,12 @@ async def store_record( @abstractmethod async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False - ) -> None: - """This method is invoked to inform the StrucuturedStorageProvider that no more + ) -> Awaitable[None]: + """This method is invoked to inform the StructuredStorageProvider that no more records for this visit_id will be submitted - It will only return once all records for this visit_id have been saved - to permanent storage. + + This method returns an awaitable that will resolve once the records have been + saved out to persistent storage """ pass diff --git a/test/conftest.py b/test/conftest.py index 9315b0082..995f4abc3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -101,6 +101,11 @@ def parameterize( @pytest.fixture() def mp_logger(tmp_path): - logger = MPLogger(tmp_path / "openwpm.log") + log_path = tmp_path / "openwpm.log" + logger = MPLogger(log_path) yield logger logger.close() + # The performance hit for this might be unacceptable but it might help us discover bugs + with log_path.open("r") as f: + for line in f: + assert "ERROR" not in line diff --git a/test/storage/__init__.py b/test/storage/__init__.py index 8b1378917..374c8f7b5 100644 --- a/test/storage/__init__.py +++ b/test/storage/__init__.py @@ -1 +1,3 @@ +"""TODO: Write tests for a correctly set interrupt status +""" diff --git a/test/storage/test_gcp.py b/test/storage/test_gcp.py index 4e5a7d419..9cc763aea 100644 --- a/test/storage/test_gcp.py +++ b/test/storage/test_gcp.py @@ -14,7 +14,10 @@ async def test_gcp_structured(mp_logger): project = "senglehardt-openwpm-test-1" bucket_name = "openwpm-test-bucket" structured_provider = GcsStructuredProvider( - project=project, bucket_name=bucket_name, base_path="visits" + project=project, + bucket_name=bucket_name, + base_path="test/visits", + token="/home/stefan/.config/gcloud/legacy_credentials/szabka@mozilla.com/adc.json", ) await structured_provider.init() visit_ids = set() diff --git a/test/storage/test_storage_controller.py b/test/storage/test_storage_controller.py index 10e80d779..98de82c6b 100644 --- a/test/storage/test_storage_controller.py +++ b/test/storage/test_storage_controller.py @@ -1,15 +1,27 @@ +import asyncio import logging import time +import pandas as pd import pytest +from multiprocess import Queue +from pandas.testing import assert_frame_equal from openwpm.mp_logger import MPLogger from openwpm.socket_interface import ClientSocket from openwpm.storage.in_memory_storage import ( + MemoryArrowProvider, MemoryStructuredProvider, MemoryUnstructuredProvider, ) -from openwpm.storage.storage_controller import StorageControllerHandle +from openwpm.storage.storage_controller import ( + ACTION_TYPE_FINALIZE, + RECORD_TYPE_META, + SHUTDOWN_SIGNAL, + StorageController, + StorageControllerHandle, +) +from test.storage.test_values import TEST_VALUES, TEST_VISIT_IDS @pytest.fixture(scope="session") @@ -26,7 +38,7 @@ def logger() -> MPLogger: def test_startup_and_shutdown(logger: MPLogger) -> None: - data = {"visit_id": 1, "asd": "dfg"} + structured = MemoryStructuredProvider() unstructured = MemoryUnstructuredProvider() agg_handle = StorageControllerHandle(structured, unstructured) @@ -34,8 +46,60 @@ def test_startup_and_shutdown(logger: MPLogger) -> None: assert agg_handle.listener_address is not None cs = ClientSocket() cs.connect(*agg_handle.listener_address) - cs.send(("test", data)) + for table, data in TEST_VALUES.items(): + cs.send((table, data)) + + for visit_id in TEST_VISIT_IDS: + cs.send( + ( + RECORD_TYPE_META, + {"action": ACTION_TYPE_FINALIZE, "visit_id": visit_id, "success": True}, + ) + ) agg_handle.shutdown() handle = structured.handle handle.poll_queue() - assert handle.storage["test"] == [data] + for table, data in TEST_VALUES.items(): + assert handle.storage[table] == [data] + + +@pytest.mark.asyncio +async def test_arrow_provider(logger: MPLogger) -> None: + structured = MemoryArrowProvider() + status_queue = Queue() + completion_queue = Queue() + shutdown_queue = Queue() + + storage_controller = StorageController( + structured, + None, + status_queue=status_queue, + completion_queue=completion_queue, + shutdown_queue=shutdown_queue, + ) + task = asyncio.create_task(storage_controller._run()) + cs = ClientSocket() + while status_queue.empty(): + await asyncio.sleep(5) + + cs.connect(*status_queue.get()) + + for table, data in TEST_VALUES.items(): + cs.send((table, data)) + + # This sleep needs to be here because otherwise it is executing blockingly on the single thread, + # so the server doesn't ever wake up + await asyncio.sleep(1) + shutdown_queue.put((SHUTDOWN_SIGNAL, True)) + await task + + handle = structured.handle + handle.poll_queue() + for table, data in TEST_VALUES.items(): + if table == "incomplete_visits": + # We currently mark all of them as failed, because we don't bother sending a finalize command + continue + t1 = handle.storage[table][0].to_pandas().drop(columns=["instance_id"]) + t2 = pd.DataFrame({k: [v] for k, v in data.items()}) + # Since t2 doesn't get created schema the inferred types are different + assert_frame_equal(t1, t2, check_dtype=False) diff --git a/test/storage/test_values.py b/test/storage/test_values.py index 2c79c20c3..4a564ec8c 100644 --- a/test/storage/test_values.py +++ b/test/storage/test_values.py @@ -223,3 +223,5 @@ def random_word(length): "time_stamp": random_word(12), } TEST_VALUES["dns_responses"] = fields + +TEST_VISIT_IDS = [d["visit_id"] for d in TEST_VALUES.values()] From 6068c696450c83a61fc5f783af24479d581589d0 Mon Sep 17 00:00:00 2001 From: vringar Date: Tue, 8 Dec 2020 00:35:55 +0100 Subject: [PATCH 053/139] Running two event loops kinda works??? --- demo.py | 2 +- openwpm/storage/cloud_storage/gcp_storage.py | 6 +----- test/storage/test_gcp.py | 16 +++++++++------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/demo.py b/demo.py index 3a54be979..5cf9d8996 100644 --- a/demo.py +++ b/demo.py @@ -56,7 +56,7 @@ GcsStructuredProvider( project=project, bucket_name=bucket_name, - base_path="visits", + base_path="demo/visits", token="/home/stefan/.config/gcloud/legacy_credentials/szabka@mozilla.com/adc.json", ), None, diff --git a/openwpm/storage/cloud_storage/gcp_storage.py b/openwpm/storage/cloud_storage/gcp_storage.py index ca6bb7eb6..74ebfefbc 100644 --- a/openwpm/storage/cloud_storage/gcp_storage.py +++ b/openwpm/storage/cloud_storage/gcp_storage.py @@ -36,11 +36,7 @@ def __init__( async def init(self) -> None: event_loop = asyncio.get_event_loop() self.file_system = GCSFileSystem( - project=self.project, - token=self.token, - access="read_write", - loop=event_loop, - asynchronous=True, + project=self.project, token=self.token, access="read_write" ) async def write_table(self, table_name: TableName, table: Table) -> None: diff --git a/test/storage/test_gcp.py b/test/storage/test_gcp.py index 9cc763aea..470cd157d 100644 --- a/test/storage/test_gcp.py +++ b/test/storage/test_gcp.py @@ -5,7 +5,7 @@ from openwpm.storage.cloud_storage.gcp_storage import GcsStructuredProvider from openwpm.storage.storage_providers import TableName from openwpm.types import VisitId -from test.storage.test_values import TEST_VALUES +from test.storage.test_values import TEST_VALUES, TEST_VISIT_IDS @pytest.mark.skip @@ -16,17 +16,19 @@ async def test_gcp_structured(mp_logger): structured_provider = GcsStructuredProvider( project=project, bucket_name=bucket_name, - base_path="test/visits", + base_path="test/2", token="/home/stefan/.config/gcloud/legacy_credentials/szabka@mozilla.com/adc.json", ) await structured_provider.init() - visit_ids = set() + for table_name, test_data in TEST_VALUES.items(): visit_id = VisitId(test_data["visit_id"]) - visit_ids.add(visit_id) await structured_provider.store_record( TableName(table_name), visit_id, test_data ) - task_list = list(structured_provider.finalize_visit_id(i) for i in visit_ids) - task_list.append(structured_provider.flush_cache()) - await asyncio.gather(*task_list) + finalize_token = [ + await structured_provider.finalize_visit_id(i) for i in TEST_VISIT_IDS + ] + await structured_provider.flush_cache() + for token in finalize_token: + await token From 17a22d31b29e5ca280ac3ba18f59abb426d2bce6 Mon Sep 17 00:00:00 2001 From: vringar Date: Tue, 8 Dec 2020 08:37:41 +0100 Subject: [PATCH 054/139] Rearming the event --- openwpm/storage/arrow_storage.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/openwpm/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py index 421eed51a..fa560ce38 100644 --- a/openwpm/storage/arrow_storage.py +++ b/openwpm/storage/arrow_storage.py @@ -94,6 +94,20 @@ def _is_cache_full(self) -> bool: async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False ) -> Awaitable[None]: + """This method is the reason the finalize_visit_id interface returns an awaitable. + This was necessary as we needed to enable the following pattern. + ``` + token = await structured_storage.finalize_visit_id(1) + structured_storage.flush_cache() + await token + ``` + If there was no token returned and the method would just block/yield after turning the + record into a batch, there would be no way to know, when it's save to flush_cache as + I couldn't find a way to run a coroutine until it yields and then run a different one. + + With the current setup `token` aka a `wait_on_condition` coroutine will only return once + the event has been set. + """ if interrupted: await self.store_record(INCOMPLETE_VISITS, visit_id, {"visit_id": visit_id}) @@ -101,6 +115,11 @@ async def finalize_visit_id( # 1. No finalize_visit_id shouldn't return unless the visit has been saved to storage # 2. No new batches should be created while saving out all the batches async with self.storing_lock: + # After flush_cache has executed the event needs to be rearmed + # so that newly created wait_on_condition don't just complete + # instantly + if self.flush_event.is_set(): + self.flush_event.clear() self._create_batch(visit_id) if self._is_cache_full(): await self.flush_cache(self.storing_lock) @@ -118,9 +137,8 @@ async def write_table(self, table_name: TableName, table: Table) -> None: async def flush_cache(self, cond: asyncio.Lock = None) -> None: """We need to hack around the fact that asyncio has no reentrant lock - and which prevents us from creating a reentrant condition - So we either grab the storing condition ourselves or the caller needs - to pass us the locked storing_condition + So we either grab the storing storing_lock ourselves or the caller needs + to pass us the locked storing_lock """ got_cond = cond is not None if not got_cond: From 3389d00ce101dd76c81d0d758492efdcb532bd05 Mon Sep 17 00:00:00 2001 From: vringar Date: Tue, 8 Dec 2020 14:22:28 +0100 Subject: [PATCH 055/139] Introduced mypy --- .pre-commit-config.yaml | 11 ++- crawler.py | 81 +++++++++++++------- demo.py | 5 +- openwpm/browser_manager.py | 21 ++--- openwpm/storage/arrow_storage.py | 2 +- openwpm/storage/cloud_storage/gcp_storage.py | 9 ++- openwpm/utilities/build_cookie_table.py | 2 +- scripts/prune-environment.py | 8 +- setup.cfg | 15 ++++ test/manual_test.py | 17 ++-- test/storage/test_memory_storage_provider.py | 6 +- test/test_js_instrument.py | 8 +- 12 files changed, 124 insertions(+), 61 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 17fa3be38..828c50433 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,9 +4,16 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 20.8b1 # Replace by any tag/version: https://github.com/psf/black/tags + rev: 20.8b1 hooks: - id: black - language_version: python3 # Should be a command that runs python3.6+ + language_version: python3 + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v0.790 + hooks: + - id: mypy + additional_dependencies: [pytest] + # We may need to add more and more dependencies here, as pre-commit + # runs in an environment without our dependencies diff --git a/crawler.py b/crawler.py index 396210e48..27d490787 100644 --- a/crawler.py +++ b/crawler.py @@ -7,36 +7,48 @@ from threading import Lock from typing import Any, Callable, List -import boto3 import sentry_sdk -from openwpm import CommandSequence, MPLogger, TaskManager +from openwpm import mp_logger +from openwpm.command_sequence import CommandSequence +from openwpm.storage.cloud_storage.gcp_storage import ( + GcsStructuredProvider, + GcsUnstructuredProvider, +) +from openwpm.task_manager import TaskManager, load_default_params from openwpm.utilities import rediswq -from test.utilities import LocalS3Session, local_s3_bucket # Configuration via environment variables +# Crawler specific config REDIS_HOST = os.getenv("REDIS_HOST", "redis-box") REDIS_QUEUE_NAME = os.getenv("REDIS_QUEUE_NAME", "crawl-queue") +MAX_JOB_RETRIES = int(os.getenv("MAX_JOB_RETRIES", "2")) +DWELL_TIME = int(os.getenv("DWELL_TIME", "10")) +TIMEOUT = int(os.getenv("TIMEOUT", "60")) + +# Storage Provider Params CRAWL_DIRECTORY = os.getenv("CRAWL_DIRECTORY", "crawl-data") -S3_BUCKET = os.getenv("S3_BUCKET", "openwpm-crawls") +GCS_BUCKET = os.getenv("GCS_BUCKET", "openwpm-crawls") +GCP_PROJECT = os.getenv("GCP_PROJECT", "senglehardt-openwpm-test-1") +AUTH_TOKEN = os.getenv("GCP_AUTH_TOKEN", "cloud") + +# Browser Params DISPLAY_MODE = os.getenv("DISPLAY_MODE", "headless") HTTP_INSTRUMENT = os.getenv("HTTP_INSTRUMENT", "1") == "1" COOKIE_INSTRUMENT = os.getenv("COOKIE_INSTRUMENT", "1") == "1" NAVIGATION_INSTRUMENT = os.getenv("NAVIGATION_INSTRUMENT", "1") == "1" JS_INSTRUMENT = os.getenv("JS_INSTRUMENT", "1") == "1" CALLSTACK_INSTRUMENT = os.getenv("CALLSTACK_INSTRUMENT", "1") == "1" -JS_INSTRUMENT_SETTINGS = os.getenv( - "JS_INSTRUMENT_SETTINGS", '["collection_fingerprinting"]' +JS_INSTRUMENT_SETTINGS = json.loads( + os.getenv("JS_INSTRUMENT_SETTINGS", '["collection_fingerprinting"]') ) + SAVE_CONTENT = os.getenv("SAVE_CONTENT", "") PREFS = os.getenv("PREFS", None) -DWELL_TIME = int(os.getenv("DWELL_TIME", "10")) -TIMEOUT = int(os.getenv("TIMEOUT", "60")) -SENTRY_DSN = os.getenv("SENTRY_DSN", None) -LOGGER_SETTINGS = MPLogger.parse_config_from_env() -MAX_JOB_RETRIES = int(os.getenv("MAX_JOB_RETRIES", "2")) -JS_INSTRUMENT_SETTINGS = json.loads(JS_INSTRUMENT_SETTINGS) + +SENTRY_DSN = os.getenv("SENTRY_DSN", None) +LOGGER_SETTINGS = mp_logger.parse_config_from_env() if CALLSTACK_INSTRUMENT is True: # Must have JS_INSTRUMENT True for CALLSTACK_INSTRUMENT to work @@ -49,7 +61,7 @@ # code below requires blocking commands. For more context see: # https://github.com/mozilla/OpenWPM/issues/470 NUM_BROWSERS = 1 -manager_params, browser_params = TaskManager.load_default_params(NUM_BROWSERS) +manager_params, browser_params = load_default_params(NUM_BROWSERS) # Browser configuration for i in range(NUM_BROWSERS): @@ -73,19 +85,29 @@ manager_params["data_directory"] = "~/Desktop/%s/" % CRAWL_DIRECTORY manager_params["log_directory"] = "~/Desktop/%s/" % CRAWL_DIRECTORY manager_params["output_format"] = "s3" -manager_params["s3_bucket"] = S3_BUCKET +manager_params["s3_bucket"] = GCS_BUCKET manager_params["s3_directory"] = CRAWL_DIRECTORY -# Allow the use of localstack's mock s3 service -S3_ENDPOINT = os.getenv("S3_ENDPOINT") -if S3_ENDPOINT: - boto3.DEFAULT_SESSION = LocalS3Session(endpoint_url=S3_ENDPOINT) - manager_params["s3_bucket"] = local_s3_bucket(boto3.resource("s3"), name=S3_BUCKET) - +structured = GcsStructuredProvider( + project=GCP_PROJECT, + bucket_name=GCS_BUCKET, + base_path=CRAWL_DIRECTORY, + token=AUTH_TOKEN, +) +unstructured = GcsUnstructuredProvider( + project=GCP_PROJECT, + bucket_name=GCS_BUCKET, + base_path=CRAWL_DIRECTORY, + token=AUTH_TOKEN, +) # Instantiates the measurement platform # Commands time out by default after 60 seconds -manager = TaskManager.TaskManager( - manager_params, browser_params, logger_kwargs=LOGGER_SETTINGS +manager = TaskManager( + manager_params, + browser_params, + structured, + unstructured, + logger_kwargs=LOGGER_SETTINGS, ) # At this point, Sentry should be initiated @@ -94,7 +116,7 @@ with sentry_sdk.configure_scope() as scope: # tags generate breakdown charts and search filters scope.set_tag("CRAWL_DIRECTORY", CRAWL_DIRECTORY) - scope.set_tag("S3_BUCKET", S3_BUCKET) + scope.set_tag("S3_BUCKET", GCS_BUCKET) scope.set_tag("DISPLAY_MODE", DISPLAY_MODE) scope.set_tag("HTTP_INSTRUMENT", HTTP_INSTRUMENT) scope.set_tag("COOKIE_INSTRUMENT", COOKIE_INSTRUMENT) @@ -106,9 +128,10 @@ scope.set_tag("DWELL_TIME", DWELL_TIME) scope.set_tag("TIMEOUT", TIMEOUT) scope.set_tag("MAX_JOB_RETRIES", MAX_JOB_RETRIES) - scope.set_tag("CRAWL_REFERENCE", "%s/%s" % (S3_BUCKET, CRAWL_DIRECTORY)) + scope.set_tag("CRAWL_REFERENCE", "%s/%s" % (GCS_BUCKET, CRAWL_DIRECTORY)) # context adds addition information that may be of interest - scope.set_context("PREFS", PREFS) + if PREFS: + scope.set_context("PREFS", json.loads(PREFS)) scope.set_context( "crawl_config", { @@ -133,7 +156,7 @@ def on_shutdown( - manager: TaskManager.TaskManager, unsaved_jobs_lock: Lock + manager: TaskManager, unsaved_jobs_lock: Lock ) -> Callable[[signal.Signals, Any], None]: def actual_callback(s: signal.Signals, __: Any) -> None: global shutting_down @@ -157,9 +180,9 @@ def get_job_completion_callback( job_queue: rediswq.RedisWQ, job: bytes, ) -> Callable[[bool], None]: - def callback(sucess: bool) -> None: + def callback(success: bool) -> None: with unsaved_jobs_lock: - if sucess: + if success: logger.info("Job %r is done", job) job_queue.complete(job) else: @@ -194,7 +217,7 @@ def callback(sucess: bool) -> None: callback = get_job_completion_callback( manager.logger, unsaved_jobs_lock, job_queue, job ) - command_sequence = CommandSequence.CommandSequence( + command_sequence = CommandSequence( site, blocking=True, reset=True, diff --git a/demo.py b/demo.py index 5cf9d8996..105478223 100644 --- a/demo.py +++ b/demo.py @@ -65,12 +65,15 @@ # Visits the sites for index, site in enumerate(sites): + def callback(success: bool, val: str = site) -> None: + print("CommandSequence {} done".format(val)) + # Parallelize sites over all number of browsers set above. command_sequence = CommandSequence( site, site_rank=index, reset=True, - callback=lambda success, val=site: print("CommandSequence {} done".format(val)), + callback=callback, ) # Start by visiting the page diff --git a/openwpm/browser_manager.py b/openwpm/browser_manager.py index 2a70b218a..433aa4be5 100644 --- a/openwpm/browser_manager.py +++ b/openwpm/browser_manager.py @@ -50,14 +50,14 @@ def __init__(self, manager_params, browser_params) -> None: self.current_profile_path = None self.db_socket_address = manager_params["aggregator_address"] self.browser_id = browser_params["browser_id"] - self.curr_visit_id: int = None + self.curr_visit_id: Optional[int] = None self.browser_params = browser_params self.manager_params = manager_params # Queues and process IDs for BrowserManager # thread to run commands issues from TaskManager - self.command_thread: threading.Thread = None + self.command_thread: Optional[threading.Thread] = None # queue for passing command tuples to BrowserManager self.command_queue: Optional[Queue] = None # queue for receiving command execution status from BrowserManager @@ -75,7 +75,7 @@ def __init__(self, manager_params, browser_params) -> None: self.restart_required = False self.current_timeout: Optional[int] = None # timeout of the current command - self.browser_manager = None # process that controls browser + self.browser_manager: Optional[Process] = None # process that controls browser self.logger = logging.getLogger("openwpm") @@ -246,6 +246,7 @@ def close_browser_manager(self, force: bool = False): If the browser manager process is unresponsive, the process is killed. """ self.logger.debug("BROWSER %i: Closing browser..." % self.browser_id) + assert self.status_queue is not None if force: self.kill_browser_manager() @@ -309,13 +310,13 @@ def close_browser_manager(self, force: bool = False): # Verify that the browser process has closed (30 second timeout) if self.browser_manager is not None: self.browser_manager.join(30) - if self.browser_manager.is_alive(): - self.logger.debug( - "BROWSER %i: Browser manager process still alive 30 seconds " - "after executing shutdown command." % self.browser_id - ) - self.kill_browser_manager() - return + if self.browser_manager.is_alive(): + self.logger.debug( + "BROWSER %i: Browser manager process still alive 30 seconds " + "after executing shutdown command." % self.browser_id + ) + self.kill_browser_manager() + return self.logger.debug( "BROWSER %i: Browser manager closed successfully." % self.browser_id diff --git a/openwpm/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py index fa560ce38..b2e36c664 100644 --- a/openwpm/storage/arrow_storage.py +++ b/openwpm/storage/arrow_storage.py @@ -144,7 +144,7 @@ async def flush_cache(self, cond: asyncio.Lock = None) -> None: if not got_cond: cond = self.storing_lock await cond.acquire() - assert cond.locked() + assert cond is not None and cond.locked() for table_name, batches in self._batches.items(): table = pa.Table.from_batches(batches) await self.write_table(table_name, table) diff --git a/openwpm/storage/cloud_storage/gcp_storage.py b/openwpm/storage/cloud_storage/gcp_storage.py index 74ebfefbc..b20fdb90d 100644 --- a/openwpm/storage/cloud_storage/gcp_storage.py +++ b/openwpm/storage/cloud_storage/gcp_storage.py @@ -1,7 +1,7 @@ import asyncio import logging from functools import partial -from typing import Any, Callable, Set +from typing import Any, Callable, Optional, Set import pyarrow.parquet as pq from gcsfs import GCSFileSystem @@ -30,16 +30,16 @@ def __init__( self.bucket_name = bucket_name self.base_path = base_path self.token = token - self.file_system = None + self.file_system: Optional[GCSFileSystem] = None self.base_path = f"{self.bucket_name}/{base_path}/{{table_name}}" async def init(self) -> None: - event_loop = asyncio.get_event_loop() self.file_system = GCSFileSystem( project=self.project, token=self.token, access="read_write" ) async def write_table(self, table_name: TableName, table: Table) -> None: + assert self.file_system is not None self.file_system.start_transaction() pq.write_to_dataset( table, @@ -71,7 +71,7 @@ def __init__( self.bucket_name = bucket_name self.base_path = base_path self.token = token - self.file_system = None + self.file_system: Optional[GCSFileSystem] = None self.base_path = f"{bucket_name}/{base_path}/{{filename}}" self.file_name_cache: Set[str] = set() @@ -84,6 +84,7 @@ async def init(self) -> None: async def store_blob( self, filename: str, blob: bytes, overwrite: bool = False ) -> None: + assert self.file_system is not None target_path = self.base_path.format(filename=filename) if not overwrite and ( filename in self.file_name_cache or self.file_system.exists(target_path) diff --git a/openwpm/utilities/build_cookie_table.py b/openwpm/utilities/build_cookie_table.py index e6faa347d..9d4130deb 100644 --- a/openwpm/utilities/build_cookie_table.py +++ b/openwpm/utilities/build_cookie_table.py @@ -8,7 +8,7 @@ # This should be the modified Cookie.py included # the standard lib Cookie.py has many bugs -from . import Cookie +from . import cookie as Cookie # Potential formats for expires timestamps DATE_FORMATS = [ diff --git a/scripts/prune-environment.py b/scripts/prune-environment.py index ab2c0abca..1134d56cc 100644 --- a/scripts/prune-environment.py +++ b/scripts/prune-environment.py @@ -1,3 +1,5 @@ +from typing import Iterable, List + import yaml with open("environment-unpinned.yaml", "r") as fp: @@ -19,15 +21,15 @@ # Only pin explicit dependencies -def iterate_deps(xs, ys, accumulator): +def iterate_deps(xs: Iterable[str], ys: Iterable[str], accumulator: List[str]) -> None: for x in xs: for y in ys: if x.split("=")[0] == y.split("=")[0]: accumulator.append(x) -deps_not_pip = [] -deps_pip = [] +deps_not_pip: List[str] = [] +deps_pip: List[str] = [] iterate_deps( env_pinned["dependencies"][:-1], diff --git a/setup.cfg b/setup.cfg index a5d6dbe86..b07805f75 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,3 +11,18 @@ python_version = 3.8 warn_unused_configs = True ignore_missing_imports = True disallow_incomplete_defs = True +disallow_untyped_defs = True + +[mypy-openwpm.*] +disallow_untyped_defs = False + +[mypy-openwpm.utilities.*,openwpm.mp_logger,openwpm.commands.browser_commands] +disallow_incomplete_defs = False + +[mypy-openwpm.browser_manager] +allow_redefinition = True +disallow_incomplete_defs = False + +[mypy-test.*] +disallow_incomplete_defs = False +disallow_untyped_defs = False \ No newline at end of file diff --git a/test/manual_test.py b/test/manual_test.py index 53b624f2e..bbcd3354a 100644 --- a/test/manual_test.py +++ b/test/manual_test.py @@ -10,10 +10,10 @@ from openwpm import js_instrumentation as jsi from openwpm.deploy_browsers import configure_firefox -from openwpm.TaskManager import load_default_params +from openwpm.task_manager import load_default_params from openwpm.utilities.platform_utils import get_firefox_binary_path -from .conftest import create_xpi +from .conftest import xpi from .utilities import BASE_TEST_URL, start_server # import commonly used modules and utilities so they can be easily accessed @@ -141,7 +141,7 @@ def cleanup_server(): if with_extension: # add openwpm extension to profile - create_xpi() + xpi() ext_xpi = join(EXT_PATH, "dist", "openwpm-1.0.zip") driver.install_addon(ext_xpi, temporary=True) @@ -178,17 +178,22 @@ def start_webext(): "--selenium", help=""" Run a selenium webdriver instance, and drop into an IPython shell""", - **flag_opts, + is_flag=True, + default=False, ) @click.option( "--no-extension", help=""" Use this to prevent the openwpm webextension being loaded. Only applies if --selenium is being used.""", - **flag_opts, + is_flag=True, + default=False, ) @click.option( - "--browser-params", help="""Set flag to load browser_params.""", **flag_opts + "--browser-params", + help="""Set flag to load browser_params.""", + is_flag=True, + default=False, ) @click.option( "--browser-params-file", diff --git a/test/storage/test_memory_storage_provider.py b/test/storage/test_memory_storage_provider.py index ba1300075..ebc5c751a 100644 --- a/test/storage/test_memory_storage_provider.py +++ b/test/storage/test_memory_storage_provider.py @@ -1,7 +1,8 @@ import asyncio -from typing import Any, Dict, List, Tuple +from typing import Any, Dict, List, Tuple, Union import pytest +from _pytest.fixtures import FixtureRequest from openwpm.storage.in_memory_storage import ( MemoryArrowProvider, @@ -41,6 +42,9 @@ def structured_provider( return SqlLiteStorageProvider(tmp_path / "test_db.sqllite") elif request.param == memory_arrow: return MemoryArrowProvider() + assert isinstance( + request, FixtureRequest + ) # See https://github.com/pytest-dev/pytest/issues/8073 for why this can't be type annotated request.raiseerror("invalid internal test config") diff --git a/test/test_js_instrument.py b/test/test_js_instrument.py index 21af91128..378ab9b2a 100644 --- a/test/test_js_instrument.py +++ b/test/test_js_instrument.py @@ -1,3 +1,5 @@ +from typing import Set, Tuple + from openwpm.utilities import db_utils from . import utilities as util @@ -21,7 +23,7 @@ class TestJSInstrumentNonExistingWindowProperty(OpenWPMJSTest): ("window.nonExisting", "get", "undefined"), } - METHOD_CALLS = set() + METHOD_CALLS: Set[Tuple[str, str, str]] = set() TEST_PAGE = "instrument_non_existing_window_property.html" TOP_URL = u"%s/js_instrument/%s" % (util.BASE_TEST_URL, TEST_PAGE) @@ -64,7 +66,7 @@ class TestJSInstrumentExistingWindowProperty(OpenWPMJSTest): # Note 1: nonExistingProp1 is not enumerable even after being set # Note 2: nonExistingMethod1 shows up as a get rather than call - METHOD_CALLS = set() # Note 2 + METHOD_CALLS: Set[Tuple[str, str, str]] = set() # Note 2 TEST_PAGE = "instrument_existing_window_property.html" TOP_URL = u"%s/js_instrument/%s" % (util.BASE_TEST_URL, TEST_PAGE) @@ -376,7 +378,7 @@ class TestJSInstrumentRecursiveProperties(OpenWPMJSTest): ("window.test.test.prop2", "get", "test_test_prop2"), } - METHOD_CALLS = set() + METHOD_CALLS: Set[Tuple[str, str, str]] = set() TEST_PAGE = "instrument_do_not_recurse_properties_to_instrument.html" TOP_URL = u"%s/js_instrument/%s" % (util.BASE_TEST_URL, TEST_PAGE) From 7343c88a2451c6355039657bd4e9816d537141c1 Mon Sep 17 00:00:00 2001 From: vringar Date: Tue, 8 Dec 2020 15:45:52 +0100 Subject: [PATCH 056/139] Downgraded black in pre-commit --- .pre-commit-config.yaml | 2 +- crawler.py | 5 +---- demo.py | 5 +---- environment.yaml | 4 ++-- openwpm/browser_manager.py | 6 +----- openwpm/commands/command_executor.py | 6 +----- openwpm/storage/cloud_storage/gcp_storage.py | 12 ++---------- openwpm/storage/leveldb.py | 5 +---- openwpm/storage/storage_providers.py | 5 +---- scripts/prune-environment.py | 15 +++------------ test/conftest.py | 5 +---- test/manual_test.py | 5 +---- test/storage/test_memory_storage_provider.py | 9 +++++---- test/test_js_instrument.py | 18 +++--------------- test/test_js_instrument_py.py | 5 +---- test/test_mp_logger.py | 3 +-- test/test_profile.py | 3 +-- 17 files changed, 27 insertions(+), 86 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 828c50433..0a9d0cbca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 20.8b1 + rev: 19.10b0 hooks: - id: black language_version: python3 diff --git a/crawler.py b/crawler.py index 27d490787..121f6f0e9 100644 --- a/crawler.py +++ b/crawler.py @@ -133,10 +133,7 @@ if PREFS: scope.set_context("PREFS", json.loads(PREFS)) scope.set_context( - "crawl_config", - { - "REDIS_QUEUE_NAME": REDIS_QUEUE_NAME, - }, + "crawl_config", {"REDIS_QUEUE_NAME": REDIS_QUEUE_NAME,}, ) # Send a sentry error message (temporarily - to easily be able # to compare error frequencies to crawl worker instance count) diff --git a/demo.py b/demo.py index 105478223..789b14b74 100644 --- a/demo.py +++ b/demo.py @@ -70,10 +70,7 @@ def callback(success: bool, val: str = site) -> None: # Parallelize sites over all number of browsers set above. command_sequence = CommandSequence( - site, - site_rank=index, - reset=True, - callback=callback, + site, site_rank=index, reset=True, callback=callback, ) # Start by visiting the page diff --git a/environment.yaml b/environment.yaml index 656185705..19aee3d11 100644 --- a/environment.yaml +++ b/environment.yaml @@ -14,10 +14,10 @@ dependencies: - multiprocess=0.70.11.1 - mypy=0.790 - nodejs=14.15.1 -- pandas=1.1.4 +- pandas=1.1.5 - pillow=8.0.1 - pip=20.3.1 -- pre-commit=2.9.2 +- pre-commit=2.9.3 - psutil=5.7.3 - pyarrow=2.0.0 - pytest-cov=2.10.1 diff --git a/openwpm/browser_manager.py b/openwpm/browser_manager.py index 433aa4be5..b28dbda01 100644 --- a/openwpm/browser_manager.py +++ b/openwpm/browser_manager.py @@ -514,11 +514,7 @@ def BrowserManager( # kill and restart its worker processes try: command_executor.execute_command( - command, - driver, - browser_params, - manager_params, - extension_socket, + command, driver, browser_params, manager_params, extension_socket, ) status_queue.put("OK") except WebDriverException: diff --git a/openwpm/commands/command_executor.py b/openwpm/commands/command_executor.py index 355a81bff..3a8c1c2f0 100644 --- a/openwpm/commands/command_executor.py +++ b/openwpm/commands/command_executor.py @@ -15,11 +15,7 @@ def execute_command( - command, - webdriver, - browser_params, - manager_params, - extension_socket, + command, webdriver, browser_params, manager_params, extension_socket, ): """Executes BrowserManager commands commands are of form (COMMAND, ARG0, ARG1, ...) diff --git a/openwpm/storage/cloud_storage/gcp_storage.py b/openwpm/storage/cloud_storage/gcp_storage.py index b20fdb90d..369a3f428 100644 --- a/openwpm/storage/cloud_storage/gcp_storage.py +++ b/openwpm/storage/cloud_storage/gcp_storage.py @@ -19,11 +19,7 @@ class GcsStructuredProvider(ArrowProvider): """ def __init__( - self, - project: str, - bucket_name: str, - base_path: str, - token: str = None, + self, project: str, bucket_name: str, base_path: str, token: str = None, ) -> None: super().__init__() self.project = project @@ -60,11 +56,7 @@ class GcsUnstructuredProvider(UnstructuredStorageProvider): """ def __init__( - self, - project: str, - bucket_name: str, - base_path: str, - token: str = None, + self, project: str, bucket_name: str, base_path: str, token: str = None, ) -> None: super().__init__() self.project = project diff --git a/openwpm/storage/leveldb.py b/openwpm/storage/leveldb.py index 96d42ef7c..91a2021b7 100644 --- a/openwpm/storage/leveldb.py +++ b/openwpm/storage/leveldb.py @@ -39,10 +39,7 @@ async def shutdown(self) -> None: print("Ldb is closed:", self.ldb.closed) async def store_blob( - self, - filename: str, - blob: bytes, - overwrite: bool = False, + self, filename: str, blob: bytes, overwrite: bool = False, ) -> None: assert self.ldb is not None assert self.content_batch is not None diff --git a/openwpm/storage/storage_providers.py b/openwpm/storage/storage_providers.py index 0a835177c..7ba21cca3 100644 --- a/openwpm/storage/storage_providers.py +++ b/openwpm/storage/storage_providers.py @@ -75,10 +75,7 @@ async def finalize_visit_id( class UnstructuredStorageProvider(StorageProvider): @abstractmethod async def store_blob( - self, - filename: str, - blob: bytes, - overwrite: bool = False, + self, filename: str, blob: bytes, overwrite: bool = False, ) -> None: """Stores the given bytes under the provided filename""" pass diff --git a/scripts/prune-environment.py b/scripts/prune-environment.py index 1134d56cc..6673814d3 100644 --- a/scripts/prune-environment.py +++ b/scripts/prune-environment.py @@ -3,20 +3,11 @@ import yaml with open("environment-unpinned.yaml", "r") as fp: - env_unpinned = yaml.load( - fp.read(), - Loader=yaml.SafeLoader, - ) + env_unpinned = yaml.load(fp.read(), Loader=yaml.SafeLoader,) with open("environment-unpinned-dev.yaml", "r") as fp: - env_unpinned_dev = yaml.load( - fp.read(), - Loader=yaml.SafeLoader, - ) + env_unpinned_dev = yaml.load(fp.read(), Loader=yaml.SafeLoader,) with open("../environment.yaml", "r") as fp: - env_pinned = yaml.load( - fp.read(), - Loader=yaml.SafeLoader, - ) + env_pinned = yaml.load(fp.read(), Loader=yaml.SafeLoader,) # Only pin explicit dependencies diff --git a/test/conftest.py b/test/conftest.py index 995f4abc3..d184538c0 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -71,10 +71,7 @@ def _create_task_manager( manager_params, browser_params = params structured_provider = SqlLiteStorageProvider(manager_params["db"]) manager = task_manager.TaskManager( - manager_params, - browser_params, - structured_provider, - None, + manager_params, browser_params, structured_provider, None, ) return manager diff --git a/test/manual_test.py b/test/manual_test.py index bbcd3354a..08e45d7f6 100644 --- a/test/manual_test.py +++ b/test/manual_test.py @@ -167,10 +167,7 @@ def start_webext(): thread.join() -flag_opts = dict( - is_flag=True, - default=False, -) +flag_opts = dict(is_flag=True, default=False,) @click.command() diff --git a/test/storage/test_memory_storage_provider.py b/test/storage/test_memory_storage_provider.py index ebc5c751a..3780faf58 100644 --- a/test/storage/test_memory_storage_provider.py +++ b/test/storage/test_memory_storage_provider.py @@ -73,13 +73,14 @@ async def test_basic_access( "site_url": "https://example.com", } + await structured_provider.init() + await structured_provider.store_record( TableName("site_visits"), VisitId(2), data ) - await asyncio.gather( - structured_provider.finalize_visit_id(VisitId(2)), - structured_provider.flush_cache(), - ) + token = await structured_provider.finalize_visit_id(VisitId(2)) + await structured_provider.flush_cache() + await token @pytest.mark.asyncio diff --git a/test/test_js_instrument.py b/test/test_js_instrument.py index 378ab9b2a..dc8e2092b 100644 --- a/test/test_js_instrument.py +++ b/test/test_js_instrument.py @@ -108,21 +108,9 @@ def get_config(self, data_dir=""): } browser_params[0]["js_instrument_settings"] = [ # Note that the string "window.document.cookie" does not work. - { - "window.document": [ - "cookie", - ] - }, - { - "window.navigator": [ - "webdriver", - ] - }, - { - "window": [ - "fetch", - ] - }, + {"window.document": ["cookie",]}, + {"window.navigator": ["webdriver",]}, + {"window": ["fetch",]}, ] return manager_params, browser_params diff --git a/test/test_js_instrument_py.py b/test/test_js_instrument_py.py index eafc6e3f3..5cf62fe7c 100644 --- a/test/test_js_instrument_py.py +++ b/test/test_js_instrument_py.py @@ -16,10 +16,7 @@ def test_python_to_js_lower_true_false(): inpy = [ { "object": "window", - "logSettings": { - "logCallStack": False, - "preventSets": True, - }, + "logSettings": {"logCallStack": False, "preventSets": True,}, } ] expected_out = _no_whitespace( diff --git a/test/test_mp_logger.py b/test/test_mp_logger.py index e37bf53dd..5cb9ebfa0 100644 --- a/test/test_mp_logger.py +++ b/test/test_mp_logger.py @@ -73,8 +73,7 @@ def child_proc_logging_exception(): raise Exception("This is my generic Test Exception") except Exception: logger.error( - "I'm logging an exception", - exc_info=True, + "I'm logging an exception", exc_info=True, ) diff --git a/test/test_profile.py b/test/test_profile.py index 847771348..a7ec4f418 100644 --- a/test/test_profile.py +++ b/test/test_profile.py @@ -118,8 +118,7 @@ def test_config_is_set(*args, **kwargs): manager.execute_command_sequence(cs) manager.close() query_result = db_utils.query_db( - manager_params["db"], - "SELECT * FROM crawl_history;", + manager_params["db"], "SELECT * FROM crawl_history;", ) assert len(query_result) > 0 for row in query_result: From babd96204f90ceb6902f2dc417096a652e209adf Mon Sep 17 00:00:00 2001 From: vringar Date: Tue, 8 Dec 2020 17:36:20 +0100 Subject: [PATCH 057/139] Modifying the database directly --- test/test_custom_function_command.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test/test_custom_function_command.py b/test/test_custom_function_command.py index 6452155af..4a033d8e7 100644 --- a/test/test_custom_function_command.py +++ b/test/test_custom_function_command.py @@ -1,3 +1,5 @@ +import sqlite3 + from openwpm import command_sequence, task_manager from openwpm.socket_interface import ClientSocket from openwpm.storage.sql_provider import SqlLiteStorageProvider @@ -61,13 +63,20 @@ def collect_links(table_name, scheme, **kwargs): sock.close() manager_params, browser_params = default_params - storage_provider = SqlLiteStorageProvider(manager_params["db"]) - storage_provider.execute_statement( + + db = sqlite3.connect(manager_params["db"]) + cur = db.cursor() + + cur.execute( """CREATE TABLE IF NOT EXISTS %s ( top_url TEXT, link TEXT, visit_id INTEGER, browser_id INTEGER);""" % table_name ) + cur.close() + db.close() + + storage_provider = SqlLiteStorageProvider(manager_params["db"]) manager = TaskManager(manager_params, browser_params, storage_provider, None) cs = command_sequence.CommandSequence(url_a) cs.get(sleep=0, timeout=60) From b3d28a06fb87cf381077dc7d52c9bad2527c1cd1 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 11 Dec 2020 10:38:46 +0100 Subject: [PATCH 058/139] Fixed formatting --- openwpm/commands/profile_commands.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/openwpm/commands/profile_commands.py b/openwpm/commands/profile_commands.py index 191571479..0b98ea847 100644 --- a/openwpm/commands/profile_commands.py +++ b/openwpm/commands/profile_commands.py @@ -61,11 +61,7 @@ def dump_profile( tar = tarfile.open(tar_location + tar_name, "w", errorlevel=1) logger.debug( "BROWSER %i: Backing up full profile from %s to %s" - % ( - browser_params.browser_id, - browser_profile_folder, - tar_location + tar_name, - ) + % (browser_params.browser_id, browser_profile_folder, tar_location + tar_name,) ) storage_vector_files = [ "cookies.sqlite", # cookies From 791d86553b61435f4173ef4b1665f22793efd8af Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 11 Dec 2020 10:52:39 +0100 Subject: [PATCH 059/139] Made mypy a lil stricter --- openwpm/storage/in_memory_storage.py | 38 +++++++++++++++++----------- openwpm/storage/sql_provider.py | 2 +- setup.cfg | 4 +++ 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/openwpm/storage/in_memory_storage.py b/openwpm/storage/in_memory_storage.py index b18e7cba0..50819b464 100644 --- a/openwpm/storage/in_memory_storage.py +++ b/openwpm/storage/in_memory_storage.py @@ -1,5 +1,6 @@ import asyncio import logging +from asyncio import Event, Lock from collections import defaultdict from typing import Any, Awaitable, DefaultDict, Dict, List, Tuple @@ -49,15 +50,19 @@ def __init__(self) -> None: """The cache for entries before they are finalized""" self.cache2: DefaultDict[TableName, List[Dict[str, Any]]] = defaultdict(list) """For all entries that have been finalized but not yet flushed out to the queue""" + self.signal: Event = asyncio.Event() + self.lock: Lock = asyncio.Lock() async def flush_cache(self) -> None: - self.logger.info("Flushing cache") + with self.lock as _: + self.logger.info("Flushing cache") - for table, record_list in self.cache2.items(): - self.logger.info(f"Saving out {len(record_list)} entries for {table}") - for record in record_list: - self.queue.put((table, record)) - self.cache2.clear() + for table, record_list in self.cache2.items(): + self.logger.info(f"Saving out {len(record_list)} entries for {table}") + for record in record_list: + self.queue.put((table, record)) + self.cache2.clear() + self.signal.set() async def store_record( self, table: TableName, visit_id: VisitId, record: Dict[str, Any] @@ -70,17 +75,20 @@ async def store_record( async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False ) -> Awaitable[None]: - self.logger.info( - f"Finalizing visit_id {visit_id} which was {'' if interrupted else 'not'} interrupted" - ) - for table, record_list in self.cache1[visit_id].items(): - self.cache2[table].extend(record_list) + with self.lock as _: + self.signal.clear() + self.logger.info( + f"Finalizing visit_id {visit_id} which was {'' if interrupted else 'not'} interrupted" + ) + for table, record_list in self.cache1[visit_id].items(): + self.cache2[table].extend(record_list) + + del self.cache1[visit_id] - del self.cache1[visit_id] + async def wait(signal: Event) -> None: + await signal.wait() - fut = asyncio.get_event_loop().create_future() - fut.set_result(None) - return fut + return wait(self.signal) async def shutdown(self) -> None: if self.cache1 != {} or self.cache2 != {}: diff --git a/openwpm/storage/sql_provider.py b/openwpm/storage/sql_provider.py index cff13ae87..a883db4e8 100644 --- a/openwpm/storage/sql_provider.py +++ b/openwpm/storage/sql_provider.py @@ -108,7 +108,7 @@ async def finalize_visit_id( self.logger.warning("Visit with visit_id %d got interrupted", visit_id) self.cur.execute("INSERT INTO incomplete_visits VALUES (?)", (visit_id,)) - async def done(): + async def done() -> None: return return done() diff --git a/setup.cfg b/setup.cfg index b07805f75..c49313ec8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,6 +13,10 @@ ignore_missing_imports = True disallow_incomplete_defs = True disallow_untyped_defs = True +[mypy-openwpm.storage.*] +disallow_incomplete_defs = True +disallow_untyped_defs = True + [mypy-openwpm.*] disallow_untyped_defs = False From 66e8caad13cfce9a6411f927ec952f07f996f422 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 11 Dec 2020 11:52:21 +0100 Subject: [PATCH 060/139] Fixing docs and config printing --- openwpm/storage/in_memory_storage.py | 16 ++++++++-------- openwpm/utilities/platform_utils.py | 22 ++++++++++++++++++---- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/openwpm/storage/in_memory_storage.py b/openwpm/storage/in_memory_storage.py index 50819b464..8930c4b0c 100644 --- a/openwpm/storage/in_memory_storage.py +++ b/openwpm/storage/in_memory_storage.py @@ -1,3 +1,11 @@ +""" +This module contains implementations for various kinds of storage providers +that store their results in memory. +These classes are designed to allow for easier parallel testing as there are +no shared resources between tests. It also makes it easier to verify results +by not having to do a round trip to a perstitent storage provider +""" + import asyncio import logging from asyncio import Event, Lock @@ -16,14 +24,6 @@ UnstructuredStorageProvider, ) -""" -This module contains implementations for various kinds of storage providers -that store their results in memory. -These classes are designed to allow for easier parallel testing as there are -no shared resources between tests. It also makes it easier to verify results -by not having to do a round trip to a perstitent storage provider -""" - class MemoryStructuredProvider(StructuredStorageProvider): """ diff --git a/openwpm/utilities/platform_utils.py b/openwpm/utilities/platform_utils.py index bf94a0d4f..65a2c6032 100644 --- a/openwpm/utilities/platform_utils.py +++ b/openwpm/utilities/platform_utils.py @@ -3,7 +3,9 @@ import subprocess from collections import OrderedDict from copy import deepcopy +from pathlib import Path from sys import platform +from typing import Any from tabulate import tabulate @@ -95,8 +97,20 @@ def get_configuration_string(manager_params, browser_params, versions): config_str = "\n\nOpenWPM Version: %s\nFirefox Version: %s\n" % versions config_str += "\n========== Manager Configuration ==========\n" + + def serialize_paths(obj: Any) -> Any: + if isinstance(obj, Path): + return str(obj) + raise TypeError( + f"Object of type {obj.__class__.__name__} is not JSON serializable" + ) + config_str += json.dumps( - manager_params.to_dict(), sort_keys=True, indent=2, separators=(",", ": ") + manager_params.to_dict(), + sort_keys=True, + indent=2, + separators=(",", ": "), + default=serialize_paths, ) config_str += "\n\n========== Browser Configuration ==========\n" @@ -116,13 +130,13 @@ def get_configuration_string(manager_params, browser_params, versions): archive_all_none = False # Separate out long profile directory strings - profile_dirs[browser_id] = item.pop("seed_tar") - archive_dirs[browser_id] = item.pop("profile_archive_dir") + profile_dirs[browser_id] = str(item.pop("seed_tar")) + archive_dirs[browser_id] = str(item.pop("profile_archive_dir")) js_config[browser_id] = item.pop("js_instrument_settings") # Copy items in sorted order dct = OrderedDict() - dct[u"browser_id"] = browser_id + dct["browser_id"] = browser_id for key in sorted(item.keys()): dct[key] = item[key] table_input.append(dct) From 70963bd54683df10c7b9f09e1f71324f50824a23 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 11 Dec 2020 12:07:57 +0100 Subject: [PATCH 061/139] Realising I've been using the wrong with --- demo.py | 8 ++------ environment.yaml | 2 +- openwpm/storage/arrow_storage.py | 16 +++++++++++----- openwpm/storage/cloud_storage/gcp_storage.py | 1 + openwpm/storage/in_memory_storage.py | 4 ++-- openwpm/storage/local_storage.py | 3 --- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/demo.py b/demo.py index 49e59f472..1578da732 100644 --- a/demo.py +++ b/demo.py @@ -8,6 +8,7 @@ GcsStructuredProvider, GcsUnstructuredProvider, ) +from openwpm.storage.local_storage import LocalArrowProvider from openwpm.storage.sql_provider import SqlLiteStorageProvider from openwpm.task_manager import TaskManager @@ -60,12 +61,7 @@ manager = TaskManager( manager_params, browser_params, - GcsStructuredProvider( - project=project, - bucket_name=bucket_name, - base_path="demo/visits", - token="/home/stefan/.config/gcloud/legacy_credentials/szabka@mozilla.com/adc.json", - ), + LocalArrowProvider(Path("./datadir/") / "parquet"), None, ) diff --git a/environment.yaml b/environment.yaml index 1cbce5455..049e17b4e 100644 --- a/environment.yaml +++ b/environment.yaml @@ -27,7 +27,7 @@ dependencies: - redis-py=3.5.3 - s3fs=0.5.1 - selenium=3.141.0 -- sentry-sdk=0.19.4 +- sentry-sdk=0.19.5 - tabulate=0.8.7 - tblib=1.6.0 - wget=1.20.1 diff --git a/openwpm/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py index b2e36c664..c1add3386 100644 --- a/openwpm/storage/arrow_storage.py +++ b/openwpm/storage/arrow_storage.py @@ -3,7 +3,7 @@ import random from abc import abstractmethod from collections import defaultdict -from typing import Any, Awaitable, DefaultDict, Dict, List +from typing import Any, Awaitable, DefaultDict, Dict, List, Optional import pandas as pd import pyarrow as pa @@ -38,11 +38,15 @@ def factory_function() -> DefaultDict[TableName, List[Dict[str, Any]]]: # Record batches by TableName self._batches: DefaultDict[TableName, List[pa.RecordBatch]] = defaultdict(list) + self._instance_id = random.getrandbits(32) + + self.storing_lock: Optional[asyncio.Lock] = None + self.flush_event: Optional[asyncio.Event] = None + async def init(self) -> None: # Used to synchronize the finalizing and the flushing self.storing_lock = asyncio.Lock() self.flush_event = asyncio.Event() - self._instance_id = random.getrandbits(32) async def store_record( self, table: TableName, visit_id: VisitId, record: Dict[str, Any] @@ -110,7 +114,8 @@ async def finalize_visit_id( """ if interrupted: await self.store_record(INCOMPLETE_VISITS, visit_id, {"visit_id": visit_id}) - + assert self.storing_lock is not None + assert self.flush_event is not None # This code is pretty tricky as there are a number of things going on # 1. No finalize_visit_id shouldn't return unless the visit has been saved to storage # 2. No new batches should be created while saving out all the batches @@ -118,8 +123,7 @@ async def finalize_visit_id( # After flush_cache has executed the event needs to be rearmed # so that newly created wait_on_condition don't just complete # instantly - if self.flush_event.is_set(): - self.flush_event.clear() + self.flush_event.clear() self._create_batch(visit_id) if self._is_cache_full(): await self.flush_cache(self.storing_lock) @@ -140,6 +144,8 @@ async def flush_cache(self, cond: asyncio.Lock = None) -> None: So we either grab the storing storing_lock ourselves or the caller needs to pass us the locked storing_lock """ + assert self.storing_lock is not None + assert self.flush_event is not None got_cond = cond is not None if not got_cond: cond = self.storing_lock diff --git a/openwpm/storage/cloud_storage/gcp_storage.py b/openwpm/storage/cloud_storage/gcp_storage.py index 369a3f428..58eeac126 100644 --- a/openwpm/storage/cloud_storage/gcp_storage.py +++ b/openwpm/storage/cloud_storage/gcp_storage.py @@ -30,6 +30,7 @@ def __init__( self.base_path = f"{self.bucket_name}/{base_path}/{{table_name}}" async def init(self) -> None: + await super(GcsStructuredProvider, self).init() self.file_system = GCSFileSystem( project=self.project, token=self.token, access="read_write" ) diff --git a/openwpm/storage/in_memory_storage.py b/openwpm/storage/in_memory_storage.py index 8930c4b0c..d4131c926 100644 --- a/openwpm/storage/in_memory_storage.py +++ b/openwpm/storage/in_memory_storage.py @@ -54,7 +54,7 @@ def __init__(self) -> None: self.lock: Lock = asyncio.Lock() async def flush_cache(self) -> None: - with self.lock as _: + async with self.lock as _: self.logger.info("Flushing cache") for table, record_list in self.cache2.items(): @@ -75,7 +75,7 @@ async def store_record( async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False ) -> Awaitable[None]: - with self.lock as _: + async with self.lock as _: self.signal.clear() self.logger.info( f"Finalizing visit_id {visit_id} which was {'' if interrupted else 'not'} interrupted" diff --git a/openwpm/storage/local_storage.py b/openwpm/storage/local_storage.py index 1ecfffe60..a8c3827b1 100644 --- a/openwpm/storage/local_storage.py +++ b/openwpm/storage/local_storage.py @@ -12,9 +12,6 @@ class LocalArrowProvider(ArrowProvider): """ Stores Parquet files under storage_path/table_name/n.parquet""" - async def init(self) -> None: - pass - def __init__(self, storage_path: Path) -> None: super().__init__() self.storage_path = storage_path From 9862bf71bcc35e7d2a4175ecb562a63e43d9d48a Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 11 Dec 2020 12:19:06 +0100 Subject: [PATCH 062/139] Trying to figure arrow_storage --- openwpm/storage/storage_controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 72bc4edac..38a504d57 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -182,7 +182,6 @@ async def _handle_meta(self, visit_id: VisitId, data: Dict[str, Any]) -> None: completion_token = await self.finalize_visit_id(visit_id, success) await completion_token self.completion_queue.put((visit_id, success)) - del self.current_tasks[visit_id] else: raise ValueError("Unexpected action: %s", action) @@ -197,6 +196,8 @@ async def finalize_visit_id( for task in self.current_tasks[visit_id]: await task + del self.current_tasks[visit_id] + self.logger.debug( "Awaited all tasks for visit_id %d while finalizing", visit_id ) @@ -318,7 +319,6 @@ async def _run(self) -> None: for visit_id, token in finalization_tokens.items(): await token self.completion_queue.put((visit_id, False)) - del self.current_tasks[visit_id] await self.shutdown() From 4a036fa4aade011731646b2aba3cd49f9e58f5d4 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 11 Dec 2020 13:04:51 +0100 Subject: [PATCH 063/139] Moving lock initialization in in_memory_storage --- openwpm/storage/in_memory_storage.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/openwpm/storage/in_memory_storage.py b/openwpm/storage/in_memory_storage.py index d4131c926..36be35f74 100644 --- a/openwpm/storage/in_memory_storage.py +++ b/openwpm/storage/in_memory_storage.py @@ -36,8 +36,8 @@ class MemoryStructuredProvider(StructuredStorageProvider): only relies on the guarantees given in the interface. """ - async def init(self) -> None: - pass + lock: Lock + signal: Event def __init__(self) -> None: super().__init__() @@ -50,8 +50,10 @@ def __init__(self) -> None: """The cache for entries before they are finalized""" self.cache2: DefaultDict[TableName, List[Dict[str, Any]]] = defaultdict(list) """For all entries that have been finalized but not yet flushed out to the queue""" - self.signal: Event = asyncio.Event() - self.lock: Lock = asyncio.Lock() + + async def init(self) -> None: + self.signal = asyncio.Event() + self.lock = asyncio.Lock() async def flush_cache(self) -> None: async with self.lock as _: From 57d8ba97b5026eb90d3adc238a8a1cf1487da9af Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 11 Dec 2020 13:59:31 +0100 Subject: [PATCH 064/139] Fixing tests --- demo.py | 2 +- openwpm/storage/cloud_storage/gcp_storage.py | 20 +++++-- test/conftest.py | 4 +- test/openwpmtest.py | 2 +- test/test_callstack_instrument.py | 2 +- test/test_custom_function_command.py | 8 +-- test/test_dns_instrument.py | 4 +- test/test_http_instrumentation.py | 28 +++++----- test/test_profile.py | 2 +- test/test_simple_commands.py | 57 ++++++++++---------- test/test_storage_vectors.py | 2 +- test/test_timer.py | 2 +- test/test_webdriver_utils.py | 2 +- 13 files changed, 75 insertions(+), 60 deletions(-) diff --git a/demo.py b/demo.py index 1578da732..e0f1d96b3 100644 --- a/demo.py +++ b/demo.py @@ -61,7 +61,7 @@ manager = TaskManager( manager_params, browser_params, - LocalArrowProvider(Path("./datadir/") / "parquet"), + GcsStructuredProvider(project, bucket_name, base_path="test3"), None, ) diff --git a/openwpm/storage/cloud_storage/gcp_storage.py b/openwpm/storage/cloud_storage/gcp_storage.py index 58eeac126..054520a7e 100644 --- a/openwpm/storage/cloud_storage/gcp_storage.py +++ b/openwpm/storage/cloud_storage/gcp_storage.py @@ -16,18 +16,27 @@ class GcsStructuredProvider(ArrowProvider): This might not actually be the thing that we want to do long term but seeing as GCS is the S3 equivalent of GCP it is the easiest way forward. + + Inspired by the old S3Aggregator structure the GcsStructuredProvider + will by default store into + base_path/visits/table_name in the given bucket. + + Pass a different sub_dir to change this. """ def __init__( - self, project: str, bucket_name: str, base_path: str, token: str = None, + self, + project: str, + bucket_name: str, + base_path: str, + token: str = None, + sub_dir: str = "visits", ) -> None: super().__init__() self.project = project - self.bucket_name = bucket_name - self.base_path = base_path self.token = token self.file_system: Optional[GCSFileSystem] = None - self.base_path = f"{self.bucket_name}/{base_path}/{{table_name}}" + self.base_path = f"{bucket_name}/{base_path}/{sub_dir}/{{table_name}}" async def init(self) -> None: await super(GcsStructuredProvider, self).init() @@ -54,6 +63,9 @@ class GcsUnstructuredProvider(UnstructuredStorageProvider): This might not actually be the thing that we want to do long term but seeing as GCS is the S3 equivalent of GCP it is the easiest way forward. + + + """ def __init__( diff --git a/test/conftest.py b/test/conftest.py index 7549b6b38..7064f804c 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -88,13 +88,13 @@ def http_params( ) -> Callable[[str], Tuple[ManagerParams, List[BrowserParams]]]: manager_params, browser_params = default_params for browser_param in browser_params: - browser_param["http_instrument"] = True + browser_param.http_instrument = True def parameterize( display_mode: str = "headless", ) -> Tuple[ManagerParams, List[BrowserParams]]: for browser_param in browser_params: - browser_param["display_mode"] = display_mode + browser_param.display_mode = display_mode return manager_params, browser_params return parameterize diff --git a/test/openwpmtest.py b/test/openwpmtest.py index 5860b1d1c..0ccb5c014 100644 --- a/test/openwpmtest.py +++ b/test/openwpmtest.py @@ -33,7 +33,7 @@ def get_config(self, data_dir) -> Tuple[ManagerParams, List[BrowserParams]]: def visit(self, page_url, data_dir="", sleep_after=0): """Visit a test page with the given parameters.""" manager_params, browser_params = self.get_config(data_dir) - structured_provider = SqlLiteStorageProvider(manager_params["db"]) + structured_provider = SqlLiteStorageProvider(manager_params.database_name) manager = task_manager.TaskManager( manager_params, browser_params, structured_provider, None ) diff --git a/test/test_callstack_instrument.py b/test/test_callstack_instrument.py index 23add8bb7..93de627e2 100644 --- a/test/test_callstack_instrument.py +++ b/test/test_callstack_instrument.py @@ -69,7 +69,7 @@ def test_http_stacktrace(default_params, task_manager_creator): test_url = utilities.BASE_TEST_URL + "/http_stacktrace.html" manager = task_manager_creator((manager_params, browser_params)) manager.get(test_url, sleep=10) - db = manager_params["db"] + db = manager_params.database_name manager.close() rows = db_utils.query_db( db, diff --git a/test/test_custom_function_command.py b/test/test_custom_function_command.py index 4d412441d..8c0b62506 100644 --- a/test/test_custom_function_command.py +++ b/test/test_custom_function_command.py @@ -64,7 +64,7 @@ def collect_links(table_name, scheme, **kwargs): manager_params, browser_params = default_params - db = sqlite3.connect(manager_params["db"]) + db = sqlite3.connect(manager_params.database_name) cur = db.cursor() cur.execute( @@ -76,7 +76,7 @@ def collect_links(table_name, scheme, **kwargs): cur.close() db.close() - storage_provider = SqlLiteStorageProvider(manager_params["db"]) + storage_provider = SqlLiteStorageProvider(manager_params.database_name) manager = TaskManager(manager_params, browser_params, storage_provider, None) cs = command_sequence.CommandSequence(url_a) cs.get(sleep=0, timeout=60) @@ -84,6 +84,8 @@ def collect_links(table_name, scheme, **kwargs): manager.execute_command_sequence(cs) manager.close() query_result = db_utils.query_db( - manager_params["db"], "SELECT top_url, link FROM page_links;", as_tuple=True + manager_params.database_name, + "SELECT top_url, link FROM page_links;", + as_tuple=True, ) assert PAGE_LINKS == set(query_result) diff --git a/test/test_dns_instrument.py b/test/test_dns_instrument.py index 6cb3ad802..a34a41111 100644 --- a/test/test_dns_instrument.py +++ b/test/test_dns_instrument.py @@ -10,7 +10,9 @@ def test_name_resolution(default_params, task_manager_creator): manager.get("http://localtest.me:8000") manager.close() - result = db_utils.query_db(manager_params["db"], "SELECT * FROM dns_responses") + result = db_utils.query_db( + manager_params.database_name, "SELECT * FROM dns_responses" + ) result = result[0] assert result["used_address"] == "127.0.0.1" assert result["addresses"] == "127.0.0.1" diff --git a/test/test_http_instrumentation.py b/test/test_http_instrumentation.py index a1e39613c..728672a42 100644 --- a/test/test_http_instrumentation.py +++ b/test/test_http_instrumentation.py @@ -893,10 +893,10 @@ def test_javascript_saving(http_params, xpi, server): manager_params, browser_params = http_params() for browser_param in browser_params: - browser_param["http_instrument"] = True - browser_param["save_content"] = "script" - structured_storage = SqlLiteStorageProvider(db_path=manager_params["db"]) - ldb_path = Path(manager_params["data_directory"]) / "content.ldb" + browser_param.http_instrument = True + browser_param.save_content = "script" + structured_storage = SqlLiteStorageProvider(db_path=manager_params.database_name) + ldb_path = Path(manager_params.data_directory) / "content.ldb" unstructured_storage = LevelDbProvider(db_path=ldb_path) manager = task_manager.TaskManager( manager_params, browser_params, structured_storage, unstructured_storage @@ -925,11 +925,11 @@ def test_document_saving(http_params, xpi, server): } manager_params, browser_params = http_params() for browser_param in browser_params: - browser_param["http_instrument"] = True - browser_param["save_content"] = "main_frame,sub_frame" + browser_param.http_instrument = True + browser_param.save_content = "main_frame,sub_frame" - structured_storage = SqlLiteStorageProvider(db_path=manager_params["db"]) - ldb_path = Path(manager_params["data_directory"]) / "content.ldb" + structured_storage = SqlLiteStorageProvider(db_path=manager_params.database_name) + ldb_path = Path(manager_params.data_directory) / "content.ldb" unstructured_storage = LevelDbProvider(db_path=ldb_path) manager = task_manager.TaskManager( manager_params, browser_params, structured_storage, unstructured_storage @@ -951,11 +951,11 @@ def test_content_saving(http_params, xpi, server): test_url = utilities.BASE_TEST_URL + "/http_test_page.html" manager_params, browser_params = http_params() for browser_param in browser_params: - browser_param["http_instrument"] = True - browser_param["save_content"] = True + browser_param.http_instrument = True + browser_param.save_content = True - structured_storage = SqlLiteStorageProvider(db_path=manager_params["db"]) - ldb_path = Path(manager_params["data_directory"]) / "content.ldb" + structured_storage = SqlLiteStorageProvider(db_path=manager_params.database_name) + ldb_path = Path(manager_params.data_directory) / "content.ldb" unstructured_storage = LevelDbProvider(db_path=ldb_path) manager = task_manager.TaskManager( manager_params, browser_params, structured_storage, unstructured_storage @@ -963,7 +963,7 @@ def test_content_saving(http_params, xpi, server): manager.get(url=test_url, sleep=1) manager.close() - db = manager_params["db"] + db = manager_params.database_name rows = db_utils.query_db(db, "SELECT * FROM http_responses;") disk_content = dict() for row in rows: @@ -1008,7 +1008,7 @@ def test_cache_hits_recorded(http_params, task_manager_creator): manager.execute_command_sequence(cs) manager.close() - db = manager_params["db"] + db = manager_params.database_name request_id_to_url = dict() diff --git a/test/test_profile.py b/test/test_profile.py index 34bdb8930..9f1fd3cfb 100644 --- a/test/test_profile.py +++ b/test/test_profile.py @@ -118,7 +118,7 @@ def test_config_is_set(*args, **kwargs): manager.execute_command_sequence(cs) manager.close() query_result = db_utils.query_db( - manager_params["db"], "SELECT * FROM crawl_history;", + manager_params.database_name, "SELECT * FROM crawl_history;", ) assert len(query_result) > 0 for row in query_result: diff --git a/test/test_simple_commands.py b/test/test_simple_commands.py index bdffe9626..0240510e2 100644 --- a/test/test_simple_commands.py +++ b/test/test_simple_commands.py @@ -112,7 +112,8 @@ def test_get_site_visits_table_valid(http_params, task_manager_creator, display_ manager.close() qry_res = db_utils.query_db( - manager_params["db"], "SELECT site_url FROM site_visits ORDER BY site_url" + manager_params.database_name, + "SELECT site_url FROM site_visits ORDER BY site_url", ) # We had two separate page visits @@ -140,7 +141,7 @@ def test_get_http_tables_valid(http_params, task_manager_creator, display_mode): manager.close() qry_res = db_utils.query_db( - manager_params["db"], "SELECT visit_id, site_url FROM site_visits" + manager_params.database_name, "SELECT visit_id, site_url FROM site_visits" ) # Construct dict mapping site_url to visit_id @@ -149,28 +150,28 @@ def test_get_http_tables_valid(http_params, task_manager_creator, display_mode): visit_ids[row[1]] = row[0] qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_requests WHERE url = ?", (url_a,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_requests WHERE url = ?", (url_b,), ) assert qry_res[0][0] == visit_ids[url_b] qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_responses WHERE url = ?", (url_a,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_responses WHERE url = ?", (url_b,), ) @@ -197,7 +198,7 @@ def test_browse_site_visits_table_valid( manager.close() qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT site_url, site_rank FROM site_visits ORDER BY site_rank", ) @@ -233,7 +234,7 @@ def test_browse_http_table_valid(http_params, task_manager_creator, display_mode manager.close() qry_res = db_utils.query_db( - manager_params["db"], "SELECT visit_id, site_url FROM site_visits" + manager_params.database_name, "SELECT visit_id, site_url FROM site_visits" ) # Construct dict mapping site_url to visit_id @@ -242,28 +243,28 @@ def test_browse_http_table_valid(http_params, task_manager_creator, display_mode visit_ids[row[1]] = row[0] qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_requests WHERE url = ?", (url_a,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_requests WHERE url = ?", (url_b,), ) assert qry_res[0][0] == visit_ids[url_b] qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_responses WHERE url = ?", (url_a,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_responses WHERE url = ?", (url_b,), ) @@ -277,13 +278,13 @@ def test_browse_http_table_valid(http_params, task_manager_creator, display_mode # 5) A link to example.com?localtest.me # We should see page visits for 1 and 2, but not 3-5. qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_responses WHERE url = ?", (url_c,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_responses WHERE url = ?", (url_d,), ) @@ -291,7 +292,7 @@ def test_browse_http_table_valid(http_params, task_manager_creator, display_mode # We expect 4 urls: a,c,d and a favicon request qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT COUNT(DISTINCT url) FROM http_responses WHERE visit_id = ?", (visit_ids[url_a],), ) @@ -319,7 +320,7 @@ def test_browse_wrapper_http_table_valid( manager.close() qry_res = db_utils.query_db( - manager_params["db"], "SELECT visit_id, site_url FROM site_visits" + manager_params.database_name, "SELECT visit_id, site_url FROM site_visits" ) # Construct dict mapping site_url to visit_id @@ -328,28 +329,28 @@ def test_browse_wrapper_http_table_valid( visit_ids[row[1]] = row[0] qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_requests WHERE url = ?", (url_a,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_requests WHERE url = ?", (url_b,), ) assert qry_res[0][0] == visit_ids[url_b] qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_responses WHERE url = ?", (url_a,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_responses WHERE url = ?", (url_b,), ) @@ -363,13 +364,13 @@ def test_browse_wrapper_http_table_valid( # 5) A link to example.com?localtest.me # We should see page visits for 1 and 2, but not 3-5. qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_responses WHERE url = ?", (url_c,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT visit_id FROM http_responses WHERE url = ?", (url_d,), ) @@ -377,7 +378,7 @@ def test_browse_wrapper_http_table_valid( # We expect 4 urls: a,c,d and a favicon request qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT COUNT(DISTINCT url) FROM http_responses WHERE visit_id = ?", (visit_ids[url_a],), ) @@ -399,9 +400,7 @@ def test_save_screenshot_valid(http_params, task_manager_creator, display_mode): manager.close() # Check that viewport image is not blank - pattern = os.path.join( - manager_params["data_directory"], "screenshots", "*-*-test.png" - ) + pattern = os.path.join(manager_params.data_directory, "screenshots", "*-*-test.png") screenshot = glob.glob(pattern)[0] im = Image.open(screenshot) bands = im.split() @@ -410,7 +409,7 @@ def test_save_screenshot_valid(http_params, task_manager_creator, display_mode): # Check that full page screenshot is not blank pattern = os.path.join( - manager_params["data_directory"], "screenshots", "*-*-test_full.png" + manager_params.data_directory, "screenshots", "*-*-test_full.png" ) screenshot = glob.glob(pattern)[0] im = Image.open(screenshot) @@ -435,7 +434,7 @@ def test_dump_page_source_valid(http_params, task_manager_creator, display_mode) # Source filename is of the follow structure: # `sources/-(-suffix).html` # thus for this test we expect `sources/1--test.html`. - outfile = os.path.join(manager_params["data_directory"], "sources", "*-*-test.html") + outfile = os.path.join(manager_params.data_directory, "sources", "*-*-test.html") source_file = glob.glob(outfile)[0] with open(source_file, "rb") as f: actual_source = f.read() @@ -459,7 +458,7 @@ def test_recursive_dump_page_source_valid( manager.execute_command_sequence(cs) manager.close() - outfile = os.path.join(manager_params["data_directory"], "sources", "*-*.json.gz") + outfile = os.path.join(manager_params.data_directory, "sources", "*-*.json.gz") src_file = glob.glob(outfile)[0] with gzip.GzipFile(src_file, "rb") as f: visit_source = json.loads(f.read().decode("utf-8")) diff --git a/test/test_storage_vectors.py b/test/test_storage_vectors.py index e4b67f035..1d945f2e1 100644 --- a/test/test_storage_vectors.py +++ b/test/test_storage_vectors.py @@ -39,7 +39,7 @@ def test_js_profile_cookies(default_params, task_manager_creator): manager.close() # Check that the JS cookie we stored is recorded qry_res = db_utils.query_db( - manager_params["db"], + manager_params.database_name, ( "SELECT record_type, change_cause, is_http_only, " "is_host_only, is_session, host, is_secure, name, path, " diff --git a/test/test_timer.py b/test/test_timer.py index f3203b85c..40979cc8f 100644 --- a/test/test_timer.py +++ b/test/test_timer.py @@ -15,7 +15,7 @@ def test_command_duration(default_params, task_manager_creator): manager.close() get_command = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT duration FROM crawl_history WHERE command = 'GetCommand'", as_tuple=True, )[0] diff --git a/test/test_webdriver_utils.py b/test/test_webdriver_utils.py index ddb913d54..76aa7124c 100644 --- a/test/test_webdriver_utils.py +++ b/test/test_webdriver_utils.py @@ -23,7 +23,7 @@ def test_parse_neterror_integration(default_params, task_manager_creator): manager.close() get_command = db_utils.query_db( - manager_params["db"], + manager_params.database_name, "SELECT command_status, error FROM crawl_history WHERE command ='GetCommand'", as_tuple=True, )[0] From 67d30701b9f995ab7c5392026a7f3901a00af3a1 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 11 Dec 2020 14:49:55 +0100 Subject: [PATCH 065/139] Fixing up tests and adding more typechecking --- openwpm/utilities/db_utils.py | 6 ++++-- test/openwpm_jstest.py | 7 ++++++- test/openwpmtest.py | 10 +++++++--- test/test_callstack_instrument.py | 6 +++--- test/test_crawl.py | 13 ++++++++++--- test/test_extension.py | 20 ++++++++++++++------ test/test_http_instrumentation.py | 11 ++++++++--- test/test_js_instrument.py | 8 ++++++-- test/test_profile.py | 7 ++++++- 9 files changed, 64 insertions(+), 24 deletions(-) diff --git a/openwpm/utilities/db_utils.py b/openwpm/utilities/db_utils.py index 4a1c7eef6..03b4dbf02 100644 --- a/openwpm/utilities/db_utils.py +++ b/openwpm/utilities/db_utils.py @@ -1,7 +1,7 @@ import os import sqlite3 from pathlib import Path -from typing import AnyStr, Iterator, Tuple +from typing import Any, AnyStr, Iterator, List, Tuple, Union import plyvel @@ -36,7 +36,9 @@ def get_content(db_name: Path) -> Iterator[Tuple[AnyStr, AnyStr]]: db.close() -def get_javascript_entries(db, all_columns=False, as_tuple=False): +def get_javascript_entries( + db: Path, all_columns: bool = False, as_tuple: bool = False +) -> List[Union[Tuple[Any, ...], sqlite3.Row]]: if all_columns: select_columns = "*" else: diff --git a/test/openwpm_jstest.py b/test/openwpm_jstest.py index f89dc7851..203d7453d 100644 --- a/test/openwpm_jstest.py +++ b/test/openwpm_jstest.py @@ -1,12 +1,17 @@ import re +from pathlib import Path +from typing import List, Optional, Tuple +from openwpm.config import BrowserParams, ManagerParams from openwpm.utilities import db_utils from .openwpmtest import OpenWPMTest class OpenWPMJSTest(OpenWPMTest): - def get_config(self, data_dir=""): + def get_config( + self, data_dir: Optional[Path] + ) -> Tuple[ManagerParams, List[BrowserParams]]: manager_params, browser_params = self.get_test_config(data_dir) browser_params[0].js_instrument = True manager_params.testing = True diff --git a/test/openwpmtest.py b/test/openwpmtest.py index 0ccb5c014..ff98b1e34 100644 --- a/test/openwpmtest.py +++ b/test/openwpmtest.py @@ -25,12 +25,16 @@ def set_tmpdir(self, tmpdir): Based on: https://mail.python.org/pipermail/pytest-dev/2014-April/002484.html """ - self.tmpdir = tmpdir + self.tmpdir = Path(tmpdir) - def get_config(self, data_dir) -> Tuple[ManagerParams, List[BrowserParams]]: + def get_config( + self, data_dir: Optional[Path] + ) -> Tuple[ManagerParams, List[BrowserParams]]: pass - def visit(self, page_url, data_dir="", sleep_after=0): + def visit( + self, page_url: str, data_dir: Optional[Path] = None, sleep_after: int = 0 + ) -> Path: """Visit a test page with the given parameters.""" manager_params, browser_params = self.get_config(data_dir) structured_provider = SqlLiteStorageProvider(manager_params.database_name) diff --git a/test/test_callstack_instrument.py b/test/test_callstack_instrument.py index 93de627e2..a5f2a698d 100644 --- a/test/test_callstack_instrument.py +++ b/test/test_callstack_instrument.py @@ -61,11 +61,11 @@ def test_http_stacktrace(default_params, task_manager_creator): manager_params, browser_params = default_params # Record HTTP Requests and Responses - browser_params[0]["http_instrument"] = True + browser_params[0].http_instrument = True # Record JS Web API calls - browser_params[0]["js_instrument"] = True + browser_params[0].js_instrument = True # Record the callstack of all WebRequests made - browser_params[0]["callstack_instrument"] = True + browser_params[0].callstack_instrument = True test_url = utilities.BASE_TEST_URL + "/http_stacktrace.html" manager = task_manager_creator((manager_params, browser_params)) manager.get(test_url, sleep=10) diff --git a/test/test_crawl.py b/test/test_crawl.py index 8bf17f5ef..046e1f558 100644 --- a/test/test_crawl.py +++ b/test/test_crawl.py @@ -1,10 +1,15 @@ +# type:ignore +# As this file is no longer maintained, mypy shouldn't check this import os import tarfile +from pathlib import Path +from typing import List, Tuple import domain_utils as du import pytest from openwpm import task_manager +from openwpm.config import BrowserParams, ManagerParams from openwpm.utilities import db_utils from .openwpmtest import OpenWPMTest @@ -46,7 +51,9 @@ class TestCrawl(OpenWPMTest): tests will be easier to debug """ - def get_config(self, data_dir=""): + def get_config( + self, data_dir: Path = None + ) -> Tuple[ManagerParams, List[BrowserParams]]: manager_params, browser_params = self.get_test_config(data_dir) browser_params[0].profile_archive_dir = os.path.join( manager_params.data_directory, "browser_profile" @@ -56,7 +63,7 @@ def get_config(self, data_dir=""): @pytest.mark.xfail(run=False) @pytest.mark.slow - def test_browser_profile_coverage(self, tmpdir): + def test_browser_profile_coverage(self, tmpdir: Path) -> None: """Test the coverage of the browser's profile This verifies that Firefox's places.sqlite database contains @@ -64,7 +71,7 @@ def test_browser_profile_coverage(self, tmpdir): it is likely the profile is lost at some point during the crawl """ # Run the test crawl - data_dir = os.path.join(str(tmpdir), "data_dir") + data_dir = tmpdir / "data_dir" manager_params, browser_params = self.get_config(data_dir) manager = task_manager.TaskManager(manager_params, browser_params) for site in TEST_SITES: diff --git a/test/test_extension.py b/test/test_extension.py index 59515d20b..49036911c 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -1,9 +1,13 @@ import os from datetime import datetime +from pathlib import Path +from sqlite3 import Row +from typing import List, Tuple import pytest from openwpm import task_manager +from openwpm.config import BrowserParams, ManagerParams from openwpm.utilities import db_utils from . import utilities @@ -248,12 +252,14 @@ class TestExtension(OpenWPMTest): - def get_config(self, data_dir=""): + def get_config( + self, data_dir: Path = None + ) -> Tuple[ManagerParams, List[BrowserParams]]: manager_params, browser_params = self.get_test_config(data_dir) browser_params[0].js_instrument = True return manager_params, browser_params - def test_property_enumeration(self): + def test_property_enumeration(self) -> None: test_url = utilities.BASE_TEST_URL + "/property_enumeration.html" db = self.visit(test_url) rows = db_utils.query_db(db, "SELECT script_url, symbol FROM javascript") @@ -263,12 +269,13 @@ def test_property_enumeration(self): observed_symbols.add(symbol) assert PROPERTIES == observed_symbols - def test_canvas_fingerprinting(self): + def test_canvas_fingerprinting(self) -> None: db = self.visit("/canvas_fingerprinting.html") # Check that all calls and methods are recorded rows = db_utils.get_javascript_entries(db) observed_rows = set() for row in rows: + assert isinstance(row, Row) item = ( row["script_url"], row["symbol"], @@ -279,7 +286,7 @@ def test_canvas_fingerprinting(self): observed_rows.add(item) assert CANVAS_CALLS == observed_rows - def test_extension_gets_correct_visit_id(self): + def test_extension_gets_correct_visit_id(self) -> None: url_a = utilities.BASE_TEST_URL + "/simple_a.html" url_b = utilities.BASE_TEST_URL + "/simple_b.html" self.visit(url_a) @@ -307,7 +314,7 @@ def test_extension_gets_correct_visit_id(self): assert visit_ids[url_a] == simple_a_visit_id[0][0] assert visit_ids[url_b] == simple_b_visit_id[0][0] - def check_webrtc_sdp_offer(self, sdp_str): + def check_webrtc_sdp_offer(self, sdp_str: str) -> None: """Make sure the SDP offer includes expected fields/strings. SDP offer contains randomly generated strings (e.g. GUID). That's why @@ -317,12 +324,13 @@ def check_webrtc_sdp_offer(self, sdp_str): for expected_str in WEBRTC_SDP_OFFER_STRINGS: assert expected_str in sdp_str - def test_webrtc_localip(self): + def test_webrtc_localip(self) -> None: db = self.visit("/webrtc_localip.html") # Check that all calls and methods are recorded rows = db_utils.get_javascript_entries(db) observed_rows = set() for row in rows: + assert isinstance(row, Row) if row["symbol"] == "RTCPeerConnection.setLocalDescription" and ( row["operation"] == "call" ): diff --git a/test/test_http_instrumentation.py b/test/test_http_instrumentation.py index 728672a42..e3b0d9fb5 100644 --- a/test/test_http_instrumentation.py +++ b/test/test_http_instrumentation.py @@ -7,13 +7,14 @@ from hashlib import sha256 from pathlib import Path from time import sleep -from typing import Set, Tuple +from typing import List, Optional, Set, Tuple from urllib.parse import urlparse import pytest from openwpm import command_sequence, task_manager from openwpm.command_sequence import CommandSequence +from openwpm.config import BrowserParams, ManagerParams from openwpm.storage.leveldb import LevelDbProvider from openwpm.storage.sql_provider import SqlLiteStorageProvider from openwpm.utilities import db_utils @@ -594,7 +595,9 @@ class TestHTTPInstrument(OpenWPMTest): - def get_config(self, data_dir=""): + def get_config( + self, data_dir: Optional[Path] + ) -> Tuple[ManagerParams, List[BrowserParams]]: manager_params, browser_params = self.get_test_config(data_dir) browser_params[0].http_instrument = True return manager_params, browser_params @@ -745,7 +748,9 @@ class TestPOSTInstrument(OpenWPMTest): "line2 line2_word2\r\n" ) - def get_config(self, data_dir=""): + def get_config( + self, data_dir: Optional[Path] + ) -> Tuple[ManagerParams, List[BrowserParams]]: manager_params, browser_params = self.get_test_config(data_dir) browser_params[0].http_instrument = True return manager_params, browser_params diff --git a/test/test_js_instrument.py b/test/test_js_instrument.py index 1d879e0c0..c4540c174 100644 --- a/test/test_js_instrument.py +++ b/test/test_js_instrument.py @@ -1,5 +1,7 @@ -from typing import Set, Tuple +from pathlib import Path +from typing import List, Optional, Set, Tuple +from openwpm.config import BrowserParams, ManagerParams from openwpm.utilities import db_utils from . import utilities as util @@ -101,7 +103,9 @@ class TestJSInstrumentByPython(OpenWPMJSTest): # noqa ("window.fetch", "call", '["https://example.org"]'), } - def get_config(self, data_dir=""): + def get_config( + self, data_dir: Optional[Path] + ) -> Tuple[ManagerParams, List[BrowserParams]]: manager_params, browser_params = super().get_config(data_dir) browser_params[0].prefs = { "network.dns.localDomains": "example.com,example.org" diff --git a/test/test_profile.py b/test/test_profile.py index 9f1fd3cfb..d875dc44d 100644 --- a/test/test_profile.py +++ b/test/test_profile.py @@ -1,9 +1,12 @@ from os.path import isfile, join +from pathlib import Path +from typing import List, Optional, Tuple import pytest from openwpm import task_manager from openwpm.command_sequence import CommandSequence +from openwpm.config import BrowserParams, ManagerParams from openwpm.errors import CommandExecutionError, ProfileLoadError from openwpm.utilities import db_utils @@ -13,7 +16,9 @@ class TestProfile(OpenWPMTest): - def get_config(self, data_dir=""): + def get_config( + self, data_dir: Optional[Path] + ) -> Tuple[ManagerParams, List[BrowserParams]]: manager_params, browser_params = self.get_test_config(data_dir) browser_params[0].profile_archive_dir = join( manager_params.data_directory, "browser_profile" From de00f94ca3b4a21f6fa70e4a5b436446572ea56f Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 11 Dec 2020 15:01:06 +0100 Subject: [PATCH 066/139] Fixed num_browsers in test_cache_hits_recorded --- test/test_http_instrumentation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_http_instrumentation.py b/test/test_http_instrumentation.py index e3b0d9fb5..a3ca32ebe 100644 --- a/test/test_http_instrumentation.py +++ b/test/test_http_instrumentation.py @@ -1006,6 +1006,7 @@ def test_cache_hits_recorded(http_params, task_manager_creator): test_url = utilities.BASE_TEST_URL + "/http_test_page.html" manager_params, browser_params = http_params() # ensuring that we only spawn one browser + manager_params.num_browsers = 1 manager = task_manager_creator((manager_params, [browser_params[0]])) for i in range(2): cs = CommandSequence(test_url, site_rank=i) From 4291ddbfb13f5cc50f9a15f82caaf0fc5295c6ad Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 11 Dec 2020 17:23:21 +0100 Subject: [PATCH 067/139] Parametrized unstructured --- test/storage/test_local_storage_provider.py | 9 ++- test/storage/test_memory_storage_provider.py | 64 ++++++++++++++------ 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/test/storage/test_local_storage_provider.py b/test/storage/test_local_storage_provider.py index 1afa01435..67fbcae2e 100644 --- a/test/storage/test_local_storage_provider.py +++ b/test/storage/test_local_storage_provider.py @@ -14,6 +14,7 @@ @pytest.mark.asyncio async def test_local_arrow_storage_provider(tmp_path, mp_logger): structured_provider = LocalArrowProvider(tmp_path) + await structured_provider.init() visit_ids = set() for table_name, test_data in TEST_VALUES.items(): visit_id = VisitId(test_data["visit_id"]) @@ -21,9 +22,11 @@ async def test_local_arrow_storage_provider(tmp_path, mp_logger): await structured_provider.store_record( TableName(table_name), visit_id, test_data ) - task_list = list(structured_provider.finalize_visit_id(i) for i in visit_ids) - task_list.append(structured_provider.flush_cache()) - await asyncio.gather(*task_list) + token_list = [] + for i in visit_ids: + token_list.append(await structured_provider.finalize_visit_id(i)) + await structured_provider.flush_cache() + await asyncio.gather(*token_list) for table_name, test_data in TEST_VALUES.items(): dataset = ParquetDataset(tmp_path / table_name) df: DataFrame = dataset.read().to_pandas() diff --git a/test/storage/test_memory_storage_provider.py b/test/storage/test_memory_storage_provider.py index 3780faf58..a5d3cde91 100644 --- a/test/storage/test_memory_storage_provider.py +++ b/test/storage/test_memory_storage_provider.py @@ -10,8 +10,13 @@ MemoryUnstructuredProvider, ) from openwpm.storage.leveldb import LevelDbProvider +from openwpm.storage.local_storage import LocalGzipProvider from openwpm.storage.sql_provider import SqlLiteStorageProvider -from openwpm.storage.storage_providers import StructuredStorageProvider, TableName +from openwpm.storage.storage_providers import ( + StructuredStorageProvider, + TableName, + UnstructuredStorageProvider, +) from openwpm.types import VisitId from ..openwpmtest import OpenWPMTest @@ -20,16 +25,6 @@ sqllite = "sqllite" memory_arrow = "memory_arrow" -# Unstructured Providers -memory_unstructured = "memory_unstructured" -leveldb = "leveldb" - -structured_scenarios: List[Tuple[str, Dict[str, Any]]] = [ - (memory_structured, {"structured_provider": memory_structured}), - (sqllite, {"structured_provider": sqllite}), - (memory_arrow, {"structured_provider": memory_arrow}), -] - @pytest.fixture def structured_provider( @@ -62,7 +57,11 @@ def pytest_generate_tests(metafunc: Any) -> Any: @pytest.mark.asyncio class TestStructuredStorageProvider: - scenarios = structured_scenarios + scenarios: List[Tuple[str, Dict[str, Any]]] = [ + (memory_structured, {"structured_provider": memory_structured}), + (sqllite, {"structured_provider": sqllite}), + (memory_arrow, {"structured_provider": memory_arrow}), + ] async def test_basic_access( self, structured_provider: StructuredStorageProvider @@ -83,12 +82,43 @@ async def test_basic_access( await token +# Unstructured Providers +memory_unstructured = "memory_unstructured" +leveldb = "leveldb" +local_gzip = "local_gzip" + + +@pytest.fixture +def unstructured_provider( + request: Any, tmp_path_factory: Any +) -> UnstructuredStorageProvider: + if request.param == memory_unstructured: + return MemoryUnstructuredProvider() + elif request.param == leveldb: + tmp_path = tmp_path_factory.mktemp(leveldb) + return LevelDbProvider(tmp_path / "content.ldb") + elif request.param == local_gzip: + tmp_path = tmp_path_factory.mktemp(local_gzip) + return LocalGzipProvider(tmp_path) + assert isinstance( + request, FixtureRequest + ) # See https://github.com/pytest-dev/pytest/issues/8073 for why this can't be type annotated + request.raiseerror("invalid internal test config") + + @pytest.mark.asyncio class TestUnstructuredStorageProvide: - scenarios: List[Tuple[str, Dict[str, Any]]] = [(memory_unstructured, {})] - - async def test_basic_unstructured_storing(self) -> None: + scenarios: List[Tuple[str, Dict[str, Any]]] = [ + (memory_unstructured, {"unstructured_provider": memory_unstructured}), + (leveldb, {"unstructured_provider": leveldb}), + (local_gzip, {"unstructured_provider": local_gzip}), + ] + + async def test_basic_unstructured_storing( + self, unstructured_provider: UnstructuredStorageProvider + ) -> None: test_string = "This is my test string" blob = test_string.encode() - prov = MemoryUnstructuredProvider() - await prov.store_blob("test", blob, compressed=False) + await unstructured_provider.init() + await unstructured_provider.store_blob("test", blob) + await unstructured_provider.flush_cache() From fa1c52f1ab13504815fa6bef4db4287f7fc92255 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 11 Dec 2020 17:25:17 +0100 Subject: [PATCH 068/139] String fix --- openwpm/storage/local_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwpm/storage/local_storage.py b/openwpm/storage/local_storage.py index a8c3827b1..82b38ab7f 100644 --- a/openwpm/storage/local_storage.py +++ b/openwpm/storage/local_storage.py @@ -38,7 +38,7 @@ def __init__(self, storage_path: Path) -> None: async def store_blob( self, filename: str, blob: bytes, overwrite: bool = False ) -> None: - path = self.storage_path / (filename + "zip") + path = self.storage_path / (filename + ".zip") if path.exists() and not overwrite: self.logger.debug( "File %s already exists on disk. Not overwriting", filename From 9aed882394bfcecfc4c7249f8ef04795495eaa0a Mon Sep 17 00:00:00 2001 From: vringar Date: Tue, 15 Dec 2020 11:34:26 +0100 Subject: [PATCH 069/139] Added failing test --- openwpm/storage/in_memory_storage.py | 7 ++--- test/storage/test_arrow_cache.py | 41 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) create mode 100644 test/storage/test_arrow_cache.py diff --git a/openwpm/storage/in_memory_storage.py b/openwpm/storage/in_memory_storage.py index 36be35f74..95193188b 100644 --- a/openwpm/storage/in_memory_storage.py +++ b/openwpm/storage/in_memory_storage.py @@ -107,9 +107,9 @@ def __init__(self, queue: Queue) -> None: self.queue = queue self.storage: DefaultDict[str, List[Any]] = defaultdict(list) - def poll_queue(self) -> None: + def poll_queue(self, *args: Any, **kwargs: Any) -> None: while not self.queue.empty(): - table, record = self.queue.get() + table, record = self.queue.get(*args, **kwargs) self.storage[table].append(record) @@ -147,9 +147,6 @@ async def shutdown(self) -> None: class MemoryArrowProvider(ArrowProvider): - async def init(self) -> None: - pass - def __init__(self) -> None: super().__init__() self.queue = Queue() diff --git a/test/storage/test_arrow_cache.py b/test/storage/test_arrow_cache.py new file mode 100644 index 000000000..4d99203e5 --- /dev/null +++ b/test/storage/test_arrow_cache.py @@ -0,0 +1,41 @@ +import asyncio +import random +from typing import Awaitable, Dict + +import pytest +from pandas import DataFrame +from pyarrow.parquet import ParquetDataset + +from openwpm.mp_logger import MPLogger +from openwpm.storage.arrow_storage import CACHE_SIZE +from openwpm.storage.in_memory_storage import MemoryArrowProvider +from openwpm.storage.storage_providers import TableName +from openwpm.types import VisitId +from test.storage.test_values import TEST_VALUES + + +@pytest.mark.asyncio +async def test_arrow_cache(mp_logger: MPLogger) -> None: + prov = MemoryArrowProvider() + await prov.init() + site_visit = TEST_VALUES["site_visits"] + d: Dict[VisitId, Awaitable[None]] = {} + for i in range(CACHE_SIZE + 1): + visit_id = VisitId(random.randint(0, 2 ** 63 - 1)) + site_visit["visit_id"] = visit_id + await prov.store_record(TableName("site_visits"), visit_id, site_visit) + d[visit_id] = await prov.finalize_visit_id(visit_id) + + handle = prov.handle + + # The queue should not be empty at this point + handle.poll_queue(block=False) + + for visit_id in d: + task = d[visit_id] + await asyncio.wait_for(task, 1) + + assert len(handle.storage["site_visits"]) == 1 + table = handle.storage["site_visits"][0] + + df: DataFrame = table.to_pandas() From ef0ba1e50a0c49e96d3f66f47168237e9f4ea405 Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 23 Dec 2020 15:22:45 +0100 Subject: [PATCH 070/139] New test --- environment.yaml | 10 +++++----- test/storage/test_arrow_cache.py | 13 +++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/environment.yaml b/environment.yaml index 049e17b4e..2dad1541f 100644 --- a/environment.yaml +++ b/environment.yaml @@ -5,7 +5,7 @@ dependencies: - beautifulsoup4=4.9.3 - black=20.8b1 - click=7.1.2 -- codecov=2.1.10 +- codecov=2.1.11 - dill=0.3.3 - gcsfs=0.7.1 - geckodriver=0.28.0 @@ -16,16 +16,16 @@ dependencies: - nodejs=14.15.1 - pandas=1.1.5 - pillow=8.0.1 -- pip=20.3.1 +- pip=20.3.3 - pre-commit=2.9.3 -- psutil=5.7.3 +- psutil=5.8.0 - pyarrow=2.0.0 - pytest-cov=2.10.1 -- pytest=6.1.2 +- pytest=6.2.1 - python=3.8.6 - pyvirtualdisplay=0.2.5 - redis-py=3.5.3 -- s3fs=0.5.1 +- s3fs=0.5.2 - selenium=3.141.0 - sentry-sdk=0.19.5 - tabulate=0.8.7 diff --git a/test/storage/test_arrow_cache.py b/test/storage/test_arrow_cache.py index 4d99203e5..09490bb4e 100644 --- a/test/storage/test_arrow_cache.py +++ b/test/storage/test_arrow_cache.py @@ -21,21 +21,22 @@ async def test_arrow_cache(mp_logger: MPLogger) -> None: site_visit = TEST_VALUES["site_visits"] d: Dict[VisitId, Awaitable[None]] = {} for i in range(CACHE_SIZE + 1): - visit_id = VisitId(random.randint(0, 2 ** 63 - 1)) + visit_id = VisitId(i) site_visit["visit_id"] = visit_id await prov.store_record(TableName("site_visits"), visit_id, site_visit) d[visit_id] = await prov.finalize_visit_id(visit_id) - handle = prov.handle - - # The queue should not be empty at this point - handle.poll_queue(block=False) - for visit_id in d: task = d[visit_id] await asyncio.wait_for(task, 1) + await asyncio.sleep(1) + handle = prov.handle + # The queue should not be empty at this point + handle.poll_queue(block=False) assert len(handle.storage["site_visits"]) == 1 table = handle.storage["site_visits"][0] df: DataFrame = table.to_pandas() + for row in df.itertuples(index=False): + del d[row["visit_id"]] From 1b14cbd22e0b0a68c6fcd886799a6a5f90b26801 Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 23 Dec 2020 18:49:22 +0100 Subject: [PATCH 071/139] Review changes with Steven --- openwpm/storage/arrow_storage.py | 55 ++++++++++++-------- openwpm/storage/cloud_storage/gcp_storage.py | 15 +++--- openwpm/storage/leveldb.py | 13 ++--- openwpm/storage/local_storage.py | 1 - openwpm/storage/sql_provider.py | 14 ++--- openwpm/storage/storage_controller.py | 55 ++++++++------------ openwpm/storage/storage_providers.py | 4 +- test/conftest.py | 4 +- test/storage/test_memory_storage_provider.py | 2 + test/storage/test_storage_controller.py | 17 +----- test/test_extension.py | 1 - test/test_timer.py | 2 - 12 files changed, 80 insertions(+), 103 deletions(-) diff --git a/openwpm/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py index c1add3386..e4526da6e 100644 --- a/openwpm/storage/arrow_storage.py +++ b/openwpm/storage/arrow_storage.py @@ -24,6 +24,9 @@ class ArrowProvider(StructuredStorageProvider): serializes records into the arrow format """ + storing_lock: asyncio.Lock + flush_event: asyncio.Event + def __init__(self) -> None: super().__init__() self.logger = logging.getLogger("openwpm") @@ -40,8 +43,7 @@ def factory_function() -> DefaultDict[TableName, List[Dict[str, Any]]]: self._batches: DefaultDict[TableName, List[pa.RecordBatch]] = defaultdict(list) self._instance_id = random.getrandbits(32) - self.storing_lock: Optional[asyncio.Lock] = None - self.flush_event: Optional[asyncio.Event] = None + self.flush_events: List[asyncio.Event] = list() async def init(self) -> None: # Used to synchronize the finalizing and the flushing @@ -65,7 +67,7 @@ def _create_batch(self, visit_id: VisitId) -> None: if visit_id not in self._records: # The batch for this `visit_id` was already created, skip self.logger.error( - "Trying to create batch for visit_id %d" "when one was already created", + "Trying to create batch for visit_id %d when one was already created", visit_id, ) return @@ -106,7 +108,7 @@ async def finalize_visit_id( await token ``` If there was no token returned and the method would just block/yield after turning the - record into a batch, there would be no way to know, when it's save to flush_cache as + record into a batch, there would be no way to know, when it's safe to flush_cache as I couldn't find a way to run a coroutine until it yields and then run a different one. With the current setup `token` aka a `wait_on_condition` coroutine will only return once @@ -114,24 +116,26 @@ async def finalize_visit_id( """ if interrupted: await self.store_record(INCOMPLETE_VISITS, visit_id, {"visit_id": visit_id}) - assert self.storing_lock is not None - assert self.flush_event is not None # This code is pretty tricky as there are a number of things going on - # 1. No finalize_visit_id shouldn't return unless the visit has been saved to storage + # 1. The awaitable returned by finalize_visit_id should only + # resolve once the data is saved to persistent storage # 2. No new batches should be created while saving out all the batches async with self.storing_lock: # After flush_cache has executed the event needs to be rearmed # so that newly created wait_on_condition don't just complete # instantly - self.flush_event.clear() self._create_batch(visit_id) + + event = asyncio.Event() + self.flush_events.append(event) + if self._is_cache_full(): await self.flush_cache(self.storing_lock) - async def wait_on_condition(event: asyncio.Event) -> None: - await event.wait() + async def wait_on_condition(e: asyncio.Event) -> None: + await e.wait() - return wait_on_condition(self.flush_event) + return wait_on_condition(event) @abstractmethod async def write_table(self, table_name: TableName, table: Table) -> None: @@ -139,21 +143,26 @@ async def write_table(self, table_name: TableName, table: Table) -> None: This should only return once it's actually saved out """ - async def flush_cache(self, cond: asyncio.Lock = None) -> None: + async def flush_cache(self, lock: asyncio.Lock = None) -> None: """We need to hack around the fact that asyncio has no reentrant lock - So we either grab the storing storing_lock ourselves or the caller needs + So we either grab the storing_lock ourselves or the caller needs to pass us the locked storing_lock """ - assert self.storing_lock is not None - assert self.flush_event is not None - got_cond = cond is not None - if not got_cond: - cond = self.storing_lock - await cond.acquire() - assert cond is not None and cond.locked() + got_lock = lock is not None + if not got_lock: + lock = self.storing_lock + await lock.acquire() + + assert lock is not None and lock.locked() + for table_name, batches in self._batches.items(): table = pa.Table.from_batches(batches) await self.write_table(table_name, table) - self.flush_event.set() - if not got_cond: - cond.release() + self._batches.clear() + + for event in self.flush_events: + event.set() + self.flush_events.clear() + + if not got_lock: + lock.release() diff --git a/openwpm/storage/cloud_storage/gcp_storage.py b/openwpm/storage/cloud_storage/gcp_storage.py index 054520a7e..795bfe1a1 100644 --- a/openwpm/storage/cloud_storage/gcp_storage.py +++ b/openwpm/storage/cloud_storage/gcp_storage.py @@ -24,6 +24,8 @@ class GcsStructuredProvider(ArrowProvider): Pass a different sub_dir to change this. """ + file_system: GCSFileSystem + def __init__( self, project: str, @@ -35,7 +37,6 @@ def __init__( super().__init__() self.project = project self.token = token - self.file_system: Optional[GCSFileSystem] = None self.base_path = f"{bucket_name}/{base_path}/{sub_dir}/{{table_name}}" async def init(self) -> None: @@ -45,7 +46,6 @@ async def init(self) -> None: ) async def write_table(self, table_name: TableName, table: Table) -> None: - assert self.file_system is not None self.file_system.start_transaction() pq.write_to_dataset( table, @@ -63,11 +63,10 @@ class GcsUnstructuredProvider(UnstructuredStorageProvider): This might not actually be the thing that we want to do long term but seeing as GCS is the S3 equivalent of GCP it is the easiest way forward. - - - """ + file_system: GCSFileSystem + def __init__( self, project: str, bucket_name: str, base_path: str, token: str = None, ) -> None: @@ -76,7 +75,6 @@ def __init__( self.bucket_name = bucket_name self.base_path = base_path self.token = token - self.file_system: Optional[GCSFileSystem] = None self.base_path = f"{bucket_name}/{base_path}/{{filename}}" self.file_name_cache: Set[str] = set() @@ -89,16 +87,19 @@ async def init(self) -> None: async def store_blob( self, filename: str, blob: bytes, overwrite: bool = False ) -> None: - assert self.file_system is not None target_path = self.base_path.format(filename=filename) if not overwrite and ( filename in self.file_name_cache or self.file_system.exists(target_path) ): self.logger.info("Not saving out file %s as it already exists", filename) return + self.file_system.start_transaction() with self.file_system.open(target_path, mode="wb") as f: f.write(blob) + + self.file_system.end_transaction() + self.file_name_cache.add(filename) async def flush_cache(self) -> None: diff --git a/openwpm/storage/leveldb.py b/openwpm/storage/leveldb.py index 91a2021b7..18c289045 100644 --- a/openwpm/storage/leveldb.py +++ b/openwpm/storage/leveldb.py @@ -10,12 +10,13 @@ class LevelDbProvider(UnstructuredStorageProvider): + ldb: plyvel.DB + content_batch: WriteBatch + def __init__(self, db_path: Path): self.db_path = db_path self._ldb_counter = 0 self._ldb_commit_time = 0 - self.ldb: Optional[plyvel.DB] = None - self.content_batch: Optional[WriteBatch] = None async def init(self) -> None: self.ldb = plyvel.DB( @@ -28,24 +29,18 @@ async def init(self) -> None: async def flush_cache(self) -> None: """Write out content batch to LevelDB database""" - assert self.content_batch is not None - assert self.ldb is not None self.content_batch.write() self.content_batch = self.ldb.write_batch() async def shutdown(self) -> None: - assert self.ldb is not None self.ldb.close() - print("Ldb is closed:", self.ldb.closed) async def store_blob( self, filename: str, blob: bytes, overwrite: bool = False, ) -> None: - assert self.ldb is not None - assert self.content_batch is not None content_hash = str(filename).encode("ascii") - if self.ldb.get(content_hash) is not None: + if self.ldb.get(content_hash) is not None and not overwrite: return self.content_batch.put(content_hash, blob) self._ldb_counter += 1 diff --git a/openwpm/storage/local_storage.py b/openwpm/storage/local_storage.py index 82b38ab7f..0ef698942 100644 --- a/openwpm/storage/local_storage.py +++ b/openwpm/storage/local_storage.py @@ -15,7 +15,6 @@ class LocalArrowProvider(ArrowProvider): def __init__(self, storage_path: Path) -> None: super().__init__() self.storage_path = storage_path - self.logger = logging.getLogger("openwpm") async def write_table(self, table_name: TableName, table: Table) -> None: pq.write_to_dataset(table, str(self.storage_path / table_name)) diff --git a/openwpm/storage/sql_provider.py b/openwpm/storage/sql_provider.py index a883db4e8..38c91440f 100644 --- a/openwpm/storage/sql_provider.py +++ b/openwpm/storage/sql_provider.py @@ -11,7 +11,7 @@ OperationalError, ProgrammingError, ) -from typing import Any, AsyncGenerator, Awaitable, Dict, List, Optional, Tuple +from typing import Any, Awaitable, Dict, List, Optional, Tuple from openwpm.types import VisitId @@ -21,14 +21,15 @@ class SqlLiteStorageProvider(StructuredStorageProvider): + db: Connection + cur: Cursor + def __init__(self, db_path: Path) -> None: super().__init__() self.db_path = db_path self._sql_counter = 0 self._sql_commit_time = 0 self.logger = logging.getLogger("openwpm") - self.db: Optional[Connection] = None - self.cur: Optional[Cursor] = None async def init(self) -> None: self.db = sqlite3.connect(str(self.db_path)) @@ -37,13 +38,11 @@ async def init(self) -> None: def _create_tables(self) -> None: """Create tables (if this is a new database)""" - assert self.db is not None with open(SCHEMA_FILE, "r") as f: self.db.executescript(f.read()) self.db.commit() async def flush_cache(self) -> None: - assert self.db is not None self.db.commit() async def store_record( @@ -94,19 +93,17 @@ def _generate_insert( return statement, values def execute_statement(self, statement: str) -> None: - assert self.cur is not None - assert self.db is not None self.cur.execute(statement) self.db.commit() async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False ) -> Awaitable[None]: - assert self.cur is not None if interrupted: self.logger.warning("Visit with visit_id %d got interrupted", visit_id) self.cur.execute("INSERT INTO incomplete_visits VALUES (?)", (visit_id,)) + self.db.commit() async def done() -> None: return @@ -114,6 +111,5 @@ async def done() -> None: return done() async def shutdown(self) -> None: - assert self.db is not None self.db.commit() self.db.close() diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 38a504d57..4437ce2ac 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -1,6 +1,7 @@ import asyncio import base64 import logging +import math import queue import random import socket @@ -165,14 +166,14 @@ async def handler( async def _handle_meta(self, visit_id: VisitId, data: Dict[str, Any]) -> None: """ - Messages for the table RECORD_TYPE_SPECIAL are metainformation + Messages for the table RECORD_TYPE_SPECIAL are meta information communicated to the aggregator Supported message types: - finalize: A message sent by the extension to signal that a visit_id is complete. - initialize: TODO: Start complaining if we receive data for a visit_id before the initialize event happened. (This might not be easy - because of `site_visits` + because of `site_visits`) """ action: str = data["action"] if action == ACTION_TYPE_INITIALIZE: @@ -191,6 +192,9 @@ async def finalize_visit_id( """Makes sure all records for a given visit_id have been processed before we invoke finalize_visit_id on the structured_storage + + See StructuredStorageProvider::finalize_visit_id for additional + documentation """ self.logger.info("Awaiting all tasks for visit_id %d", visit_id) @@ -231,8 +235,19 @@ async def update_status_queue(self) -> NoReturn: ) async def shutdown(self) -> None: + completion_tokens = {} + visit_ids = list(self.current_tasks.keys()) + for visit_id in visit_ids: + completion_tokens[visit_id] = await self.finalize_visit_id( + visit_id, success=False + ) await self.structured_storage.flush_cache() + for visit_id, token in completion_tokens.items(): + await token + self.completion_queue.put((visit_id, False)) + await self.structured_storage.shutdown() + if self.unstructured_storage is not None: await self.unstructured_storage.flush_cache() await self.unstructured_storage.shutdown() @@ -257,32 +272,22 @@ async def save_batch_if_past_timeout(self) -> NoReturn: await asyncio.sleep(BATCH_COMMIT_TIMEOUT) continue - time_until_timeout = ( - time.time() - self._last_record_received - BATCH_COMMIT_TIMEOUT - ) - if time_until_timeout > 0: + diff = time.time() - self._last_record_received + if diff < BATCH_COMMIT_TIMEOUT: + time_until_timeout = BATCH_COMMIT_TIMEOUT - diff await asyncio.sleep(time_until_timeout) continue self.logger.debug( "Saving current records since no new data has " - "been written for %d seconds." - % (time.time() - self._last_record_received) + "been written for %d seconds." % diff ) await self.structured_storage.flush_cache() if self.unstructured_storage: await self.unstructured_storage.flush_cache() self._last_record_received = None - async def finish_tasks(self) -> None: - self.logger.info("Awaiting unfinished tasks before shutting down") - for visit_id, tasks in self.current_tasks.items(): - self.logger.debug("Awaiting tasks for visit_id %d", visit_id) - for task in tasks: - await task - async def _run(self) -> None: - await self.structured_storage.init() if self.unstructured_storage: await self.unstructured_storage.init() @@ -306,20 +311,6 @@ async def _run(self) -> None: status_queue_update.cancel() timeout_check.cancel() await server.wait_closed() - - await self.finish_tasks() - - finalization_tokens = {} - visit_ids = list(self.current_tasks.keys()) - for visit_id in visit_ids: - finalization_tokens[visit_id] = await self.finalize_visit_id( - visit_id, success=False - ) - await self.structured_storage.flush_cache() - for visit_id, token in finalization_tokens.items(): - await token - self.completion_queue.put((visit_id, False)) - await self.shutdown() def run(self) -> None: @@ -346,7 +337,7 @@ def __init__( self._last_status = None self._last_status_received: Optional[float] = None self.logger = logging.getLogger("openwpm") - self.aggregator = StorageController( + self.storage_controller = StorageController( structured_storage, unstructured_storage, status_queue=self.status_queue, @@ -383,7 +374,7 @@ def launch(self) -> None: self.storage_controller = Process( name="StorageController", target=StorageController.run, - args=(self.aggregator,), + args=(self.storage_controller,), ) self.storage_controller.daemon = True self.storage_controller.start() diff --git a/openwpm/storage/storage_providers.py b/openwpm/storage/storage_providers.py index 7ba21cca3..ccb83c5c2 100644 --- a/openwpm/storage/storage_providers.py +++ b/openwpm/storage/storage_providers.py @@ -1,7 +1,7 @@ import gzip import io from abc import ABC, abstractmethod -from typing import Any, AsyncGenerator, Awaitable, Dict, List, NewType, Tuple +from typing import Any, AsyncGenerator, Awaitable, Dict, List, NewType, Optional, Tuple from openwpm.types import VisitId @@ -36,7 +36,7 @@ async def flush_cache(self) -> None: @abstractmethod async def shutdown(self) -> None: - """Close all open ressources + """Close all open resources After this method has been called no further calls should be made to the object """ pass diff --git a/test/conftest.py b/test/conftest.py index 7064f804c..af1111955 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -31,7 +31,7 @@ def xpi(): @pytest.fixture(scope="session") -def server(request): +def server(): """Run an HTTP server during the tests.""" print("Starting local_http_server") server, server_thread = utilities.start_server() @@ -46,7 +46,7 @@ def default_params( tmp_path: Path, num_browsers: int = NUM_BROWSERS ) -> Tuple[ManagerParams, List[BrowserParams]]: """Just a simple wrapper around task_manager.load_default_params""" - assert len(list(tmp_path.iterdir())) == 0 + manager_params = ManagerParams( num_browsers=NUM_BROWSERS ) # num_browsers is necessary to let TaskManager know how many browsers to spawn diff --git a/test/storage/test_memory_storage_provider.py b/test/storage/test_memory_storage_provider.py index a5d3cde91..7a29b493b 100644 --- a/test/storage/test_memory_storage_provider.py +++ b/test/storage/test_memory_storage_provider.py @@ -80,6 +80,7 @@ async def test_basic_access( token = await structured_provider.finalize_visit_id(VisitId(2)) await structured_provider.flush_cache() await token + await structured_provider.shutdown() # Unstructured Providers @@ -122,3 +123,4 @@ async def test_basic_unstructured_storing( await unstructured_provider.init() await unstructured_provider.store_blob("test", blob) await unstructured_provider.flush_cache() + await unstructured_provider.shutdown() diff --git a/test/storage/test_storage_controller.py b/test/storage/test_storage_controller.py index 98de82c6b..048f5c593 100644 --- a/test/storage/test_storage_controller.py +++ b/test/storage/test_storage_controller.py @@ -24,20 +24,7 @@ from test.storage.test_values import TEST_VALUES, TEST_VISIT_IDS -@pytest.fixture(scope="session") -def logger() -> MPLogger: - """PyTest only captures logging events in the Main Process - so we need to log everything to console to have it show - up in our tests - """ - return MPLogger( - "/dev/null", - None, # We have no manager params here - log_level_console=logging.DEBUG, - ) - - -def test_startup_and_shutdown(logger: MPLogger) -> None: +def test_startup_and_shutdown(mp_logger: MPLogger) -> None: structured = MemoryStructuredProvider() unstructured = MemoryUnstructuredProvider() @@ -64,7 +51,7 @@ def test_startup_and_shutdown(logger: MPLogger) -> None: @pytest.mark.asyncio -async def test_arrow_provider(logger: MPLogger) -> None: +async def test_arrow_provider(mp_logger: MPLogger) -> None: structured = MemoryArrowProvider() status_queue = Queue() completion_queue = Queue() diff --git a/test/test_extension.py b/test/test_extension.py index 49036911c..f40865a2c 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -6,7 +6,6 @@ import pytest -from openwpm import task_manager from openwpm.config import BrowserParams, ManagerParams from openwpm.utilities import db_utils diff --git a/test/test_timer.py b/test/test_timer.py index 40979cc8f..2f72dc3b5 100644 --- a/test/test_timer.py +++ b/test/test_timer.py @@ -1,7 +1,5 @@ -from openwpm import task_manager from openwpm.utilities import db_utils -from .openwpmtest import OpenWPMTest from .utilities import BASE_TEST_URL TEST_FILE = "canvas_fingerprinting.html" From 8eb6ef08174727bdebc8e602540097511eeb4de1 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 8 Jan 2021 17:50:06 +0100 Subject: [PATCH 072/139] Fixed repin.sh and test_arrow_cache --- environment.yaml | 5 +++-- scripts/prune-environment.py | 2 +- test/storage/test_arrow_cache.py | 4 +++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/environment.yaml b/environment.yaml index 2dad1541f..39b8950a2 100644 --- a/environment.yaml +++ b/environment.yaml @@ -14,12 +14,13 @@ dependencies: - multiprocess=0.70.11.1 - mypy=0.790 - nodejs=14.15.1 -- pandas=1.1.5 -- pillow=8.0.1 +- pandas=1.2.0 +- pillow=8.1.0 - pip=20.3.3 - pre-commit=2.9.3 - psutil=5.8.0 - pyarrow=2.0.0 +- pytest-asyncio=0.14.0 - pytest-cov=2.10.1 - pytest=6.2.1 - python=3.8.6 diff --git a/scripts/prune-environment.py b/scripts/prune-environment.py index 6673814d3..bf4b3b8ab 100644 --- a/scripts/prune-environment.py +++ b/scripts/prune-environment.py @@ -24,7 +24,7 @@ def iterate_deps(xs: Iterable[str], ys: Iterable[str], accumulator: List[str]) - iterate_deps( env_pinned["dependencies"][:-1], - env_unpinned["dependencies"][:-1] + env_unpinned_dev["dependencies"][:-1], + env_unpinned["dependencies"][:-1] + env_unpinned_dev["dependencies"], deps_not_pip, ) diff --git a/test/storage/test_arrow_cache.py b/test/storage/test_arrow_cache.py index 09490bb4e..add3ab845 100644 --- a/test/storage/test_arrow_cache.py +++ b/test/storage/test_arrow_cache.py @@ -39,4 +39,6 @@ async def test_arrow_cache(mp_logger: MPLogger) -> None: df: DataFrame = table.to_pandas() for row in df.itertuples(index=False): - del d[row["visit_id"]] + del d[row.visit_id] + + assert len(d) == 0 From 24fc5d2e24c8d7b683650cd5bdd360d7ca363a5a Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 15 Jan 2021 14:48:47 +0100 Subject: [PATCH 073/139] Minor change --- openwpm/commands/browser_commands.py | 53 +++++----------------------- 1 file changed, 9 insertions(+), 44 deletions(-) diff --git a/openwpm/commands/browser_commands.py b/openwpm/commands/browser_commands.py index cbfde6661..aa90f1c2b 100644 --- a/openwpm/commands/browser_commands.py +++ b/openwpm/commands/browser_commands.py @@ -124,11 +124,7 @@ def __repr__(self): return "GetCommand({},{})".format(self.url, self.sleep) def execute( - self, - webdriver, - browser_params, - manager_params, - extension_socket, + self, webdriver, browser_params, manager_params, extension_socket, ): tab_restart_browser(webdriver) @@ -169,11 +165,7 @@ def __repr__(self): return "BrowseCommand({},{},{})".format(self.url, self.num_links, self.sleep) def execute( - self, - webdriver, - browser_params, - manager_params, - extension_socket, + self, webdriver, browser_params, manager_params, extension_socket, ): """Calls get_website before visiting present on the page. @@ -184,10 +176,7 @@ def execute( get_command = GetCommand(self.url, self.sleep) get_command.set_visit_browser_id(self.visit_id, self.browser_id) get_command.execute( - webdriver, - browser_params, - manager_params, - extension_socket, + webdriver, browser_params, manager_params, extension_socket, ) # Then visit a few subpages @@ -231,11 +220,7 @@ def __repr__(self): return "SaveScreenshotCommand({})".format(self.suffix) def execute( - self, - webdriver, - browser_params, - manager_params, - extension_socket, + self, webdriver, browser_params, manager_params, extension_socket, ): if self.suffix != "": self.suffix = "-" + self.suffix @@ -318,11 +303,7 @@ def __repr__(self): return "ScreenshotFullPageCommand({})".format(self.suffix) def execute( - self, - webdriver, - browser_params, - manager_params, - extension_socket, + self, webdriver, browser_params, manager_params, extension_socket, ): self.outdir = os.path.join(manager_params.screenshot_path, "parts") if not os.path.isdir(self.outdir): @@ -390,11 +371,7 @@ def __repr__(self): return "DumpPageSourceCommand({})".format(self.suffix) def execute( - self, - webdriver, - browser_params, - manager_params, - extension_socket, + self, webdriver, browser_params, manager_params, extension_socket, ): if self.suffix != "": @@ -419,11 +396,7 @@ def __repr__(self): return "RecursiveDumpPageSourceCommand({})".format(self.suffix) def execute( - self, - webdriver, - browser_params, - manager_params, - extension_socket, + self, webdriver, browser_params, manager_params, extension_socket, ): """Dump a compressed html tree for the current page visit""" @@ -480,11 +453,7 @@ def __repr__(self): return f"FinalizeCommand({self.sleep})" def execute( - self, - webdriver, - browser_params, - manager_params, - extension_socket, + self, webdriver, browser_params, manager_params, extension_socket, ): """ Informs the extension that a visit is done """ @@ -507,11 +476,7 @@ def __repr__(self): return "IntitializeCommand()" def execute( - self, - webdriver, - browser_params, - manager_params, - extension_socket, + self, webdriver, browser_params, manager_params, extension_socket, ): msg = {"action": "Initialize", "visit_id": self.visit_id} From 0096007b5089c0d2588b942f9a7618aef2ebbda3 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 15 Jan 2021 15:36:45 +0100 Subject: [PATCH 074/139] Fixed prune-environment.py --- environment.yaml | 8 ++++++++ scripts/prune-environment.py | 15 ++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/environment.yaml b/environment.yaml index 50b28ba5b..07461aa65 100644 --- a/environment.yaml +++ b/environment.yaml @@ -3,18 +3,26 @@ channels: - main dependencies: - beautifulsoup4=4.9.3 +- black=20.8b1 - click=7.1.2 +- codecov=2.1.11 - dill=0.3.3 - gcsfs=0.7.1 - geckodriver=0.28.0 +- ipython=7.19.0 - leveldb=1.22 - multiprocess=0.70.11.1 +- mypy=0.790 - nodejs=14.15.1 - pandas=1.2.0 - pillow=8.1.0 - pip=20.3.3 +- pre-commit=2.9.3 - psutil=5.8.0 - pyarrow=2.0.0 +- pytest-asyncio=0.14.0 +- pytest-cov=2.10.1 +- pytest=6.2.1 - python=3.9.1 - pyvirtualdisplay=1.3.2 - redis-py=3.5.3 diff --git a/scripts/prune-environment.py b/scripts/prune-environment.py index 2ba67bfc1..880bcb90f 100644 --- a/scripts/prune-environment.py +++ b/scripts/prune-environment.py @@ -26,11 +26,16 @@ def iterate_deps(xs: Iterable[str], ys: Iterable[str], accumulator: List[str]) - env_unpinned_dev_contains_pip = "pip" in env_unpinned_dev["dependencies"][-1] iterate_deps( env_pinned["dependencies"][:-1], - env_unpinned["dependencies"][:-1] - if env_unpinned_contains_pip - else env_unpinned["dependencies"] + env_unpinned_dev["dependencies"][:-1] - if env_unpinned_dev_contains_pip - else env_unpinned_dev["dependencies"], + ( + env_unpinned["dependencies"][:-1] + if env_unpinned_contains_pip + else env_unpinned["dependencies"] + ) + + ( + env_unpinned_dev["dependencies"][:-1] + if env_unpinned_dev_contains_pip + else env_unpinned_dev["dependencies"] + ), deps_not_pip, ) From 962af535ae5f0c15b1716e82a782c17833eb7ee4 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 15 Jan 2021 15:51:34 +0100 Subject: [PATCH 075/139] Removing references to DataAggregator --- openwpm/browser_manager.py | 2 +- openwpm/config.py | 2 +- openwpm/deploy_browsers/deploy_firefox.py | 4 +++- openwpm/storage/storage_providers.py | 2 ++ openwpm/task_manager.py | 18 +++++++++--------- setup.cfg | 4 ++++ test/storage/test_storage_controller.py | 10 +++++----- test/test_callback.py | 2 +- test/test_custom_function_command.py | 4 ++-- 9 files changed, 28 insertions(+), 20 deletions(-) diff --git a/openwpm/browser_manager.py b/openwpm/browser_manager.py index 03f67b86e..a526a06e9 100644 --- a/openwpm/browser_manager.py +++ b/openwpm/browser_manager.py @@ -47,7 +47,7 @@ def __init__(self, manager_params, browser_params) -> None: # manager parameters self.current_profile_path = None - self.db_socket_address = manager_params.aggregator_address + self.db_socket_address = manager_params.storage_controller_address self.browser_id = browser_params.browser_id self.curr_visit_id: Optional[int] = None self.browser_params = browser_params diff --git a/openwpm/config.py b/openwpm/config.py index 5c392b044..f08fac1fe 100644 --- a/openwpm/config.py +++ b/openwpm/config.py @@ -118,7 +118,7 @@ class BrowserParamsInternal(BrowserParams): @dataclass_json @dataclass class ManagerParamsInternal(ManagerParams): - aggregator_address: Optional[Tuple[str, int]] = None + storage_controller_address: Optional[Tuple[str, int]] = None logger_address: Optional[Tuple[str, ...]] = None ldb_address: Optional[Tuple[str, ...]] = None diff --git a/openwpm/deploy_browsers/deploy_firefox.py b/openwpm/deploy_browsers/deploy_firefox.py index f88f5709b..737491590 100755 --- a/openwpm/deploy_browsers/deploy_firefox.py +++ b/openwpm/deploy_browsers/deploy_firefox.py @@ -94,7 +94,9 @@ def deploy_firefox( extension_config = dict() extension_config.update(browser_params.to_dict()) # type: ignore extension_config["logger_address"] = manager_params.logger_address - extension_config["aggregator_address"] = manager_params.aggregator_address + extension_config[ + "aggregator_address" + ] = manager_params.storage_controller_address if manager_params.ldb_address: extension_config["leveldb_address"] = manager_params.ldb_address else: diff --git a/openwpm/storage/storage_providers.py b/openwpm/storage/storage_providers.py index ccb83c5c2..cfe434991 100644 --- a/openwpm/storage/storage_providers.py +++ b/openwpm/storage/storage_providers.py @@ -3,6 +3,8 @@ from abc import ABC, abstractmethod from typing import Any, AsyncGenerator, Awaitable, Dict, List, NewType, Optional, Tuple +from pyarrow.lib import Table + from openwpm.types import VisitId """ diff --git a/openwpm/task_manager.py b/openwpm/task_manager.py index 1fb83c99f..de296a189 100644 --- a/openwpm/task_manager.py +++ b/openwpm/task_manager.py @@ -45,15 +45,15 @@ SLEEP_CONS = 0.1 # command sleep constant (in seconds) BROWSER_MEMORY_LIMIT = 1500 # in MB -AGGREGATOR_QUEUE_LIMIT = 10000 # number of records in the queue +STORAGE_CONTROLLER_JOB_LIMIT = 10000 # number of records in the queue class TaskManager: """User-facing Class for interfacing with OpenWPM The TaskManager spawns several child processes to run the automation tasks. - - DataAggregator to aggregate data across browsers and save to the - database. + - StorageController to receive data from across browsers and save it to + the provided StorageProviders - MPLogger to aggregate logs across processes - BrowserManager processes to isolate Browsers in a separate process """ @@ -291,14 +291,14 @@ def _launch_aggregators( structured_storage_provider, unstructured_storage_provider ) self.storage_controler_handle.launch() - self.manager_params.aggregator_address = ( + self.manager_params.storage_controller_address = ( self.storage_controler_handle.listener_address ) # open connection to aggregator for saving crawl details self.sock = ClientSocket(serialization="dill") - assert self.manager_params.aggregator_address is not None - self.sock.connect(*self.manager_params.aggregator_address) + assert self.manager_params.storage_controller_address is not None + self.sock.connect(*self.manager_params.storage_controller_address) def _shutdown_manager( self, during_init: bool = False, relaxed: bool = True @@ -633,12 +633,12 @@ def execute_command_sequence( # Block if the aggregator queue is too large agg_queue_size = self.storage_controler_handle.get_most_recent_status() - if agg_queue_size >= AGGREGATOR_QUEUE_LIMIT: - while agg_queue_size >= AGGREGATOR_QUEUE_LIMIT: + if agg_queue_size >= STORAGE_CONTROLLER_JOB_LIMIT: + while agg_queue_size >= STORAGE_CONTROLLER_JOB_LIMIT: self.logger.info( "Blocking command submission until the DataAggregator " "is below the max queue size of %d. Current queue " - "length %d. " % (AGGREGATOR_QUEUE_LIMIT, agg_queue_size) + "length %d. " % (STORAGE_CONTROLLER_JOB_LIMIT, agg_queue_size) ) agg_queue_size = self.storage_controler_handle.get_status() diff --git a/setup.cfg b/setup.cfg index c49313ec8..9b05cbd1c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,6 +17,10 @@ disallow_untyped_defs = True disallow_incomplete_defs = True disallow_untyped_defs = True +[mypy-openwpm.commands.*] +disallow_incomplete_defs = True +disallow_untyped_defs = True + [mypy-openwpm.*] disallow_untyped_defs = False diff --git a/test/storage/test_storage_controller.py b/test/storage/test_storage_controller.py index 048f5c593..b1ae7f738 100644 --- a/test/storage/test_storage_controller.py +++ b/test/storage/test_storage_controller.py @@ -28,11 +28,11 @@ def test_startup_and_shutdown(mp_logger: MPLogger) -> None: structured = MemoryStructuredProvider() unstructured = MemoryUnstructuredProvider() - agg_handle = StorageControllerHandle(structured, unstructured) - agg_handle.launch() - assert agg_handle.listener_address is not None + controller_handle = StorageControllerHandle(structured, unstructured) + controller_handle.launch() + assert controller_handle.listener_address is not None cs = ClientSocket() - cs.connect(*agg_handle.listener_address) + cs.connect(*controller_handle.listener_address) for table, data in TEST_VALUES.items(): cs.send((table, data)) @@ -43,7 +43,7 @@ def test_startup_and_shutdown(mp_logger: MPLogger) -> None: {"action": ACTION_TYPE_FINALIZE, "visit_id": visit_id, "success": True}, ) ) - agg_handle.shutdown() + controller_handle.shutdown() handle = structured.handle handle.poll_queue() for table, data in TEST_VALUES.items(): diff --git a/test/test_callback.py b/test/test_callback.py index 70953befc..e943de5fd 100644 --- a/test/test_callback.py +++ b/test/test_callback.py @@ -9,7 +9,7 @@ def test_local_callbacks(default_params, task_manager_creator) -> None: - """Test test the Aggregators as well as the entire callback machinery + """Test test the storage controller as well as the entire callback machinery to see if all callbacks get correctly called""" manager = task_manager_creator(default_params) TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html" diff --git a/test/test_custom_function_command.py b/test/test_custom_function_command.py index 76e50e0ec..565a09f90 100644 --- a/test/test_custom_function_command.py +++ b/test/test_custom_function_command.py @@ -58,8 +58,8 @@ def execute( current_url = webdriver.current_url sock = ClientSocket() - assert manager_params.aggregator_address is not None - sock.connect(*manager_params.aggregator_address) + assert manager_params.storage_controller_address is not None + sock.connect(*manager_params.storage_controller_address) for link in link_urls: query = ( From 902e4ed81d4dc2d83b7b001f53a7d352377c8c86 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 15 Jan 2021 16:04:34 +0100 Subject: [PATCH 076/139] Fixed test_seed_persistance --- test/test_profile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_profile.py b/test/test_profile.py index 7c2ab4501..7c35b7d34 100644 --- a/test/test_profile.py +++ b/test/test_profile.py @@ -93,14 +93,14 @@ def test_profile_saved_when_launch_crashes(self): def test_seed_persistance(default_params, task_manager_creator): manager_params, browser_params = default_params for browser_param in browser_params: - browser_param.seed_tar = "." + browser_param.seed_tar = Path(".") manager = task_manager_creator(default_params) command_sequences = [] for _ in range(2): cs = CommandSequence(url="https://example.com", reset=True) cs.get() - cs.append_command(TestConfigSetCommand("test_pref", True)) + cs.append_command(AssertConfigSetCommand("test_pref", True)) command_sequences.append(cs) for cs in command_sequences: @@ -114,7 +114,7 @@ def test_seed_persistance(default_params, task_manager_creator): assert row["command_status"] == "ok", f"Command {tuple(row)} was not ok" -class TestConfigSetCommand(BaseCommand): +class AssertConfigSetCommand(BaseCommand): def __init__(self, pref_name: str, expected_value: Any) -> None: self.pref_name = pref_name self.expected_value = expected_value From 25cd9cf7800a5c4c372c591f63a7e45f42797624 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 15 Jan 2021 17:48:32 +0100 Subject: [PATCH 077/139] More paths --- openwpm/commands/profile_commands.py | 6 +++--- openwpm/config.py | 8 ++++++++ openwpm/deploy_browsers/deploy_firefox.py | 9 +++++---- setup.cfg | 4 ---- test/test_profile.py | 5 +++-- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/openwpm/commands/profile_commands.py b/openwpm/commands/profile_commands.py index bef535694..e04d52077 100644 --- a/openwpm/commands/profile_commands.py +++ b/openwpm/commands/profile_commands.py @@ -36,7 +36,7 @@ def __init__(self, tar_path: Path, close_webdriver: bool, compress: bool) -> Non "See: https://github.com/mozilla/OpenWPM/projects/2." % self.browser_id ) - def __repr__(self): + def __repr__(self) -> str: return "DumpProfCommand({},{},{})".format( self.tar_path, self.close_webdriver, self.compress ) @@ -136,14 +136,14 @@ def load_profile( % (browser_params.browser_id, tar_location, browser_profile_folder,) ) shutil.copy(tar_location, browser_profile_folder) - + tar_location = browser_profile_folder / tar_location.name if tar_location.name.endswith("tar.gz"): f = tarfile.open(tar_location, "r:gz", errorlevel=1) else: f = tarfile.open(tar_location, "r", errorlevel=1) f.extractall(browser_profile_folder) f.close() - tar_location.unlink + tar_location.unlink() logger.debug("BROWSER %i: Tarfile extracted" % browser_params.browser_id) except Exception as ex: diff --git a/openwpm/config.py b/openwpm/config.py index f08fac1fe..37dc00ceb 100644 --- a/openwpm/config.py +++ b/openwpm/config.py @@ -1,5 +1,6 @@ import os from dataclasses import dataclass, field +from json import JSONEncoder from pathlib import Path from typing import List, Optional, Tuple, Union @@ -277,3 +278,10 @@ def validate_crawl_configs( "as manager_params.num_browsers. Make sure you are assigning number of browsers " "to be used to manager_params.num_browsers in your entry file" ) + + +class ConfigEncoder(JSONEncoder): + def default(self, obj): + if isinstance(obj, Path): + return str(obj.resolve()) + return JSONEncoder.default(self, obj) diff --git a/openwpm/deploy_browsers/deploy_firefox.py b/openwpm/deploy_browsers/deploy_firefox.py index 737491590..53d36d37d 100755 --- a/openwpm/deploy_browsers/deploy_firefox.py +++ b/openwpm/deploy_browsers/deploy_firefox.py @@ -1,6 +1,7 @@ import json import logging import os.path +from pathlib import Path from typing import Any, List, Optional, Tuple from easyprocess import EasyProcessError @@ -10,7 +11,7 @@ from selenium.webdriver.firefox.firefox_profile import FirefoxProfile from ..commands.profile_commands import load_profile -from ..config import BrowserParamsInternal, ManagerParamsInternal +from ..config import BrowserParamsInternal, ConfigEncoder, ManagerParamsInternal from ..utilities.platform_utils import get_firefox_binary_path from . import configure_firefox from .selenium_firefox import FirefoxBinary, FirefoxLogInterceptor, Options @@ -33,7 +34,7 @@ def deploy_firefox( root_dir = os.path.dirname(__file__) # directory of this file fp = FirefoxProfile() - browser_profile_path = fp.path + "/" + browser_profile_path = Path(fp.path) status_queue.put(("STATUS", "Profile Created", browser_profile_path)) # Use Options instead of FirefoxProfile to set preferences since the @@ -102,9 +103,9 @@ def deploy_firefox( else: extension_config["leveldb_address"] = None extension_config["testing"] = manager_params.testing - ext_config_file = browser_profile_path + "browser_params.json" + ext_config_file = browser_profile_path / "browser_params.json" with open(ext_config_file, "w") as f: - json.dump(extension_config, f) + json.dump(extension_config, f, cls=ConfigEncoder) logger.debug( "BROWSER %i: Saved extension config file to: %s" % (browser_params.browser_id, ext_config_file) diff --git a/setup.cfg b/setup.cfg index 9b05cbd1c..c49313ec8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,10 +17,6 @@ disallow_untyped_defs = True disallow_incomplete_defs = True disallow_untyped_defs = True -[mypy-openwpm.commands.*] -disallow_incomplete_defs = True -disallow_untyped_defs = True - [mypy-openwpm.*] disallow_untyped_defs = False diff --git a/test/test_profile.py b/test/test_profile.py index 7c35b7d34..f6563a41f 100644 --- a/test/test_profile.py +++ b/test/test_profile.py @@ -79,7 +79,7 @@ def test_profile_saved_when_launch_crashes(self): manager.ldb_status_queue.put("DIE") manager.browsers[0]._SPAWN_TIMEOUT = 2 # Have timeout occur quickly manager.browsers[0]._UNSUCCESSFUL_SPAWN_LIMIT = 2 # Quick timeout - manager.get("example.com") # Cause a selenium crasht + manager.get("example.com") # Cause a selenium crash # The browser will fail to launch due to the proxy crashes try: @@ -92,8 +92,9 @@ def test_profile_saved_when_launch_crashes(self): def test_seed_persistance(default_params, task_manager_creator): manager_params, browser_params = default_params + p = Path("profile.tar.gz") for browser_param in browser_params: - browser_param.seed_tar = Path(".") + browser_param.seed_tar = p manager = task_manager_creator(default_params) command_sequences = [] From dcb9a6a3da827d325293eb405d046c2285d379fb Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 18 Jan 2021 12:36:31 +0100 Subject: [PATCH 078/139] Fixed test display shutdown --- test/storage/test_arrow_cache.py | 1 - test/test_xvfb_browser.py | 49 +++++++++++++++++--------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/test/storage/test_arrow_cache.py b/test/storage/test_arrow_cache.py index add3ab845..e337a19ae 100644 --- a/test/storage/test_arrow_cache.py +++ b/test/storage/test_arrow_cache.py @@ -4,7 +4,6 @@ import pytest from pandas import DataFrame -from pyarrow.parquet import ParquetDataset from openwpm.mp_logger import MPLogger from openwpm.storage.arrow_storage import CACHE_SIZE diff --git a/test/test_xvfb_browser.py b/test/test_xvfb_browser.py index 24d0371ff..77f463200 100644 --- a/test/test_xvfb_browser.py +++ b/test/test_xvfb_browser.py @@ -1,35 +1,38 @@ import os -from functools import partial -from typing import List + +from selenium.webdriver import Firefox from openwpm.command_sequence import CommandSequence from openwpm.commands.types import BaseCommand -from openwpm.task_manager import TaskManager +from openwpm.config import BrowserParamsInternal, ManagerParamsInternal +from openwpm.socket_interface import ClientSocket -from .openwpmtest import OpenWPMTest from .utilities import BASE_TEST_URL class ExceptionCommand(BaseCommand): - def execute(self): - raise Exception + def execute( + self, + webdriver: Firefox, + browser_params: BrowserParamsInternal, + manager_params: ManagerParamsInternal, + extension_socket: ClientSocket, + ) -> None: + raise RuntimeError("We simulate a Command failing") -class TestXVFBDisplay(OpenWPMTest): +def test_display_shutdown(task_manager_creator, default_params): """Test the XVFB display option to see if it runs and deletes the lockfile upon shutdown""" - - def get_config(self, data_dir=""): - return self.get_test_config(data_dir, display_mode="xvfb") - - def test_display_shutdown(self): - manager_params, browser_params = self.get_config() - TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html" - manager = TaskManager(manager_params, browser_params) - port = manager.browsers[0].display_port - - sequence = CommandSequence(TEST_SITE) - sequence.get() - sequence.append_command(ExceptionCommand) - manager.execute_command_sequence(sequence) - manager.close() - assert not os.path.exists("/tmp/.X%s-lock" % port) + manager_params, browser_params = default_params + for browser_param in browser_params: + browser_param.display_mode = "xvfb" + TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html" + manager = task_manager_creator((manager_params, browser_params)) + port = manager.browsers[0].display_port + + sequence = CommandSequence(TEST_SITE) + sequence.get() + sequence.append_command(ExceptionCommand()) + manager.execute_command_sequence(sequence) + manager.close() + assert not os.path.exists("/tmp/.X%s-lock" % port) From e91aba7337acbb27ed7a00f77090c9199cf40b57 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 18 Jan 2021 12:57:42 +0100 Subject: [PATCH 079/139] Made cache test more robust --- test/storage/test_arrow_cache.py | 48 +++++++++++++++++--------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/test/storage/test_arrow_cache.py b/test/storage/test_arrow_cache.py index e337a19ae..95b5a0337 100644 --- a/test/storage/test_arrow_cache.py +++ b/test/storage/test_arrow_cache.py @@ -18,26 +18,28 @@ async def test_arrow_cache(mp_logger: MPLogger) -> None: prov = MemoryArrowProvider() await prov.init() site_visit = TEST_VALUES["site_visits"] - d: Dict[VisitId, Awaitable[None]] = {} - for i in range(CACHE_SIZE + 1): - visit_id = VisitId(i) - site_visit["visit_id"] = visit_id - await prov.store_record(TableName("site_visits"), visit_id, site_visit) - d[visit_id] = await prov.finalize_visit_id(visit_id) - - for visit_id in d: - task = d[visit_id] - await asyncio.wait_for(task, 1) - await asyncio.sleep(1) - handle = prov.handle - # The queue should not be empty at this point - handle.poll_queue(block=False) - - assert len(handle.storage["site_visits"]) == 1 - table = handle.storage["site_visits"][0] - - df: DataFrame = table.to_pandas() - for row in df.itertuples(index=False): - del d[row.visit_id] - - assert len(d) == 0 + for j in range(5): # Testing that the cache works repeatedly + d: Dict[VisitId, Awaitable[None]] = {} + for i in range(CACHE_SIZE + 1): + visit_id = VisitId(i + j * 1000) + site_visit["visit_id"] = visit_id + await prov.store_record(TableName("site_visits"), visit_id, site_visit) + d[visit_id] = await prov.finalize_visit_id(visit_id) + + for visit_id in d: + await d[visit_id] + + await asyncio.sleep(1) + handle = prov.handle + # The queue should not be empty at this point + handle.poll_queue(block=False) + + assert len(handle.storage["site_visits"]) == j + 1 + table = handle.storage["site_visits"][j] + + df: DataFrame = table.to_pandas() + for row in df.itertuples(index=False): + del d[row.visit_id] + + assert len(d) == 0 + await prov.shutdown() From e4c9bb89a55334b18ab7320511afea1766573dfe Mon Sep 17 00:00:00 2001 From: Stefan Zabka Date: Mon, 18 Jan 2021 15:18:15 +0100 Subject: [PATCH 080/139] Update crawler.py Co-authored-by: Steven Englehardt --- crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawler.py b/crawler.py index 0d25c0996..1e5439d36 100644 --- a/crawler.py +++ b/crawler.py @@ -31,7 +31,7 @@ # Storage Provider Params CRAWL_DIRECTORY = os.getenv("CRAWL_DIRECTORY", "crawl-data") GCS_BUCKET = os.getenv("GCS_BUCKET", "openwpm-crawls") -GCP_PROJECT = os.getenv("GCP_PROJECT", "senglehardt-openwpm-test-1") +GCP_PROJECT = os.getenv("GCP_PROJECT", "") AUTH_TOKEN = os.getenv("GCP_AUTH_TOKEN", "cloud") # Browser Params From 247a69c386761383a4b1db7c7c47478cdbbd0100 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 18 Jan 2021 17:10:57 +0100 Subject: [PATCH 081/139] Slimming down ManagerParams --- .pre-commit-config.yaml | 2 +- crawler.py | 8 +-- demo.py | 5 +- openwpm/browser_manager.py | 5 +- openwpm/commands/browser_commands.py | 53 ++++++++++++--- openwpm/commands/profile_commands.py | 12 +++- openwpm/config.py | 43 ------------- openwpm/storage/cloud_storage/gcp_storage.py | 6 +- openwpm/storage/leveldb.py | 5 +- openwpm/storage/storage_providers.py | 5 +- openwpm/task_manager.py | 4 -- openwpm/utilities/db_utils.py | 8 ++- scripts/prune-environment.py | 15 ++++- test/conftest.py | 18 ++++-- test/manual_test.py | 5 +- test/openwpmtest.py | 15 ++--- test/test_callback.py | 2 +- test/test_callstack_instrument.py | 3 +- test/test_custom_function_command.py | 8 +-- test/test_dataclass_validations.py | 19 ------ test/test_dns_instrument.py | 6 +- test/test_http_instrumentation.py | 23 ++++--- test/test_js_instrument.py | 18 +++++- test/test_js_instrument_py.py | 5 +- test/test_mp_logger.py | 3 +- test/test_profile.py | 11 +++- test/test_simple_commands.py | 68 +++++++++----------- test/test_storage_vectors.py | 2 +- test/test_webdriver_utils.py | 2 +- test/test_xvfb_browser.py | 2 +- 30 files changed, 206 insertions(+), 175 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0a9d0cbca..828c50433 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: hooks: - id: isort - repo: https://github.com/psf/black - rev: 19.10b0 + rev: 20.8b1 hooks: - id: black language_version: python3 diff --git a/crawler.py b/crawler.py index 0d25c0996..111046a82 100644 --- a/crawler.py +++ b/crawler.py @@ -87,9 +87,6 @@ # Manager configuration manager_params.data_directory = Path("~/Desktop/") / CRAWL_DIRECTORY manager_params.log_directory = Path("~/Desktop/") / CRAWL_DIRECTORY -manager_params.output_format = "s3" -manager_params.s3_bucket = GCS_BUCKET -manager_params.s3_directory = CRAWL_DIRECTORY structured = GcsStructuredProvider( project=GCP_PROJECT, @@ -136,7 +133,10 @@ if PREFS: scope.set_context("PREFS", json.loads(PREFS)) scope.set_context( - "crawl_config", {"REDIS_QUEUE_NAME": REDIS_QUEUE_NAME,}, + "crawl_config", + { + "REDIS_QUEUE_NAME": REDIS_QUEUE_NAME, + }, ) # Send a sentry error message (temporarily - to easily be able # to compare error frequencies to crawl worker instance count) diff --git a/demo.py b/demo.py index 35c2e5f76..f0be0e300 100644 --- a/demo.py +++ b/demo.py @@ -65,7 +65,10 @@ def callback(success: bool, val: str = site) -> None: # Parallelize sites over all number of browsers set above. command_sequence = CommandSequence( - site, site_rank=index, reset=True, callback=callback, + site, + site_rank=index, + reset=True, + callback=callback, ) # Start by visiting the page diff --git a/openwpm/browser_manager.py b/openwpm/browser_manager.py index a526a06e9..d2cf3157e 100644 --- a/openwpm/browser_manager.py +++ b/openwpm/browser_manager.py @@ -511,7 +511,10 @@ def BrowserManager( # kill and restart its worker processes try: command.execute( - driver, browser_params, manager_params, extension_socket, + driver, + browser_params, + manager_params, + extension_socket, ) status_queue.put("OK") except WebDriverException: diff --git a/openwpm/commands/browser_commands.py b/openwpm/commands/browser_commands.py index aa90f1c2b..cbfde6661 100644 --- a/openwpm/commands/browser_commands.py +++ b/openwpm/commands/browser_commands.py @@ -124,7 +124,11 @@ def __repr__(self): return "GetCommand({},{})".format(self.url, self.sleep) def execute( - self, webdriver, browser_params, manager_params, extension_socket, + self, + webdriver, + browser_params, + manager_params, + extension_socket, ): tab_restart_browser(webdriver) @@ -165,7 +169,11 @@ def __repr__(self): return "BrowseCommand({},{},{})".format(self.url, self.num_links, self.sleep) def execute( - self, webdriver, browser_params, manager_params, extension_socket, + self, + webdriver, + browser_params, + manager_params, + extension_socket, ): """Calls get_website before visiting present on the page. @@ -176,7 +184,10 @@ def execute( get_command = GetCommand(self.url, self.sleep) get_command.set_visit_browser_id(self.visit_id, self.browser_id) get_command.execute( - webdriver, browser_params, manager_params, extension_socket, + webdriver, + browser_params, + manager_params, + extension_socket, ) # Then visit a few subpages @@ -220,7 +231,11 @@ def __repr__(self): return "SaveScreenshotCommand({})".format(self.suffix) def execute( - self, webdriver, browser_params, manager_params, extension_socket, + self, + webdriver, + browser_params, + manager_params, + extension_socket, ): if self.suffix != "": self.suffix = "-" + self.suffix @@ -303,7 +318,11 @@ def __repr__(self): return "ScreenshotFullPageCommand({})".format(self.suffix) def execute( - self, webdriver, browser_params, manager_params, extension_socket, + self, + webdriver, + browser_params, + manager_params, + extension_socket, ): self.outdir = os.path.join(manager_params.screenshot_path, "parts") if not os.path.isdir(self.outdir): @@ -371,7 +390,11 @@ def __repr__(self): return "DumpPageSourceCommand({})".format(self.suffix) def execute( - self, webdriver, browser_params, manager_params, extension_socket, + self, + webdriver, + browser_params, + manager_params, + extension_socket, ): if self.suffix != "": @@ -396,7 +419,11 @@ def __repr__(self): return "RecursiveDumpPageSourceCommand({})".format(self.suffix) def execute( - self, webdriver, browser_params, manager_params, extension_socket, + self, + webdriver, + browser_params, + manager_params, + extension_socket, ): """Dump a compressed html tree for the current page visit""" @@ -453,7 +480,11 @@ def __repr__(self): return f"FinalizeCommand({self.sleep})" def execute( - self, webdriver, browser_params, manager_params, extension_socket, + self, + webdriver, + browser_params, + manager_params, + extension_socket, ): """ Informs the extension that a visit is done """ @@ -476,7 +507,11 @@ def __repr__(self): return "IntitializeCommand()" def execute( - self, webdriver, browser_params, manager_params, extension_socket, + self, + webdriver, + browser_params, + manager_params, + extension_socket, ): msg = {"action": "Initialize", "visit_id": self.visit_id} diff --git a/openwpm/commands/profile_commands.py b/openwpm/commands/profile_commands.py index e04d52077..1dff208fd 100644 --- a/openwpm/commands/profile_commands.py +++ b/openwpm/commands/profile_commands.py @@ -70,7 +70,11 @@ def execute( tar = tarfile.open(self.tar_path, "w", errorlevel=1) logger.debug( "BROWSER %i: Backing up full profile from %s to %s" - % (self.browser_id, browser_profile_folder, self.tar_path,) + % ( + self.browser_id, + browser_profile_folder, + self.tar_path, + ) ) storage_vector_files = [ "cookies.sqlite", # cookies @@ -133,7 +137,11 @@ def load_profile( # Copy and untar the loaded profile logger.debug( "BROWSER %i: Copying profile tar from %s to %s" - % (browser_params.browser_id, tar_location, browser_profile_folder,) + % ( + browser_params.browser_id, + tar_location, + browser_profile_folder, + ) ) shutil.copy(tar_location, browser_profile_folder) tar_location = browser_profile_folder / tar_location.name diff --git a/openwpm/config.py b/openwpm/config.py index 37dc00ceb..03daa1754 100644 --- a/openwpm/config.py +++ b/openwpm/config.py @@ -14,7 +14,6 @@ "firefox" ] # Using List instead of a str type to future proof the logic as OpenWPM may add support for more browsers in future TP_COOKIES_OPTIONALS_LIST = ["always", "never", "from_visited"] -DB_EXTENSION_TYPE_LIST = [".db", ".sqlite"] LOG_EXTENSION_TYPE_LIST = [".log"] CONFIG_ERROR_STRING = ( "Found {value} as value for {parameter_name} in BrowserParams. " @@ -30,7 +29,6 @@ "Found invalid value `{value}` for {parameter_name} in {params_type}. " "Please look at docs/Configuration.md for more information" ) -OUTPUT_FORMAT_LIST = ["local", "s3"] ALL_RESOURCE_TYPES = { "beacon", @@ -97,13 +95,9 @@ class ManagerParams: log_directory: Path = Path("~/openwpm/") screenshot_path: Optional[Path] = None source_dump_path: Optional[Path] = None - output_format: str = "local" - database_name: Path = Path("crawl-data.sqlite") log_file: Path = Path("openwpm.log") failure_limit: Optional[int] = None testing: bool = False - s3_bucket: Optional[str] = None - s3_directory: Optional[str] = None memory_watchdog: bool = False process_watchdog: bool = False num_browsers: int = 1 @@ -213,25 +207,6 @@ def validate_manager_params(manager_params: ManagerParams) -> None: ) ) - try: - database_extension = os.path.splitext(manager_params.database_name)[1] - if database_extension.lower() not in DB_EXTENSION_TYPE_LIST: - raise ConfigError( - EXTENSION_ERROR_STRING.format( - extension=database_extension or "no", - value_list=DB_EXTENSION_TYPE_LIST, - parameter_name="database_name", - ) - ) - except (TypeError, AttributeError): - raise ConfigError( - GENERAL_ERROR_STRING.format( - value=manager_params.database_name, - parameter_name="database_name", - params_type="ManagerParams", - ) - ) - # This check is necessary to not cause any internal error because # failure_limit gets set in TaskManager if its value is anything other than None if ( @@ -249,24 +224,6 @@ def validate_manager_params(manager_params: ManagerParams) -> None: ) ) - try: - if manager_params.output_format.lower() not in OUTPUT_FORMAT_LIST: - raise ConfigError( - CONFIG_ERROR_STRING.format( - value=manager_params.output_format, - parameter_name="output_format", - value_list=OUTPUT_FORMAT_LIST, - ).replace( - "Please look at docs/Configuration.md#browser-configuration-options for more information", - "Please look at docs/Configuration.md for more information", - ) - ) - except: - raise ConfigError( - "Something went wrong while validating ManagerParams. " - "Please check values provided for ManagerParams are of expected types" - ) - def validate_crawl_configs( manager_params: ManagerParams, browser_params: List[BrowserParams] diff --git a/openwpm/storage/cloud_storage/gcp_storage.py b/openwpm/storage/cloud_storage/gcp_storage.py index 795bfe1a1..a40908031 100644 --- a/openwpm/storage/cloud_storage/gcp_storage.py +++ b/openwpm/storage/cloud_storage/gcp_storage.py @@ -68,7 +68,11 @@ class GcsUnstructuredProvider(UnstructuredStorageProvider): file_system: GCSFileSystem def __init__( - self, project: str, bucket_name: str, base_path: str, token: str = None, + self, + project: str, + bucket_name: str, + base_path: str, + token: str = None, ) -> None: super().__init__() self.project = project diff --git a/openwpm/storage/leveldb.py b/openwpm/storage/leveldb.py index 18c289045..b3107ff26 100644 --- a/openwpm/storage/leveldb.py +++ b/openwpm/storage/leveldb.py @@ -36,7 +36,10 @@ async def shutdown(self) -> None: self.ldb.close() async def store_blob( - self, filename: str, blob: bytes, overwrite: bool = False, + self, + filename: str, + blob: bytes, + overwrite: bool = False, ) -> None: content_hash = str(filename).encode("ascii") diff --git a/openwpm/storage/storage_providers.py b/openwpm/storage/storage_providers.py index cfe434991..3be8c4836 100644 --- a/openwpm/storage/storage_providers.py +++ b/openwpm/storage/storage_providers.py @@ -77,7 +77,10 @@ async def finalize_visit_id( class UnstructuredStorageProvider(StorageProvider): @abstractmethod async def store_blob( - self, filename: str, blob: bytes, overwrite: bool = False, + self, + filename: str, + blob: bytes, + overwrite: bool = False, ) -> None: """Stores the given bytes under the provided filename""" pass diff --git a/openwpm/task_manager.py b/openwpm/task_manager.py index de296a189..a8696c45d 100644 --- a/openwpm/task_manager.py +++ b/openwpm/task_manager.py @@ -96,10 +96,6 @@ def __init__( if manager_params.log_directory: manager_params.log_directory = manager_params.log_directory.expanduser() - manager_params.database_name = ( - manager_params.data_directory / manager_params.database_name - ) - manager_params.log_file = manager_params.log_directory / manager_params.log_file manager_params.screenshot_path = manager_params.data_directory / "screenshots" diff --git a/openwpm/utilities/db_utils.py b/openwpm/utilities/db_utils.py index 03b4dbf02..129b83bfe 100644 --- a/openwpm/utilities/db_utils.py +++ b/openwpm/utilities/db_utils.py @@ -1,12 +1,15 @@ import os import sqlite3 +from collections import Iterable from pathlib import Path from typing import Any, AnyStr, Iterator, List, Tuple, Union import plyvel -def query_db(db, query, params=None, as_tuple=False): +def query_db( + db: Path, query: str, params: Iterable = None, as_tuple: bool = False +) -> List[Union[sqlite3.Row, tuple]]: """Run a query against the given db. If params is not None, securely construct a query from the given @@ -47,10 +50,11 @@ def get_javascript_entries( return query_db(db, f"SELECT {select_columns} FROM javascript", as_tuple=as_tuple) -def any_command_failed(db): +def any_command_failed(db: Path) -> bool: """Returns True if any command in a given database failed""" rows = query_db(db, "SELECT * FROM crawl_history;") for row in rows: + assert isinstance(row, sqlite3.Row) if row["command_status"] != "ok": return True return False diff --git a/scripts/prune-environment.py b/scripts/prune-environment.py index 880bcb90f..9f81d0b26 100644 --- a/scripts/prune-environment.py +++ b/scripts/prune-environment.py @@ -3,11 +3,20 @@ import yaml with open("environment-unpinned.yaml", "r") as fp: - env_unpinned = yaml.load(fp.read(), Loader=yaml.SafeLoader,) + env_unpinned = yaml.load( + fp.read(), + Loader=yaml.SafeLoader, + ) with open("environment-unpinned-dev.yaml", "r") as fp: - env_unpinned_dev = yaml.load(fp.read(), Loader=yaml.SafeLoader,) + env_unpinned_dev = yaml.load( + fp.read(), + Loader=yaml.SafeLoader, + ) with open("../environment.yaml", "r") as fp: - env_pinned = yaml.load(fp.read(), Loader=yaml.SafeLoader,) + env_pinned = yaml.load( + fp.read(), + Loader=yaml.SafeLoader, + ) # Only pin explicit dependencies diff --git a/test/conftest.py b/test/conftest.py index af1111955..b0001f675 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -58,26 +58,30 @@ def default_params( manager_params.log_directory = tmp_path for i in range(num_browsers): browser_params[i].display_mode = "headless" - manager_params.database_name = tmp_path / manager_params.database_name return manager_params, browser_params @pytest.fixture() def task_manager_creator( - server, xpi -) -> Callable[[Tuple[ManagerParams, List[BrowserParams]]], TaskManager]: + server, + xpi, +) -> Callable[[Tuple[ManagerParams, List[BrowserParams]]], Tuple[TaskManager, Path]]: """We create a callable that returns a TaskManager that has been configured with the Manager and BrowserParams""" def _create_task_manager( params: Tuple[ManagerParams, List[BrowserParams]] - ) -> TaskManager: + ) -> Tuple[TaskManager, Path]: manager_params, browser_params = params - structured_provider = SqlLiteStorageProvider(manager_params.database_name) + db_path = manager_params.data_directory / "crawl-data.sqlite" + structured_provider = SqlLiteStorageProvider(db_path) manager = TaskManager( - manager_params, browser_params, structured_provider, None, + manager_params, + browser_params, + structured_provider, + None, ) - return manager + return manager, db_path return _create_task_manager diff --git a/test/manual_test.py b/test/manual_test.py index 40c1472f6..026ca967b 100644 --- a/test/manual_test.py +++ b/test/manual_test.py @@ -166,7 +166,10 @@ def start_webext(): thread.join() -flag_opts = dict(is_flag=True, default=False,) +flag_opts = dict( + is_flag=True, + default=False, +) @click.command() diff --git a/test/openwpmtest.py b/test/openwpmtest.py index ff98b1e34..2491eb2a4 100644 --- a/test/openwpmtest.py +++ b/test/openwpmtest.py @@ -30,14 +30,18 @@ def set_tmpdir(self, tmpdir): def get_config( self, data_dir: Optional[Path] ) -> Tuple[ManagerParams, List[BrowserParams]]: - pass + raise NotImplementedError() def visit( self, page_url: str, data_dir: Optional[Path] = None, sleep_after: int = 0 ) -> Path: """Visit a test page with the given parameters.""" manager_params, browser_params = self.get_config(data_dir) - structured_provider = SqlLiteStorageProvider(manager_params.database_name) + if data_dir: + db_path = data_dir / "crawl-data.sqlite" + else: + db_path = self.tmpdir / "crawl-data.sqlite" + structured_provider = SqlLiteStorageProvider(db_path) manager = task_manager.TaskManager( manager_params, browser_params, structured_provider, None ) @@ -45,7 +49,7 @@ def visit( page_url = utilities.BASE_TEST_URL + page_url manager.get(url=page_url, sleep=sleep_after) manager.close() - return manager_params.database_name + return db_path def get_test_config( self, @@ -59,13 +63,8 @@ def get_test_config( assert data_dir is not None # Mypy doesn't understand this without help manager_params = ManagerParams(num_browsers=num_browsers) browser_params = [BrowserParams() for _ in range(num_browsers)] - manager_params.data_directory = data_dir manager_params.log_directory = data_dir manager_params.num_browsers = num_browsers for i in range(num_browsers): browser_params[i].display_mode = display_mode - manager_params.database_name = ( - manager_params.data_directory / manager_params.database_name - ) - return manager_params, browser_params diff --git a/test/test_callback.py b/test/test_callback.py index e943de5fd..53bb9bb6e 100644 --- a/test/test_callback.py +++ b/test/test_callback.py @@ -11,7 +11,7 @@ def test_local_callbacks(default_params, task_manager_creator) -> None: """Test test the storage controller as well as the entire callback machinery to see if all callbacks get correctly called""" - manager = task_manager_creator(default_params) + manager, _ = task_manager_creator(default_params) TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html" def callback(argument: List[int], success: bool) -> None: diff --git a/test/test_callstack_instrument.py b/test/test_callstack_instrument.py index a5f2a698d..32cfcdc20 100644 --- a/test/test_callstack_instrument.py +++ b/test/test_callstack_instrument.py @@ -67,9 +67,8 @@ def test_http_stacktrace(default_params, task_manager_creator): # Record the callstack of all WebRequests made browser_params[0].callstack_instrument = True test_url = utilities.BASE_TEST_URL + "/http_stacktrace.html" - manager = task_manager_creator((manager_params, browser_params)) + manager, db = task_manager_creator((manager_params, browser_params)) manager.get(test_url, sleep=10) - db = manager_params.database_name manager.close() rows = db_utils.query_db( db, diff --git a/test/test_custom_function_command.py b/test/test_custom_function_command.py index 565a09f90..2820d8229 100644 --- a/test/test_custom_function_command.py +++ b/test/test_custom_function_command.py @@ -80,8 +80,8 @@ def test_custom_function(default_params, xpi, server): table_name = TableName("page_links") manager_params, browser_params = default_params - - db = sqlite3.connect(manager_params.database_name) + path = manager_params.data_directory / "crawl-data.sqlite" + db = sqlite3.connect(path) cur = db.cursor() cur.execute( @@ -93,7 +93,7 @@ def test_custom_function(default_params, xpi, server): cur.close() db.close() - storage_provider = SqlLiteStorageProvider(manager_params.database_name) + storage_provider = SqlLiteStorageProvider(path) manager = TaskManager(manager_params, browser_params, storage_provider, None) cs = command_sequence.CommandSequence(url_a) cs.get(sleep=0, timeout=60) @@ -101,7 +101,7 @@ def test_custom_function(default_params, xpi, server): manager.execute_command_sequence(cs) manager.close() query_result = db_utils.query_db( - manager_params.database_name, + path, "SELECT top_url, link FROM page_links;", as_tuple=True, ) diff --git a/test/test_dataclass_validations.py b/test/test_dataclass_validations.py index 9f32da3ad..1a8386a51 100644 --- a/test/test_dataclass_validations.py +++ b/test/test_dataclass_validations.py @@ -77,14 +77,6 @@ def test_log_file_extension(): validate_manager_params(manager_params) -def test_database_file_extension(): - manager_params = ManagerParams() - - manager_params.database_name = "something.unsupported" - with pytest.raises(ConfigError): - validate_manager_params(manager_params) - - def test_failure_limit(): manager_params = ManagerParams() @@ -99,17 +91,6 @@ def test_failure_limit(): validate_manager_params(manager_params) -def test_output_format(): - manager_params = ManagerParams() - - manager_params.output_format = "not None and not int" - with pytest.raises(ConfigError): - validate_manager_params(manager_params) - - manager_params.output_format = "s3" - validate_manager_params(manager_params) - - def test_num_browser_crawl_config(): manager_params = ManagerParams(num_browsers=2) browser_params = [BrowserParams()] diff --git a/test/test_dns_instrument.py b/test/test_dns_instrument.py index a34a41111..48f183292 100644 --- a/test/test_dns_instrument.py +++ b/test/test_dns_instrument.py @@ -6,13 +6,11 @@ def test_name_resolution(default_params, task_manager_creator): for browser_param in browser_params: browser_param.dns_instrument = True - manager = task_manager_creator((manager_params, browser_params)) + manager, db = task_manager_creator((manager_params, browser_params)) manager.get("http://localtest.me:8000") manager.close() - result = db_utils.query_db( - manager_params.database_name, "SELECT * FROM dns_responses" - ) + result = db_utils.query_db(db, "SELECT * FROM dns_responses") result = result[0] assert result["used_address"] == "127.0.0.1" assert result["addresses"] == "127.0.0.1" diff --git a/test/test_http_instrumentation.py b/test/test_http_instrumentation.py index 033199804..824ec99a3 100644 --- a/test/test_http_instrumentation.py +++ b/test/test_http_instrumentation.py @@ -892,7 +892,10 @@ def test_javascript_saving(http_params, xpi, server): for browser_param in browser_params: browser_param.http_instrument = True browser_param.save_content = "script" - structured_storage = SqlLiteStorageProvider(db_path=manager_params.database_name) + + structured_storage = SqlLiteStorageProvider( + db_path=manager_params.data_directory / "crawl-data.sqlite" + ) ldb_path = Path(manager_params.data_directory) / "content.ldb" unstructured_storage = LevelDbProvider(db_path=ldb_path) manager = task_manager.TaskManager( @@ -925,7 +928,9 @@ def test_document_saving(http_params, xpi, server): browser_param.http_instrument = True browser_param.save_content = "main_frame,sub_frame" - structured_storage = SqlLiteStorageProvider(db_path=manager_params.database_name) + structured_storage = SqlLiteStorageProvider( + db_path=manager_params.data_directory / "crawl-data.sqlite" + ) ldb_path = Path(manager_params.data_directory) / "content.ldb" unstructured_storage = LevelDbProvider(db_path=ldb_path) manager = task_manager.TaskManager( @@ -950,8 +955,8 @@ def test_content_saving(http_params, xpi, server): for browser_param in browser_params: browser_param.http_instrument = True browser_param.save_content = True - - structured_storage = SqlLiteStorageProvider(db_path=manager_params.database_name) + db = manager_params.data_directory / "crawl-data.sqlite" + structured_storage = SqlLiteStorageProvider(db_path=db) ldb_path = Path(manager_params.data_directory) / "content.ldb" unstructured_storage = LevelDbProvider(db_path=ldb_path) manager = task_manager.TaskManager( @@ -960,7 +965,6 @@ def test_content_saving(http_params, xpi, server): manager.get(url=test_url, sleep=1) manager.close() - db = manager_params.database_name rows = db_utils.query_db(db, "SELECT * FROM http_responses;") disk_content = dict() for row in rows: @@ -999,14 +1003,13 @@ def test_cache_hits_recorded(http_params, task_manager_creator): manager_params, browser_params = http_params() # ensuring that we only spawn one browser manager_params.num_browsers = 1 - manager = task_manager_creator((manager_params, [browser_params[0]])) + manager, db = task_manager_creator((manager_params, [browser_params[0]])) for i in range(2): cs = CommandSequence(test_url, site_rank=i) cs.get(sleep=5) manager.execute_command_sequence(cs) manager.close() - db = manager_params.database_name request_id_to_url = dict() @@ -1092,7 +1095,11 @@ def __init__(self, img_file_path, css_file_path) -> None: self.css_file_path = css_file_path def execute( - self, webdriver, browser_params, manager_params, extension_socket, + self, + webdriver, + browser_params, + manager_params, + extension_socket, ) -> None: img_file_upload_element = webdriver.find_element_by_id("upload-img") css_file_upload_element = webdriver.find_element_by_id("upload-css") diff --git a/test/test_js_instrument.py b/test/test_js_instrument.py index c4540c174..70d84b744 100644 --- a/test/test_js_instrument.py +++ b/test/test_js_instrument.py @@ -112,9 +112,21 @@ def get_config( } browser_params[0].js_instrument_settings = [ # Note that the string "window.document.cookie" does not work. - {"window.document": ["cookie",]}, - {"window.navigator": ["webdriver",]}, - {"window": ["fetch",]}, + { + "window.document": [ + "cookie", + ] + }, + { + "window.navigator": [ + "webdriver", + ] + }, + { + "window": [ + "fetch", + ] + }, ] return manager_params, browser_params diff --git a/test/test_js_instrument_py.py b/test/test_js_instrument_py.py index 5cf62fe7c..eafc6e3f3 100644 --- a/test/test_js_instrument_py.py +++ b/test/test_js_instrument_py.py @@ -16,7 +16,10 @@ def test_python_to_js_lower_true_false(): inpy = [ { "object": "window", - "logSettings": {"logCallStack": False, "preventSets": True,}, + "logSettings": { + "logCallStack": False, + "preventSets": True, + }, } ] expected_out = _no_whitespace( diff --git a/test/test_mp_logger.py b/test/test_mp_logger.py index b91ae6949..72182a39c 100644 --- a/test/test_mp_logger.py +++ b/test/test_mp_logger.py @@ -73,7 +73,8 @@ def child_proc_logging_exception(): raise Exception("This is my generic Test Exception") except Exception: logger.error( - "I'm logging an exception", exc_info=True, + "I'm logging an exception", + exc_info=True, ) diff --git a/test/test_profile.py b/test/test_profile.py index f6563a41f..c30b26979 100644 --- a/test/test_profile.py +++ b/test/test_profile.py @@ -95,7 +95,7 @@ def test_seed_persistance(default_params, task_manager_creator): p = Path("profile.tar.gz") for browser_param in browser_params: browser_param.seed_tar = p - manager = task_manager_creator(default_params) + manager, db = task_manager_creator(default_params) command_sequences = [] for _ in range(2): @@ -108,7 +108,8 @@ def test_seed_persistance(default_params, task_manager_creator): manager.execute_command_sequence(cs) manager.close() query_result = db_utils.query_db( - manager_params.database_name, "SELECT * FROM crawl_history;", + db, + "SELECT * FROM crawl_history;", ) assert len(query_result) > 0 for row in query_result: @@ -121,7 +122,11 @@ def __init__(self, pref_name: str, expected_value: Any) -> None: self.expected_value = expected_value def execute( - self, webdriver, browser_params, manager_params, extension_socket, + self, + webdriver, + browser_params, + manager_params, + extension_socket, ) -> None: webdriver.get("about:config") result = webdriver.execute_script( diff --git a/test/test_simple_commands.py b/test/test_simple_commands.py index 0240510e2..9c7e0ce5d 100644 --- a/test/test_simple_commands.py +++ b/test/test_simple_commands.py @@ -98,7 +98,7 @@ def test_get_site_visits_table_valid(http_params, task_manager_creator, display_ """Check that get works and populates db correctly.""" # Run the test crawl manager_params, browser_params = http_params(display_mode) - manager = task_manager_creator((manager_params, browser_params)) + manager, db = task_manager_creator((manager_params, browser_params)) # Set up two sequential get commands to two URLS cs_a = command_sequence.CommandSequence(url_a) @@ -112,7 +112,7 @@ def test_get_site_visits_table_valid(http_params, task_manager_creator, display_ manager.close() qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT site_url FROM site_visits ORDER BY site_url", ) @@ -128,7 +128,7 @@ def test_get_http_tables_valid(http_params, task_manager_creator, display_mode): """Check that get works and populates http tables correctly.""" # Run the test crawl manager_params, browser_params = http_params(display_mode) - manager = task_manager_creator((manager_params, browser_params)) + manager, db = task_manager_creator((manager_params, browser_params)) # Set up two sequential get commands to two URLS cs_a = command_sequence.CommandSequence(url_a) @@ -140,9 +140,7 @@ def test_get_http_tables_valid(http_params, task_manager_creator, display_mode): manager.execute_command_sequence(cs_b) manager.close() - qry_res = db_utils.query_db( - manager_params.database_name, "SELECT visit_id, site_url FROM site_visits" - ) + qry_res = db_utils.query_db(db, "SELECT visit_id, site_url FROM site_visits") # Construct dict mapping site_url to visit_id visit_ids = dict() @@ -150,28 +148,28 @@ def test_get_http_tables_valid(http_params, task_manager_creator, display_mode): visit_ids[row[1]] = row[0] qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_requests WHERE url = ?", (url_a,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_requests WHERE url = ?", (url_b,), ) assert qry_res[0][0] == visit_ids[url_b] qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_responses WHERE url = ?", (url_a,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_responses WHERE url = ?", (url_b,), ) @@ -185,7 +183,7 @@ def test_browse_site_visits_table_valid( """Check that CommandSequence.browse() populates db correctly.""" # Run the test crawl manager_params, browser_params = http_params(display_mode) - manager = task_manager_creator((manager_params, browser_params)) + manager, db = task_manager_creator((manager_params, browser_params)) # Set up two sequential browse commands to two URLS cs_a = command_sequence.CommandSequence(url_a, site_rank=0) @@ -198,7 +196,7 @@ def test_browse_site_visits_table_valid( manager.close() qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT site_url, site_rank FROM site_visits ORDER BY site_rank", ) @@ -221,7 +219,7 @@ def test_browse_http_table_valid(http_params, task_manager_creator, display_mode """ # Run the test crawl manager_params, browser_params = http_params(display_mode) - manager = task_manager_creator((manager_params, browser_params)) + manager, db = task_manager_creator((manager_params, browser_params)) # Set up two sequential browse commands to two URLS cs_a = command_sequence.CommandSequence(url_a) @@ -233,9 +231,7 @@ def test_browse_http_table_valid(http_params, task_manager_creator, display_mode manager.execute_command_sequence(cs_b) manager.close() - qry_res = db_utils.query_db( - manager_params.database_name, "SELECT visit_id, site_url FROM site_visits" - ) + qry_res = db_utils.query_db(db, "SELECT visit_id, site_url FROM site_visits") # Construct dict mapping site_url to visit_id visit_ids = dict() @@ -243,28 +239,28 @@ def test_browse_http_table_valid(http_params, task_manager_creator, display_mode visit_ids[row[1]] = row[0] qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_requests WHERE url = ?", (url_a,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_requests WHERE url = ?", (url_b,), ) assert qry_res[0][0] == visit_ids[url_b] qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_responses WHERE url = ?", (url_a,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_responses WHERE url = ?", (url_b,), ) @@ -278,13 +274,13 @@ def test_browse_http_table_valid(http_params, task_manager_creator, display_mode # 5) A link to example.com?localtest.me # We should see page visits for 1 and 2, but not 3-5. qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_responses WHERE url = ?", (url_c,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_responses WHERE url = ?", (url_d,), ) @@ -292,7 +288,7 @@ def test_browse_http_table_valid(http_params, task_manager_creator, display_mode # We expect 4 urls: a,c,d and a favicon request qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT COUNT(DISTINCT url) FROM http_responses WHERE visit_id = ?", (visit_ids[url_a],), ) @@ -312,16 +308,14 @@ def test_browse_wrapper_http_table_valid( """ # Run the test crawl manager_params, browser_params = http_params(display_mode) - manager = task_manager_creator((manager_params, browser_params)) + manager, db = task_manager_creator((manager_params, browser_params)) # Set up two sequential browse commands to two URLS manager.browse(url_a, num_links=20, sleep=1) manager.browse(url_b, num_links=1, sleep=1) manager.close() - qry_res = db_utils.query_db( - manager_params.database_name, "SELECT visit_id, site_url FROM site_visits" - ) + qry_res = db_utils.query_db(db, "SELECT visit_id, site_url FROM site_visits") # Construct dict mapping site_url to visit_id visit_ids = dict() @@ -329,28 +323,28 @@ def test_browse_wrapper_http_table_valid( visit_ids[row[1]] = row[0] qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_requests WHERE url = ?", (url_a,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_requests WHERE url = ?", (url_b,), ) assert qry_res[0][0] == visit_ids[url_b] qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_responses WHERE url = ?", (url_a,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_responses WHERE url = ?", (url_b,), ) @@ -364,13 +358,13 @@ def test_browse_wrapper_http_table_valid( # 5) A link to example.com?localtest.me # We should see page visits for 1 and 2, but not 3-5. qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_responses WHERE url = ?", (url_c,), ) assert qry_res[0][0] == visit_ids[url_a] qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT visit_id FROM http_responses WHERE url = ?", (url_d,), ) @@ -378,7 +372,7 @@ def test_browse_wrapper_http_table_valid( # We expect 4 urls: a,c,d and a favicon request qry_res = db_utils.query_db( - manager_params.database_name, + db, "SELECT COUNT(DISTINCT url) FROM http_responses WHERE visit_id = ?", (visit_ids[url_a],), ) @@ -390,7 +384,7 @@ def test_save_screenshot_valid(http_params, task_manager_creator, display_mode): """Check that 'save_screenshot' works""" # Run the test crawl manager_params, browser_params = http_params(display_mode) - manager = task_manager_creator((manager_params, browser_params)) + manager, _ = task_manager_creator((manager_params, browser_params)) cs = command_sequence.CommandSequence(url_a) cs.get(sleep=1) @@ -423,7 +417,7 @@ def test_dump_page_source_valid(http_params, task_manager_creator, display_mode) """Check that 'dump_page_source' works and source is saved properly.""" # Run the test crawl manager_params, browser_params = http_params(display_mode) - manager = task_manager_creator((manager_params, browser_params)) + manager, db = task_manager_creator((manager_params, browser_params)) cs = command_sequence.CommandSequence(url_a) cs.get(sleep=1) @@ -451,7 +445,7 @@ def test_recursive_dump_page_source_valid( """Check that 'recursive_dump_page_source' works""" # Run the test crawl manager_params, browser_params = http_params(display_mode) - manager = task_manager_creator((manager_params, browser_params)) + manager, db = task_manager_creator((manager_params, browser_params)) cs = command_sequence.CommandSequence(NESTED_FRAMES_URL) cs.get(sleep=1) cs.recursive_dump_page_source() diff --git a/test/test_storage_vectors.py b/test/test_storage_vectors.py index 1d945f2e1..fa9191755 100644 --- a/test/test_storage_vectors.py +++ b/test/test_storage_vectors.py @@ -31,7 +31,7 @@ def test_js_profile_cookies(default_params, task_manager_creator): manager_params, browser_params = default_params for browser_param in browser_params: browser_param.cookie_instrument = True - manager = task_manager_creator((manager_params, browser_params)) + manager, db = task_manager_creator((manager_params, browser_params)) url = utilities.BASE_TEST_URL + "/js_cookie.html" cs = command_sequence.CommandSequence(url) cs.get(sleep=3, timeout=120) diff --git a/test/test_webdriver_utils.py b/test/test_webdriver_utils.py index 76aa7124c..0512059e4 100644 --- a/test/test_webdriver_utils.py +++ b/test/test_webdriver_utils.py @@ -23,7 +23,7 @@ def test_parse_neterror_integration(default_params, task_manager_creator): manager.close() get_command = db_utils.query_db( - manager_params.database_name, + db, "SELECT command_status, error FROM crawl_history WHERE command ='GetCommand'", as_tuple=True, )[0] diff --git a/test/test_xvfb_browser.py b/test/test_xvfb_browser.py index 77f463200..dc5ec5350 100644 --- a/test/test_xvfb_browser.py +++ b/test/test_xvfb_browser.py @@ -27,7 +27,7 @@ def test_display_shutdown(task_manager_creator, default_params): for browser_param in browser_params: browser_param.display_mode = "xvfb" TEST_SITE = BASE_TEST_URL + "/test_pages/simple_a.html" - manager = task_manager_creator((manager_params, browser_params)) + manager, db = task_manager_creator((manager_params, browser_params)) port = manager.browsers[0].display_port sequence = CommandSequence(TEST_SITE) From 41e59ad8be0f99dd0e10701aece64a2453546ac3 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 18 Jan 2021 18:33:37 +0100 Subject: [PATCH 082/139] Fixing more tests --- openwpm/utilities/db_utils.py | 2 +- test/test_storage_vectors.py | 2 +- test/test_timer.py | 5 ++--- test/test_webdriver_utils.py | 3 +-- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/openwpm/utilities/db_utils.py b/openwpm/utilities/db_utils.py index 129b83bfe..3cb9f01f4 100644 --- a/openwpm/utilities/db_utils.py +++ b/openwpm/utilities/db_utils.py @@ -1,6 +1,6 @@ import os import sqlite3 -from collections import Iterable +from collections.abc import Iterable from pathlib import Path from typing import Any, AnyStr, Iterator, List, Tuple, Union diff --git a/test/test_storage_vectors.py b/test/test_storage_vectors.py index fa9191755..73755c980 100644 --- a/test/test_storage_vectors.py +++ b/test/test_storage_vectors.py @@ -39,7 +39,7 @@ def test_js_profile_cookies(default_params, task_manager_creator): manager.close() # Check that the JS cookie we stored is recorded qry_res = db_utils.query_db( - manager_params.database_name, + db, ( "SELECT record_type, change_cause, is_http_only, " "is_host_only, is_session, host, is_secure, name, path, " diff --git a/test/test_timer.py b/test/test_timer.py index 2f72dc3b5..0eab01202 100644 --- a/test/test_timer.py +++ b/test/test_timer.py @@ -7,13 +7,12 @@ def test_command_duration(default_params, task_manager_creator): - manager_params = default_params[0] - manager = task_manager_creator(default_params) + manager, db = task_manager_creator(default_params) manager.get(url=TEST_URL, sleep=5) manager.close() get_command = db_utils.query_db( - manager_params.database_name, + db, "SELECT duration FROM crawl_history WHERE command = 'GetCommand'", as_tuple=True, )[0] diff --git a/test/test_webdriver_utils.py b/test/test_webdriver_utils.py index 0512059e4..998a8250e 100644 --- a/test/test_webdriver_utils.py +++ b/test/test_webdriver_utils.py @@ -17,8 +17,7 @@ def test_parse_neterror(): def test_parse_neterror_integration(default_params, task_manager_creator): - manager_params = default_params[0] - manager = task_manager_creator(default_params) + manager, db = task_manager_creator(default_params) manager.get("http://website.invalid") manager.close() From 7acb6243f658ffe4d633a80a8ced7f0bbb61c215 Mon Sep 17 00:00:00 2001 From: Stefan Zabka Date: Tue, 19 Jan 2021 16:36:58 +0100 Subject: [PATCH 083/139] Update test/storage/test_storage_controller.py Co-authored-by: Steven Englehardt --- test/storage/test_storage_controller.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/storage/test_storage_controller.py b/test/storage/test_storage_controller.py index b1ae7f738..a77c3e343 100644 --- a/test/storage/test_storage_controller.py +++ b/test/storage/test_storage_controller.py @@ -25,7 +25,6 @@ def test_startup_and_shutdown(mp_logger: MPLogger) -> None: - structured = MemoryStructuredProvider() unstructured = MemoryUnstructuredProvider() controller_handle = StorageControllerHandle(structured, unstructured) From db0d27f0b88f25519d4b72eef0806c3e987422af Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 22 Jan 2021 10:07:32 +0100 Subject: [PATCH 084/139] Purging references to DataAggregator --- docs/Platform-Architecture.md | 19 ++++++++------ openwpm/Extension/firefox/feature.js/index.js | 2 +- .../Extension/firefox/feature.js/loggingdb.js | 26 +++++++++---------- openwpm/commands/browser_commands.py | 2 +- openwpm/config.py | 8 ++++++ openwpm/deploy_browsers/deploy_firefox.py | 6 ++--- openwpm/storage/arrow_storage.py | 3 +-- openwpm/storage/storage_controller.py | 21 +++------------ openwpm/storage/storage_providers.py | 20 ++++++-------- openwpm/utilities/db_utils.py | 1 - 10 files changed, 50 insertions(+), 58 deletions(-) diff --git a/docs/Platform-Architecture.md b/docs/Platform-Architecture.md index c6b0c541c..277625daa 100644 --- a/docs/Platform-Architecture.md +++ b/docs/Platform-Architecture.md @@ -10,13 +10,13 @@ All automation code is contained within the `openwpm` folder; the Task Manager c Task Managers can be instantiated in the following way: ```python -from opemwpm.task_manager import TaskManager +from openwpm.task_manager import TaskManager from openwpm.config import ( BrowserParams, ManagerParams, ) -number_of_browser = 5 # Number of browsers to spawn +number_of_browsers = 5 # Number of browsers to spawn # Instantiating Browser and Manager Params with default values. manager_params = ManagerParams(num_browsers = number_of_browsers) @@ -37,13 +37,14 @@ manager = TaskManager(manager_params, browser_params) To learn more about the `manager_params` and `browser_params` have a look at [Configuration.md](Configuration.md) ## Watchdogs -In OpenWPM we have a so called watchdog that tries to ensure two things. +In OpenWPM we have a watchdog thread that tries to ensure two things. - `process_watchdog` * It is part of default manager_params. It is set to false by default which can manually be set to true. - * It is used to create another thread that kills off `GeckoDriver` (or `Xvfb`) instances that aren't currently controlled by OpenWPM. (GeckoDriver is used by Selenium to control Firefox and Xvfb a "virtual display" so we simulate having graphics when running on a server). + * It is used to create another thread that kills off `GeckoDriver` (or `Xvfb`) instances that aren't currently controlled by OpenWPM. + (GeckoDriver is used by Selenium to control Firefox and Xvfb is a "virtual display" we use to simulate having graphics when running on a server). - `memory_watchdog` * It is part of default manager_params. It is set to false by default which can manually be set to true. - * It is a watchdog that tries to ensure that no Firefox instance takes up to much memory. + * It is a watchdog that tries to ensure that no Firefox instance takes up too much memory. * It is mostly useful for long running cloud crawls. ## Issuing commands @@ -60,8 +61,8 @@ For example you could wire up a `CommandSequence` to go to a given url and take command_sequence.save_screenshot() ``` -But this on it's own would do nothing, because `CommandSequence`s are not automatically scheduled. -Instead you need to submit them to a `TaskManager` by calling: +But this on its own would do nothing, because `CommandSequence`s are not automatically scheduled. +Instead, you need to submit them to a `TaskManager` by calling: ```python manager.execute_command_sequence(command_sequence) manager.close() @@ -94,7 +95,9 @@ The Browser class, contained in the same file, is the Task Manager's wrapper aro ## Browser Information Logging -Throughout the course of a measurement, the Browser Managers' commands (along with timestamps and the status of the commands) are logged by the Task Manager, which contributes the the reproducibility of individual experiments. The data are sent to the Data Aggregator process, which provides stability in logging data despite the possibility of individual browser crashes. +Throughout the course of a measurement, the Browser Managers' commands (along with timestamps and the status of the commands) +are logged by the Task Manager, which contributes to the reproducibility of individual experiments. +The data is sent to the Storage Controller process, which provides stability in logging data despite the possibility of individual browser crashes. # The WebExtension diff --git a/openwpm/Extension/firefox/feature.js/index.js b/openwpm/Extension/firefox/feature.js/index.js index 183b95b39..4454fcc1d 100644 --- a/openwpm/Extension/firefox/feature.js/index.js +++ b/openwpm/Extension/firefox/feature.js/index.js @@ -50,7 +50,7 @@ async function main() { "the extension. Outputting all queries to console.", {config}); } - await loggingDB.open(config['aggregator_address'], + await loggingDB.open(config['storage_controller_address'], config['logger_address'], config['browser_id']); diff --git a/openwpm/Extension/firefox/feature.js/loggingdb.js b/openwpm/Extension/firefox/feature.js/loggingdb.js index 4626b34c5..b8b7caa01 100644 --- a/openwpm/Extension/firefox/feature.js/loggingdb.js +++ b/openwpm/Extension/firefox/feature.js/loggingdb.js @@ -3,7 +3,7 @@ import * as socket from "./socket.js"; let crawlID = null; let visitID = null; let debugging = false; -let dataAggregator = null; +let storageController = null; let logAggregator = null; let listeningSocket = null; @@ -19,19 +19,19 @@ let listeningSocketCallback = async (data) => { } visitID = _visitID; data["browser_id"] = crawlID; - dataAggregator.send(JSON.stringify(["meta_information", data])); + storageController.send(JSON.stringify(["meta_information", data])); break; case "Finalize": if (!visitID) { logWarn("Send Finalize while no visit_id was set") } - if (_visitID != visitID ) { + if (_visitID !== visitID ) { logError("Send Finalize but visit_id didn't match. " + `Current visit_id ${visit_id}, sent visit_id ${_visit_id}.`); } data["browser_id"] = crawlID; data["success"] = true; - dataAggregator.send(JSON.stringify(["meta_information", data])); + storageController.send(JSON.stringify(["meta_information", data])); visitID = null; break; default: @@ -43,8 +43,8 @@ let listeningSocketCallback = async (data) => { } } -export let open = async function(aggregatorAddress, logAddress, curr_crawlID) { - if (aggregatorAddress == null && logAddress == null && curr_crawlID == '') { +export let open = async function(storageControllerAddress, logAddress, curr_crawlID) { + if (storageControllerAddress == null && logAddress == null && curr_crawlID === '') { console.log("Debugging, everything will output to console"); debugging = true; return; @@ -61,9 +61,9 @@ export let open = async function(aggregatorAddress, logAddress, curr_crawlID) { } // Connect to databases for saving data - if (aggregatorAddress != null) { - dataAggregator = new socket.SendingSocket(); - let rv = await dataAggregator.connect(aggregatorAddress[0], aggregatorAddress[1]); + if (storageControllerAddress != null) { + storageController = new socket.SendingSocket(); + let rv = await storageController.connect(storageControllerAddress[0], storageControllerAddress[1]); console.log("StorageController started?",rv); } @@ -76,8 +76,8 @@ export let open = async function(aggregatorAddress, logAddress, curr_crawlID) { }; export let close = function() { - if (dataAggregator != null) { - dataAggregator.close(); + if (storageController != null) { + storageController.close(); } if (logAggregator != null) { logAggregator.close(); @@ -189,7 +189,7 @@ export let saveRecord = function(instrument, record) { console.log("EXTENSION", instrument, record); return; } - dataAggregator.send(JSON.stringify([instrument, record])); + storageController.send(JSON.stringify([instrument, record])); }; // Stub for now @@ -203,7 +203,7 @@ export let saveContent = async function(content, contentHash) { // Since the content might not be a valid utf8 string and it needs to be // json encoded later, it is encoded using base64 first. const b64 = Uint8ToBase64(content); - dataAggregator.send(JSON.stringify(['page_content', [b64, contentHash]])); + storageController.send(JSON.stringify(['page_content', [b64, contentHash]])); }; function encode_utf8(s) { diff --git a/openwpm/commands/browser_commands.py b/openwpm/commands/browser_commands.py index cbfde6661..e1692f83b 100644 --- a/openwpm/commands/browser_commands.py +++ b/openwpm/commands/browser_commands.py @@ -500,7 +500,7 @@ class InitializeCommand(BaseCommand): """The command is automatically prepended to the beginning of a CommandSequence It initializes state both in the extensions as well in as the - Aggregator + StorageController """ def __repr__(self): diff --git a/openwpm/config.py b/openwpm/config.py index 03daa1754..7cfbf475f 100644 --- a/openwpm/config.py +++ b/openwpm/config.py @@ -91,6 +91,14 @@ class BrowserParams: @dataclass_json @dataclass class ManagerParams: + """ + Configuration for the TaskManager + The configuration will be the same for all browsers running on the same + TaskManager. + It can be used to control storage locations or which watchdogs should + run + """ + data_directory: Path = Path("~/openwpm/") log_directory: Path = Path("~/openwpm/") screenshot_path: Optional[Path] = None diff --git a/openwpm/deploy_browsers/deploy_firefox.py b/openwpm/deploy_browsers/deploy_firefox.py index 53d36d37d..6deaa46eb 100755 --- a/openwpm/deploy_browsers/deploy_firefox.py +++ b/openwpm/deploy_browsers/deploy_firefox.py @@ -2,7 +2,7 @@ import logging import os.path from pathlib import Path -from typing import Any, List, Optional, Tuple +from typing import Optional, Tuple from easyprocess import EasyProcessError from multiprocess import Queue @@ -71,7 +71,7 @@ def deploy_firefox( display_port = None display = None if display_mode == "headless": - fo.set_headless(True) + fo.headless = True fo.add_argument("--width={}".format(DEFAULT_SCREEN_RES[0])) fo.add_argument("--height={}".format(DEFAULT_SCREEN_RES[1])) if display_mode == "xvfb": @@ -96,7 +96,7 @@ def deploy_firefox( extension_config.update(browser_params.to_dict()) # type: ignore extension_config["logger_address"] = manager_params.logger_address extension_config[ - "aggregator_address" + "storage_controller_address" ] = manager_params.storage_controller_address if manager_params.ldb_address: extension_config["leveldb_address"] = manager_params.ldb_address diff --git a/openwpm/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py index e4526da6e..a48b4501b 100644 --- a/openwpm/storage/arrow_storage.py +++ b/openwpm/storage/arrow_storage.py @@ -3,11 +3,10 @@ import random from abc import abstractmethod from collections import defaultdict -from typing import Any, Awaitable, DefaultDict, Dict, List, Optional +from typing import Any, Awaitable, DefaultDict, Dict, List import pandas as pd import pyarrow as pa -import pyarrow.parquet as pq from pyarrow import Table from openwpm.types import VisitId diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 4437ce2ac..d53a672e1 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -1,24 +1,12 @@ import asyncio import base64 import logging -import math import queue import random import socket -import threading import time from collections import defaultdict -from typing import ( - Any, - Awaitable, - DefaultDict, - Dict, - List, - Literal, - NoReturn, - Optional, - Tuple, -) +from typing import Any, Awaitable, DefaultDict, Dict, List, NoReturn, Optional, Tuple from multiprocess import Queue @@ -95,7 +83,7 @@ async def _handler( ) async def handler( - self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter + self, reader: asyncio.StreamReader, _: asyncio.StreamWriter ) -> None: """Created for every new connection to the Server""" self.logger.debug("Initializing new handler") @@ -121,9 +109,8 @@ async def handler( if record_type == RECORD_TYPE_CREATE: raise RuntimeError( f"""{RECORD_TYPE_CREATE} is no longer supported. - since the user now has access to the DB before it - goes into use, they should set up all schemas before - launching the DataAggregator + Please change the schema before starting the StorageController. + For an example of that see test/test_custom_function.py """ ) diff --git a/openwpm/storage/storage_providers.py b/openwpm/storage/storage_providers.py index 3be8c4836..dd8706a60 100644 --- a/openwpm/storage/storage_providers.py +++ b/openwpm/storage/storage_providers.py @@ -1,18 +1,17 @@ +""" +This module contains all base classes of the storage provider hierarchy +Any subclass of these classes should be able to be used in OpenWPM +without any changes to the rest of the code base +""" import gzip import io from abc import ABC, abstractmethod -from typing import Any, AsyncGenerator, Awaitable, Dict, List, NewType, Optional, Tuple +from typing import Any, Awaitable, Dict, NewType from pyarrow.lib import Table from openwpm.types import VisitId -""" -This module contains all base classes of the storage provider hierachy -Any subclass of these classes should be able to be used in OpenWPM -without any changes to the rest of the code base -""" - TableName = NewType("TableName", str) @@ -48,10 +47,6 @@ class StructuredStorageProvider(StorageProvider): def __init__(self) -> None: super().__init__() - # TODO: Discuss if we want visit_id here - # It will always be part of the record and make the interface bigger - # than it needs to be - # But not having to access the data is also convenient @abstractmethod async def store_record( self, table: TableName, visit_id: VisitId, record: Dict[str, Any] @@ -68,7 +63,8 @@ async def finalize_visit_id( """This method is invoked to inform the StructuredStorageProvider that no more records for this visit_id will be submitted - This method returns an awaitable that will resolve once the records have been + This method returns once the data is ready to be written out. + The returned awaitable will resolve once the records have been saved out to persistent storage """ pass diff --git a/openwpm/utilities/db_utils.py b/openwpm/utilities/db_utils.py index 3cb9f01f4..2ae04dcf1 100644 --- a/openwpm/utilities/db_utils.py +++ b/openwpm/utilities/db_utils.py @@ -1,4 +1,3 @@ -import os import sqlite3 from collections.abc import Iterable from pathlib import Path From abe4a01d9f5092e8001ccb9697e87028969376b0 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 22 Jan 2021 10:08:07 +0100 Subject: [PATCH 085/139] Reverted changes to .travis.yml --- .travis.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index cd57240de..acd3be359 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,11 +6,11 @@ env: # Once we add and remove tests, this distribution may become unbalanced. # Feel free to move tests around to make the running time of the jobs # as close as possible. - - TESTS=test/test_[a-e]* - - TESTS=test/test_[f-h]* - - TESTS=test/test_[i-r,t-z]* + - TESTS=test_[a-e]* + - TESTS=test_[f-h]* + - TESTS=test_[i-r,t-z]* # test_simple_commands.py is slow due to parametrization. - - TESTS=test/test_[s]* + - TESTS=test_[s]* - TESTS=webextension git: depth: 3 @@ -36,11 +36,15 @@ after_success: jobs: include: - - language: "python" + - language: + python: env: - TESTS="Docker" services: - docker + before_install: + before_script: + install: script: - docker build -f Dockerfile -t openwpm . - ./scripts/deploy-to-dockerhub.sh From d7400d21f20ff8dae976b9d19f112c38cfc2c1fa Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 22 Jan 2021 10:15:08 +0100 Subject: [PATCH 086/139] Demo.py saves locally again --- demo.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/demo.py b/demo.py index f0be0e300..9cdc75bba 100644 --- a/demo.py +++ b/demo.py @@ -5,8 +5,6 @@ from openwpm.command_sequence import CommandSequence from openwpm.commands.browser_commands import GetCommand from openwpm.config import BrowserParams, ManagerParams -from openwpm.storage.cloud_storage.gcp_storage import GcsStructuredProvider -from openwpm.storage.local_storage import LocalArrowProvider from openwpm.storage.sql_provider import SqlLiteStorageProvider from openwpm.task_manager import TaskManager @@ -40,20 +38,17 @@ manager_params.data_directory = Path("./datadir/") manager_params.log_directory = Path("./datadir/") -logging_params = {"log_level_console": logging.DEBUG} # memory_watchdog and process_watchdog are useful for large scale cloud crawls. # Please refer to docs/Configuration.md#platform-configuration-options for more information # manager_params.memory_watchdog = True # manager_params.process_watchdog = True -# Instantiates the measurement platform -project = "senglehardt-openwpm-test-1" -bucket_name = "openwpm-test-bucket" + # Commands time out by default after 60 seconds manager = TaskManager( manager_params, browser_params, - GcsStructuredProvider(project, bucket_name, base_path="test3"), + SqlLiteStorageProvider(Path("./datadir/crawl-data.sqllite")), None, ) From 645240be48cb3c8602c37daa46690ad0240f55f5 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 22 Jan 2021 10:43:53 +0100 Subject: [PATCH 087/139] Readjusting test paths --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index acd3be359..f1516cdb0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,11 +6,11 @@ env: # Once we add and remove tests, this distribution may become unbalanced. # Feel free to move tests around to make the running time of the jobs # as close as possible. - - TESTS=test_[a-e]* - - TESTS=test_[f-h]* - - TESTS=test_[i-r,t-z]* + - TESTS=test/test_[a-e]* + - TESTS=test/test_[f-h]* + - TESTS=test/test_[i-r,t-z]* # test_simple_commands.py is slow due to parametrization. - - TESTS=test_[s]* + - TESTS=test/test_[s]* - TESTS=webextension git: depth: 3 From ecb87f03c910ff344251884c1a297c0cfd7913e7 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 22 Jan 2021 11:19:46 +0100 Subject: [PATCH 088/139] Expanded comment on initialize to reference #846 --- openwpm/storage/storage_controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index d53a672e1..253610195 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -159,8 +159,8 @@ async def _handle_meta(self, visit_id: VisitId, data: Dict[str, Any]) -> None: - finalize: A message sent by the extension to signal that a visit_id is complete. - initialize: TODO: Start complaining if we receive data for a visit_id - before the initialize event happened. (This might not be easy - because of `site_visits`) + before the initialize event happened. + See also https://github.com/mozilla/OpenWPM/issues/846 """ action: str = data["action"] if action == ACTION_TYPE_INITIALIZE: From 8629538a32c0a555a8975831696bf684a05c213c Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 22 Jan 2021 11:35:51 +0100 Subject: [PATCH 089/139] Made token optional in finalize_visit_id --- openwpm/storage/storage_controller.py | 11 ++++++----- openwpm/storage/storage_providers.py | 7 ++++--- test/manual_test.py | 6 ------ test/storage/test_memory_storage_provider.py | 3 ++- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 253610195..993e4a5c2 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -168,14 +168,15 @@ async def _handle_meta(self, visit_id: VisitId, data: Dict[str, Any]) -> None: elif action == ACTION_TYPE_FINALIZE: success = data["success"] completion_token = await self.finalize_visit_id(visit_id, success) - await completion_token + if completion_token is not None: + await completion_token self.completion_queue.put((visit_id, success)) else: raise ValueError("Unexpected action: %s", action) async def finalize_visit_id( self, visit_id: VisitId, success: bool - ) -> Awaitable[None]: + ) -> Optional[Awaitable[None]]: """Makes sure all records for a given visit_id have been processed before we invoke finalize_visit_id on the structured_storage @@ -225,9 +226,9 @@ async def shutdown(self) -> None: completion_tokens = {} visit_ids = list(self.current_tasks.keys()) for visit_id in visit_ids: - completion_tokens[visit_id] = await self.finalize_visit_id( - visit_id, success=False - ) + t = await self.finalize_visit_id(visit_id, success=False) + if t is not None: + completion_tokens[visit_id] = t await self.structured_storage.flush_cache() for visit_id, token in completion_tokens.items(): await token diff --git a/openwpm/storage/storage_providers.py b/openwpm/storage/storage_providers.py index dd8706a60..7ec081a45 100644 --- a/openwpm/storage/storage_providers.py +++ b/openwpm/storage/storage_providers.py @@ -6,7 +6,7 @@ import gzip import io from abc import ABC, abstractmethod -from typing import Any, Awaitable, Dict, NewType +from typing import Any, Awaitable, Dict, NewType, Optional from pyarrow.lib import Table @@ -59,12 +59,13 @@ async def store_record( @abstractmethod async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False - ) -> Awaitable[None]: + ) -> Optional[Awaitable[None]]: """This method is invoked to inform the StructuredStorageProvider that no more records for this visit_id will be submitted This method returns once the data is ready to be written out. - The returned awaitable will resolve once the records have been + If the data is immediately written out nothing will be returned. + Otherwise an awaitable will returned that resolve onces the records have been saved out to persistent storage """ pass diff --git a/test/manual_test.py b/test/manual_test.py index 026ca967b..5d923f7ac 100644 --- a/test/manual_test.py +++ b/test/manual_test.py @@ -166,12 +166,6 @@ def start_webext(): thread.join() -flag_opts = dict( - is_flag=True, - default=False, -) - - @click.command() @click.option( "--selenium", diff --git a/test/storage/test_memory_storage_provider.py b/test/storage/test_memory_storage_provider.py index 7a29b493b..8ba1c2018 100644 --- a/test/storage/test_memory_storage_provider.py +++ b/test/storage/test_memory_storage_provider.py @@ -79,7 +79,8 @@ async def test_basic_access( ) token = await structured_provider.finalize_visit_id(VisitId(2)) await structured_provider.flush_cache() - await token + if token is not None: + await token await structured_provider.shutdown() From 99833621aa1d6a06a79bfd114ad5f3b4dde551ac Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 22 Jan 2021 11:56:53 +0100 Subject: [PATCH 090/139] Simplified test paramtetrization --- test/storage/test_memory_storage_provider.py | 86 ++++++++------------ 1 file changed, 34 insertions(+), 52 deletions(-) diff --git a/test/storage/test_memory_storage_provider.py b/test/storage/test_memory_storage_provider.py index 8ba1c2018..3f58b583a 100644 --- a/test/storage/test_memory_storage_provider.py +++ b/test/storage/test_memory_storage_provider.py @@ -43,45 +43,30 @@ def structured_provider( request.raiseerror("invalid internal test config") -def pytest_generate_tests(metafunc: Any) -> Any: - # Source: https://docs.pytest.org/en/latest/example/parametrize.html#a-quick-port-of-testscenarios # noqa - idlist = [] - argvalues = [] - for scenario in metafunc.cls.scenarios: - idlist.append(scenario[0]) - items = scenario[1].items() - argnames = [x[0] for x in items] - argvalues.append([x[1] for x in items]) - metafunc.parametrize(argnames, argvalues, ids=idlist, scope="class", indirect=True) +structured_scenarios: List[str] = [ + memory_structured, + sqllite, + memory_arrow, +] +@pytest.mark.parametrize("structured_provider", structured_scenarios, indirect=True) @pytest.mark.asyncio -class TestStructuredStorageProvider: - scenarios: List[Tuple[str, Dict[str, Any]]] = [ - (memory_structured, {"structured_provider": memory_structured}), - (sqllite, {"structured_provider": sqllite}), - (memory_arrow, {"structured_provider": memory_arrow}), - ] - - async def test_basic_access( - self, structured_provider: StructuredStorageProvider - ) -> None: - data = { - "visit_id": 2, - "browser_id": 3, - "site_url": "https://example.com", - } - - await structured_provider.init() - - await structured_provider.store_record( - TableName("site_visits"), VisitId(2), data - ) - token = await structured_provider.finalize_visit_id(VisitId(2)) - await structured_provider.flush_cache() - if token is not None: - await token - await structured_provider.shutdown() +async def test_basic_access(structured_provider: StructuredStorageProvider) -> None: + data = { + "visit_id": 2, + "browser_id": 3, + "site_url": "https://example.com", + } + + await structured_provider.init() + + await structured_provider.store_record(TableName("site_visits"), VisitId(2), data) + token = await structured_provider.finalize_visit_id(VisitId(2)) + await structured_provider.flush_cache() + if token is not None: + await token + await structured_provider.shutdown() # Unstructured Providers @@ -108,20 +93,17 @@ def unstructured_provider( request.raiseerror("invalid internal test config") +unstructured_scenarios: List[str] = [memory_unstructured, leveldb, local_gzip] + + +@pytest.mark.parametrize("unstructured_provider", unstructured_scenarios, indirect=True) @pytest.mark.asyncio -class TestUnstructuredStorageProvide: - scenarios: List[Tuple[str, Dict[str, Any]]] = [ - (memory_unstructured, {"unstructured_provider": memory_unstructured}), - (leveldb, {"unstructured_provider": leveldb}), - (local_gzip, {"unstructured_provider": local_gzip}), - ] - - async def test_basic_unstructured_storing( - self, unstructured_provider: UnstructuredStorageProvider - ) -> None: - test_string = "This is my test string" - blob = test_string.encode() - await unstructured_provider.init() - await unstructured_provider.store_blob("test", blob) - await unstructured_provider.flush_cache() - await unstructured_provider.shutdown() +async def test_basic_unstructured_storing( + unstructured_provider: UnstructuredStorageProvider, +) -> None: + test_string = "This is my test string" + blob = test_string.encode() + await unstructured_provider.init() + await unstructured_provider.store_blob("test", blob) + await unstructured_provider.flush_cache() + await unstructured_provider.shutdown() From 105c73b005d66b5c9e785c5ebaf2e4035f6ba509 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 22 Jan 2021 13:10:44 +0100 Subject: [PATCH 091/139] Fixed callback semantics change --- demo.py | 4 +++- openwpm/storage/storage_controller.py | 2 +- openwpm/task_manager.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/demo.py b/demo.py index 9cdc75bba..538b2202b 100644 --- a/demo.py +++ b/demo.py @@ -56,7 +56,9 @@ for index, site in enumerate(sites): def callback(success: bool, val: str = site) -> None: - print("CommandSequence {} done".format(val)) + print( + f"CommandSequence for {val} ran {'successfully' if success else 'unsuccessfully'}" + ) # Parallelize sites over all number of browsers set above. command_sequence = CommandSequence( diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 993e4a5c2..62fb58f9d 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -373,7 +373,7 @@ def get_new_completed_visits(self) -> List[Tuple[int, bool]]: """ Returns a list of all visit ids that have been processed since the last time the method was called and whether or not they - have been interrupted. + ran successfully. This method will return an empty list in case no visit ids have been processed since the last time this method was called diff --git a/openwpm/task_manager.py b/openwpm/task_manager.py index a8696c45d..1d0640f36 100644 --- a/openwpm/task_manager.py +++ b/openwpm/task_manager.py @@ -415,11 +415,11 @@ def _mark_command_sequences_complete(self) -> None: time.sleep(1) continue - for visit_id, interrupted in visit_id_list: + for visit_id, successful in visit_id_list: self.logger.debug("Invoking callback of visit_id %d", visit_id) cs = self.unsaved_command_sequences.pop(visit_id, None) if cs: - cs.mark_done(not interrupted) + cs.mark_done(successful) def _unpack_picked_error(self, pickled_error: bytes) -> Tuple[str, str]: """Unpacks `pickled_error` into and error `message` and `tb` string.""" From f5a0abdbb31608ce41884c3efcc87a9bafce1c09 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 22 Jan 2021 17:19:21 +0100 Subject: [PATCH 092/139] Removed test_parse_http_stack_trace_str --- test/test_callstack_instrument.py | 41 ------------------------------- 1 file changed, 41 deletions(-) diff --git a/test/test_callstack_instrument.py b/test/test_callstack_instrument.py index 32cfcdc20..a31f7e61d 100644 --- a/test/test_callstack_instrument.py +++ b/test/test_callstack_instrument.py @@ -32,30 +32,6 @@ STACK_TRACE_INJECT_PIXEL, STACK_TRACE_INJECT_JS, } -# parsed HTTP call stack dict -CALL_STACK_INJECT_IMAGE = [ - { - "func_name": "inject_image", - "filename": HTTP_STACKTRACE_TEST_URL, - "line_no": "18", - "col_no": "7", - "async_cause": "null", - }, - { - "func_name": "inject_all", - "filename": HTTP_STACKTRACE_TEST_URL, - "line_no": "22", - "col_no": "7", - "async_cause": "null", - }, - { - "func_name": "onload", - "filename": HTTP_STACKTRACE_TEST_URL, - "line_no": "1", - "col_no": "1", - "async_cause": "null", - }, -] def test_http_stacktrace(default_params, task_manager_creator): @@ -94,20 +70,3 @@ def test_http_stacktrace(default_params, task_manager_creator): if url.endswith(test_urls): observed_records.add(call_stack) assert HTTP_STACKTRACES == observed_records - - -def test_parse_http_stack_trace_str(): - stacktrace = STACK_TRACE_INJECT_IMAGE - stack_frames = parse_http_stack_trace_str(stacktrace) - assert stack_frames == CALL_STACK_INJECT_IMAGE - - # TODO: webext instrumentation doesn't support req_call_stack yet. - # def test_http_stacktrace_nonjs_loads(self): - # # stacktrace should be empty for requests NOT triggered by scripts - # test_url = utilities.BASE_TEST_URL + '/http_test_page.html' - # db = self.visit(test_url, sleep_after=3) - # rows = db_utils.query_db(db, ( - # "SELECT url, req_call_stack FROM http_requests")) - # for row in rows: - # _, stacktrace = row - # assert stacktrace == "" From e6175db76c75efcdf1dc52a08e44307c687328f1 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 22 Jan 2021 17:55:13 +0100 Subject: [PATCH 093/139] Added DataSocket --- openwpm/browser_manager.py | 13 ++- openwpm/config.py | 3 +- openwpm/storage/cloud_storage/gcp_storage.py | 10 +- openwpm/storage/sql_provider.py | 8 +- openwpm/storage/storage_controller.py | 90 +++++++++++++-- openwpm/task_manager.py | 112 ++++++++----------- 6 files changed, 144 insertions(+), 92 deletions(-) diff --git a/openwpm/browser_manager.py b/openwpm/browser_manager.py index d2cf3157e..ed8bce976 100644 --- a/openwpm/browser_manager.py +++ b/openwpm/browser_manager.py @@ -17,9 +17,11 @@ from tblib import pickling_support from .commands.types import BaseCommand, ShutdownSignal +from .config import BrowserParamsInternal, ManagerParamsInternal from .deploy_browsers import deploy_firefox from .errors import BrowserConfigError, BrowserCrashError, ProfileLoadError from .socket_interface import ClientSocket +from .types import BrowserId, VisitId from .utilities.multiprocess_utils import ( Process, kill_process_and_children, @@ -40,7 +42,11 @@ class Browser: this browser is headless, etc.) """ - def __init__(self, manager_params, browser_params) -> None: + def __init__( + self, + manager_params: ManagerParamsInternal, + browser_params: BrowserParamsInternal, + ) -> None: # Constants self._SPAWN_TIMEOUT = 120 # seconds self._UNSUCCESSFUL_SPAWN_LIMIT = 4 @@ -48,8 +54,9 @@ def __init__(self, manager_params, browser_params) -> None: # manager parameters self.current_profile_path = None self.db_socket_address = manager_params.storage_controller_address - self.browser_id = browser_params.browser_id - self.curr_visit_id: Optional[int] = None + assert browser_params.browser_id is not None + self.browser_id: BrowserId = browser_params.browser_id + self.curr_visit_id: Optional[VisitId] = None self.browser_params = browser_params self.manager_params = manager_params diff --git a/openwpm/config.py b/openwpm/config.py index 7cfbf475f..588c674a1 100644 --- a/openwpm/config.py +++ b/openwpm/config.py @@ -7,6 +7,7 @@ from dataclasses_json import dataclass_json from .errors import ConfigError +from .types import BrowserId BOOL_TYPE_VALIDATION_LIST = [True, False] DISPLAY_MODE_VALIDATION_LIST = ["native", "headless", "xvfb"] @@ -114,7 +115,7 @@ class ManagerParams: @dataclass_json @dataclass class BrowserParamsInternal(BrowserParams): - browser_id: Optional[int] = None + browser_id: Optional[BrowserId] = None profile_path: Optional[Path] = None diff --git a/openwpm/storage/cloud_storage/gcp_storage.py b/openwpm/storage/cloud_storage/gcp_storage.py index a40908031..119c1de16 100644 --- a/openwpm/storage/cloud_storage/gcp_storage.py +++ b/openwpm/storage/cloud_storage/gcp_storage.py @@ -1,7 +1,5 @@ -import asyncio import logging -from functools import partial -from typing import Any, Callable, Optional, Set +from typing import Set import pyarrow.parquet as pq from gcsfs import GCSFileSystem @@ -59,10 +57,8 @@ async def shutdown(self) -> None: class GcsUnstructuredProvider(UnstructuredStorageProvider): - """This class allows you to upload Parquet files to GCS. - This might not actually be the thing that we want to do - long term but seeing as GCS is the S3 equivalent of GCP - it is the easiest way forward. + """This class allows you to upload arbitrary bytes to GCS. + They will be stored under bucket_name/base_path/filename """ file_system: GCSFileSystem diff --git a/openwpm/storage/sql_provider.py b/openwpm/storage/sql_provider.py index 38c91440f..47b7d6bb3 100644 --- a/openwpm/storage/sql_provider.py +++ b/openwpm/storage/sql_provider.py @@ -98,18 +98,12 @@ def execute_statement(self, statement: str) -> None: async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False - ) -> Awaitable[None]: - + ) -> None: if interrupted: self.logger.warning("Visit with visit_id %d got interrupted", visit_id) self.cur.execute("INSERT INTO incomplete_visits VALUES (?)", (visit_id,)) self.db.commit() - async def done() -> None: - return - - return done() - async def shutdown(self) -> None: self.db.commit() self.db.close() diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 62fb58f9d..735fc51c7 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -12,7 +12,8 @@ from openwpm.utilities.multiprocess_utils import Process -from ..socket_interface import get_message_from_reader +from ..config import BrowserParamsInternal, ManagerParamsInternal +from ..socket_interface import ClientSocket, get_message_from_reader from ..types import BrowserId, VisitId from .storage_providers import ( StructuredStorageProvider, @@ -32,6 +33,7 @@ STATUS_UPDATE_INTERVAL = 5 # seconds +FAKE_VISIT_ID = VisitId(-1) class StorageController: @@ -142,14 +144,23 @@ async def handler( continue table_name = TableName(record_type) - # Turning these into task to be able to verify - self.current_tasks[visit_id].append( - asyncio.create_task( - self.structured_storage.store_record( - table=table_name, visit_id=visit_id, record=data - ) + await self.store_record(table_name, visit_id, data) + + async def store_record( + self, table_name: TableName, visit_id: VisitId, data: Dict[str, Any] + ) -> None: + + if visit_id == FAKE_VISIT_ID: + # Hacking around the fact that task and crawl don't have a VisitID + del data["visit_id"] + # Turning these into task to be able to have them complete without blocking the socket + self.current_tasks[visit_id].append( + asyncio.create_task( + self.structured_storage.store_record( + table=table_name, visit_id=visit_id, record=data ) ) + ) async def _handle_meta(self, visit_id: VisitId, data: Dict[str, Any]) -> None: """ @@ -306,6 +317,36 @@ def run(self) -> None: asyncio.run(self._run(), debug=True) +class DataSocket: + """Wrapper around ClientSocket to make sending records to the StorageController more convenient""" + + def __init__(self, listener_address: Tuple[str, int]) -> None: + self.socket = ClientSocket(serialization="dill") + self.socket.connect(*listener_address) + + def store_record( + self, table_name: TableName, visit_id: VisitId, data: Dict[str, Any] + ) -> None: + data["visit_id"] = visit_id + self.socket.send( + ( + table_name, + data, + ) + ) + + def finalize_visit_id(self, visit_id: VisitId, success: bool) -> None: + self.socket.send( + ( + RECORD_TYPE_META, + {"action": ACTION_TYPE_FINALIZE, visit_id: visit_id, success: success}, + ) + ) + + def close(self) -> None: + self.socket.close() + + class StorageControllerHandle: """This class contains all methods relevant for the TaskManager to interact with the DataAggregator @@ -351,11 +392,38 @@ def get_next_browser_id(self) -> BrowserId: """ return BrowserId(random.getrandbits(32)) - def save_configuration(self, openwpm_version: str, browser_version: str) -> None: - # FIXME I need to find a solution for this - self.logger.error( - "Can't log config as of yet, because it's still not implemented" + def save_configuration( + self, + manager_params: ManagerParamsInternal, + browser_params: List[BrowserParamsInternal], + openwpm_version: str, + browser_version: str, + ) -> None: + assert self.listener_address is not None + sock = DataSocket(self.listener_address) + task_id = random.getrandbits(32) + sock.store_record( + TableName("task"), + FAKE_VISIT_ID, + { + "task_id": task_id, + "manager_params": manager_params.to_json(), # type:ignore + "openwpm_version": openwpm_version, + "browser_version": browser_version, + }, ) + # Record browser details for each brower + for browser_param in browser_params: + sock.store_record( + TableName("crawl"), + FAKE_VISIT_ID, + { + "browser_id": browser_param.browser_id, + "task_id": task_id, + "browser_params": browser_param, + }, + ) + sock.finalize_visit_id(FAKE_VISIT_ID, success=True) def launch(self) -> None: """Starts the data aggregator""" diff --git a/openwpm/task_manager.py b/openwpm/task_manager.py index 1d0640f36..fdec33351 100644 --- a/openwpm/task_manager.py +++ b/openwpm/task_manager.py @@ -31,10 +31,12 @@ from .storage.storage_controller import ( ACTION_TYPE_FINALIZE, RECORD_TYPE_META, + DataSocket, StorageControllerHandle, ) from .storage.storage_providers import ( StructuredStorageProvider, + TableName, UnstructuredStorageProvider, ) from .utilities.multiprocess_utils import kill_process_and_children @@ -155,7 +157,9 @@ def __init__( # Save crawl config information to database openwpm_v, browser_v = get_version() - self.storage_controler_handle.save_configuration(openwpm_v, browser_v) + self.storage_controller_handle.save_configuration( + manager_params, browser_params, openwpm_v, browser_v + ) self.logger.info( get_configuration_string( self.manager_params, browser_params, (openwpm_v, browser_v) @@ -174,7 +178,7 @@ def __enter__(self): """ return self - def __exit__(self): + def __exit__(self) -> None: """ Execute shutdown procedure for TaskManager """ @@ -188,7 +192,7 @@ def _initialize_browsers( for i in range(self.num_browsers): browser_params[ i - ].browser_id = self.storage_controler_handle.get_next_browser_id() + ].browser_id = self.storage_controller_handle.get_next_browser_id() browsers.append(Browser(self.manager_params, browser_params[i])) return browsers @@ -283,18 +287,16 @@ def _launch_aggregators( unstructured_storage_provider: Optional[UnstructuredStorageProvider], ) -> None: """Launch the necessary data aggregators""" - self.storage_controler_handle = StorageControllerHandle( + self.storage_controller_handle = StorageControllerHandle( structured_storage_provider, unstructured_storage_provider ) - self.storage_controler_handle.launch() + self.storage_controller_handle.launch() self.manager_params.storage_controller_address = ( - self.storage_controler_handle.listener_address + self.storage_controller_handle.listener_address ) - - # open connection to aggregator for saving crawl details - self.sock = ClientSocket(serialization="dill") assert self.manager_params.storage_controller_address is not None - self.sock.connect(*self.manager_params.storage_controller_address) + # open connection to storage controller for saving crawl details + self.sock = DataSocket(self.manager_params.storage_controller_address) def _shutdown_manager( self, during_init: bool = False, relaxed: bool = True @@ -327,7 +329,7 @@ def _shutdown_manager( browser.shutdown_browser(during_init, force=not relaxed) self.sock.close() # close socket to data aggregator - self.storage_controler_handle.shutdown(relaxed=relaxed) + self.storage_controller_handle.shutdown(relaxed=relaxed) self.logging_server.close() if hasattr(self, "callback_thread"): self.callback_thread.join() @@ -376,21 +378,20 @@ def _start_thread( "Attempted to execute" " command on a closed TaskManager" ) self._check_failure_status() - visit_id = self.storage_controler_handle.get_next_visit_id() + visit_id = self.storage_controller_handle.get_next_visit_id() browser.set_visit_id(visit_id) if command_sequence.callback: self.unsaved_command_sequences[visit_id] = command_sequence - self.sock.send( - ( - "site_visits", - { - "visit_id": visit_id, - "browser_id": browser.browser_id, - "site_url": command_sequence.url, - "site_rank": command_sequence.site_rank, - }, - ) + self.sock.store_record( + TableName("site_visits"), + visit_id, + { + "visit_id": visit_id, + "browser_id": browser.browser_id, + "site_url": command_sequence.url, + "site_rank": command_sequence.site_rank, + }, ) # Start command execution thread @@ -410,7 +411,7 @@ def _mark_command_sequences_complete(self) -> None: # we're shutting down and have no unprocessed callbacks break - visit_id_list = self.storage_controler_handle.get_new_completed_visits() + visit_id_list = self.storage_controller_handle.get_new_completed_visits() if not visit_id_list: time.sleep(1) continue @@ -435,7 +436,8 @@ def _issue_command( Sends CommandSequence to the BrowserManager one command at a time """ browser.is_fresh = False - + assert browser.browser_id is not None + assert browser.curr_visit_id is not None reset = command_sequence.reset if not reset: self.logger.warning( @@ -519,36 +521,28 @@ def _issue_command( else: raise ValueError("Unknown browser status message %s" % status) - self.sock.send( - ( - "crawl_history", - { - "browser_id": browser.browser_id, - "visit_id": browser.curr_visit_id, - "command": type(command).__name__, - "arguments": json.dumps( - command.__dict__, default=lambda x: repr(x) - ).encode("utf-8"), - "retry_number": command_sequence.retry_number, - "command_status": command_status, - "error": error_text, - "traceback": tb, - "duration": int((time.time_ns() - t1) / 1000000), - }, - ) + self.sock.store_record( + TableName("crawl_history"), + browser.curr_visit_id, + { + "browser_id": browser.browser_id, + "visit_id": browser.curr_visit_id, + "command": type(command).__name__, + "arguments": json.dumps( + command.__dict__, default=lambda x: repr(x) + ).encode("utf-8"), + "retry_number": command_sequence.retry_number, + "command_status": command_status, + "error": error_text, + "traceback": tb, + "duration": int((time.time_ns() - t1) / 1000000), + }, ) if command_status == "critical": - self.sock.send( - ( - RECORD_TYPE_META, - { - "browser_id": browser.browser_id, - "success": False, - "action": ACTION_TYPE_FINALIZE, - "visit_id": browser.curr_visit_id, - }, - ) + self.sock.finalize_visit_id( + success=False, + visit_id=browser.curr_visit_id, ) return @@ -576,16 +570,8 @@ def _issue_command( self.failurecount = 0 if browser.restart_required: - self.sock.send( - ( - RECORD_TYPE_META, - { - "browser_id": browser.browser_id, - "success": False, - "action": ACTION_TYPE_FINALIZE, - "visit_id": browser.curr_visit_id, - }, - ) + self.sock.finalize_visit_id( + success=False, visit_id=browser.curr_visit_id ) break @@ -628,7 +614,7 @@ def execute_command_sequence( """ # Block if the aggregator queue is too large - agg_queue_size = self.storage_controler_handle.get_most_recent_status() + agg_queue_size = self.storage_controller_handle.get_most_recent_status() if agg_queue_size >= STORAGE_CONTROLLER_JOB_LIMIT: while agg_queue_size >= STORAGE_CONTROLLER_JOB_LIMIT: self.logger.info( @@ -636,7 +622,7 @@ def execute_command_sequence( "is below the max queue size of %d. Current queue " "length %d. " % (STORAGE_CONTROLLER_JOB_LIMIT, agg_queue_size) ) - agg_queue_size = self.storage_controler_handle.get_status() + agg_queue_size = self.storage_controller_handle.get_status() # Distribute command if index is None: From 173de3a33c1d904765bd9906130274bafef30c5c Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 22 Jan 2021 18:21:12 +0100 Subject: [PATCH 094/139] WIP need to fix path encoding --- openwpm/storage/storage_controller.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 735fc51c7..96d88a824 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -1,5 +1,6 @@ import asyncio import base64 +import json import logging import queue import random @@ -12,7 +13,7 @@ from openwpm.utilities.multiprocess_utils import Process -from ..config import BrowserParamsInternal, ManagerParamsInternal +from ..config import BrowserParamsInternal, ConfigEncoder, ManagerParamsInternal from ..socket_interface import ClientSocket, get_message_from_reader from ..types import BrowserId, VisitId from .storage_providers import ( @@ -420,7 +421,7 @@ def save_configuration( { "browser_id": browser_param.browser_id, "task_id": task_id, - "browser_params": browser_param, + "browser_params": browser_param.to_json(), # type:ignore }, ) sock.finalize_visit_id(FAKE_VISIT_ID, success=True) From 501dc5c175e6a54e18018b90e12a51ff8b8d626b Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 25 Jan 2021 11:13:35 +0100 Subject: [PATCH 095/139] Fixed path encoding --- openwpm/config.py | 52 +++++++++++++++++------ openwpm/deploy_browsers/deploy_firefox.py | 6 +-- openwpm/storage/storage_controller.py | 12 ++++-- openwpm/task_manager.py | 4 +- 4 files changed, 51 insertions(+), 23 deletions(-) diff --git a/openwpm/config.py b/openwpm/config.py index 588c674a1..0f57da2c4 100644 --- a/openwpm/config.py +++ b/openwpm/config.py @@ -4,7 +4,8 @@ from pathlib import Path from typing import List, Optional, Tuple, Union -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin +from dataclasses_json import config as DCJConfig from .errors import ConfigError from .types import BrowserId @@ -55,9 +56,20 @@ } -@dataclass_json +def str_to_path(string: Optional[str]) -> Optional[Path]: + if string is not None: + return Path(string) + return None + + +def path_to_str(path: Optional[Path]) -> Optional[str]: + if path is not None: + return str(path.resolve()) + return None + + @dataclass -class BrowserParams: +class BrowserParams(DataClassJsonMixin): """ Configuration that might differ per browser @@ -69,7 +81,7 @@ class BrowserParams: extension_enabled: bool = True cookie_instrument: bool = True js_instrument: bool = False - js_instrument_settings: List = field( + js_instrument_settings: List[Union[str, dict]] = field( default_factory=lambda: ["collection_fingerprinting"] ) http_instrument: bool = False @@ -77,7 +89,9 @@ class BrowserParams: save_content: Union[bool, str] = False callstack_instrument: bool = False dns_instrument: bool = False - seed_tar: Optional[Path] = None + seed_tar: Optional[Path] = field( + default=None, metadata=DCJConfig(encoder=path_to_str, decoder=str_to_path) + ) display_mode: str = "native" browser: str = "firefox" prefs: dict = field(default_factory=dict) @@ -89,9 +103,8 @@ class BrowserParams: tracking_protection: bool = False -@dataclass_json @dataclass -class ManagerParams: +class ManagerParams(DataClassJsonMixin): """ Configuration for the TaskManager The configuration will be the same for all browsers running on the same @@ -100,11 +113,24 @@ class ManagerParams: run """ - data_directory: Path = Path("~/openwpm/") - log_directory: Path = Path("~/openwpm/") - screenshot_path: Optional[Path] = None - source_dump_path: Optional[Path] = None - log_file: Path = Path("openwpm.log") + data_directory: Path = field( + default=Path("~/openwpm/"), + metadata=DCJConfig(encoder=path_to_str, decoder=str_to_path), + ) + log_directory: Path = field( + default=Path("~/openwpm/"), + metadata=DCJConfig(encoder=path_to_str, decoder=str_to_path), + ) + screenshot_path: Optional[Path] = field( + default=None, metadata=DCJConfig(encoder=path_to_str, decoder=str_to_path) + ) + source_dump_path: Optional[Path] = field( + default=None, metadata=DCJConfig(encoder=path_to_str, decoder=str_to_path) + ) + log_file: Path = field( + default=Path("openwpm.log"), + metadata=DCJConfig(encoder=path_to_str, decoder=str_to_path), + ) failure_limit: Optional[int] = None testing: bool = False memory_watchdog: bool = False @@ -112,14 +138,12 @@ class ManagerParams: num_browsers: int = 1 -@dataclass_json @dataclass class BrowserParamsInternal(BrowserParams): browser_id: Optional[BrowserId] = None profile_path: Optional[Path] = None -@dataclass_json @dataclass class ManagerParamsInternal(ManagerParams): storage_controller_address: Optional[Tuple[str, int]] = None diff --git a/openwpm/deploy_browsers/deploy_firefox.py b/openwpm/deploy_browsers/deploy_firefox.py index 6deaa46eb..e9ec4a242 100755 --- a/openwpm/deploy_browsers/deploy_firefox.py +++ b/openwpm/deploy_browsers/deploy_firefox.py @@ -2,7 +2,7 @@ import logging import os.path from pathlib import Path -from typing import Optional, Tuple +from typing import Any, Dict, Optional, Tuple from easyprocess import EasyProcessError from multiprocess import Queue @@ -92,8 +92,8 @@ def deploy_firefox( if browser_params.extension_enabled: # Write config file - extension_config = dict() - extension_config.update(browser_params.to_dict()) # type: ignore + extension_config: Dict[str, Any] = dict() + extension_config.update(browser_params.to_dict()) extension_config["logger_address"] = manager_params.logger_address extension_config[ "storage_controller_address" diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 96d88a824..c4a996340 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -340,7 +340,11 @@ def finalize_visit_id(self, visit_id: VisitId, success: bool) -> None: self.socket.send( ( RECORD_TYPE_META, - {"action": ACTION_TYPE_FINALIZE, visit_id: visit_id, success: success}, + { + "action": ACTION_TYPE_FINALIZE, + "visit_id": visit_id, + "success": success, + }, ) ) @@ -408,12 +412,12 @@ def save_configuration( FAKE_VISIT_ID, { "task_id": task_id, - "manager_params": manager_params.to_json(), # type:ignore + "manager_params": manager_params.to_json(), "openwpm_version": openwpm_version, "browser_version": browser_version, }, ) - # Record browser details for each brower + # Record browser details for each browser for browser_param in browser_params: sock.store_record( TableName("crawl"), @@ -421,7 +425,7 @@ def save_configuration( { "browser_id": browser_param.browser_id, "task_id": task_id, - "browser_params": browser_param.to_json(), # type:ignore + "browser_params": browser_param.to_json(), }, ) sock.finalize_visit_id(FAKE_VISIT_ID, success=True) diff --git a/openwpm/task_manager.py b/openwpm/task_manager.py index fdec33351..883f0e240 100644 --- a/openwpm/task_manager.py +++ b/openwpm/task_manager.py @@ -86,9 +86,9 @@ def __init__( validate_browser_params(bp) validate_crawl_configs(manager_params_temp, browser_params_temp) # Mypy doesn't see the methods generated by dataclass_json - manager_params = ManagerParamsInternal(**manager_params_temp.to_dict()) # type: ignore + manager_params = ManagerParamsInternal.from_dict(manager_params_temp.to_dict()) browser_params = [ - BrowserParamsInternal(**bp.to_dict()) for bp in browser_params_temp # type: ignore + BrowserParamsInternal.from_dict(bp.to_dict()) for bp in browser_params_temp ] # Make paths absolute in manager_params From 6bd55751c05aa3bb6b9c26c374312e7c94d9cc5d Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 25 Jan 2021 12:12:49 +0100 Subject: [PATCH 096/139] Added task and crawl to schema --- openwpm/storage/parquet_schema.py | 17 +++++++++++++++++ openwpm/storage/storage_controller.py | 1 - test/storage/test_values.py | 18 ++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/openwpm/storage/parquet_schema.py b/openwpm/storage/parquet_schema.py index d33d61af1..496bb8d21 100644 --- a/openwpm/storage/parquet_schema.py +++ b/openwpm/storage/parquet_schema.py @@ -10,6 +10,23 @@ PQ_SCHEMAS = dict() +fields = [ + pa.field("task_id", pa.int64(), nullable=False), + pa.field("manager_params", pa.string(), nullable=False), + pa.field("openwpm_version", pa.string(), nullable=False), + pa.field("browser_version", pa.string(), nullable=False) + # Omitting start_time as I couldn't figure out how to implement it +] +PQ_SCHEMAS["task"] = pa.schema(fields) + +fields = [ + pa.field("browser_id", pa.uint32(), nullable=False), + pa.field("task_id", pa.int64(), nullable=False), + pa.field("browser_params", pa.string(), nullable=False), + # Omitting start_time as I couldn't figure out how to implement it +] +PQ_SCHEMAS["crawl"] = pa.schema(fields) + # site_visits fields = [ pa.field("visit_id", pa.int64(), nullable=False), diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index c4a996340..92ddafcbb 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -1,6 +1,5 @@ import asyncio import base64 -import json import logging import queue import random diff --git a/test/storage/test_values.py b/test/storage/test_values.py index 4a564ec8c..cc74eb1cf 100644 --- a/test/storage/test_values.py +++ b/test/storage/test_values.py @@ -17,6 +17,24 @@ def random_word(length): return "".join(random.choice(letters) for _ in range(length)) +# task +fields = { + "task_id": random.randint(0, 2 ** 63 - 1), + "manager_params": random_word(12), + "openwpm_version": random_word(12), + "browser_version": random_word(12), +} +TEST_VALUES["task"] = fields + + +# crawl +fields = { + "browser_id": random.randint(0, 2 ** 31 - 1), + "task_id": random.randint(0, 2 ** 63 - 1), + "browser_params": random_word(12), +} +TEST_VALUES["crawl"] = fields + # site_visits fields = { "visit_id": random.randint(0, 2 ** 63 - 1), From eeceaa30bf131b338a58c9c62ded35ea54de4ed1 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 29 Jan 2021 12:40:23 +0100 Subject: [PATCH 097/139] Fixed paths in GitHub actions --- .github/workflows/run-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index fd26569c3..2789f249f 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -46,7 +46,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - test-groups: ["test_[a-e]*", "test_[f-h]*", "test_[i-r,t-z]*", "test_[s]*"] + test-groups: ["test/test_[a-e]*", "test/test_[f-h]*", "test/test_[i-r,t-z]*", "test/test_[s]*"] fail-fast: false steps: # All of these steps are just setup, maybe we should wrap them in an action From d7db8ca1c463c3b62e182d0c9af9179534ae4fd4 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 29 Jan 2021 16:24:34 +0100 Subject: [PATCH 098/139] Refactored completion handling --- .github/workflows/run-tests.yaml | 2 +- openwpm/browser_manager.py | 2 +- openwpm/storage/arrow_storage.py | 20 +++-- openwpm/storage/in_memory_storage.py | 8 +- openwpm/storage/local_storage.py | 3 - openwpm/storage/parquet_schema.py | 4 +- openwpm/storage/storage_controller.py | 64 +++++++++++----- openwpm/storage/storage_providers.py | 4 +- openwpm/task_manager.py | 4 +- test/conftest.py | 1 + ...memory_storage_provider.py => fixtures.py} | 40 +--------- test/storage/test_arrow_cache.py | 2 +- test/storage/test_local_storage_provider.py | 35 --------- test/storage/test_storage_controller.py | 59 ++++++-------- test/storage/test_storage_providers.py | 76 +++++++++++++++++++ test/storage/test_values.py | 35 +++++---- 16 files changed, 193 insertions(+), 166 deletions(-) rename test/storage/{test_memory_storage_provider.py => fixtures.py} (63%) delete mode 100644 test/storage/test_local_storage_provider.py create mode 100644 test/storage/test_storage_providers.py diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index c19cb5c9d..bd84abdf9 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -48,7 +48,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - test-groups: ["test/test_[a-e]*", "test/test_[f-h]*", "test/test_[i-r,t-z]*", "test/test_[s]*"] + test-groups: ["test/test_[a-e]*", "test/test_[f-h]*", "test/test_[i-r,t-z]*", "test/test_[s]*", "test/storage/*"] fail-fast: false steps: # All of these steps are just setup, maybe we should wrap them in an action diff --git a/openwpm/browser_manager.py b/openwpm/browser_manager.py index ed8bce976..ff8d84ef7 100644 --- a/openwpm/browser_manager.py +++ b/openwpm/browser_manager.py @@ -294,7 +294,7 @@ def close_browser_manager(self, force: bool = False): # Send the shutdown command command = ShutdownSignal() - self.command_queue.put((command)) + self.command_queue.put(command) # Verify that webdriver has closed (30 second timeout) try: diff --git a/openwpm/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py index a48b4501b..1ea9c10b5 100644 --- a/openwpm/storage/arrow_storage.py +++ b/openwpm/storage/arrow_storage.py @@ -2,8 +2,9 @@ import logging import random from abc import abstractmethod +from asyncio import Task from collections import defaultdict -from typing import Any, Awaitable, DefaultDict, Dict, List +from typing import Any, DefaultDict, Dict, List import pandas as pd import pyarrow as pa @@ -12,9 +13,8 @@ from openwpm.types import VisitId from .parquet_schema import PQ_SCHEMAS -from .storage_providers import StructuredStorageProvider, TableName +from .storage_providers import INCOMPLETE_VISITS, StructuredStorageProvider, TableName -INCOMPLETE_VISITS = TableName("incomplete_visits") CACHE_SIZE = 500 @@ -47,7 +47,6 @@ def factory_function() -> DefaultDict[TableName, List[Dict[str, Any]]]: async def init(self) -> None: # Used to synchronize the finalizing and the flushing self.storing_lock = asyncio.Lock() - self.flush_event = asyncio.Event() async def store_record( self, table: TableName, visit_id: VisitId, record: Dict[str, Any] @@ -98,7 +97,7 @@ def _is_cache_full(self) -> bool: async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False - ) -> Awaitable[None]: + ) -> Task[None]: """This method is the reason the finalize_visit_id interface returns an awaitable. This was necessary as we needed to enable the following pattern. ``` @@ -134,7 +133,7 @@ async def finalize_visit_id( async def wait_on_condition(e: asyncio.Event) -> None: await e.wait() - return wait_on_condition(event) + return asyncio.create_task(wait_on_condition(event)) @abstractmethod async def write_table(self, table_name: TableName, table: Table) -> None: @@ -165,3 +164,12 @@ async def flush_cache(self, lock: asyncio.Lock = None) -> None: if not got_lock: lock.release() + + async def shutdown(self) -> None: + for table_name, batches in self._batches.items(): + if len(batches) != 0: + self.logger.error( + "While shutting down there were %d cached entries for table %s", + len(batches), + table_name, + ) diff --git a/openwpm/storage/in_memory_storage.py b/openwpm/storage/in_memory_storage.py index 95193188b..371ef65e1 100644 --- a/openwpm/storage/in_memory_storage.py +++ b/openwpm/storage/in_memory_storage.py @@ -8,9 +8,9 @@ import asyncio import logging -from asyncio import Event, Lock +from asyncio import Event, Lock, Task from collections import defaultdict -from typing import Any, Awaitable, DefaultDict, Dict, List, Tuple +from typing import Any, DefaultDict, Dict, List from multiprocess import Queue from pyarrow import Table @@ -76,7 +76,7 @@ async def store_record( async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False - ) -> Awaitable[None]: + ) -> Task[None]: async with self.lock as _: self.signal.clear() self.logger.info( @@ -90,7 +90,7 @@ async def finalize_visit_id( async def wait(signal: Event) -> None: await signal.wait() - return wait(self.signal) + return asyncio.create_task(wait(self.signal)) async def shutdown(self) -> None: if self.cache1 != {} or self.cache2 != {}: diff --git a/openwpm/storage/local_storage.py b/openwpm/storage/local_storage.py index 0ef698942..89aa84e2d 100644 --- a/openwpm/storage/local_storage.py +++ b/openwpm/storage/local_storage.py @@ -19,9 +19,6 @@ def __init__(self, storage_path: Path) -> None: async def write_table(self, table_name: TableName, table: Table) -> None: pq.write_to_dataset(table, str(self.storage_path / table_name)) - async def shutdown(self) -> None: - pass - class LocalGzipProvider(UnstructuredStorageProvider): """ Stores files as storage_path/hash.zip """ diff --git a/openwpm/storage/parquet_schema.py b/openwpm/storage/parquet_schema.py index 496bb8d21..7c11c8302 100644 --- a/openwpm/storage/parquet_schema.py +++ b/openwpm/storage/parquet_schema.py @@ -14,7 +14,8 @@ pa.field("task_id", pa.int64(), nullable=False), pa.field("manager_params", pa.string(), nullable=False), pa.field("openwpm_version", pa.string(), nullable=False), - pa.field("browser_version", pa.string(), nullable=False) + pa.field("browser_version", pa.string(), nullable=False), + pa.field("instance_id", pa.uint32(), nullable=False), # Omitting start_time as I couldn't figure out how to implement it ] PQ_SCHEMAS["task"] = pa.schema(fields) @@ -23,6 +24,7 @@ pa.field("browser_id", pa.uint32(), nullable=False), pa.field("task_id", pa.int64(), nullable=False), pa.field("browser_params", pa.string(), nullable=False), + pa.field("instance_id", pa.uint32(), nullable=False), # Omitting start_time as I couldn't figure out how to implement it ] PQ_SCHEMAS["crawl"] = pa.schema(fields) diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 92ddafcbb..991b86acb 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -5,6 +5,7 @@ import random import socket import time +from asyncio import Task from collections import defaultdict from typing import Any, Awaitable, DefaultDict, Dict, List, NoReturn, Optional, Tuple @@ -33,7 +34,7 @@ STATUS_UPDATE_INTERVAL = 5 # seconds -FAKE_VISIT_ID = VisitId(-1) +INVALID_VISIT_ID = VisitId(-1) class StorageController: @@ -65,7 +66,12 @@ def __init__( self._shutdown_flag = False self._relaxed = False self.logger = logging.getLogger("openwpm") - self.current_tasks: DefaultDict[VisitId, List[asyncio.Task]] = defaultdict(list) + self.store_record_tasks: DefaultDict[VisitId, List[Task[None]]] = defaultdict( + list + ) + """Contains all store_record tasks for a given visit_id""" + self.finalize_tasks: List[Tuple[VisitId, Optional[Task[None]], bool]] = [] + """Contains all information required for update_completion_queue to work""" self.structured_storage = structured_storage self.unstructured_storage = unstructured_storage self._last_record_received: Optional[float] = None @@ -106,8 +112,6 @@ async def handler( self._last_record_received = time.time() record_type, data = record - self.logger.debug("Received record for record_type %s", record_type) - if record_type == RECORD_TYPE_CREATE: raise RuntimeError( f"""{RECORD_TYPE_CREATE} is no longer supported. @@ -150,11 +154,11 @@ async def store_record( self, table_name: TableName, visit_id: VisitId, data: Dict[str, Any] ) -> None: - if visit_id == FAKE_VISIT_ID: + if visit_id == INVALID_VISIT_ID: # Hacking around the fact that task and crawl don't have a VisitID del data["visit_id"] # Turning these into task to be able to have them complete without blocking the socket - self.current_tasks[visit_id].append( + self.store_record_tasks[visit_id].append( asyncio.create_task( self.structured_storage.store_record( table=table_name, visit_id=visit_id, record=data @@ -177,17 +181,15 @@ async def _handle_meta(self, visit_id: VisitId, data: Dict[str, Any]) -> None: if action == ACTION_TYPE_INITIALIZE: return elif action == ACTION_TYPE_FINALIZE: - success = data["success"] + success: bool = data["success"] completion_token = await self.finalize_visit_id(visit_id, success) - if completion_token is not None: - await completion_token - self.completion_queue.put((visit_id, success)) + self.finalize_tasks.append((visit_id, completion_token, success)) else: raise ValueError("Unexpected action: %s", action) async def finalize_visit_id( self, visit_id: VisitId, success: bool - ) -> Optional[Awaitable[None]]: + ) -> Optional[Task[None]]: """Makes sure all records for a given visit_id have been processed before we invoke finalize_visit_id on the structured_storage @@ -197,9 +199,9 @@ async def finalize_visit_id( """ self.logger.info("Awaiting all tasks for visit_id %d", visit_id) - for task in self.current_tasks[visit_id]: + for task in self.store_record_tasks[visit_id]: await task - del self.current_tasks[visit_id] + del self.store_record_tasks[visit_id] self.logger.debug( "Awaited all tasks for visit_id %d while finalizing", visit_id @@ -217,9 +219,9 @@ async def update_status_queue(self) -> NoReturn: """ while True: await asyncio.sleep(STATUS_UPDATE_INTERVAL) - visit_id_count = len(self.current_tasks.keys()) + visit_id_count = len(self.store_record_tasks.keys()) task_count = 0 - for task_list in self.current_tasks.values(): + for task_list in self.store_record_tasks.values(): for task in task_list: if not task.done(): task_count += 1 @@ -233,14 +235,15 @@ async def update_status_queue(self) -> NoReturn: visit_id_count, ) - async def shutdown(self) -> None: + async def shutdown(self, completion_queue_task: Task[None]) -> None: completion_tokens = {} - visit_ids = list(self.current_tasks.keys()) + visit_ids = list(self.store_record_tasks.keys()) for visit_id in visit_ids: t = await self.finalize_visit_id(visit_id, success=False) if t is not None: completion_tokens[visit_id] = t await self.structured_storage.flush_cache() + await completion_queue_task for visit_id, token in completion_tokens.items(): await token self.completion_queue.put((visit_id, False)) @@ -286,6 +289,20 @@ async def save_batch_if_past_timeout(self) -> NoReturn: await self.unstructured_storage.flush_cache() self._last_record_received = None + async def update_completion_queue(self) -> None: + """ All completed finalize_visit_id tasks get put into the completion_queue here """ + while not (self._shutdown_flag and len(self.finalize_tasks) == 0): + # This list is needed because iterating over a list and changing it at the same time + # is forbidden + new_finalize_tasks: List[Tuple[VisitId, Optional[Task[None]], bool]] = [] + for visit_id, token, success in self.finalize_tasks: + if not token or token.done(): + self.completion_queue.put((visit_id, success)) + else: + new_finalize_tasks.append((visit_id, token, success)) + self.finalize_tasks = new_finalize_tasks + await asyncio.sleep(5) + async def _run(self) -> None: await self.structured_storage.init() if self.unstructured_storage: @@ -303,6 +320,10 @@ async def _run(self) -> None: timeout_check = asyncio.create_task( self.save_batch_if_past_timeout(), name="TimeoutCheck" ) + + update_completion_queue = asyncio.create_task( + self.update_completion_queue(), name="CompletionQueueFeeder" + ) # Blocks until we should shutdown await self.should_shutdown() @@ -310,7 +331,7 @@ async def _run(self) -> None: status_queue_update.cancel() timeout_check.cancel() await server.wait_closed() - await self.shutdown() + await self.shutdown(update_completion_queue) def run(self) -> None: logging.getLogger("asyncio").setLevel(logging.WARNING) @@ -323,6 +344,7 @@ class DataSocket: def __init__(self, listener_address: Tuple[str, int]) -> None: self.socket = ClientSocket(serialization="dill") self.socket.connect(*listener_address) + self.logger = logging.getLogger("openwpm") def store_record( self, table_name: TableName, visit_id: VisitId, data: Dict[str, Any] @@ -408,7 +430,7 @@ def save_configuration( task_id = random.getrandbits(32) sock.store_record( TableName("task"), - FAKE_VISIT_ID, + INVALID_VISIT_ID, { "task_id": task_id, "manager_params": manager_params.to_json(), @@ -420,14 +442,14 @@ def save_configuration( for browser_param in browser_params: sock.store_record( TableName("crawl"), - FAKE_VISIT_ID, + INVALID_VISIT_ID, { "browser_id": browser_param.browser_id, "task_id": task_id, "browser_params": browser_param.to_json(), }, ) - sock.finalize_visit_id(FAKE_VISIT_ID, success=True) + sock.finalize_visit_id(INVALID_VISIT_ID, success=True) def launch(self) -> None: """Starts the data aggregator""" diff --git a/openwpm/storage/storage_providers.py b/openwpm/storage/storage_providers.py index 7ec081a45..bcc9e94b2 100644 --- a/openwpm/storage/storage_providers.py +++ b/openwpm/storage/storage_providers.py @@ -6,6 +6,7 @@ import gzip import io from abc import ABC, abstractmethod +from asyncio import Task from typing import Any, Awaitable, Dict, NewType, Optional from pyarrow.lib import Table @@ -13,6 +14,7 @@ from openwpm.types import VisitId TableName = NewType("TableName", str) +INCOMPLETE_VISITS = TableName("incomplete_visits") class StorageProvider(ABC): @@ -59,7 +61,7 @@ async def store_record( @abstractmethod async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False - ) -> Optional[Awaitable[None]]: + ) -> Optional[Task[None]]: """This method is invoked to inform the StructuredStorageProvider that no more records for this visit_id will be submitted diff --git a/openwpm/task_manager.py b/openwpm/task_manager.py index 883f0e240..a9cfd727b 100644 --- a/openwpm/task_manager.py +++ b/openwpm/task_manager.py @@ -98,7 +98,9 @@ def __init__( if manager_params.log_directory: manager_params.log_directory = manager_params.log_directory.expanduser() - manager_params.log_file = manager_params.log_directory / manager_params.log_file + manager_params.log_file = ( + manager_params.log_directory / manager_params.log_file.name + ) manager_params.screenshot_path = manager_params.data_directory / "screenshots" manager_params.source_dump_path = manager_params.data_directory / "sources" diff --git a/test/conftest.py b/test/conftest.py index b0001f675..0b31b48d8 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -21,6 +21,7 @@ "Extension", "firefox", ) +pytest_plugins = "test.storage.fixtures" @pytest.fixture(scope="session") diff --git a/test/storage/test_memory_storage_provider.py b/test/storage/fixtures.py similarity index 63% rename from test/storage/test_memory_storage_provider.py rename to test/storage/fixtures.py index 3f58b583a..89f719001 100644 --- a/test/storage/test_memory_storage_provider.py +++ b/test/storage/fixtures.py @@ -1,5 +1,4 @@ -import asyncio -from typing import Any, Dict, List, Tuple, Union +from typing import Any, List import pytest from _pytest.fixtures import FixtureRequest @@ -14,12 +13,8 @@ from openwpm.storage.sql_provider import SqlLiteStorageProvider from openwpm.storage.storage_providers import ( StructuredStorageProvider, - TableName, UnstructuredStorageProvider, ) -from openwpm.types import VisitId - -from ..openwpmtest import OpenWPMTest memory_structured = "memory_structured" sqllite = "sqllite" @@ -49,26 +44,6 @@ def structured_provider( memory_arrow, ] - -@pytest.mark.parametrize("structured_provider", structured_scenarios, indirect=True) -@pytest.mark.asyncio -async def test_basic_access(structured_provider: StructuredStorageProvider) -> None: - data = { - "visit_id": 2, - "browser_id": 3, - "site_url": "https://example.com", - } - - await structured_provider.init() - - await structured_provider.store_record(TableName("site_visits"), VisitId(2), data) - token = await structured_provider.finalize_visit_id(VisitId(2)) - await structured_provider.flush_cache() - if token is not None: - await token - await structured_provider.shutdown() - - # Unstructured Providers memory_unstructured = "memory_unstructured" leveldb = "leveldb" @@ -94,16 +69,3 @@ def unstructured_provider( unstructured_scenarios: List[str] = [memory_unstructured, leveldb, local_gzip] - - -@pytest.mark.parametrize("unstructured_provider", unstructured_scenarios, indirect=True) -@pytest.mark.asyncio -async def test_basic_unstructured_storing( - unstructured_provider: UnstructuredStorageProvider, -) -> None: - test_string = "This is my test string" - blob = test_string.encode() - await unstructured_provider.init() - await unstructured_provider.store_blob("test", blob) - await unstructured_provider.flush_cache() - await unstructured_provider.shutdown() diff --git a/test/storage/test_arrow_cache.py b/test/storage/test_arrow_cache.py index 95b5a0337..ec8ccfa29 100644 --- a/test/storage/test_arrow_cache.py +++ b/test/storage/test_arrow_cache.py @@ -17,7 +17,7 @@ async def test_arrow_cache(mp_logger: MPLogger) -> None: prov = MemoryArrowProvider() await prov.init() - site_visit = TEST_VALUES["site_visits"] + site_visit = TEST_VALUES[TableName("site_visits")] for j in range(5): # Testing that the cache works repeatedly d: Dict[VisitId, Awaitable[None]] = {} for i in range(CACHE_SIZE + 1): diff --git a/test/storage/test_local_storage_provider.py b/test/storage/test_local_storage_provider.py deleted file mode 100644 index 67fbcae2e..000000000 --- a/test/storage/test_local_storage_provider.py +++ /dev/null @@ -1,35 +0,0 @@ -import asyncio - -import pytest -from pandas import DataFrame -from pyarrow.parquet import ParquetDataset - -from openwpm.storage.local_storage import LocalArrowProvider -from openwpm.storage.storage_providers import TableName -from openwpm.types import VisitId - -from .test_values import TEST_VALUES - - -@pytest.mark.asyncio -async def test_local_arrow_storage_provider(tmp_path, mp_logger): - structured_provider = LocalArrowProvider(tmp_path) - await structured_provider.init() - visit_ids = set() - for table_name, test_data in TEST_VALUES.items(): - visit_id = VisitId(test_data["visit_id"]) - visit_ids.add(visit_id) - await structured_provider.store_record( - TableName(table_name), visit_id, test_data - ) - token_list = [] - for i in visit_ids: - token_list.append(await structured_provider.finalize_visit_id(i)) - await structured_provider.flush_cache() - await asyncio.gather(*token_list) - for table_name, test_data in TEST_VALUES.items(): - dataset = ParquetDataset(tmp_path / table_name) - df: DataFrame = dataset.read().to_pandas() - assert df.shape[0] == 1 - for row in df.itertuples(index=False): - assert row._asdict() == test_data diff --git a/test/storage/test_storage_controller.py b/test/storage/test_storage_controller.py index a77c3e343..4a142c148 100644 --- a/test/storage/test_storage_controller.py +++ b/test/storage/test_storage_controller.py @@ -16,8 +16,10 @@ ) from openwpm.storage.storage_controller import ( ACTION_TYPE_FINALIZE, + INVALID_VISIT_ID, RECORD_TYPE_META, SHUTDOWN_SIGNAL, + DataSocket, StorageController, StorageControllerHandle, ) @@ -26,22 +28,18 @@ def test_startup_and_shutdown(mp_logger: MPLogger) -> None: structured = MemoryStructuredProvider() - unstructured = MemoryUnstructuredProvider() - controller_handle = StorageControllerHandle(structured, unstructured) + controller_handle = StorageControllerHandle(structured, None) controller_handle.launch() assert controller_handle.listener_address is not None - cs = ClientSocket() - cs.connect(*controller_handle.listener_address) + cs = DataSocket(controller_handle.listener_address) for table, data in TEST_VALUES.items(): - cs.send((table, data)) + visit_id = data["visit_id"] if "visit_id" in data else INVALID_VISIT_ID + cs.store_record( + table, visit_id, dict(**data) + ) # cloning to avoid the modifications in store_record - for visit_id in TEST_VISIT_IDS: - cs.send( - ( - RECORD_TYPE_META, - {"action": ACTION_TYPE_FINALIZE, "visit_id": visit_id, "success": True}, - ) - ) + for visit_id in [*TEST_VISIT_IDS, INVALID_VISIT_ID]: + cs.finalize_visit_id(visit_id, True) controller_handle.shutdown() handle = structured.handle handle.poll_queue() @@ -49,42 +47,29 @@ def test_startup_and_shutdown(mp_logger: MPLogger) -> None: assert handle.storage[table] == [data] -@pytest.mark.asyncio -async def test_arrow_provider(mp_logger: MPLogger) -> None: +def test_arrow_provider(mp_logger: MPLogger) -> None: structured = MemoryArrowProvider() - status_queue = Queue() - completion_queue = Queue() - shutdown_queue = Queue() - - storage_controller = StorageController( - structured, - None, - status_queue=status_queue, - completion_queue=completion_queue, - shutdown_queue=shutdown_queue, - ) - task = asyncio.create_task(storage_controller._run()) - cs = ClientSocket() - while status_queue.empty(): - await asyncio.sleep(5) + controller_handle = StorageControllerHandle(structured, None) + controller_handle.launch() - cs.connect(*status_queue.get()) + assert controller_handle.listener_address is not None + cs = DataSocket(controller_handle.listener_address) for table, data in TEST_VALUES.items(): - cs.send((table, data)) + visit_id = data["visit_id"] if "visit_id" in data else INVALID_VISIT_ID + cs.store_record( + table, visit_id, dict(**data) + ) # cloning to avoid the modifications in store_record # This sleep needs to be here because otherwise it is executing blockingly on the single thread, # so the server doesn't ever wake up - await asyncio.sleep(1) - shutdown_queue.put((SHUTDOWN_SIGNAL, True)) - await task + cs.close() + controller_handle.shutdown() handle = structured.handle handle.poll_queue() for table, data in TEST_VALUES.items(): - if table == "incomplete_visits": - # We currently mark all of them as failed, because we don't bother sending a finalize command - continue + t1 = handle.storage[table][0].to_pandas().drop(columns=["instance_id"]) t2 = pd.DataFrame({k: [v] for k, v in data.items()}) # Since t2 doesn't get created schema the inferred types are different diff --git a/test/storage/test_storage_providers.py b/test/storage/test_storage_providers.py new file mode 100644 index 000000000..ecb712cba --- /dev/null +++ b/test/storage/test_storage_providers.py @@ -0,0 +1,76 @@ +import asyncio + +import pytest +from pandas import DataFrame +from pyarrow.parquet import ParquetDataset + +from openwpm.storage.local_storage import LocalArrowProvider +from openwpm.storage.storage_controller import INVALID_VISIT_ID +from openwpm.storage.storage_providers import ( + StructuredStorageProvider, + TableName, + UnstructuredStorageProvider, +) +from openwpm.types import VisitId + +from .fixtures import structured_scenarios, unstructured_scenarios +from .test_values import TEST_VALUES + + +@pytest.mark.asyncio +async def test_local_arrow_storage_provider(tmp_path, mp_logger): + structured_provider = LocalArrowProvider(tmp_path) + await structured_provider.init() + visit_ids = set() + for table_name, test_data in TEST_VALUES.items(): + try: + visit_id = VisitId(test_data["visit_id"]) + except KeyError: + visit_id = INVALID_VISIT_ID + visit_ids.add(visit_id) + await structured_provider.store_record( + TableName(table_name), visit_id, test_data + ) + token_list = [] + for i in visit_ids: + token_list.append(await structured_provider.finalize_visit_id(i)) + await structured_provider.flush_cache() + await asyncio.gather(*token_list) + for table_name, test_data in TEST_VALUES.items(): + dataset = ParquetDataset(tmp_path / table_name) + df: DataFrame = dataset.read().to_pandas() + assert df.shape[0] == 1 + for row in df.itertuples(index=False): + assert row._asdict() == test_data + + +@pytest.mark.parametrize("structured_provider", structured_scenarios, indirect=True) +@pytest.mark.asyncio +async def test_basic_access(structured_provider: StructuredStorageProvider) -> None: + data = { + "visit_id": 2, + "browser_id": 3, + "site_url": "https://example.com", + } + + await structured_provider.init() + + await structured_provider.store_record(TableName("site_visits"), VisitId(2), data) + token = await structured_provider.finalize_visit_id(VisitId(2)) + await structured_provider.flush_cache() + if token is not None: + await token + await structured_provider.shutdown() + + +@pytest.mark.parametrize("unstructured_provider", unstructured_scenarios, indirect=True) +@pytest.mark.asyncio +async def test_basic_unstructured_storing( + unstructured_provider: UnstructuredStorageProvider, +) -> None: + test_string = "This is my test string" + blob = test_string.encode() + await unstructured_provider.init() + await unstructured_provider.store_blob("test", blob) + await unstructured_provider.flush_cache() + await unstructured_provider.shutdown() diff --git a/test/storage/test_values.py b/test/storage/test_values.py index cc74eb1cf..461147b2b 100644 --- a/test/storage/test_values.py +++ b/test/storage/test_values.py @@ -8,8 +8,11 @@ import random import string +from typing import Any, Dict -TEST_VALUES = dict() +from openwpm.storage.storage_providers import TableName + +TEST_VALUES: Dict[TableName, Dict[str, Any]] = dict() def random_word(length): @@ -24,7 +27,7 @@ def random_word(length): "openwpm_version": random_word(12), "browser_version": random_word(12), } -TEST_VALUES["task"] = fields +TEST_VALUES[TableName("task")] = fields # crawl @@ -33,7 +36,7 @@ def random_word(length): "task_id": random.randint(0, 2 ** 63 - 1), "browser_params": random_word(12), } -TEST_VALUES["crawl"] = fields +TEST_VALUES[TableName("crawl")] = fields # site_visits fields = { @@ -42,7 +45,7 @@ def random_word(length): "site_url": random_word(12), "site_rank": random.randint(0, 2 ** 31 - 1), } -TEST_VALUES["site_visits"] = fields +TEST_VALUES[TableName("site_visits")] = fields # crawl_history fields = { @@ -56,7 +59,7 @@ def random_word(length): "traceback": random_word(12), "duration": random.randint(0, 2 ** 63 - 1), } -TEST_VALUES["crawl_history"] = fields +TEST_VALUES[TableName("crawl_history")] = fields # http_requests fields = { @@ -88,7 +91,7 @@ def random_word(length): "post_body_raw": random_word(12), "time_stamp": random_word(12), } -TEST_VALUES["http_requests"] = fields +TEST_VALUES[TableName("http_requests")] = fields # http_responses fields = { @@ -111,7 +114,7 @@ def random_word(length): "time_stamp": random_word(12), "content_hash": random_word(12), } -TEST_VALUES["http_responses"] = fields +TEST_VALUES[TableName("http_responses")] = fields # http_redirects fields = { @@ -132,7 +135,7 @@ def random_word(length): "headers": random_word(12), "time_stamp": random_word(12), } -TEST_VALUES["http_redirects"] = fields +TEST_VALUES[TableName("http_redirects")] = fields # javascript fields = { @@ -159,7 +162,7 @@ def random_word(length): "arguments": random_word(12), "time_stamp": random_word(12), } -TEST_VALUES["javascript"] = fields +TEST_VALUES[TableName("javascript")] = fields # javascript_cookies fields = { @@ -183,7 +186,7 @@ def random_word(length): "store_id": random_word(12), "time_stamp": random_word(12), } -TEST_VALUES["javascript_cookies"] = fields +TEST_VALUES[TableName("javascript_cookies")] = fields # navigations fields = { @@ -212,7 +215,7 @@ def random_word(length): "committed_event_ordinal": random.randint(0, 2 ** 63 - 1), "time_stamp": random_word(12), } -TEST_VALUES["navigations"] = fields +TEST_VALUES[TableName("navigations")] = fields # callstacks fields = { @@ -221,13 +224,13 @@ def random_word(length): "browser_id": random.randint(0, 2 ** 31 - 1), "call_stack": random_word(12), } -TEST_VALUES["callstacks"] = fields +TEST_VALUES[TableName("callstacks")] = fields # incomplete_visits fields = { "visit_id": random.randint(0, 2 ** 63 - 1), } -TEST_VALUES["incomplete_visits"] = fields +TEST_VALUES[TableName("incomplete_visits")] = fields # dns_responses fields = { @@ -240,6 +243,8 @@ def random_word(length): "is_TRR": random.choice([True, False]), "time_stamp": random_word(12), } -TEST_VALUES["dns_responses"] = fields +TEST_VALUES[TableName("dns_responses")] = fields -TEST_VISIT_IDS = [d["visit_id"] for d in TEST_VALUES.values()] +TEST_VISIT_IDS = [ + d["visit_id"] for d in filter(lambda d: "visit_id" in d, TEST_VALUES.values()) +] From d5733db960fbf12ae660edc8082ae14636671d94 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 29 Jan 2021 17:41:50 +0100 Subject: [PATCH 099/139] Fix tests --- test/storage/test_storage_controller.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/storage/test_storage_controller.py b/test/storage/test_storage_controller.py index 4a142c148..e3ac03ead 100644 --- a/test/storage/test_storage_controller.py +++ b/test/storage/test_storage_controller.py @@ -40,7 +40,9 @@ def test_startup_and_shutdown(mp_logger: MPLogger) -> None: for visit_id in [*TEST_VISIT_IDS, INVALID_VISIT_ID]: cs.finalize_visit_id(visit_id, True) + cs.close() controller_handle.shutdown() + handle = structured.handle handle.poll_queue() for table, data in TEST_VALUES.items(): @@ -61,8 +63,8 @@ def test_arrow_provider(mp_logger: MPLogger) -> None: table, visit_id, dict(**data) ) # cloning to avoid the modifications in store_record - # This sleep needs to be here because otherwise it is executing blockingly on the single thread, - # so the server doesn't ever wake up + for visit_id in [*TEST_VISIT_IDS, INVALID_VISIT_ID]: + cs.finalize_visit_id(visit_id, True) cs.close() controller_handle.shutdown() From 6ee997277d1a13acdc0a999f494b70063de0cabb Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 1 Feb 2021 10:49:46 +0100 Subject: [PATCH 100/139] Trying to fix tests on CI --- test/storage/test_values.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/storage/test_values.py b/test/storage/test_values.py index 461147b2b..8ae915fcc 100644 --- a/test/storage/test_values.py +++ b/test/storage/test_values.py @@ -245,6 +245,6 @@ def random_word(length): } TEST_VALUES[TableName("dns_responses")] = fields -TEST_VISIT_IDS = [ +TEST_VISIT_IDS = set( d["visit_id"] for d in filter(lambda d: "visit_id" in d, TEST_VALUES.values()) -] +) From 89635c26b915d9fa0116450fea7fb6f48c10d336 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 1 Feb 2021 10:55:58 +0100 Subject: [PATCH 101/139] Removed redundant setting of tag --- openwpm/mp_logger.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/openwpm/mp_logger.py b/openwpm/mp_logger.py index fd6a84596..2d0f64483 100644 --- a/openwpm/mp_logger.py +++ b/openwpm/mp_logger.py @@ -207,16 +207,6 @@ def _initialize_sentry(self): ) self._event_handler = EventHandler(level=self._log_level_sentry_event) sentry_sdk.init(dsn=self._sentry_dsn, before_send=self._sentry_before_send) - with sentry_sdk.configure_scope() as scope: - if self._crawl_context: - scope.set_tag( - "CRAWL_REFERENCE", - "%s/%s" - % ( - self._crawl_context.get("s3_bucket", "UNKNOWN"), - self._crawl_context.get("s3_directory", "UNKNOWN"), - ), - ) def _start_listener(self): """Start listening socket for remote logs from extension""" From d4a391d4b0eaae58f07b9371a39089d6e38160da Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 1 Feb 2021 11:30:51 +0100 Subject: [PATCH 102/139] Removing references to S3 --- crawler.py | 2 +- scripts/ci.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crawler.py b/crawler.py index b8e394072..04b405faa 100644 --- a/crawler.py +++ b/crawler.py @@ -116,7 +116,7 @@ with sentry_sdk.configure_scope() as scope: # tags generate breakdown charts and search filters scope.set_tag("CRAWL_DIRECTORY", CRAWL_DIRECTORY) - scope.set_tag("S3_BUCKET", GCS_BUCKET) + scope.set_tag("GCS_BUCKET", GCS_BUCKET) scope.set_tag("DISPLAY_MODE", DISPLAY_MODE) scope.set_tag("HTTP_INSTRUMENT", HTTP_INSTRUMENT) scope.set_tag("COOKIE_INSTRUMENT", COOKIE_INSTRUMENT) diff --git a/scripts/ci.sh b/scripts/ci.sh index 7ad64ecee..e192d9b72 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -1,6 +1,6 @@ #!/bin/bash -python -m pytest --cov=../openwpm --cov-report=xml $TESTS -s -v --durations=10; +python -m pytest --cov=openwpm --cov-report=xml $TESTS -s -v --durations=10; exit_code=$?; if [[ "$exit_code" -ne 0 ]]; then exit $exit_code; From ffbb346f66d8029fb2dcc019d962c656d0502b06 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 1 Feb 2021 11:46:48 +0100 Subject: [PATCH 103/139] Purging more DataAggregator references --- openwpm/storage/in_memory_storage.py | 14 +++++++++----- openwpm/storage/storage_controller.py | 12 ++++++------ openwpm/task_manager.py | 16 ++++++++-------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/openwpm/storage/in_memory_storage.py b/openwpm/storage/in_memory_storage.py index 371ef65e1..02c13a8eb 100644 --- a/openwpm/storage/in_memory_storage.py +++ b/openwpm/storage/in_memory_storage.py @@ -37,7 +37,6 @@ class MemoryStructuredProvider(StructuredStorageProvider): """ lock: Lock - signal: Event def __init__(self) -> None: super().__init__() @@ -50,9 +49,9 @@ def __init__(self) -> None: """The cache for entries before they are finalized""" self.cache2: DefaultDict[TableName, List[Dict[str, Any]]] = defaultdict(list) """For all entries that have been finalized but not yet flushed out to the queue""" + self.signal_list: List[Event] = [] async def init(self) -> None: - self.signal = asyncio.Event() self.lock = asyncio.Lock() async def flush_cache(self) -> None: @@ -64,7 +63,8 @@ async def flush_cache(self) -> None: for record in record_list: self.queue.put((table, record)) self.cache2.clear() - self.signal.set() + for ev in self.signal_list: + ev.set() async def store_record( self, table: TableName, visit_id: VisitId, record: Dict[str, Any] @@ -78,7 +78,6 @@ async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False ) -> Task[None]: async with self.lock as _: - self.signal.clear() self.logger.info( f"Finalizing visit_id {visit_id} which was {'' if interrupted else 'not'} interrupted" ) @@ -90,7 +89,9 @@ async def finalize_visit_id( async def wait(signal: Event) -> None: await signal.wait() - return asyncio.create_task(wait(self.signal)) + ev = Event() + self.signal_list.append(ev) + return asyncio.create_task(wait(ev)) async def shutdown(self) -> None: if self.cache1 != {} or self.cache2 != {}: @@ -124,6 +125,8 @@ async def init(self) -> None: def __init__(self) -> None: self.storage: Dict[str, bytes] = {} + self.queue = Queue() + self.handle = MemoryProviderHandle(self.queue) async def store_blob( self, @@ -138,6 +141,7 @@ async def store_blob( bytesIO = self._compress(blob) blob = bytesIO.getvalue() self.storage[filename] = blob + self.queue.put((filename, blob)) async def flush_cache(self) -> None: pass diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 991b86acb..b661424db 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -169,7 +169,7 @@ async def store_record( async def _handle_meta(self, visit_id: VisitId, data: Dict[str, Any]) -> None: """ Messages for the table RECORD_TYPE_SPECIAL are meta information - communicated to the aggregator + communicated to the storage controller Supported message types: - finalize: A message sent by the extension to signal that a visit_id is complete. @@ -375,7 +375,7 @@ def close(self) -> None: class StorageControllerHandle: """This class contains all methods relevant for the TaskManager - to interact with the DataAggregator + to interact with the StorageController """ def __init__( @@ -452,7 +452,7 @@ def save_configuration( sock.finalize_visit_id(INVALID_VISIT_ID, success=True) def launch(self) -> None: - """Starts the data aggregator""" + """Starts the storage controller""" self.storage_controller = Process( name="StorageController", target=StorageController.run, @@ -478,7 +478,7 @@ def get_new_completed_visits(self) -> List[Tuple[int, bool]]: return finished_visit_ids def shutdown(self, relaxed: bool = True) -> None: - """ Terminate the aggregator listener process""" + """ Terminate the storage controller process""" assert isinstance(self.storage_controller, Process) self.logger.debug("Sending the shutdown signal to the Storage Controller...") self.shutdown_queue.put((SHUTDOWN_SIGNAL, relaxed)) @@ -504,7 +504,7 @@ def get_most_recent_status(self) -> int: # Check last status signal if (time.time() - self._last_status_received) > STATUS_TIMEOUT: raise RuntimeError( - "No status update from DataAggregator listener process " + "No status update from the storage controller process " "for %d seconds." % (time.time() - self._last_status_received) ) @@ -520,7 +520,7 @@ def get_status(self) -> int: except queue.Empty: assert self._last_status_received is not None raise RuntimeError( - "No status update from DataAggregator listener process " + "No status update from the storage controller process " "for %d seconds." % (time.time() - self._last_status_received) ) assert isinstance(self._last_status, int) diff --git a/openwpm/task_manager.py b/openwpm/task_manager.py index a9cfd727b..f18c2a814 100644 --- a/openwpm/task_manager.py +++ b/openwpm/task_manager.py @@ -142,8 +142,8 @@ def __init__( self.manager_params.logger_address = self.logging_server.logger_address self.logger = logging.getLogger("openwpm") - # Initialize the data aggregators - self._launch_aggregators( + # Initialize the storage controller + self._launch_storage_controller( structured_storage_provider, unstructured_storage_provider ) @@ -283,12 +283,11 @@ def _manager_watchdog(self) -> None: ) kill_process_and_children(process, self.logger) - def _launch_aggregators( + def _launch_storage_controller( self, structured_storage_provider: StructuredStorageProvider, unstructured_storage_provider: Optional[UnstructuredStorageProvider], ) -> None: - """Launch the necessary data aggregators""" self.storage_controller_handle = StorageControllerHandle( structured_storage_provider, unstructured_storage_provider ) @@ -330,7 +329,7 @@ def _shutdown_manager( browser.command_thread.join() browser.shutdown_browser(during_init, force=not relaxed) - self.sock.close() # close socket to data aggregator + self.sock.close() # close socket to storage controller self.storage_controller_handle.shutdown(relaxed=relaxed) self.logging_server.close() if hasattr(self, "callback_thread"): @@ -405,7 +404,7 @@ def _start_thread( return thread def _mark_command_sequences_complete(self) -> None: - """Polls the data aggregator for saved records + """Polls the storage controller for saved records and calls their callbacks """ while True: @@ -615,12 +614,13 @@ def execute_command_sequence( int -> index of browser to send command to """ - # Block if the aggregator queue is too large + # Block if the storage controller has too many unfinished records + # TODO create tests that these numbers are still meaningful agg_queue_size = self.storage_controller_handle.get_most_recent_status() if agg_queue_size >= STORAGE_CONTROLLER_JOB_LIMIT: while agg_queue_size >= STORAGE_CONTROLLER_JOB_LIMIT: self.logger.info( - "Blocking command submission until the DataAggregator " + "Blocking command submission until the storage controller " "is below the max queue size of %d. Current queue " "length %d. " % (STORAGE_CONTROLLER_JOB_LIMIT, agg_queue_size) ) From 379af2d19c06db3aa81b7b733c5f7e76f9eec0e9 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 1 Feb 2021 11:54:58 +0100 Subject: [PATCH 104/139] Craking up logging to figure out test failure --- test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index 0b31b48d8..af8a95b6a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -108,7 +108,7 @@ def parameterize( @pytest.fixture() def mp_logger(tmp_path): log_path = tmp_path / "openwpm.log" - logger = MPLogger(log_path) + logger = MPLogger(log_path, log_level_console=logging.DEBUG) yield logger logger.close() # The performance hit for this might be unacceptable but it might help us discover bugs From e5c897bab8c1b351058f4b97e31dce2c4977b8a5 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 1 Feb 2021 12:32:57 +0100 Subject: [PATCH 105/139] Moved test_values into a fixture --- test/conftest.py | 6 +- test/storage/fixtures.py | 17 +- test/storage/test_arrow_cache.py | 7 +- test/storage/test_gcp.py | 10 +- test/storage/test_storage_controller.py | 47 +-- test/storage/test_storage_providers.py | 21 +- test/storage/test_values.py | 466 ++++++++++++------------ 7 files changed, 283 insertions(+), 291 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index af8a95b6a..7d9ee1c83 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -2,7 +2,7 @@ import os import subprocess from pathlib import Path -from typing import Callable, List, Tuple +from typing import Any, Callable, Generator, List, Tuple import pytest @@ -89,7 +89,7 @@ def _create_task_manager( @pytest.fixture() def http_params( - default_params, + default_params: Tuple[ManagerParams, List[BrowserParams]], ) -> Callable[[str], Tuple[ManagerParams, List[BrowserParams]]]: manager_params, browser_params = default_params for browser_param in browser_params: @@ -106,7 +106,7 @@ def parameterize( @pytest.fixture() -def mp_logger(tmp_path): +def mp_logger(tmp_path: Path) -> Generator[MPLogger, Any, None]: log_path = tmp_path / "openwpm.log" logger = MPLogger(log_path, log_level_console=logging.DEBUG) yield logger diff --git a/test/storage/fixtures.py b/test/storage/fixtures.py index 89f719001..793f745e4 100644 --- a/test/storage/fixtures.py +++ b/test/storage/fixtures.py @@ -1,4 +1,4 @@ -from typing import Any, List +from typing import Any, Dict, List, Set, Tuple import pytest from _pytest.fixtures import FixtureRequest @@ -11,10 +11,14 @@ from openwpm.storage.leveldb import LevelDbProvider from openwpm.storage.local_storage import LocalGzipProvider from openwpm.storage.sql_provider import SqlLiteStorageProvider +from openwpm.storage.storage_controller import INVALID_VISIT_ID from openwpm.storage.storage_providers import ( StructuredStorageProvider, + TableName, UnstructuredStorageProvider, ) +from openwpm.types import VisitId +from test.storage.test_values import dt_test_values, generate_test_values memory_structured = "memory_structured" sqllite = "sqllite" @@ -69,3 +73,14 @@ def unstructured_provider( unstructured_scenarios: List[str] = [memory_unstructured, leveldb, local_gzip] + + +@pytest.fixture +def test_values() -> dt_test_values: + data, visit_ids = generate_test_values() + for table in data.values(): + table["visit_id"] = ( + table["visit_id"] if "visit_id" in table else INVALID_VISIT_ID + ) + visit_ids.add(INVALID_VISIT_ID) + return data, visit_ids diff --git a/test/storage/test_arrow_cache.py b/test/storage/test_arrow_cache.py index ec8ccfa29..b728b1256 100644 --- a/test/storage/test_arrow_cache.py +++ b/test/storage/test_arrow_cache.py @@ -1,5 +1,4 @@ import asyncio -import random from typing import Awaitable, Dict import pytest @@ -10,14 +9,14 @@ from openwpm.storage.in_memory_storage import MemoryArrowProvider from openwpm.storage.storage_providers import TableName from openwpm.types import VisitId -from test.storage.test_values import TEST_VALUES +from test.storage.test_values import dt_test_values @pytest.mark.asyncio -async def test_arrow_cache(mp_logger: MPLogger) -> None: +async def test_arrow_cache(mp_logger: MPLogger, test_values: dt_test_values) -> None: prov = MemoryArrowProvider() await prov.init() - site_visit = TEST_VALUES[TableName("site_visits")] + site_visit = test_values[0][TableName("site_visits")] for j in range(5): # Testing that the cache works repeatedly d: Dict[VisitId, Awaitable[None]] = {} for i in range(CACHE_SIZE + 1): diff --git a/test/storage/test_gcp.py b/test/storage/test_gcp.py index 470cd157d..d2a53c511 100644 --- a/test/storage/test_gcp.py +++ b/test/storage/test_gcp.py @@ -5,12 +5,12 @@ from openwpm.storage.cloud_storage.gcp_storage import GcsStructuredProvider from openwpm.storage.storage_providers import TableName from openwpm.types import VisitId -from test.storage.test_values import TEST_VALUES, TEST_VISIT_IDS @pytest.mark.skip @pytest.mark.asyncio -async def test_gcp_structured(mp_logger): +async def test_gcp_structured(mp_logger, test_values): + tables, visit_ids = test_values project = "senglehardt-openwpm-test-1" bucket_name = "openwpm-test-bucket" structured_provider = GcsStructuredProvider( @@ -21,14 +21,12 @@ async def test_gcp_structured(mp_logger): ) await structured_provider.init() - for table_name, test_data in TEST_VALUES.items(): + for table_name, test_data in tables.items(): visit_id = VisitId(test_data["visit_id"]) await structured_provider.store_record( TableName(table_name), visit_id, test_data ) - finalize_token = [ - await structured_provider.finalize_visit_id(i) for i in TEST_VISIT_IDS - ] + finalize_token = [await structured_provider.finalize_visit_id(i) for i in visit_ids] await structured_provider.flush_cache() for token in finalize_token: await token diff --git a/test/storage/test_storage_controller.py b/test/storage/test_storage_controller.py index e3ac03ead..98eefcd07 100644 --- a/test/storage/test_storage_controller.py +++ b/test/storage/test_storage_controller.py @@ -1,55 +1,47 @@ -import asyncio -import logging -import time - import pandas as pd -import pytest -from multiprocess import Queue from pandas.testing import assert_frame_equal from openwpm.mp_logger import MPLogger -from openwpm.socket_interface import ClientSocket from openwpm.storage.in_memory_storage import ( MemoryArrowProvider, MemoryStructuredProvider, - MemoryUnstructuredProvider, ) from openwpm.storage.storage_controller import ( - ACTION_TYPE_FINALIZE, INVALID_VISIT_ID, - RECORD_TYPE_META, - SHUTDOWN_SIGNAL, DataSocket, - StorageController, StorageControllerHandle, ) -from test.storage.test_values import TEST_VALUES, TEST_VISIT_IDS +from test.storage.fixtures import dt_test_values -def test_startup_and_shutdown(mp_logger: MPLogger) -> None: +def test_startup_and_shutdown(mp_logger: MPLogger, test_values: dt_test_values) -> None: + test_table, visit_ids = test_values structured = MemoryStructuredProvider() controller_handle = StorageControllerHandle(structured, None) controller_handle.launch() assert controller_handle.listener_address is not None cs = DataSocket(controller_handle.listener_address) - for table, data in TEST_VALUES.items(): - visit_id = data["visit_id"] if "visit_id" in data else INVALID_VISIT_ID + for table, data in test_table.items(): + visit_id = data["visit_id"] cs.store_record( - table, visit_id, dict(**data) + table, visit_id, data ) # cloning to avoid the modifications in store_record - for visit_id in [*TEST_VISIT_IDS, INVALID_VISIT_ID]: + for visit_id in visit_ids: cs.finalize_visit_id(visit_id, True) cs.close() controller_handle.shutdown() handle = structured.handle handle.poll_queue() - for table, data in TEST_VALUES.items(): + for table, data in test_table.items(): + if data["visit_id"] == INVALID_VISIT_ID: + del data["visit_id"] assert handle.storage[table] == [data] -def test_arrow_provider(mp_logger: MPLogger) -> None: +def test_arrow_provider(mp_logger: MPLogger, test_values: dt_test_values) -> None: + test_table, visit_ids = test_values structured = MemoryArrowProvider() controller_handle = StorageControllerHandle(structured, None) controller_handle.launch() @@ -57,22 +49,21 @@ def test_arrow_provider(mp_logger: MPLogger) -> None: assert controller_handle.listener_address is not None cs = DataSocket(controller_handle.listener_address) - for table, data in TEST_VALUES.items(): - visit_id = data["visit_id"] if "visit_id" in data else INVALID_VISIT_ID - cs.store_record( - table, visit_id, dict(**data) - ) # cloning to avoid the modifications in store_record + for table, data in test_table.items(): + visit_id = data["visit_id"] + cs.store_record(table, visit_id, data) - for visit_id in [*TEST_VISIT_IDS, INVALID_VISIT_ID]: + for visit_id in visit_ids: cs.finalize_visit_id(visit_id, True) cs.close() controller_handle.shutdown() handle = structured.handle handle.poll_queue() - for table, data in TEST_VALUES.items(): - + for table, data in test_table.items(): t1 = handle.storage[table][0].to_pandas().drop(columns=["instance_id"]) + if data["visit_id"] == INVALID_VISIT_ID: + del data["visit_id"] t2 = pd.DataFrame({k: [v] for k, v in data.items()}) # Since t2 doesn't get created schema the inferred types are different assert_frame_equal(t1, t2, check_dtype=False) diff --git a/test/storage/test_storage_providers.py b/test/storage/test_storage_providers.py index ecb712cba..200689c3b 100644 --- a/test/storage/test_storage_providers.py +++ b/test/storage/test_storage_providers.py @@ -14,33 +14,32 @@ from openwpm.types import VisitId from .fixtures import structured_scenarios, unstructured_scenarios -from .test_values import TEST_VALUES +from .test_values import dt_test_values @pytest.mark.asyncio -async def test_local_arrow_storage_provider(tmp_path, mp_logger): +async def test_local_arrow_storage_provider( + tmp_path, mp_logger, test_values: dt_test_values +): + test_table, visit_ids = test_values structured_provider = LocalArrowProvider(tmp_path) await structured_provider.init() - visit_ids = set() - for table_name, test_data in TEST_VALUES.items(): - try: - visit_id = VisitId(test_data["visit_id"]) - except KeyError: - visit_id = INVALID_VISIT_ID - visit_ids.add(visit_id) + for table_name, test_data in test_table.items(): await structured_provider.store_record( - TableName(table_name), visit_id, test_data + TableName(table_name), test_data["visit_id"], test_data ) token_list = [] for i in visit_ids: token_list.append(await structured_provider.finalize_visit_id(i)) await structured_provider.flush_cache() await asyncio.gather(*token_list) - for table_name, test_data in TEST_VALUES.items(): + for table_name, test_data in test_table.items(): dataset = ParquetDataset(tmp_path / table_name) df: DataFrame = dataset.read().to_pandas() assert df.shape[0] == 1 for row in df.itertuples(index=False): + if test_data["visit_id"] == INVALID_VISIT_ID: + del test_data["visit_id"] assert row._asdict() == test_data diff --git a/test/storage/test_values.py b/test/storage/test_values.py index 8ae915fcc..30006fc9c 100644 --- a/test/storage/test_values.py +++ b/test/storage/test_values.py @@ -8,243 +8,233 @@ import random import string -from typing import Any, Dict +from typing import Any, Dict, Set, Tuple from openwpm.storage.storage_providers import TableName - -TEST_VALUES: Dict[TableName, Dict[str, Any]] = dict() - - -def random_word(length): - letters = string.ascii_lowercase - return "".join(random.choice(letters) for _ in range(length)) - - -# task -fields = { - "task_id": random.randint(0, 2 ** 63 - 1), - "manager_params": random_word(12), - "openwpm_version": random_word(12), - "browser_version": random_word(12), -} -TEST_VALUES[TableName("task")] = fields - - -# crawl -fields = { - "browser_id": random.randint(0, 2 ** 31 - 1), - "task_id": random.randint(0, 2 ** 63 - 1), - "browser_params": random_word(12), -} -TEST_VALUES[TableName("crawl")] = fields - -# site_visits -fields = { - "visit_id": random.randint(0, 2 ** 63 - 1), - "browser_id": random.randint(0, 2 ** 31 - 1), - "site_url": random_word(12), - "site_rank": random.randint(0, 2 ** 31 - 1), -} -TEST_VALUES[TableName("site_visits")] = fields - -# crawl_history -fields = { - "browser_id": random.randint(0, 2 ** 31 - 1), - "visit_id": random.randint(0, 2 ** 63 - 1), - "command": random_word(12), - "arguments": random_word(12), - "retry_number": random.randint(0, 2 ** 7 - 1), - "command_status": random_word(12), - "error": random_word(12), - "traceback": random_word(12), - "duration": random.randint(0, 2 ** 63 - 1), -} -TEST_VALUES[TableName("crawl_history")] = fields - -# http_requests -fields = { - "incognito": random.randint(0, 2 ** 31 - 1), - "browser_id": random.randint(0, 2 ** 31 - 1), - "visit_id": random.randint(0, 2 ** 63 - 1), - "extension_session_uuid": random_word(12), - "event_ordinal": random.randint(0, 2 ** 63 - 1), - "window_id": random.randint(0, 2 ** 63 - 1), - "tab_id": random.randint(0, 2 ** 63 - 1), - "frame_id": random.randint(0, 2 ** 63 - 1), - "url": random_word(12), - "top_level_url": random_word(12), - "parent_frame_id": random.randint(0, 2 ** 63 - 1), - "frame_ancestors": random_word(12), - "method": random_word(12), - "referrer": random_word(12), - "headers": random_word(12), - "request_id": random.randint(0, 2 ** 63 - 1), - "is_XHR": random.choice([True, False]), - "is_third_party_channel": random.choice([True, False]), - "is_third_party_to_top_window": random.choice([True, False]), - "triggering_origin": random_word(12), - "loading_origin": random_word(12), - "loading_href": random_word(12), - "req_call_stack": random_word(12), - "resource_type": random_word(12), - "post_body": random_word(12), - "post_body_raw": random_word(12), - "time_stamp": random_word(12), -} -TEST_VALUES[TableName("http_requests")] = fields - -# http_responses -fields = { - "incognito": random.randint(0, 2 ** 31 - 1), - "browser_id": random.randint(0, 2 ** 31 - 1), - "visit_id": random.randint(0, 2 ** 63 - 1), - "extension_session_uuid": random_word(12), - "event_ordinal": random.randint(0, 2 ** 63 - 1), - "window_id": random.randint(0, 2 ** 63 - 1), - "tab_id": random.randint(0, 2 ** 63 - 1), - "frame_id": random.randint(0, 2 ** 63 - 1), - "url": random_word(12), - "method": random_word(12), - "response_status": random.randint(0, 2 ** 63 - 1), - "response_status_text": random_word(12), - "is_cached": random.choice([True, False]), - "headers": random_word(12), - "request_id": random.randint(0, 2 ** 63 - 1), - "location": random_word(12), - "time_stamp": random_word(12), - "content_hash": random_word(12), -} -TEST_VALUES[TableName("http_responses")] = fields - -# http_redirects -fields = { - "incognito": random.randint(0, 2 ** 31 - 1), - "browser_id": random.randint(0, 2 ** 31 - 1), - "visit_id": random.randint(0, 2 ** 63 - 1), - "old_request_url": random_word(12), - "old_request_id": random_word(12), - "new_request_url": random_word(12), - "new_request_id": random_word(12), - "extension_session_uuid": random_word(12), - "event_ordinal": random.randint(0, 2 ** 63 - 1), - "window_id": random.randint(0, 2 ** 63 - 1), - "tab_id": random.randint(0, 2 ** 63 - 1), - "frame_id": random.randint(0, 2 ** 63 - 1), - "response_status": random.randint(0, 2 ** 63 - 1), - "response_status_text": random_word(12), - "headers": random_word(12), - "time_stamp": random_word(12), -} -TEST_VALUES[TableName("http_redirects")] = fields - -# javascript -fields = { - "incognito": random.randint(0, 2 ** 31 - 1), - "browser_id": random.randint(0, 2 ** 31 - 1), - "visit_id": random.randint(0, 2 ** 63 - 1), - "extension_session_uuid": random_word(12), - "event_ordinal": random.randint(0, 2 ** 63 - 1), - "page_scoped_event_ordinal": random.randint(0, 2 ** 63 - 1), - "window_id": random.randint(0, 2 ** 63 - 1), - "tab_id": random.randint(0, 2 ** 63 - 1), - "frame_id": random.randint(0, 2 ** 63 - 1), - "script_url": random_word(12), - "script_line": random_word(12), - "script_col": random_word(12), - "func_name": random_word(12), - "script_loc_eval": random_word(12), - "document_url": random_word(12), - "top_level_url": random_word(12), - "call_stack": random_word(12), - "symbol": random_word(12), - "operation": random_word(12), - "value": random_word(12), - "arguments": random_word(12), - "time_stamp": random_word(12), -} -TEST_VALUES[TableName("javascript")] = fields - -# javascript_cookies -fields = { - "browser_id": random.randint(0, 2 ** 31 - 1), - "visit_id": random.randint(0, 2 ** 63 - 1), - "extension_session_uuid": random_word(12), - "event_ordinal": random.randint(0, 2 ** 63 - 1), - "record_type": random_word(12), - "change_cause": random_word(12), - "expiry": random_word(12), - "is_http_only": random.choice([True, False]), - "is_host_only": random.choice([True, False]), - "is_session": random.choice([True, False]), - "host": random_word(12), - "is_secure": random.choice([True, False]), - "name": random_word(12), - "path": random_word(12), - "value": random_word(12), - "same_site": random_word(12), - "first_party_domain": random_word(12), - "store_id": random_word(12), - "time_stamp": random_word(12), -} -TEST_VALUES[TableName("javascript_cookies")] = fields - -# navigations -fields = { - "incognito": random.randint(0, 2 ** 31 - 1), - "browser_id": random.randint(0, 2 ** 31 - 1), - "visit_id": random.randint(0, 2 ** 63 - 1), - "extension_session_uuid": random_word(12), - "process_id": random.randint(0, 2 ** 63 - 1), - "window_id": random.randint(0, 2 ** 63 - 1), - "tab_id": random.randint(0, 2 ** 63 - 1), - "tab_opener_tab_id": random.randint(0, 2 ** 63 - 1), - "frame_id": random.randint(0, 2 ** 63 - 1), - "parent_frame_id": random.randint(0, 2 ** 63 - 1), - "window_width": random.randint(0, 2 ** 63 - 1), - "window_height": random.randint(0, 2 ** 63 - 1), - "window_type": random_word(12), - "tab_width": random.randint(0, 2 ** 63 - 1), - "tab_height": random.randint(0, 2 ** 63 - 1), - "tab_cookie_store_id": random_word(12), - "uuid": random_word(12), - "url": random_word(12), - "transition_qualifiers": random_word(12), - "transition_type": random_word(12), - "before_navigate_event_ordinal": random.randint(0, 2 ** 63 - 1), - "before_navigate_time_stamp": random_word(12), - "committed_event_ordinal": random.randint(0, 2 ** 63 - 1), - "time_stamp": random_word(12), -} -TEST_VALUES[TableName("navigations")] = fields - -# callstacks -fields = { - "visit_id": random.randint(0, 2 ** 63 - 1), - "request_id": random.randint(0, 2 ** 63 - 1), - "browser_id": random.randint(0, 2 ** 31 - 1), - "call_stack": random_word(12), -} -TEST_VALUES[TableName("callstacks")] = fields - -# incomplete_visits -fields = { - "visit_id": random.randint(0, 2 ** 63 - 1), -} -TEST_VALUES[TableName("incomplete_visits")] = fields - -# dns_responses -fields = { - "request_id": random.randint(0, 2 ** 63 - 1), - "browser_id": random.randint(0, 2 ** 31 - 1), - "visit_id": random.randint(0, 2 ** 63 - 1), - "hostname": random_word(12), - "addresses": random_word(12), - "canonical_name": random_word(12), - "is_TRR": random.choice([True, False]), - "time_stamp": random_word(12), -} -TEST_VALUES[TableName("dns_responses")] = fields - -TEST_VISIT_IDS = set( - d["visit_id"] for d in filter(lambda d: "visit_id" in d, TEST_VALUES.values()) -) +from openwpm.types import VisitId + +dt_test_values = Tuple[Dict[TableName, Dict[str, Any]], Set[VisitId]] + + +def generate_test_values() -> dt_test_values: + test_values: Dict[TableName, Dict[str, Any]] = dict() + + def random_word(length): + letters = string.ascii_lowercase + return "".join(random.choice(letters) for _ in range(length)) + + # task + fields = { + "task_id": random.randint(0, 2 ** 63 - 1), + "manager_params": random_word(12), + "openwpm_version": random_word(12), + "browser_version": random_word(12), + } + test_values[TableName("task")] = fields + # crawl + fields = { + "browser_id": random.randint(0, 2 ** 31 - 1), + "task_id": random.randint(0, 2 ** 63 - 1), + "browser_params": random_word(12), + } + test_values[TableName("crawl")] = fields + # site_visits + fields = { + "visit_id": random.randint(0, 2 ** 63 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "site_url": random_word(12), + "site_rank": random.randint(0, 2 ** 31 - 1), + } + test_values[TableName("site_visits")] = fields + # crawl_history + fields = { + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "command": random_word(12), + "arguments": random_word(12), + "retry_number": random.randint(0, 2 ** 7 - 1), + "command_status": random_word(12), + "error": random_word(12), + "traceback": random_word(12), + "duration": random.randint(0, 2 ** 63 - 1), + } + test_values[TableName("crawl_history")] = fields + # http_requests + fields = { + "incognito": random.randint(0, 2 ** 31 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "extension_session_uuid": random_word(12), + "event_ordinal": random.randint(0, 2 ** 63 - 1), + "window_id": random.randint(0, 2 ** 63 - 1), + "tab_id": random.randint(0, 2 ** 63 - 1), + "frame_id": random.randint(0, 2 ** 63 - 1), + "url": random_word(12), + "top_level_url": random_word(12), + "parent_frame_id": random.randint(0, 2 ** 63 - 1), + "frame_ancestors": random_word(12), + "method": random_word(12), + "referrer": random_word(12), + "headers": random_word(12), + "request_id": random.randint(0, 2 ** 63 - 1), + "is_XHR": random.choice([True, False]), + "is_third_party_channel": random.choice([True, False]), + "is_third_party_to_top_window": random.choice([True, False]), + "triggering_origin": random_word(12), + "loading_origin": random_word(12), + "loading_href": random_word(12), + "req_call_stack": random_word(12), + "resource_type": random_word(12), + "post_body": random_word(12), + "post_body_raw": random_word(12), + "time_stamp": random_word(12), + } + test_values[TableName("http_requests")] = fields + # http_responses + fields = { + "incognito": random.randint(0, 2 ** 31 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "extension_session_uuid": random_word(12), + "event_ordinal": random.randint(0, 2 ** 63 - 1), + "window_id": random.randint(0, 2 ** 63 - 1), + "tab_id": random.randint(0, 2 ** 63 - 1), + "frame_id": random.randint(0, 2 ** 63 - 1), + "url": random_word(12), + "method": random_word(12), + "response_status": random.randint(0, 2 ** 63 - 1), + "response_status_text": random_word(12), + "is_cached": random.choice([True, False]), + "headers": random_word(12), + "request_id": random.randint(0, 2 ** 63 - 1), + "location": random_word(12), + "time_stamp": random_word(12), + "content_hash": random_word(12), + } + test_values[TableName("http_responses")] = fields + # http_redirects + fields = { + "incognito": random.randint(0, 2 ** 31 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "old_request_url": random_word(12), + "old_request_id": random_word(12), + "new_request_url": random_word(12), + "new_request_id": random_word(12), + "extension_session_uuid": random_word(12), + "event_ordinal": random.randint(0, 2 ** 63 - 1), + "window_id": random.randint(0, 2 ** 63 - 1), + "tab_id": random.randint(0, 2 ** 63 - 1), + "frame_id": random.randint(0, 2 ** 63 - 1), + "response_status": random.randint(0, 2 ** 63 - 1), + "response_status_text": random_word(12), + "headers": random_word(12), + "time_stamp": random_word(12), + } + test_values[TableName("http_redirects")] = fields + # javascript + fields = { + "incognito": random.randint(0, 2 ** 31 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "extension_session_uuid": random_word(12), + "event_ordinal": random.randint(0, 2 ** 63 - 1), + "page_scoped_event_ordinal": random.randint(0, 2 ** 63 - 1), + "window_id": random.randint(0, 2 ** 63 - 1), + "tab_id": random.randint(0, 2 ** 63 - 1), + "frame_id": random.randint(0, 2 ** 63 - 1), + "script_url": random_word(12), + "script_line": random_word(12), + "script_col": random_word(12), + "func_name": random_word(12), + "script_loc_eval": random_word(12), + "document_url": random_word(12), + "top_level_url": random_word(12), + "call_stack": random_word(12), + "symbol": random_word(12), + "operation": random_word(12), + "value": random_word(12), + "arguments": random_word(12), + "time_stamp": random_word(12), + } + test_values[TableName("javascript")] = fields + # javascript_cookies + fields = { + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "extension_session_uuid": random_word(12), + "event_ordinal": random.randint(0, 2 ** 63 - 1), + "record_type": random_word(12), + "change_cause": random_word(12), + "expiry": random_word(12), + "is_http_only": random.choice([True, False]), + "is_host_only": random.choice([True, False]), + "is_session": random.choice([True, False]), + "host": random_word(12), + "is_secure": random.choice([True, False]), + "name": random_word(12), + "path": random_word(12), + "value": random_word(12), + "same_site": random_word(12), + "first_party_domain": random_word(12), + "store_id": random_word(12), + "time_stamp": random_word(12), + } + test_values[TableName("javascript_cookies")] = fields + # navigations + fields = { + "incognito": random.randint(0, 2 ** 31 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "extension_session_uuid": random_word(12), + "process_id": random.randint(0, 2 ** 63 - 1), + "window_id": random.randint(0, 2 ** 63 - 1), + "tab_id": random.randint(0, 2 ** 63 - 1), + "tab_opener_tab_id": random.randint(0, 2 ** 63 - 1), + "frame_id": random.randint(0, 2 ** 63 - 1), + "parent_frame_id": random.randint(0, 2 ** 63 - 1), + "window_width": random.randint(0, 2 ** 63 - 1), + "window_height": random.randint(0, 2 ** 63 - 1), + "window_type": random_word(12), + "tab_width": random.randint(0, 2 ** 63 - 1), + "tab_height": random.randint(0, 2 ** 63 - 1), + "tab_cookie_store_id": random_word(12), + "uuid": random_word(12), + "url": random_word(12), + "transition_qualifiers": random_word(12), + "transition_type": random_word(12), + "before_navigate_event_ordinal": random.randint(0, 2 ** 63 - 1), + "before_navigate_time_stamp": random_word(12), + "committed_event_ordinal": random.randint(0, 2 ** 63 - 1), + "time_stamp": random_word(12), + } + test_values[TableName("navigations")] = fields + # callstacks + fields = { + "visit_id": random.randint(0, 2 ** 63 - 1), + "request_id": random.randint(0, 2 ** 63 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "call_stack": random_word(12), + } + test_values[TableName("callstacks")] = fields + # incomplete_visits + fields = { + "visit_id": random.randint(0, 2 ** 63 - 1), + } + test_values[TableName("incomplete_visits")] = fields + # dns_responses + fields = { + "request_id": random.randint(0, 2 ** 63 - 1), + "browser_id": random.randint(0, 2 ** 31 - 1), + "visit_id": random.randint(0, 2 ** 63 - 1), + "hostname": random_word(12), + "addresses": random_word(12), + "canonical_name": random_word(12), + "is_TRR": random.choice([True, False]), + "time_stamp": random_word(12), + } + test_values[TableName("dns_responses")] = fields + visit_id_set = set( + d["visit_id"] for d in filter(lambda d: "visit_id" in d, test_values.values()) + ) + return test_values, visit_id_set From 8520be609c95cacc1fbbd8d06bdadba4783280c6 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 1 Feb 2021 20:19:26 +0100 Subject: [PATCH 106/139] Fixing GcpUnstructuredProvider --- openwpm/storage/cloud_storage/gcp_storage.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/openwpm/storage/cloud_storage/gcp_storage.py b/openwpm/storage/cloud_storage/gcp_storage.py index 119c1de16..d1c42666a 100644 --- a/openwpm/storage/cloud_storage/gcp_storage.py +++ b/openwpm/storage/cloud_storage/gcp_storage.py @@ -82,7 +82,10 @@ def __init__( self.logger = logging.getLogger("openwpm") async def init(self) -> None: - pass + await super(GcsUnstructuredProvider, self).init() + self.file_system = GCSFileSystem( + project=self.project, token=self.token, access="read_write" + ) async def store_blob( self, filename: str, blob: bytes, overwrite: bool = False From 6527179d11ef4f96eeed5ee752a02455d64be0bd Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 1 Feb 2021 21:03:49 +0100 Subject: [PATCH 107/139] Fixed paths for future crawls --- crawler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crawler.py b/crawler.py index 04b405faa..4b156531a 100644 --- a/crawler.py +++ b/crawler.py @@ -97,7 +97,7 @@ unstructured = GcsUnstructuredProvider( project=GCP_PROJECT, bucket_name=GCS_BUCKET, - base_path=CRAWL_DIRECTORY, + base_path=CRAWL_DIRECTORY + "/data", token=AUTH_TOKEN, ) # Instantiates the measurement platform From 1ca2739b0891d2684e2f5b10d1bf26cc9d11636f Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 3 Feb 2021 12:08:21 +0100 Subject: [PATCH 108/139] Renamed sqllite to official sqlite --- openwpm/storage/sql_provider.py | 2 +- openwpm/storage/storage_controller.py | 10 ++++++++-- test/conftest.py | 4 ++-- test/openwpmtest.py | 4 ++-- test/storage/fixtures.py | 12 ++++++------ test/test_custom_function_command.py | 4 ++-- test/test_http_instrumentation.py | 8 ++++---- 7 files changed, 25 insertions(+), 19 deletions(-) diff --git a/openwpm/storage/sql_provider.py b/openwpm/storage/sql_provider.py index 47b7d6bb3..f05bd0e58 100644 --- a/openwpm/storage/sql_provider.py +++ b/openwpm/storage/sql_provider.py @@ -20,7 +20,7 @@ SCHEMA_FILE = os.path.join(os.path.dirname(__file__), "schema.sql") -class SqlLiteStorageProvider(StructuredStorageProvider): +class SQLiteStorageProvider(StructuredStorageProvider): db: Connection cur: Cursor diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index b661424db..180c03390 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -197,15 +197,21 @@ async def finalize_visit_id( See StructuredStorageProvider::finalize_visit_id for additional documentation """ - self.logger.info("Awaiting all tasks for visit_id %d", visit_id) + if visit_id not in self.store_record_tasks: + self.logger.error( + "Visit_id %d got finalized multiple times, skipping...", visit_id + ) + return None + + self.logger.info("Awaiting all tasks for visit_id %d", visit_id) for task in self.store_record_tasks[visit_id]: await task del self.store_record_tasks[visit_id] - self.logger.debug( "Awaited all tasks for visit_id %d while finalizing", visit_id ) + completion_token = await self.structured_storage.finalize_visit_id( visit_id, interrupted=not success ) diff --git a/test/conftest.py b/test/conftest.py index 7d9ee1c83..c335deaf4 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -8,7 +8,7 @@ from openwpm.config import BrowserParams, ManagerParams from openwpm.mp_logger import MPLogger -from openwpm.storage.sql_provider import SqlLiteStorageProvider +from openwpm.storage.sql_provider import SQLiteStorageProvider from openwpm.task_manager import TaskManager from . import utilities @@ -75,7 +75,7 @@ def _create_task_manager( ) -> Tuple[TaskManager, Path]: manager_params, browser_params = params db_path = manager_params.data_directory / "crawl-data.sqlite" - structured_provider = SqlLiteStorageProvider(db_path) + structured_provider = SQLiteStorageProvider(db_path) manager = TaskManager( manager_params, browser_params, diff --git a/test/openwpmtest.py b/test/openwpmtest.py index 2491eb2a4..0840870b3 100644 --- a/test/openwpmtest.py +++ b/test/openwpmtest.py @@ -9,7 +9,7 @@ from openwpm import task_manager from openwpm.config import BrowserParams, ManagerParams -from openwpm.storage.sql_provider import SqlLiteStorageProvider +from openwpm.storage.sql_provider import SQLiteStorageProvider from . import utilities @@ -41,7 +41,7 @@ def visit( db_path = data_dir / "crawl-data.sqlite" else: db_path = self.tmpdir / "crawl-data.sqlite" - structured_provider = SqlLiteStorageProvider(db_path) + structured_provider = SQLiteStorageProvider(db_path) manager = task_manager.TaskManager( manager_params, browser_params, structured_provider, None ) diff --git a/test/storage/fixtures.py b/test/storage/fixtures.py index 793f745e4..67550cec2 100644 --- a/test/storage/fixtures.py +++ b/test/storage/fixtures.py @@ -10,7 +10,7 @@ ) from openwpm.storage.leveldb import LevelDbProvider from openwpm.storage.local_storage import LocalGzipProvider -from openwpm.storage.sql_provider import SqlLiteStorageProvider +from openwpm.storage.sql_provider import SQLiteStorageProvider from openwpm.storage.storage_controller import INVALID_VISIT_ID from openwpm.storage.storage_providers import ( StructuredStorageProvider, @@ -21,7 +21,7 @@ from test.storage.test_values import dt_test_values, generate_test_values memory_structured = "memory_structured" -sqllite = "sqllite" +sqlite = "sqlite" memory_arrow = "memory_arrow" @@ -31,9 +31,9 @@ def structured_provider( ) -> StructuredStorageProvider: if request.param == memory_structured: return MemoryStructuredProvider() - elif request.param == sqllite: - tmp_path = tmp_path_factory.mktemp("sqllite") - return SqlLiteStorageProvider(tmp_path / "test_db.sqllite") + elif request.param == sqlite: + tmp_path = tmp_path_factory.mktemp("sqlite") + return SQLiteStorageProvider(tmp_path / "test_db.sqlite") elif request.param == memory_arrow: return MemoryArrowProvider() assert isinstance( @@ -44,7 +44,7 @@ def structured_provider( structured_scenarios: List[str] = [ memory_structured, - sqllite, + sqlite, memory_arrow, ] diff --git a/test/test_custom_function_command.py b/test/test_custom_function_command.py index 2820d8229..2217be6d8 100644 --- a/test/test_custom_function_command.py +++ b/test/test_custom_function_command.py @@ -6,7 +6,7 @@ from openwpm.commands.types import BaseCommand from openwpm.config import BrowserParams, ManagerParamsInternal from openwpm.socket_interface import ClientSocket -from openwpm.storage.sql_provider import SqlLiteStorageProvider +from openwpm.storage.sql_provider import SQLiteStorageProvider from openwpm.storage.storage_providers import TableName from openwpm.task_manager import TaskManager from openwpm.utilities import db_utils @@ -93,7 +93,7 @@ def test_custom_function(default_params, xpi, server): cur.close() db.close() - storage_provider = SqlLiteStorageProvider(path) + storage_provider = SQLiteStorageProvider(path) manager = TaskManager(manager_params, browser_params, storage_provider, None) cs = command_sequence.CommandSequence(url_a) cs.get(sleep=0, timeout=60) diff --git a/test/test_http_instrumentation.py b/test/test_http_instrumentation.py index 824ec99a3..3ca494de0 100644 --- a/test/test_http_instrumentation.py +++ b/test/test_http_instrumentation.py @@ -17,7 +17,7 @@ from openwpm.commands.types import BaseCommand from openwpm.config import BrowserParams, ManagerParams from openwpm.storage.leveldb import LevelDbProvider -from openwpm.storage.sql_provider import SqlLiteStorageProvider +from openwpm.storage.sql_provider import SQLiteStorageProvider from openwpm.utilities import db_utils from . import utilities @@ -893,7 +893,7 @@ def test_javascript_saving(http_params, xpi, server): browser_param.http_instrument = True browser_param.save_content = "script" - structured_storage = SqlLiteStorageProvider( + structured_storage = SQLiteStorageProvider( db_path=manager_params.data_directory / "crawl-data.sqlite" ) ldb_path = Path(manager_params.data_directory) / "content.ldb" @@ -928,7 +928,7 @@ def test_document_saving(http_params, xpi, server): browser_param.http_instrument = True browser_param.save_content = "main_frame,sub_frame" - structured_storage = SqlLiteStorageProvider( + structured_storage = SQLiteStorageProvider( db_path=manager_params.data_directory / "crawl-data.sqlite" ) ldb_path = Path(manager_params.data_directory) / "content.ldb" @@ -956,7 +956,7 @@ def test_content_saving(http_params, xpi, server): browser_param.http_instrument = True browser_param.save_content = True db = manager_params.data_directory / "crawl-data.sqlite" - structured_storage = SqlLiteStorageProvider(db_path=db) + structured_storage = SQLiteStorageProvider(db_path=db) ldb_path = Path(manager_params.data_directory) / "content.ldb" unstructured_storage = LevelDbProvider(db_path=ldb_path) manager = task_manager.TaskManager( From 7cf48a19de8efade7f87ea8cf4d2d45b1732685e Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 3 Feb 2021 12:11:03 +0100 Subject: [PATCH 109/139] Restored demo.py --- demo.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/demo.py b/demo.py index 538b2202b..eecda75e9 100644 --- a/demo.py +++ b/demo.py @@ -5,7 +5,7 @@ from openwpm.command_sequence import CommandSequence from openwpm.commands.browser_commands import GetCommand from openwpm.config import BrowserParams, ManagerParams -from openwpm.storage.sql_provider import SqlLiteStorageProvider +from openwpm.storage.sql_provider import SQLiteStorageProvider from openwpm.task_manager import TaskManager # The list of sites that we wish to crawl @@ -16,7 +16,10 @@ "http://citp.princeton.edu/", ] -manager_params = ManagerParams() +# Loads the default ManagerParams +# and NUM_BROWSERS copies of the default BrowserParams + +manager_params = ManagerParams(num_browsers=NUM_BROWSERS) browser_params = [BrowserParams(display_mode="headless") for _ in range(NUM_BROWSERS)] # Update browser configuration (use this for per-browser settings) @@ -48,7 +51,7 @@ manager = TaskManager( manager_params, browser_params, - SqlLiteStorageProvider(Path("./datadir/crawl-data.sqllite")), + SQLiteStorageProvider(Path("./datadir/crawl-data.sqlite")), None, ) From 29d3e276da866e5713a054db53b37bface52ffc3 Mon Sep 17 00:00:00 2001 From: Stefan Zabka Date: Wed, 3 Feb 2021 12:19:40 +0100 Subject: [PATCH 110/139] Update openwpm/commands/profile_commands.py Co-authored-by: Georgia Kokkinou --- openwpm/commands/profile_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwpm/commands/profile_commands.py b/openwpm/commands/profile_commands.py index 1dff208fd..76bd52aad 100644 --- a/openwpm/commands/profile_commands.py +++ b/openwpm/commands/profile_commands.py @@ -51,7 +51,7 @@ def execute( browser_profile_folder = browser_params.profile_path assert browser_profile_folder is not None - # Creating the all folders if need be + # Creating the folders if need be self.tar_path.parent.mkdir(exist_ok=True, parents=True) # see if this file exists first From 5b5f229256220972c00fe6dbd44dc1ca56293ca4 Mon Sep 17 00:00:00 2001 From: Stefan Zabka Date: Wed, 3 Feb 2021 12:21:30 +0100 Subject: [PATCH 111/139] Restored previous behaviour of DumpProfileCommand Co-authored-by: Georgia Kokkinou --- openwpm/commands/profile_commands.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openwpm/commands/profile_commands.py b/openwpm/commands/profile_commands.py index 76bd52aad..56a2ce13b 100644 --- a/openwpm/commands/profile_commands.py +++ b/openwpm/commands/profile_commands.py @@ -24,7 +24,7 @@ class DumpProfileCommand(BaseCommand): """ dumps a browser profile currently stored in to - in which both folders are absolute paths. + , where both paths are absolute. """ def __init__(self, tar_path: Path, close_webdriver: bool, compress: bool) -> None: @@ -95,15 +95,15 @@ def execute( full_path = browser_profile_folder / item if ( not full_path.is_file() - and full_path.name != "shm" - and full_path.name != "wal" + and not full_path.name.endswith("shm") + and not full_path.name.endswith("wal") ): logger.critical( "BROWSER %i: %s NOT FOUND IN profile folder, skipping." % (self.browser_id, full_path) ) elif not full_path.is_file() and ( - full_path.name == "shm" or full_path.name == "wal" + full_path.name.endswith("shm") or full_path.name.endswith("wal") ): continue # These are just checkpoint files tar.add(full_path, arcname=item) From 73f08504d5e20b90038dda8a9e216111963238fb Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 3 Feb 2021 12:40:44 +0100 Subject: [PATCH 112/139] Removed leftovers --- openwpm/storage/arrow_storage.py | 1 - openwpm/storage/storage_controller.py | 4 ++-- test/utilities.py | 5 ----- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/openwpm/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py index 1ea9c10b5..38a0c5e37 100644 --- a/openwpm/storage/arrow_storage.py +++ b/openwpm/storage/arrow_storage.py @@ -24,7 +24,6 @@ class ArrowProvider(StructuredStorageProvider): """ storing_lock: asyncio.Lock - flush_event: asyncio.Event def __init__(self) -> None: super().__init__() diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 180c03390..c34da4959 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -7,13 +7,13 @@ import time from asyncio import Task from collections import defaultdict -from typing import Any, Awaitable, DefaultDict, Dict, List, NoReturn, Optional, Tuple +from typing import Any, DefaultDict, Dict, List, NoReturn, Optional, Tuple from multiprocess import Queue from openwpm.utilities.multiprocess_utils import Process -from ..config import BrowserParamsInternal, ConfigEncoder, ManagerParamsInternal +from ..config import BrowserParamsInternal, ManagerParamsInternal from ..socket_interface import ClientSocket, get_message_from_reader from ..types import BrowserId, VisitId from .storage_providers import ( diff --git a/test/utilities.py b/test/utilities.py index 476b916e4..621d17ef4 100644 --- a/test/utilities.py +++ b/test/utilities.py @@ -5,11 +5,6 @@ from os.path import dirname, realpath from urllib.parse import parse_qs, urlparse -import pyarrow.parquet as pq -import s3fs -from botocore.credentials import Credentials -from pyarrow.filesystem import S3FSWrapper # noqa - LOCAL_WEBSERVER_PORT = 8000 BASE_TEST_URL_DOMAIN = "localtest.me" BASE_TEST_URL_NOPATH = "http://%s:%s" % (BASE_TEST_URL_DOMAIN, LOCAL_WEBSERVER_PORT) From a4a75ff6d6bbd0bb41b9c0bcd5a22a0d0b333fcc Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 3 Feb 2021 12:43:58 +0100 Subject: [PATCH 113/139] Cleaned up comments --- openwpm/storage/arrow_storage.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/openwpm/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py index 38a0c5e37..868811aab 100644 --- a/openwpm/storage/arrow_storage.py +++ b/openwpm/storage/arrow_storage.py @@ -97,19 +97,19 @@ def _is_cache_full(self) -> bool: async def finalize_visit_id( self, visit_id: VisitId, interrupted: bool = False ) -> Task[None]: - """This method is the reason the finalize_visit_id interface returns an awaitable. + """This method is the reason the finalize_visit_id interface returns a task. This was necessary as we needed to enable the following pattern. ``` token = await structured_storage.finalize_visit_id(1) structured_storage.flush_cache() await token ``` - If there was no token returned and the method would just block/yield after turning the + If there was no task returned and the method would just block/yield after turning the record into a batch, there would be no way to know, when it's safe to flush_cache as I couldn't find a way to run a coroutine until it yields and then run a different one. With the current setup `token` aka a `wait_on_condition` coroutine will only return once - the event has been set. + it's respective event has been set. """ if interrupted: await self.store_record(INCOMPLETE_VISITS, visit_id, {"visit_id": visit_id}) @@ -118,9 +118,6 @@ async def finalize_visit_id( # resolve once the data is saved to persistent storage # 2. No new batches should be created while saving out all the batches async with self.storing_lock: - # After flush_cache has executed the event needs to be rearmed - # so that newly created wait_on_condition don't just complete - # instantly self._create_batch(visit_id) event = asyncio.Event() From 41f66565640793064cfe25a5e7d8a8e15457917a Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 3 Feb 2021 12:48:04 +0100 Subject: [PATCH 114/139] Expanded lock check --- openwpm/storage/arrow_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwpm/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py index 868811aab..4f4f091b8 100644 --- a/openwpm/storage/arrow_storage.py +++ b/openwpm/storage/arrow_storage.py @@ -147,7 +147,7 @@ async def flush_cache(self, lock: asyncio.Lock = None) -> None: lock = self.storing_lock await lock.acquire() - assert lock is not None and lock.locked() + assert lock == self.storing_lock and lock.locked() for table_name, batches in self._batches.items(): table = pa.Table.from_batches(batches) From 9046d0dac3152254230f5ee54de2986bcd82355b Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 3 Feb 2021 12:52:48 +0100 Subject: [PATCH 115/139] Fixed more stuff --- openwpm/storage/in_memory_storage.py | 6 +++--- openwpm/storage/leveldb.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/openwpm/storage/in_memory_storage.py b/openwpm/storage/in_memory_storage.py index 02c13a8eb..1da7a9fce 100644 --- a/openwpm/storage/in_memory_storage.py +++ b/openwpm/storage/in_memory_storage.py @@ -3,7 +3,7 @@ that store their results in memory. These classes are designed to allow for easier parallel testing as there are no shared resources between tests. It also makes it easier to verify results -by not having to do a round trip to a perstitent storage provider +by not having to do a round trip through a persistent storage provider """ import asyncio @@ -27,10 +27,10 @@ class MemoryStructuredProvider(StructuredStorageProvider): """ - This storage provider passes all it's data to the MemoryStructuredProviderHandle in + This storage provider passes all it's data to the MemoryStructuredProviderHandle in a process safe way. - This makes it ideal for testing and for small crawls where no persistence is required + This makes it ideal for testing It also aims to only save out data as late as possible to ensure that storage_controller only relies on the guarantees given in the interface. diff --git a/openwpm/storage/leveldb.py b/openwpm/storage/leveldb.py index b3107ff26..63796b3b3 100644 --- a/openwpm/storage/leveldb.py +++ b/openwpm/storage/leveldb.py @@ -50,3 +50,4 @@ async def store_blob( if self._ldb_counter >= LDB_BATCH_SIZE: await self.flush_cache() + self._ldb_counter = 0 From 0ec3353f9d635dab7d9df600303ae2552db4256e Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 3 Feb 2021 12:57:24 +0100 Subject: [PATCH 116/139] More comment updates --- openwpm/storage/storage_controller.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index c34da4959..1514ba6fc 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -38,6 +38,13 @@ class StorageController: + """ + Manages incoming data and it's saving to disk + + Provides it's status to the task manager via the completion and status queue. + Can be shut down via a shutdown signal in the shutdown queue + """ + def __init__( self, structured_storage: StructuredStorageProvider, @@ -47,8 +54,6 @@ def __init__( shutdown_queue: Queue, ) -> None: """ - Creates a BaseListener instance - Parameters ---------- status_queue From c1a6038a343527c4a18260f43fa46bf1fe0852a7 Mon Sep 17 00:00:00 2001 From: Stefan Zabka Date: Wed, 3 Feb 2021 18:06:27 +0100 Subject: [PATCH 117/139] Update openwpm/socket_interface.py Co-authored-by: Georgia Kokkinou --- openwpm/socket_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwpm/socket_interface.py b/openwpm/socket_interface.py index 8ca76c124..95f82bb32 100644 --- a/openwpm/socket_interface.py +++ b/openwpm/socket_interface.py @@ -190,7 +190,7 @@ def _parse(serialization: bytes, msg: bytes) -> Any: return json.loads(msg.decode("utf-8")) if serialization == b"u": # utf-8 serialization return msg.decode("utf-8") - raise ValueError("Unkown Encoding") + raise ValueError("Unknown Encoding") def main(): From ae25bfa75738278145f539770b6a0d05d9e46a20 Mon Sep 17 00:00:00 2001 From: vringar Date: Thu, 4 Feb 2021 12:48:40 +0100 Subject: [PATCH 118/139] Removed outdated comment --- openwpm/task_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openwpm/task_manager.py b/openwpm/task_manager.py index f18c2a814..5c1999f0d 100644 --- a/openwpm/task_manager.py +++ b/openwpm/task_manager.py @@ -85,7 +85,6 @@ def __init__( for bp in browser_params_temp: validate_browser_params(bp) validate_crawl_configs(manager_params_temp, browser_params_temp) - # Mypy doesn't see the methods generated by dataclass_json manager_params = ManagerParamsInternal.from_dict(manager_params_temp.to_dict()) browser_params = [ BrowserParamsInternal.from_dict(bp.to_dict()) for bp in browser_params_temp From 4e898064e00f5b00fbc091b34d104355feddb561 Mon Sep 17 00:00:00 2001 From: vringar Date: Thu, 4 Feb 2021 12:51:43 +0100 Subject: [PATCH 119/139] Using config_encoder --- openwpm/utilities/platform_utils.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/openwpm/utilities/platform_utils.py b/openwpm/utilities/platform_utils.py index 65a2c6032..e61dbd510 100644 --- a/openwpm/utilities/platform_utils.py +++ b/openwpm/utilities/platform_utils.py @@ -9,6 +9,8 @@ from tabulate import tabulate +from openwpm.config import ConfigEncoder + def parse_http_stack_trace_str(trace_str): """Parse a stacktrace string and return an array of dict.""" @@ -98,19 +100,12 @@ def get_configuration_string(manager_params, browser_params, versions): config_str = "\n\nOpenWPM Version: %s\nFirefox Version: %s\n" % versions config_str += "\n========== Manager Configuration ==========\n" - def serialize_paths(obj: Any) -> Any: - if isinstance(obj, Path): - return str(obj) - raise TypeError( - f"Object of type {obj.__class__.__name__} is not JSON serializable" - ) - config_str += json.dumps( manager_params.to_dict(), sort_keys=True, indent=2, separators=(",", ": "), - default=serialize_paths, + cls=ConfigEncoder, ) config_str += "\n\n========== Browser Configuration ==========\n" From 85a4c2d624334ae5e7cf8e9489ffbab05595dde6 Mon Sep 17 00:00:00 2001 From: vringar Date: Thu, 4 Feb 2021 13:06:12 +0100 Subject: [PATCH 120/139] Renamed tar_location to tar_path --- openwpm/commands/profile_commands.py | 34 ++++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/openwpm/commands/profile_commands.py b/openwpm/commands/profile_commands.py index 56a2ce13b..87c49b969 100644 --- a/openwpm/commands/profile_commands.py +++ b/openwpm/commands/profile_commands.py @@ -23,17 +23,17 @@ class DumpProfileCommand(BaseCommand): """ - dumps a browser profile currently stored in to - , where both paths are absolute. + Dumps a browser profile currently stored in to + """ def __init__(self, tar_path: Path, close_webdriver: bool, compress: bool) -> None: self.tar_path = tar_path self.close_webdriver = close_webdriver self.compress = compress - raise ConfigError( - "BROWSER %i: Profile dumping is currently unsupported. " - "See: https://github.com/mozilla/OpenWPM/projects/2." % self.browser_id + raise NotImplementedError( + "Profile dumping is currently unsupported. " + "See: https://github.com/mozilla/OpenWPM/projects/2." ) def __repr__(self) -> str: @@ -123,15 +123,15 @@ def load_profile( browser_profile_folder: Path, manager_params: ManagerParamsInternal, browser_params: BrowserParamsInternal, - tar_location: Path, + tar_path: Path, ) -> None: """ - loads a zipped cookie-based profile stored in and - unzips it to . This will load whatever profile - is in the folder, either full_profile.tar.gz or profile.tar.gz + loads a zipped cookie-based profile stored at and + unzips it to . + The tar will remain unmodified. """ - assert tar_location.is_file() + assert tar_path.is_file() assert browser_params.browser_id is not None try: # Copy and untar the loaded profile @@ -139,19 +139,19 @@ def load_profile( "BROWSER %i: Copying profile tar from %s to %s" % ( browser_params.browser_id, - tar_location, + tar_path, browser_profile_folder, ) ) - shutil.copy(tar_location, browser_profile_folder) - tar_location = browser_profile_folder / tar_location.name - if tar_location.name.endswith("tar.gz"): - f = tarfile.open(tar_location, "r:gz", errorlevel=1) + shutil.copy(tar_path, browser_profile_folder) + tar_path = browser_profile_folder / tar_path.name + if tar_path.name.endswith("tar.gz"): + f = tarfile.open(tar_path, "r:gz", errorlevel=1) else: - f = tarfile.open(tar_location, "r", errorlevel=1) + f = tarfile.open(tar_path, "r", errorlevel=1) f.extractall(browser_profile_folder) f.close() - tar_location.unlink() + tar_path.unlink() logger.debug("BROWSER %i: Tarfile extracted" % browser_params.browser_id) except Exception as ex: From 4c03174d6d7689deafed1893e5377a49a861a1e2 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 5 Feb 2021 11:45:06 +0100 Subject: [PATCH 121/139] Removed references to database_name in docs --- docs/Configuration.md | 6 ++--- docs/Platform-Architecture.md | 42 +++++------------------------------ 2 files changed, 8 insertions(+), 40 deletions(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index d49f9fe15..7ca5044cf 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -58,15 +58,13 @@ validate_crawl_configs(manager_params, browser_params) # Platform Configuration Options * `data_directory` - * The directory in which to output the crawl database and related files. The - directory given will be created if it does not exist. + * The directory into which screenshots and page dumps will be saved + * [Intended to be removed by #232](https://github.com/mozilla/OpenWPM/issues/232) * `log_directory` * The directory in which to output platform logs. The directory given will be created if it does not exist. * `log_file` -> supported file extensions are `.log` * The name of the log file to be written to `log_directory`. -* `database_name` -> supported file extensions are `.db`, `.sqlite` - * The name of the database file to be written to `data_directory` * `failure_limit` -> has to be either of type `int` or `None` * The number of successive command failures the platform will tolerate before raising a `CommandExecutionError` exception. Otherwise the default is set diff --git a/docs/Platform-Architecture.md b/docs/Platform-Architecture.md index 277625daa..1cc2938d1 100644 --- a/docs/Platform-Architecture.md +++ b/docs/Platform-Architecture.md @@ -2,39 +2,11 @@ ## Overview -The user-facing component of the OpenWPM platform is the Task Manager. The Task Manager oversees multiple browser instances and passes them commands. The Task Manager also ensures that crawls continue despite browser crashes for freezes. In particular, it checks whether a given browser fails to complete a command within a given timeout (or has died) and kills/restarts this browser as necessary. - -## Instantiating a Task Manager - -All automation code is contained within the `openwpm` folder; the Task Manager code is contained in `openwpm/task_manager.py`. - -Task Managers can be instantiated in the following way: -```python -from openwpm.task_manager import TaskManager -from openwpm.config import ( - BrowserParams, - ManagerParams, -) - -number_of_browsers = 5 # Number of browsers to spawn - -# Instantiating Browser and Manager Params with default values. -manager_params = ManagerParams(num_browsers = number_of_browsers) -browser_params = [BrowserParams() for bp in range(manager_params.num_browsers)] - -# These instances can be used to modify default values of both browser and manager params. -manager_params.data_directory = '~/Documents' -manager_params.database_name = 'custom_name.sqlite' - -for i in range(len(browser_params)): - browser_params[i].display_mode = 'headless' # all 5 browsers will spawn in headless mode - -# Instantiating TaskManager -manager = TaskManager(manager_params, browser_params) - -``` - -To learn more about the `manager_params` and `browser_params` have a look at [Configuration.md](Configuration.md) +The user-facing component of the OpenWPM platform is the Task Manager. +The Task Manager oversees multiple browser instances and passes them commands. +The Task Manager also ensures that crawls continue despite browser crashes for freezes. +In particular, it checks whether a given browser fails to complete a command within a given timeout (or has died) and +kills/restarts this browser as necessary. ## Watchdogs In OpenWPM we have a watchdog thread that tries to ensure two things. @@ -77,9 +49,7 @@ Please note that you need to close the manager, because by default CommandSequen ## Adding new commands -Currently the easiest way to execute a user defined function as part of a CommandSequence is to use the -`run_custom_function` method on the CommandSequence, however we hope to significantly improve this process -with https://github.com/mozilla/OpenWPM/issues/743. +Have a look at [`custom_command.py`](../custom_command.py) # Browser Manager From f565507da5cb9870c1cdfbfaad8209a815789ee1 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 5 Feb 2021 13:36:15 +0100 Subject: [PATCH 122/139] Cleanup --- setup.cfg | 3 +-- test/conftest.py | 5 ++--- test/storage/test_storage_providers.py | 6 ++++-- test/test_callback.py | 2 +- test/test_crawl.py | 5 ++--- test/test_http_instrumentation.py | 12 ++++++------ test/test_profile.py | 2 +- 7 files changed, 17 insertions(+), 18 deletions(-) diff --git a/setup.cfg b/setup.cfg index c49313ec8..3f0295559 100644 --- a/setup.cfg +++ b/setup.cfg @@ -28,5 +28,4 @@ allow_redefinition = True disallow_incomplete_defs = False [mypy-test.*] -disallow_incomplete_defs = False -disallow_untyped_defs = False \ No newline at end of file +allow_untyped_defs = True \ No newline at end of file diff --git a/test/conftest.py b/test/conftest.py index c335deaf4..e33801938 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -64,12 +64,11 @@ def default_params( @pytest.fixture() def task_manager_creator( - server, - xpi, + server: None, xpi: None ) -> Callable[[Tuple[ManagerParams, List[BrowserParams]]], Tuple[TaskManager, Path]]: """We create a callable that returns a TaskManager that has been configured with the Manager and BrowserParams""" - + # We need to create the fixtures like this because usefixtures doesn't work on fixtures def _create_task_manager( params: Tuple[ManagerParams, List[BrowserParams]] ) -> Tuple[TaskManager, Path]: diff --git a/test/storage/test_storage_providers.py b/test/storage/test_storage_providers.py index 200689c3b..7471335bb 100644 --- a/test/storage/test_storage_providers.py +++ b/test/storage/test_storage_providers.py @@ -1,4 +1,5 @@ import asyncio +from pathlib import Path import pytest from pandas import DataFrame @@ -17,10 +18,11 @@ from .test_values import dt_test_values +@pytest.mark.usefixtures("mp_logger") @pytest.mark.asyncio async def test_local_arrow_storage_provider( - tmp_path, mp_logger, test_values: dt_test_values -): + tmp_path: Path, test_values: dt_test_values +) -> None: test_table, visit_ids = test_values structured_provider = LocalArrowProvider(tmp_path) await structured_provider.init() diff --git a/test/test_callback.py b/test/test_callback.py index 53bb9bb6e..026edbded 100644 --- a/test/test_callback.py +++ b/test/test_callback.py @@ -8,7 +8,7 @@ from .utilities import BASE_TEST_URL -def test_local_callbacks(default_params, task_manager_creator) -> None: +def test_local_callbacks(default_params, task_manager_creator): """Test test the storage controller as well as the entire callback machinery to see if all callbacks get correctly called""" manager, _ = task_manager_creator(default_params) diff --git a/test/test_crawl.py b/test/test_crawl.py index 046e1f558..f8f1df183 100644 --- a/test/test_crawl.py +++ b/test/test_crawl.py @@ -63,7 +63,7 @@ def get_config( @pytest.mark.xfail(run=False) @pytest.mark.slow - def test_browser_profile_coverage(self, tmpdir: Path) -> None: + def test_browser_profile_coverage(self, tmpdir: Path, task_manager_creator) -> None: """Test the coverage of the browser's profile This verifies that Firefox's places.sqlite database contains @@ -73,7 +73,7 @@ def test_browser_profile_coverage(self, tmpdir: Path) -> None: # Run the test crawl data_dir = tmpdir / "data_dir" manager_params, browser_params = self.get_config(data_dir) - manager = task_manager.TaskManager(manager_params, browser_params) + manager, crawl_db = task_manager_creator((manager_params, browser_params)) for site in TEST_SITES: manager.get(site) ff_db_tar = os.path.join( @@ -87,7 +87,6 @@ def test_browser_profile_coverage(self, tmpdir: Path) -> None: # Output databases ff_db = os.path.join(browser_params[0].profile_archive_dir, "places.sqlite") - crawl_db = manager_params.database_name # Grab urls from crawl database rows = db_utils.query_db(crawl_db, "SELECT url FROM http_requests") diff --git a/test/test_http_instrumentation.py b/test/test_http_instrumentation.py index 3ca494de0..ac954d951 100644 --- a/test/test_http_instrumentation.py +++ b/test/test_http_instrumentation.py @@ -750,7 +750,7 @@ class TestPOSTInstrument(OpenWPMTest): ) def get_config( - self, data_dir: Optional[Path] + self, data_dir: Optional[Path] = None ) -> Tuple[ManagerParams, List[BrowserParams]]: manager_params, browser_params = self.get_test_config(data_dir) browser_params[0].http_instrument = True @@ -839,7 +839,7 @@ def test_record_binary_post_data(self): reason="Firefox is currently not able to return the " "file content for an upload, only the filename" ) - def test_record_file_upload(self): + def test_record_file_upload(self, task_manager_creator): """Test that we correctly capture the uploaded file contents. We upload a CSS file and a PNG file to test both text based and @@ -860,7 +860,7 @@ def test_record_file_upload(self): css_file_path = os.path.abspath("test_pages/shared/test_style.css") manager_params, browser_params = self.get_config() - manager = task_manager.TaskManager(manager_params, browser_params) + manager, db_path = task_manager_creator((manager_params, browser_params)) test_url = utilities.BASE_TEST_URL + "/post_file_upload.html" cs = command_sequence.CommandSequence(test_url) cs.get(sleep=0, timeout=60) @@ -868,7 +868,7 @@ def test_record_file_upload(self): manager.execute_command_sequence(cs) manager.close() - post_body = self.get_post_request_body_from_db(manager_params.database_name) + post_body = self.get_post_request_body_from_db(db_path) # Binary strings get put into the database as-if they were latin-1. with open(img_file_path, "rb") as f: img_file_content = f.read().strip().decode("latin-1") @@ -1090,7 +1090,7 @@ def test_cache_hits_recorded(http_params, task_manager_creator): class FilenamesIntoFormCommand(BaseCommand): - def __init__(self, img_file_path, css_file_path) -> None: + def __init__(self, img_file_path: str, css_file_path: str) -> None: self.img_file_path = img_file_path self.css_file_path = css_file_path @@ -1100,7 +1100,7 @@ def execute( browser_params, manager_params, extension_socket, - ) -> None: + ): img_file_upload_element = webdriver.find_element_by_id("upload-img") css_file_upload_element = webdriver.find_element_by_id("upload-css") img_file_upload_element.send_keys(self.img_file_path) diff --git a/test/test_profile.py b/test/test_profile.py index c30b26979..3a4b6e002 100644 --- a/test/test_profile.py +++ b/test/test_profile.py @@ -127,7 +127,7 @@ def execute( browser_params, manager_params, extension_socket, - ) -> None: + ): webdriver.get("about:config") result = webdriver.execute_script( f""" From 2553a094102f4d95c2d697cbde05c293778009b2 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 5 Feb 2021 14:15:11 +0100 Subject: [PATCH 123/139] Moved screenshot_path and source_dump_path to ManagerParamsInternal --- CONTRIBUTING.md | 1 - README.md | 108 ++++++++++++---------- openwpm/config.py | 13 ++- openwpm/deploy_browsers/deploy_firefox.py | 4 - 4 files changed, 66 insertions(+), 60 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ed334c6c2..c1612a6e9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -85,7 +85,6 @@ OpenWPM's tests are build on [pytest](https://docs.pytest.org/en/latest/). Execu in the test directory to run all tests: $ conda activate openwpm - $ cd test $ py.test -vv See the [pytest docs](https://docs.pytest.org/en/latest/) for more information on selecting diff --git a/README.md b/README.md index c63d694c3..b83715e19 100644 --- a/README.md +++ b/README.md @@ -12,25 +12,25 @@ the instrumentation section below for more details. Table of Contents ------------------ -* [Installation](#installation) - * [Pre-requisites](#pre-requisites) - * [Install](#install) - * [Mac OSX](#mac-osx) -* [Quick Start](#quick-start) -* [Troubleshooting](#troubleshooting) -* [Advice for Measurement Researchers](#advice-for-measurement-researchers) -* [Developer instructions](#developer-instructions) -* [Instrumentation and Configuration](#instrumentation-and-configuration) -* [Persistence Types](#persistence-types) - * [Local Databases](#local-databases) - * [Parquet on Amazon S3](#parquet-on-amazon-s3) -* [Docker Deployment for OpenWPM](#docker-deployment-for-openwpm) - * [Building the Docker Container](#building-the-docker-container) - * [Running Measurements from inside the Container](#running-measurements-from-inside-the-container) - * [MacOS GUI applications in Docker](#macos-gui-applications-in-docker) -* [Citation](#citation) -* [License](#license) +- [Installation](#installation) + - [Pre-requisites](#pre-requisites) + - [Install](#install) + - [Mac OSX](#mac-osx) +- [Quick Start](#quick-start) +- [Troubleshooting](#troubleshooting) +- [Advice for Measurement Researchers](#advice-for-measurement-researchers) +- [Developer instructions](#developer-instructions) +- [Instrumentation and Configuration](#instrumentation-and-configuration) +- [Storage](#storage) + - [Local Storage](#local-storage) + - [Remote storage](#remote-storage) +- [Docker Deployment for OpenWPM](#docker-deployment-for-openwpm) + - [Building the Docker Container](#building-the-docker-container) + - [Running Measurements from inside the Container](#running-measurements-from-inside-the-container) + - [MacOS GUI applications in Docker](#macos-gui-applications-in-docker) +- [Citation](#citation) +- [License](#license) Installation ------------ @@ -178,40 +178,52 @@ If you want to contribute to OpenWPM have a look at our [CONTRIBUTING.md](./CONT Instrumentation and Configuration ------------------------------- + OpenWPM provides a breadth of configuration options which can be found in [Configuration.md](docs/Configuration.md) More detail on the output is available [below](#persistence-types). -Persistence Types +Storage ------------ -#### Local Databases -By default OpenWPM saves all data locally on disk in a variety of formats. -Most of the instrumentation saves to a SQLite database specified -by `manager_params.database_name` in the main output directory. Response -bodies are saved in a LevelDB database named `content.ldb`, and are keyed by -the hash of the content. In addition, the browser commands that dump page -source and save screenshots save them in the `sources` and `screenshots` -subdirectories of the main output directory. The SQLite schema -specified by: `openwpm/DataAggregator/schema.sql`. You can specify additional tables -inline by sending a `create_table` message to the data aggregator. - -#### Parquet on Amazon S3 -As an option, OpenWPM can save data directly to an Amazon S3 bucket as a -Parquet Dataset. This is currently experimental and hasn't been thoroughly -tested. Screenshots, and page source saving is not currently supported and -will still be stored in local databases and directories. To enable S3 -saving specify the following configuration parameters in `manager_params`: -* Persistence Type: `manager_params.output_format = 's3'` -* S3 bucket name: `manager_params.s3_bucket = 'openwpm-test-crawl'` -* Directory within S3 bucket: `manager_params.s3_directory = '2018-09-09_test-crawl-new'` - -In order to save to S3 you must have valid access credentials stored in -`~/.aws`. We do not currently allow you to specify an alternate storage -location. - -**NOTE:** The schemas should be kept in sync with the exception of -output-specific columns (e.g., `instance_id` in the S3 output). You can compare +OpenWPM distinguishes between two types of data, structured and unstructured. +Structured data is all data captured by the instrumentation or emitted by the platform. +Generally speaking all data you download is unstructured data. + +For each of the data classes we offer a variety of storage providers, and you are encouraged +to implement your own, should the provided backends not be enough for you. + +We have an outstanding issue to enable saving content generated by commands, such as +screenshots and page dumps to unstructured storage (see [#232](https://github.com/mozilla/OpenWPM/issues/232)). +For now, they get saved to `manager_params.data_directory`. + +### Local Storage + +For storing structured data locally we offer two StorageProviders: + +- The SQLiteStorageProvider which writes all data into a SQLite database + - This is the recommended approach for getting started as the data is easily explorable +- The LocalArrowProvider which stores the data into Parquet files. + - This method integrates well with NumPy/Pandas + - It might be harder to ad-hoc process + +For storing unstructured data locally we also offer two solutions: + +- The LevelDBProvider which stores all data into a LevelDB + - This is the recommended approach +- The LocalGzipProvider that gzips and stores the files individually on disk + - Please note that file systems usually don't like thousands of files in one folder + - Use with care or for single site visits + +### Remote storage + +When running in the cloud, saving records to disk is not a reasonable thing to do. +So we offer a remote StorageProviders for S3 (See [#823](https://github.com/mozilla/OpenWPM/issues/823)) and GCP. +Currently, all remote StorageProviders write to the respective object storage service (S3/GCS). +The structured providers use the Parquet format. + +**NOTE:** The Parquet and SQL schemas should be kept in sync except +output-specific columns (e.g., `instance_id` in the Parquet output). You can compare the two schemas by running `diff -y openwpm/DataAggregator/schema.sql openwpm/DataAggregator/parquet_schema.py`. @@ -238,7 +250,7 @@ Docker service. __Step 2:__ to build the image, run the following command from a terminal within the root OpenWPM directory: -``` +```bash docker build -f Dockerfile -t openwpm . ``` @@ -253,7 +265,7 @@ X-server. You can do this by running: `xhost +local:docker` Then you can run the demo script using: -``` +```bash mkdir -p docker-volume && docker run -v $PWD/docker-volume:/opt/Desktop \ -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix --shm-size=2g \ -it openwpm python3 /opt/OpenWPM/demo.py diff --git a/openwpm/config.py b/openwpm/config.py index 0f57da2c4..e7e16c7be 100644 --- a/openwpm/config.py +++ b/openwpm/config.py @@ -121,12 +121,6 @@ class ManagerParams(DataClassJsonMixin): default=Path("~/openwpm/"), metadata=DCJConfig(encoder=path_to_str, decoder=str_to_path), ) - screenshot_path: Optional[Path] = field( - default=None, metadata=DCJConfig(encoder=path_to_str, decoder=str_to_path) - ) - source_dump_path: Optional[Path] = field( - default=None, metadata=DCJConfig(encoder=path_to_str, decoder=str_to_path) - ) log_file: Path = field( default=Path("openwpm.log"), metadata=DCJConfig(encoder=path_to_str, decoder=str_to_path), @@ -148,7 +142,12 @@ class BrowserParamsInternal(BrowserParams): class ManagerParamsInternal(ManagerParams): storage_controller_address: Optional[Tuple[str, int]] = None logger_address: Optional[Tuple[str, ...]] = None - ldb_address: Optional[Tuple[str, ...]] = None + screenshot_path: Optional[Path] = field( + default=None, metadata=DCJConfig(encoder=path_to_str, decoder=str_to_path) + ) + source_dump_path: Optional[Path] = field( + default=None, metadata=DCJConfig(encoder=path_to_str, decoder=str_to_path) + ) def validate_browser_params(browser_params: BrowserParams) -> None: diff --git a/openwpm/deploy_browsers/deploy_firefox.py b/openwpm/deploy_browsers/deploy_firefox.py index e9ec4a242..862525689 100755 --- a/openwpm/deploy_browsers/deploy_firefox.py +++ b/openwpm/deploy_browsers/deploy_firefox.py @@ -98,10 +98,6 @@ def deploy_firefox( extension_config[ "storage_controller_address" ] = manager_params.storage_controller_address - if manager_params.ldb_address: - extension_config["leveldb_address"] = manager_params.ldb_address - else: - extension_config["leveldb_address"] = None extension_config["testing"] = manager_params.testing ext_config_file = browser_profile_path / "browser_params.json" with open(ext_config_file, "w") as f: From fceaee0f85f72ee230d26f35c1bee29c43613c14 Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 12 Feb 2021 10:42:24 +0100 Subject: [PATCH 124/139] Fixed imports --- environment.yaml | 22 +++++----- openwpm/Extension/firefox/content.js/index.js | 2 +- openwpm/Extension/firefox/feature.js/index.js | 12 ++--- .../src/background/cookie-instrument.ts | 8 ++-- .../src/background/dns-instrument.ts | 44 +++++++++---------- .../src/background/http-instrument.ts | 24 +++++----- .../src/background/javascript-instrument.ts | 8 ++-- .../src/background/navigation-instrument.ts | 12 ++--- .../javascript-instrument-content-scope.ts | 4 +- .../src/lib/extension-session-uuid.ts | 2 +- .../src/lib/http-post-parser.ts | 8 +--- .../src/lib/number.spec.ts | 2 +- .../src/lib/pending-navigation.ts | 2 +- .../src/lib/pending-response.ts | 2 +- .../src/lib/response-body-listener.ts | 4 +- openwpm/commands/profile_commands.py | 10 +---- openwpm/socket_interface.py | 16 +------ openwpm/storage/cloud_storage/s3_storage.py | 1 - openwpm/storage/leveldb.py | 1 - openwpm/storage/local_storage.py | 1 - openwpm/storage/sql_provider.py | 2 +- openwpm/storage/storage_controller.py | 8 ++-- openwpm/storage/storage_providers.py | 4 +- openwpm/task_manager.py | 8 +--- openwpm/utilities/platform_utils.py | 2 - 25 files changed, 85 insertions(+), 124 deletions(-) diff --git a/environment.yaml b/environment.yaml index 07461aa65..cb230478d 100644 --- a/environment.yaml +++ b/environment.yaml @@ -7,28 +7,28 @@ dependencies: - click=7.1.2 - codecov=2.1.11 - dill=0.3.3 -- gcsfs=0.7.1 -- geckodriver=0.28.0 -- ipython=7.19.0 +- gcsfs=0.7.2 +- geckodriver=0.29.0 +- ipython=7.20.0 - leveldb=1.22 - multiprocess=0.70.11.1 -- mypy=0.790 +- mypy=0.800 - nodejs=14.15.1 -- pandas=1.2.0 +- pandas=1.2.2 - pillow=8.1.0 -- pip=20.3.3 -- pre-commit=2.9.3 +- pip=21.0.1 +- pre-commit=2.10.1 - psutil=5.8.0 -- pyarrow=2.0.0 +- pyarrow=3.0.0 - pytest-asyncio=0.14.0 -- pytest-cov=2.10.1 -- pytest=6.2.1 +- pytest-cov=2.11.1 +- pytest=6.2.2 - python=3.9.1 - pyvirtualdisplay=1.3.2 - redis-py=3.5.3 - s3fs=0.5.2 - selenium=3.141.0 -- sentry-sdk=0.19.5 +- sentry-sdk=0.20.0 - tabulate=0.8.7 - tblib=1.6.0 - wget=1.20.1 diff --git a/openwpm/Extension/firefox/content.js/index.js b/openwpm/Extension/firefox/content.js/index.js index d925d3bbc..c60740d92 100644 --- a/openwpm/Extension/firefox/content.js/index.js +++ b/openwpm/Extension/firefox/content.js/index.js @@ -1,4 +1,4 @@ -import { injectJavascriptInstrumentPageScript } from "openwpm-webext-instrumentation"; +import {injectJavascriptInstrumentPageScript} from "openwpm-webext-instrumentation"; injectJavascriptInstrumentPageScript(window.openWpmContentScriptConfig || {}); delete window.openWpmContentScriptConfig; diff --git a/openwpm/Extension/firefox/feature.js/index.js b/openwpm/Extension/firefox/feature.js/index.js index 4454fcc1d..daf634bcb 100644 --- a/openwpm/Extension/firefox/feature.js/index.js +++ b/openwpm/Extension/firefox/feature.js/index.js @@ -1,13 +1,13 @@ import { - CookieInstrument, - JavascriptInstrument, - HttpInstrument, - NavigationInstrument, - DnsInstrument + CookieInstrument, + DnsInstrument, + HttpInstrument, + JavascriptInstrument, + NavigationInstrument } from "openwpm-webext-instrumentation"; import * as loggingDB from "./loggingdb.js"; -import { CallstackInstrument } from "./callstack-instrument.js"; +import {CallstackInstrument} from "./callstack-instrument.js"; async function main() { // Read the browser configuration from file diff --git a/openwpm/Extension/webext-instrumentation/src/background/cookie-instrument.ts b/openwpm/Extension/webext-instrumentation/src/background/cookie-instrument.ts index b9d356b48..72d6aa7ea 100644 --- a/openwpm/Extension/webext-instrumentation/src/background/cookie-instrument.ts +++ b/openwpm/Extension/webext-instrumentation/src/background/cookie-instrument.ts @@ -1,9 +1,9 @@ -import { incrementedEventOrdinal } from "../lib/extension-session-event-ordinal"; -import { extensionSessionUuid } from "../lib/extension-session-uuid"; -import { boolToInt, escapeString } from "../lib/string-utils"; +import {incrementedEventOrdinal} from "../lib/extension-session-event-ordinal"; +import {extensionSessionUuid} from "../lib/extension-session-uuid"; +import {boolToInt, escapeString} from "../lib/string-utils"; +import {JavascriptCookie, JavascriptCookieRecord} from "../schema"; import Cookie = browser.cookies.Cookie; import OnChangedCause = browser.cookies.OnChangedCause; -import { JavascriptCookie, JavascriptCookieRecord } from "../schema"; export const transformCookieObjectToMatchOpenWPMSchema = (cookie: Cookie) => { const javascriptCookie = {} as JavascriptCookie; diff --git a/openwpm/Extension/webext-instrumentation/src/background/dns-instrument.ts b/openwpm/Extension/webext-instrumentation/src/background/dns-instrument.ts index b59ad9daf..728f7b01a 100755 --- a/openwpm/Extension/webext-instrumentation/src/background/dns-instrument.ts +++ b/openwpm/Extension/webext-instrumentation/src/background/dns-instrument.ts @@ -1,12 +1,8 @@ -import { PendingResponse } from "../lib/pending-response"; -import { DnsResolved } from "../schema"; +import {PendingResponse} from "../lib/pending-response"; +import {DnsResolved} from "../schema"; +import {WebRequestOnCompletedEventDetails,} from "../types/browser-web-request-event-details"; +import {allTypes} from "./http-instrument"; import RequestFilter = browser.webRequest.RequestFilter; -import { - WebRequestOnCompletedEventDetails, -} from "../types/browser-web-request-event-details"; -import { - allTypes -} from "./http-instrument"; export class DnsInstrument { @@ -15,48 +11,48 @@ export class DnsInstrument { private pendingResponses: { [requestId: number]: PendingResponse; } = {}; - + constructor(dataReceiver) { this.dataReceiver = dataReceiver; } - + public run(crawlID){ const filter: RequestFilter = { urls: [""], types: allTypes }; - + const requestStemsFromExtension = details => { return ( - details.originUrl && - details.originUrl.indexOf("moz-extension://") > -1 && + details.originUrl && + details.originUrl.indexOf("moz-extension://") > -1 && details.originUrl.includes("fakeRequest") ); }; - + /* * Attach handlers to event listeners */ this.onCompleteListener = ( details: WebRequestOnCompletedEventDetails, - ) => { + ) => { // Ignore requests made by extensions if (requestStemsFromExtension(details)) { return; } const pendingResponse = this.getPendingResponse(details.requestId); - pendingResponse.resolveOnCompletedEventDetails(details); - + pendingResponse.resolveOnCompletedEventDetails(details); + this.onCompleteDnsHandler( details, crawlID, ); }; - + browser.webRequest.onCompleted.addListener( this.onCompleteListener, filter, ); } - + public cleanup() { if (this.onCompleteListener) { browser.webRequest.onCompleted.removeListener( @@ -64,7 +60,7 @@ export class DnsInstrument { ); } } - + private getPendingResponse(requestId): PendingResponse { if (!this.pendingResponses[requestId]) { this.pendingResponses[requestId] = new PendingResponse(); @@ -77,14 +73,14 @@ export class DnsInstrument { return function(record) { // Get data from API call dnsRecordObj.addresses = record.addresses.toString() - dnsRecordObj.canonical_name = record.canonicalName + dnsRecordObj.canonical_name = record.canonicalName dnsRecordObj.is_TRR = record.isTRR // Send data to main OpenWPM data aggregator. dataReceiver.saveRecord("dns_responses", dnsRecordObj); } } - + private async onCompleteDnsHandler( details: WebRequestOnCompletedEventDetails, crawlID, @@ -96,12 +92,12 @@ export class DnsInstrument { dnsRecord.used_address = details.ip; const currentTime = new Date(details.timeStamp); dnsRecord.time_stamp = currentTime.toISOString(); - + // Query DNS API const url = new URL(details.url); dnsRecord.hostname = url.hostname; const dnsResolve = browser.dns.resolve(dnsRecord.hostname, ["canonical_name"]); dnsResolve.then(this.handleResolvedDnsData(dnsRecord, this.dataReceiver)); } - + } diff --git a/openwpm/Extension/webext-instrumentation/src/background/http-instrument.ts b/openwpm/Extension/webext-instrumentation/src/background/http-instrument.ts index 2eb08efaa..658d08417 100644 --- a/openwpm/Extension/webext-instrumentation/src/background/http-instrument.ts +++ b/openwpm/Extension/webext-instrumentation/src/background/http-instrument.ts @@ -1,20 +1,20 @@ -import { incrementedEventOrdinal } from "../lib/extension-session-event-ordinal"; -import { extensionSessionUuid } from "../lib/extension-session-uuid"; -import { HttpPostParser, ParsedPostRequest } from "../lib/http-post-parser"; -import { PendingRequest } from "../lib/pending-request"; -import { PendingResponse } from "../lib/pending-response"; -import ResourceType = browser.webRequest.ResourceType; -import RequestFilter = browser.webRequest.RequestFilter; -import BlockingResponse = browser.webRequest.BlockingResponse; -import HttpHeaders = browser.webRequest.HttpHeaders; -import { boolToInt, escapeString, escapeUrl } from "../lib/string-utils"; -import { HttpRedirect, HttpRequest, HttpResponse } from "../schema"; +import {incrementedEventOrdinal} from "../lib/extension-session-event-ordinal"; +import {extensionSessionUuid} from "../lib/extension-session-uuid"; +import {HttpPostParser, ParsedPostRequest} from "../lib/http-post-parser"; +import {PendingRequest} from "../lib/pending-request"; +import {PendingResponse} from "../lib/pending-response"; +import {boolToInt, escapeString, escapeUrl} from "../lib/string-utils"; +import {HttpRedirect, HttpRequest, HttpResponse} from "../schema"; import { WebRequestOnBeforeRedirectEventDetails, WebRequestOnBeforeRequestEventDetails, WebRequestOnBeforeSendHeadersEventDetails, WebRequestOnCompletedEventDetails, } from "../types/browser-web-request-event-details"; +import ResourceType = browser.webRequest.ResourceType; +import RequestFilter = browser.webRequest.RequestFilter; +import BlockingResponse = browser.webRequest.BlockingResponse; +import HttpHeaders = browser.webRequest.HttpHeaders; type SaveContentOption = boolean | string; @@ -25,7 +25,7 @@ type SaveContentOption = boolean | string; * redirect = original request headers+body, followed by a onBeforeRedirect and then a new set of request headers+body and response headers+body * Docs: https://developer.mozilla.org/en-US/docs/User:wbamberg/webRequest.RequestDetails */ - + const allTypes: ResourceType[] = [ "beacon", "csp_report", diff --git a/openwpm/Extension/webext-instrumentation/src/background/javascript-instrument.ts b/openwpm/Extension/webext-instrumentation/src/background/javascript-instrument.ts index 82cb04e78..f49b54e30 100644 --- a/openwpm/Extension/webext-instrumentation/src/background/javascript-instrument.ts +++ b/openwpm/Extension/webext-instrumentation/src/background/javascript-instrument.ts @@ -1,8 +1,8 @@ import MessageSender = browser.runtime.MessageSender; -import { incrementedEventOrdinal } from "../lib/extension-session-event-ordinal"; -import { extensionSessionUuid } from "../lib/extension-session-uuid"; -import { boolToInt, escapeString, escapeUrl } from "../lib/string-utils"; -import { JavascriptOperation } from "../schema"; +import {incrementedEventOrdinal} from "../lib/extension-session-event-ordinal"; +import {extensionSessionUuid} from "../lib/extension-session-uuid"; +import {boolToInt, escapeString, escapeUrl} from "../lib/string-utils"; +import {JavascriptOperation} from "../schema"; export class JavascriptInstrument { /** diff --git a/openwpm/Extension/webext-instrumentation/src/background/navigation-instrument.ts b/openwpm/Extension/webext-instrumentation/src/background/navigation-instrument.ts index 223f5a74d..5fee53877 100644 --- a/openwpm/Extension/webext-instrumentation/src/background/navigation-instrument.ts +++ b/openwpm/Extension/webext-instrumentation/src/background/navigation-instrument.ts @@ -1,9 +1,9 @@ -import { incrementedEventOrdinal } from "../lib/extension-session-event-ordinal"; -import { extensionSessionUuid } from "../lib/extension-session-uuid"; -import { PendingNavigation } from "../lib/pending-navigation"; -import { boolToInt, escapeString, escapeUrl } from "../lib/string-utils"; -import { makeUUID } from "../lib/uuid"; -import { Navigation } from "../schema"; +import {incrementedEventOrdinal} from "../lib/extension-session-event-ordinal"; +import {extensionSessionUuid} from "../lib/extension-session-uuid"; +import {PendingNavigation} from "../lib/pending-navigation"; +import {boolToInt, escapeString, escapeUrl} from "../lib/string-utils"; +import {makeUUID} from "../lib/uuid"; +import {Navigation} from "../schema"; import { WebNavigationBaseEventDetails, WebNavigationOnBeforeNavigateEventDetails, diff --git a/openwpm/Extension/webext-instrumentation/src/content/javascript-instrument-content-scope.ts b/openwpm/Extension/webext-instrumentation/src/content/javascript-instrument-content-scope.ts index b5aa4a06f..b63e4c80c 100755 --- a/openwpm/Extension/webext-instrumentation/src/content/javascript-instrument-content-scope.ts +++ b/openwpm/Extension/webext-instrumentation/src/content/javascript-instrument-content-scope.ts @@ -1,5 +1,5 @@ -import { getInstrumentJS } from "../lib/js-instruments"; -import { pageScript } from "./javascript-instrument-page-scope"; +import {getInstrumentJS} from "../lib/js-instruments"; +import {pageScript} from "./javascript-instrument-page-scope"; function getPageScriptAsString( jsInstrumentationSettingsString: string, diff --git a/openwpm/Extension/webext-instrumentation/src/lib/extension-session-uuid.ts b/openwpm/Extension/webext-instrumentation/src/lib/extension-session-uuid.ts index 1e5d246ab..7b76e3cfb 100644 --- a/openwpm/Extension/webext-instrumentation/src/lib/extension-session-uuid.ts +++ b/openwpm/Extension/webext-instrumentation/src/lib/extension-session-uuid.ts @@ -1,4 +1,4 @@ -import { makeUUID } from "./uuid"; +import {makeUUID} from "./uuid"; /** * This enables us to access a unique reference to this browser diff --git a/openwpm/Extension/webext-instrumentation/src/lib/http-post-parser.ts b/openwpm/Extension/webext-instrumentation/src/lib/http-post-parser.ts index 5566eac58..90916b5ad 100644 --- a/openwpm/Extension/webext-instrumentation/src/lib/http-post-parser.ts +++ b/openwpm/Extension/webext-instrumentation/src/lib/http-post-parser.ts @@ -1,12 +1,8 @@ // Incorporates code from: https://github.com/redline13/selenium-jmeter/blob/6966d4b326cd78261e31e6e317076569051cac37/content/library/recorder/HttpPostParser.js -import { - WebRequestOnBeforeRequestEventDetails, - // WebRequestOnBeforeSendHeadersEventDetails, -} from "../types/browser-web-request-event-details"; +import {WebRequestOnBeforeRequestEventDetails,} from "../types/browser-web-request-event-details"; // import { escapeString, escapeUrl } from "./string-utils"; - -import { escapeString, Uint8ToBase64 } from "./string-utils"; +import {escapeString, Uint8ToBase64} from "./string-utils"; // const components: any = {}; diff --git a/openwpm/Extension/webext-instrumentation/src/lib/number.spec.ts b/openwpm/Extension/webext-instrumentation/src/lib/number.spec.ts index 132254333..16dcf81c0 100644 --- a/openwpm/Extension/webext-instrumentation/src/lib/number.spec.ts +++ b/openwpm/Extension/webext-instrumentation/src/lib/number.spec.ts @@ -1,6 +1,6 @@ // tslint:disable:no-expression-statement import test from "ava"; -import { double, power } from "./number"; +import {double, power} from "./number"; test("double", t => { t.is(double(2), 4); diff --git a/openwpm/Extension/webext-instrumentation/src/lib/pending-navigation.ts b/openwpm/Extension/webext-instrumentation/src/lib/pending-navigation.ts index d7a014bc6..e6f9ea242 100644 --- a/openwpm/Extension/webext-instrumentation/src/lib/pending-navigation.ts +++ b/openwpm/Extension/webext-instrumentation/src/lib/pending-navigation.ts @@ -1,4 +1,4 @@ -import { Navigation } from "../schema"; +import {Navigation} from "../schema"; /** * Ties together the two separate navigation events that together holds information about both parent frame id and transition-related attributes diff --git a/openwpm/Extension/webext-instrumentation/src/lib/pending-response.ts b/openwpm/Extension/webext-instrumentation/src/lib/pending-response.ts index 5edf7f642..7ac0607da 100644 --- a/openwpm/Extension/webext-instrumentation/src/lib/pending-response.ts +++ b/openwpm/Extension/webext-instrumentation/src/lib/pending-response.ts @@ -2,7 +2,7 @@ import { WebRequestOnBeforeRequestEventDetails, WebRequestOnCompletedEventDetails, } from "../types/browser-web-request-event-details"; -import { ResponseBodyListener } from "./response-body-listener"; +import {ResponseBodyListener} from "./response-body-listener"; /** * Ties together the two separate events that together holds information about both response headers and body diff --git a/openwpm/Extension/webext-instrumentation/src/lib/response-body-listener.ts b/openwpm/Extension/webext-instrumentation/src/lib/response-body-listener.ts index 6cd86350f..4fc492a2e 100644 --- a/openwpm/Extension/webext-instrumentation/src/lib/response-body-listener.ts +++ b/openwpm/Extension/webext-instrumentation/src/lib/response-body-listener.ts @@ -1,5 +1,5 @@ -import { WebRequestOnBeforeRequestEventDetails } from "../types/browser-web-request-event-details"; -import { digestMessage } from "./sha256"; +import {WebRequestOnBeforeRequestEventDetails} from "../types/browser-web-request-event-details"; +import {digestMessage} from "./sha256"; export class ResponseBodyListener { private readonly responseBody: Promise; diff --git a/openwpm/commands/profile_commands.py b/openwpm/commands/profile_commands.py index 87c49b969..d44781d49 100644 --- a/openwpm/commands/profile_commands.py +++ b/openwpm/commands/profile_commands.py @@ -2,18 +2,12 @@ import shutil import tarfile from pathlib import Path -from typing import Any, Dict from selenium.webdriver import Firefox -from openwpm.config import ( - BrowserParams, - BrowserParamsInternal, - ManagerParams, - ManagerParamsInternal, -) +from openwpm.config import BrowserParamsInternal, ManagerParamsInternal -from ..errors import ConfigError, ProfileLoadError +from ..errors import ProfileLoadError from ..socket_interface import ClientSocket from .types import BaseCommand from .utils.firefox_profile import sleep_until_sqlite_checkpoint diff --git a/openwpm/socket_interface.py b/openwpm/socket_interface.py index 95f82bb32..55219b221 100644 --- a/openwpm/socket_interface.py +++ b/openwpm/socket_interface.py @@ -163,24 +163,12 @@ def close(self): async def get_message_from_reader(reader: asyncio.StreamReader) -> Any: - msg = await get_n_bytes_from_reader(reader, 5) + msg = await reader.readexactly(5) msglen, serialization = struct.unpack(">Lc", msg) - msg = await get_n_bytes_from_reader(reader, msglen) + msg = await reader.readexactly(msglen) return _parse(serialization, msg) -async def get_n_bytes_from_reader(reader: asyncio.StreamReader, n: int) -> bytes: - b = b"" - while True: - try: - return await reader.readexactly(n) - except IncompleteReadError as e: - b += e.partial - n -= len(e.partial) - if reader.at_eof(): - raise IOError("Socket Connection closed") - - def _parse(serialization: bytes, msg: bytes) -> Any: if serialization == b"n": return msg diff --git a/openwpm/storage/cloud_storage/s3_storage.py b/openwpm/storage/cloud_storage/s3_storage.py index 68fa8d845..788d5d126 100644 --- a/openwpm/storage/cloud_storage/s3_storage.py +++ b/openwpm/storage/cloud_storage/s3_storage.py @@ -1,5 +1,4 @@ from ..arrow_storage import ArrowProvider -from ..storage_providers import UnstructuredStorageProvider class S3StorageProvider(ArrowProvider): diff --git a/openwpm/storage/leveldb.py b/openwpm/storage/leveldb.py index 63796b3b3..14a629831 100644 --- a/openwpm/storage/leveldb.py +++ b/openwpm/storage/leveldb.py @@ -1,5 +1,4 @@ from pathlib import Path -from typing import Optional import plyvel from plyvel._plyvel import WriteBatch diff --git a/openwpm/storage/local_storage.py b/openwpm/storage/local_storage.py index 89aa84e2d..e958a3d63 100644 --- a/openwpm/storage/local_storage.py +++ b/openwpm/storage/local_storage.py @@ -1,5 +1,4 @@ import logging -from abc import ABC from pathlib import Path import pyarrow.parquet as pq diff --git a/openwpm/storage/sql_provider.py b/openwpm/storage/sql_provider.py index f05bd0e58..0e9d66900 100644 --- a/openwpm/storage/sql_provider.py +++ b/openwpm/storage/sql_provider.py @@ -11,7 +11,7 @@ OperationalError, ProgrammingError, ) -from typing import Any, Awaitable, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Tuple from openwpm.types import VisitId diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 1514ba6fc..f6ae479d9 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -101,15 +101,15 @@ async def handler( """Created for every new connection to the Server""" self.logger.debug("Initializing new handler") while True: - try: - record: Tuple[str, Any] = await get_message_from_reader(reader) - except IOError as e: + if reader.at_eof(): + # the socket is closed self.logger.debug( "Terminating handler, because the underlying socket closed", - exc_info=e, ) break + record: Tuple[str, Any] = await get_message_from_reader(reader) + if len(record) != 2: self.logger.error("Query is not the correct length %s", repr(record)) continue diff --git a/openwpm/storage/storage_providers.py b/openwpm/storage/storage_providers.py index bcc9e94b2..184fb96a2 100644 --- a/openwpm/storage/storage_providers.py +++ b/openwpm/storage/storage_providers.py @@ -7,9 +7,7 @@ import io from abc import ABC, abstractmethod from asyncio import Task -from typing import Any, Awaitable, Dict, NewType, Optional - -from pyarrow.lib import Table +from typing import Any, Dict, NewType, Optional from openwpm.types import VisitId diff --git a/openwpm/task_manager.py b/openwpm/task_manager.py index 5c1999f0d..0da2ba126 100644 --- a/openwpm/task_manager.py +++ b/openwpm/task_manager.py @@ -27,13 +27,7 @@ from .errors import CommandExecutionError from .js_instrumentation import clean_js_instrumentation_settings from .mp_logger import MPLogger -from .socket_interface import ClientSocket -from .storage.storage_controller import ( - ACTION_TYPE_FINALIZE, - RECORD_TYPE_META, - DataSocket, - StorageControllerHandle, -) +from .storage.storage_controller import DataSocket, StorageControllerHandle from .storage.storage_providers import ( StructuredStorageProvider, TableName, diff --git a/openwpm/utilities/platform_utils.py b/openwpm/utilities/platform_utils.py index e61dbd510..cee880cc0 100644 --- a/openwpm/utilities/platform_utils.py +++ b/openwpm/utilities/platform_utils.py @@ -3,9 +3,7 @@ import subprocess from collections import OrderedDict from copy import deepcopy -from pathlib import Path from sys import platform -from typing import Any from tabulate import tabulate From 1846d2541d9968ff24d8ab00876947796ddff41e Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 12 Feb 2021 12:31:51 +0100 Subject: [PATCH 125/139] Fixing up comments --- openwpm/storage/storage_controller.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index f6ae479d9..8d5aa917d 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -57,13 +57,13 @@ def __init__( Parameters ---------- status_queue - queue that the current amount of records to be processed will - be sent to - also used for initialization + queue through which the StorageControllerHandler + receives updates on the current amount of records to be processed. + Also used for initialization completion_queue - queue containing the visitIDs of saved records + queue containing the visit_ids of saved records shutdown_queue - queue that the main process can use to shut down the listener + queue that the main process can use to shut down the StorageController """ self.status_queue = status_queue self.completion_queue = completion_queue @@ -92,7 +92,7 @@ async def _handler( await self.handler(reader, writer) except Exception as e: self.logger.error( - "An exception occurred while processing for records", exc_info=e + "An exception occurred while processing records", exc_info=e ) async def handler( @@ -205,7 +205,8 @@ async def finalize_visit_id( if visit_id not in self.store_record_tasks: self.logger.error( - "Visit_id %d got finalized multiple times, skipping...", visit_id + "There are no records to be stored for visit_id %d, skipping...", + visit_id, ) return None From dfdc34dbe45c6575dd7fa0a4dbda7064d7d20bbf Mon Sep 17 00:00:00 2001 From: vringar Date: Fri, 12 Feb 2021 12:46:29 +0100 Subject: [PATCH 126/139] Fixing up comments --- openwpm/storage/storage_controller.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 8d5aa917d..ee66a3bfa 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -76,7 +76,9 @@ def __init__( ) """Contains all store_record tasks for a given visit_id""" self.finalize_tasks: List[Tuple[VisitId, Optional[Task[None]], bool]] = [] - """Contains all information required for update_completion_queue to work""" + """Contains all information required for update_completion_queue to work + Tuple structure is: VisitId, optional completion token, success + """ self.structured_storage = structured_storage self.unstructured_storage = unstructured_storage self._last_record_received: Optional[float] = None @@ -280,7 +282,11 @@ async def save_batch_if_past_timeout(self) -> NoReturn: """Save the current batch of records if no new data has been received. If we aren't receiving new data for this batch we commit early - regardless of the current batch size.""" + regardless of the current batch size. + + This coroutine will get cancelled with an exception + so there is no need for an orderly return + """ while True: if self._last_record_received is None: await asyncio.sleep(BATCH_COMMIT_TIMEOUT) @@ -308,7 +314,9 @@ async def update_completion_queue(self) -> None: # is forbidden new_finalize_tasks: List[Tuple[VisitId, Optional[Task[None]], bool]] = [] for visit_id, token, success in self.finalize_tasks: - if not token or token.done(): + if ( + not token or token.done() + ): # Either way all data for the visit_id was saved out self.completion_queue.put((visit_id, success)) else: new_finalize_tasks.append((visit_id, token, success)) @@ -502,7 +510,7 @@ def shutdown(self, relaxed: bool = True) -> None: ) def get_most_recent_status(self) -> int: - """Return the most recent queue size sent from the listener process""" + """Return the most recent queue size sent from the Storage Controller process""" # Block until we receive the first status update if self._last_status is None: From b92277474057e6f841aa53f140ee0324a21e528c Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 15 Feb 2021 16:39:30 +0100 Subject: [PATCH 127/139] More docs --- openwpm/storage/storage_providers.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/openwpm/storage/storage_providers.py b/openwpm/storage/storage_providers.py index 184fb96a2..2a2bc6aa6 100644 --- a/openwpm/storage/storage_providers.py +++ b/openwpm/storage/storage_providers.py @@ -44,6 +44,17 @@ async def shutdown(self) -> None: class StructuredStorageProvider(StorageProvider): + """Structured Storage Providers are responsible for handling all structured data + that OpenWPM emits. + This includes: + - All data that is collected by the WebExtension instrumentation + - Data about browser configuration and + - Any data that custom commands send to the Storage Controller + + See docs/Schema-Documentation.md to see what an unmodified OpenWPM will attempt + to store + """ + def __init__(self) -> None: super().__init__() @@ -72,6 +83,14 @@ async def finalize_visit_id( class UnstructuredStorageProvider(StorageProvider): + """Unstructured Storage Providers are responsible for handling the unstructured data + that OpenWPM emits. + This is primarily content loaded by websites. + Don't make any assumptions about the data (especially don't assume it's valid unicode) + + In the future this interface will be expanded to address the needs of https://github.com/mozilla/OpenWPM/issues/232 + """ + @abstractmethod async def store_blob( self, From 55f6cdbf5b4c287338bce238a4220efa6c138cdd Mon Sep 17 00:00:00 2001 From: vringar Date: Tue, 16 Feb 2021 13:02:23 +0100 Subject: [PATCH 128/139] updated dependencies --- environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yaml b/environment.yaml index cb230478d..d55d90c7c 100644 --- a/environment.yaml +++ b/environment.yaml @@ -28,7 +28,7 @@ dependencies: - redis-py=3.5.3 - s3fs=0.5.2 - selenium=3.141.0 -- sentry-sdk=0.20.0 +- sentry-sdk=0.20.2 - tabulate=0.8.7 - tblib=1.6.0 - wget=1.20.1 From 65c6edaca43af609e3bf245048b55878b62d21e9 Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 17 Feb 2021 11:56:19 +0100 Subject: [PATCH 129/139] Fixed test_task_manager --- test/test_task_manager.py | 69 ++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/test/test_task_manager.py b/test/test_task_manager.py index 67ed79074..d27e84745 100644 --- a/test/test_task_manager.py +++ b/test/test_task_manager.py @@ -1,43 +1,44 @@ +"""Test TaskManager functionality.""" import pytest from openwpm.errors import CommandExecutionError from openwpm.task_manager import TaskManager -from .openwpmtest import OpenWPMTest from .utilities import BASE_TEST_URL -class TestTaskManager(OpenWPMTest): - """Test TaskManager functionality.""" - - def get_config(self, data_dir=""): - return self.get_test_config(data_dir) - - def test_failure_limit_value(self): - manager_params, _ = self.get_config() - # The default value for failure_limit is 2 * num_browsers + 10 - assert manager_params.failure_limit == 12 - manager_params.failure_limit = 2 - # Test that the chosen value is not overwritten by the default - assert manager_params.failure_limit == 2 - - def test_failure_limit_exceeded(self): - manager_params, browser_params = self.get_config() - manager_params.failure_limit = 0 - manager = TaskManager(manager_params, browser_params) - with pytest.raises(CommandExecutionError): - manager.get("example.com") # Selenium requires scheme prefix - manager.get("example.com") # Requires two commands to shut down - - def test_failure_limit_reset(self): - """Test that failure_count is reset on command sequence completion.""" - manager_params, browser_params = self.get_config() - manager_params.failure_limit = 1 - manager = TaskManager(manager_params, browser_params) - manager.get("example.com") # Selenium requires scheme prefix - manager.get(BASE_TEST_URL) # Successful command sequence - # Now failure_count should be reset to 0 and the following command - # failure should not raise a CommandExecutionError +def test_failure_limit_value(default_params): + manager_params, _ = default_params + manager_params.num_browsers = 1 + # The default value for failure_limit is 2 * num_browsers + 10 + assert manager_params.failure_limit == 12 + manager_params.failure_limit = 2 + # Test that the chosen value is not overwritten by the default + assert manager_params.failure_limit == 2 + + +def test_failure_limit_exceeded(task_manager_creator, default_params): + manager_params, browser_params = default_params + manager_params.num_browsers = 1 + manager_params.failure_limit = 0 + manager, _ = task_manager_creator((manager_params, browser_params[:1])) + + with pytest.raises(CommandExecutionError): manager.get("example.com") # Selenium requires scheme prefix - manager.get(BASE_TEST_URL) # Requires two commands to shut down - manager.close() + manager.get("example.com") # Requires two commands to shut down + manager.close() + + +def test_failure_limit_reset(task_manager_creator, default_params): + """Test that failure_count is reset on command sequence completion.""" + manager_params, browser_params = default_params + manager_params.num_browsers = 1 + manager_params.failure_limit = 1 + manager, _ = task_manager_creator((manager_params, browser_params[:1])) + manager.get("example.com") # Selenium requires scheme prefix + manager.get(BASE_TEST_URL) # Successful command sequence + # Now failure_count should be reset to 0 and the following command + # failure should not raise a CommandExecutionError + manager.get("example.com") # Selenium requires scheme prefix + manager.get(BASE_TEST_URL) # Requires two commands to shut down + manager.close() From 048546df360bb9147dc805ddb23599b550b360b2 Mon Sep 17 00:00:00 2001 From: vringar Date: Wed, 17 Feb 2021 12:38:46 +0100 Subject: [PATCH 130/139] Reupgraded to python 3.9.1 --- environment.yaml | 2 +- scripts/environment-unpinned.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/environment.yaml b/environment.yaml index f5ac7e64e..d55d90c7c 100644 --- a/environment.yaml +++ b/environment.yaml @@ -23,7 +23,7 @@ dependencies: - pytest-asyncio=0.14.0 - pytest-cov=2.11.1 - pytest=6.2.2 -- python=3.8.6 +- python=3.9.1 - pyvirtualdisplay=1.3.2 - redis-py=3.5.3 - s3fs=0.5.2 diff --git a/scripts/environment-unpinned.yaml b/scripts/environment-unpinned.yaml index dbed2d21b..529ef937b 100644 --- a/scripts/environment-unpinned.yaml +++ b/scripts/environment-unpinned.yaml @@ -17,7 +17,7 @@ dependencies: - pillow - psutil - pyarrow - - python==3.8.6 + - python - pyvirtualdisplay - redis-py - s3fs From 59484ded6d2894960b44b6985b57806c94ec4931 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 22 Feb 2021 12:56:18 +0100 Subject: [PATCH 131/139] Restoring crawl_reference in mp_logger --- openwpm/mp_logger.py | 13 +++++++++---- openwpm/storage/cloud_storage/gcp_storage.py | 3 +++ openwpm/task_manager.py | 5 +++-- setup.cfg | 2 +- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/openwpm/mp_logger.py b/openwpm/mp_logger.py index 2d0f64483..7975c80fb 100644 --- a/openwpm/mp_logger.py +++ b/openwpm/mp_logger.py @@ -16,9 +16,8 @@ from sentry_sdk.integrations.logging import BreadcrumbHandler, EventHandler from tblib import pickling_support -from openwpm.config import ManagerParams - from .commands.utils.webdriver_utils import parse_neterror +from .config import ManagerParamsInternal from .socket_interface import ServerSocket pickling_support.install() @@ -101,13 +100,13 @@ class MPLogger(object): def __init__( self, log_file, - crawl_context: ManagerParams = None, + crawl_reference: str = None, log_level_console=logging.INFO, log_level_file=logging.DEBUG, log_level_sentry_breadcrumb=logging.DEBUG, log_level_sentry_event=logging.ERROR, ) -> None: - self._crawl_context = crawl_context + self._crawl_reference = crawl_reference self._log_level_console = log_level_console self._log_level_file = log_level_file self._log_level_sentry_breadcrumb = log_level_sentry_breadcrumb @@ -207,6 +206,12 @@ def _initialize_sentry(self): ) self._event_handler = EventHandler(level=self._log_level_sentry_event) sentry_sdk.init(dsn=self._sentry_dsn, before_send=self._sentry_before_send) + with sentry_sdk.configure_scope() as scope: + if self._crawl_reference: + scope.set_tag( + "CRAWL_REFERENCE", + self._crawl_reference, + ) def _start_listener(self): """Start listening socket for remote logs from extension""" diff --git a/openwpm/storage/cloud_storage/gcp_storage.py b/openwpm/storage/cloud_storage/gcp_storage.py index d1c42666a..09810f23e 100644 --- a/openwpm/storage/cloud_storage/gcp_storage.py +++ b/openwpm/storage/cloud_storage/gcp_storage.py @@ -37,6 +37,9 @@ def __init__( self.token = token self.base_path = f"{bucket_name}/{base_path}/{sub_dir}/{{table_name}}" + def __str__(self) -> str: + return f"GCS:{self.base_path.removesuffix('/{table_name}')}" + async def init(self) -> None: await super(GcsStructuredProvider, self).init() self.file_system = GCSFileSystem( diff --git a/openwpm/task_manager.py b/openwpm/task_manager.py index b5ffbacc3..2404e1b78 100644 --- a/openwpm/task_manager.py +++ b/openwpm/task_manager.py @@ -125,10 +125,11 @@ def __init__( self.failure_count = 0 self.failure_limit = manager_params.failure_limit - # Start logging server thread self.logging_server = MPLogger( - self.manager_params.log_file, self.manager_params, **self._logger_kwargs + self.manager_params.log_file, + str(structured_storage_provider), + **self._logger_kwargs ) self.manager_params.logger_address = self.logging_server.logger_address self.logger = logging.getLogger("openwpm") diff --git a/setup.cfg b/setup.cfg index 3f0295559..37cb0b7c8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,7 +7,7 @@ skip = venv,openwpm/Extension,firefox-bin [mypy] follow_imports = silent -python_version = 3.8 +python_version = 3.9 warn_unused_configs = True ignore_missing_imports = True disallow_incomplete_defs = True From 937c8fe1cbb952259346d5de57ffe0a2ab79d824 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 22 Feb 2021 13:16:54 +0100 Subject: [PATCH 132/139] Removed unused imports --- demo.py | 1 - openwpm/browser_manager.py | 2 +- openwpm/mp_logger.py | 1 - openwpm/socket_interface.py | 1 - test/manual_test.py | 2 +- test/openwpmtest.py | 4 ---- test/storage/fixtures.py | 4 +--- test/storage/test_gcp.py | 2 -- test/test_callback.py | 2 -- test/test_callstack_instrument.py | 3 --- test/test_crawl.py | 1 - test/test_custom_function_command.py | 2 +- test/test_mp_logger.py | 4 ---- test/test_simple_commands.py | 4 +--- test/test_task_manager.py | 1 - test/test_webdriver_utils.py | 3 --- 16 files changed, 5 insertions(+), 32 deletions(-) diff --git a/demo.py b/demo.py index eecda75e9..a7e3902c1 100644 --- a/demo.py +++ b/demo.py @@ -1,4 +1,3 @@ -import logging from pathlib import Path from custom_command import LinkCountingCommand diff --git a/openwpm/browser_manager.py b/openwpm/browser_manager.py index ff8d84ef7..544b8d817 100644 --- a/openwpm/browser_manager.py +++ b/openwpm/browser_manager.py @@ -113,7 +113,7 @@ def launch_browser_manager(self): ) # make sure browser loads crashed profile self.browser_params.recovery_tar = tempdir - + crash_recovery = True else: """ diff --git a/openwpm/mp_logger.py b/openwpm/mp_logger.py index 7975c80fb..7b9f46f35 100644 --- a/openwpm/mp_logger.py +++ b/openwpm/mp_logger.py @@ -17,7 +17,6 @@ from tblib import pickling_support from .commands.utils.webdriver_utils import parse_neterror -from .config import ManagerParamsInternal from .socket_interface import ServerSocket pickling_support.install() diff --git a/openwpm/socket_interface.py b/openwpm/socket_interface.py index 55219b221..ab2a4d064 100644 --- a/openwpm/socket_interface.py +++ b/openwpm/socket_interface.py @@ -4,7 +4,6 @@ import struct import threading import traceback -from asyncio import IncompleteReadError from queue import Queue from typing import Any diff --git a/test/manual_test.py b/test/manual_test.py index 5d923f7ac..0f5afb347 100644 --- a/test/manual_test.py +++ b/test/manual_test.py @@ -19,8 +19,8 @@ # in the interactive session from openwpm.commands.utils import webdriver_utils as wd_util # noqa isort:skip import domain_utils as du # noqa isort:skip -from selenium.webdriver.common.keys import Keys # noqa isort:skip from selenium.common.exceptions import * # noqa isort:skip +from selenium.webdriver.common.keys import Keys # noqa isort:skip OPENWPM_LOG_PREFIX = "console.log: openwpm: " INSERT_PREFIX = "Array" diff --git a/test/openwpmtest.py b/test/openwpmtest.py index 0840870b3..cb42810bd 100644 --- a/test/openwpmtest.py +++ b/test/openwpmtest.py @@ -1,7 +1,3 @@ -import logging -import os -from abc import ABCMeta, abstractmethod -from os.path import isfile, join from pathlib import Path from typing import List, Optional, Tuple diff --git a/test/storage/fixtures.py b/test/storage/fixtures.py index 67550cec2..a25423ca9 100644 --- a/test/storage/fixtures.py +++ b/test/storage/fixtures.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Set, Tuple +from typing import Any, List import pytest from _pytest.fixtures import FixtureRequest @@ -14,10 +14,8 @@ from openwpm.storage.storage_controller import INVALID_VISIT_ID from openwpm.storage.storage_providers import ( StructuredStorageProvider, - TableName, UnstructuredStorageProvider, ) -from openwpm.types import VisitId from test.storage.test_values import dt_test_values, generate_test_values memory_structured = "memory_structured" diff --git a/test/storage/test_gcp.py b/test/storage/test_gcp.py index d2a53c511..ac63e7c90 100644 --- a/test/storage/test_gcp.py +++ b/test/storage/test_gcp.py @@ -1,5 +1,3 @@ -import asyncio - import pytest from openwpm.storage.cloud_storage.gcp_storage import GcsStructuredProvider diff --git a/test/test_callback.py b/test/test_callback.py index 026edbded..472307632 100644 --- a/test/test_callback.py +++ b/test/test_callback.py @@ -2,9 +2,7 @@ from typing import List from openwpm.command_sequence import CommandSequence -from openwpm.task_manager import TaskManager -from .openwpmtest import OpenWPMTest from .utilities import BASE_TEST_URL diff --git a/test/test_callstack_instrument.py b/test/test_callstack_instrument.py index a31f7e61d..2b6cad08f 100644 --- a/test/test_callstack_instrument.py +++ b/test/test_callstack_instrument.py @@ -1,9 +1,6 @@ -from openwpm import task_manager from openwpm.utilities import db_utils -from openwpm.utilities.platform_utils import parse_http_stack_trace_str from . import utilities -from .openwpmtest import OpenWPMTest # HTTP request call stack instrumentation # Expected stack frames diff --git a/test/test_crawl.py b/test/test_crawl.py index f8f1df183..c65a2ab2b 100644 --- a/test/test_crawl.py +++ b/test/test_crawl.py @@ -8,7 +8,6 @@ import domain_utils as du import pytest -from openwpm import task_manager from openwpm.config import BrowserParams, ManagerParams from openwpm.utilities import db_utils diff --git a/test/test_custom_function_command.py b/test/test_custom_function_command.py index 2217be6d8..6768268b5 100644 --- a/test/test_custom_function_command.py +++ b/test/test_custom_function_command.py @@ -2,7 +2,7 @@ from selenium.webdriver import Firefox -from openwpm import command_sequence, task_manager +from openwpm import command_sequence from openwpm.commands.types import BaseCommand from openwpm.config import BrowserParams, ManagerParamsInternal from openwpm.socket_interface import ClientSocket diff --git a/test/test_mp_logger.py b/test/test_mp_logger.py index 72182a39c..8cdba99a8 100644 --- a/test/test_mp_logger.py +++ b/test/test_mp_logger.py @@ -2,13 +2,9 @@ import os import time -import pytest - from openwpm import mp_logger from openwpm.utilities.multiprocess_utils import Process -from .openwpmtest import OpenWPMTest - CHILD_INFO_STR_1 = "Child %d - INFO1" CHILD_INFO_STR_2 = "Child %d - INFO2" CHILD_DEBUG_STR = "Child %d - DEBUG" diff --git a/test/test_simple_commands.py b/test/test_simple_commands.py index 9c7e0ce5d..46b7787eb 100644 --- a/test/test_simple_commands.py +++ b/test/test_simple_commands.py @@ -10,17 +10,15 @@ import json import os import re -from typing import Callable, List, Tuple from urllib.parse import urlparse import pytest from PIL import Image -from openwpm import command_sequence, task_manager +from openwpm import command_sequence from openwpm.utilities import db_utils from . import utilities -from .openwpmtest import OpenWPMTest url_a = utilities.BASE_TEST_URL + "/simple_a.html" url_b = utilities.BASE_TEST_URL + "/simple_b.html" diff --git a/test/test_task_manager.py b/test/test_task_manager.py index d27e84745..40b0c743d 100644 --- a/test/test_task_manager.py +++ b/test/test_task_manager.py @@ -2,7 +2,6 @@ import pytest from openwpm.errors import CommandExecutionError -from openwpm.task_manager import TaskManager from .utilities import BASE_TEST_URL diff --git a/test/test_webdriver_utils.py b/test/test_webdriver_utils.py index 998a8250e..a9f518f8f 100644 --- a/test/test_webdriver_utils.py +++ b/test/test_webdriver_utils.py @@ -1,9 +1,6 @@ -from openwpm import task_manager from openwpm.commands.utils.webdriver_utils import parse_neterror from openwpm.utilities import db_utils -from .openwpmtest import OpenWPMTest - def test_parse_neterror(): text = ( From 1c819f7b6541d753f10d3cf2f2d7688c8253ed34 Mon Sep 17 00:00:00 2001 From: Stefan Zabka Date: Mon, 22 Feb 2021 13:28:31 +0100 Subject: [PATCH 133/139] Apply suggestions from code review Co-authored-by: Steven Englehardt --- openwpm/storage/arrow_storage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openwpm/storage/arrow_storage.py b/openwpm/storage/arrow_storage.py index 4f4f091b8..78be54b9c 100644 --- a/openwpm/storage/arrow_storage.py +++ b/openwpm/storage/arrow_storage.py @@ -142,8 +142,8 @@ async def flush_cache(self, lock: asyncio.Lock = None) -> None: So we either grab the storing_lock ourselves or the caller needs to pass us the locked storing_lock """ - got_lock = lock is not None - if not got_lock: + has_lock_arg = lock is not None + if not has_lock_arg: lock = self.storing_lock await lock.acquire() @@ -158,7 +158,7 @@ async def flush_cache(self, lock: asyncio.Lock = None) -> None: event.set() self.flush_events.clear() - if not got_lock: + if not has_lock_arg: lock.release() async def shutdown(self) -> None: From 2cbb801f9eb42a17554a9db41256881f7adbd2d3 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 22 Feb 2021 14:08:57 +0100 Subject: [PATCH 134/139] Cleaned up socket handling --- openwpm/socket_interface.py | 17 ++++++++++++++++- openwpm/storage/storage_controller.py | 13 ++++++------- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/openwpm/socket_interface.py b/openwpm/socket_interface.py index ab2a4d064..bf3862f80 100644 --- a/openwpm/socket_interface.py +++ b/openwpm/socket_interface.py @@ -144,7 +144,9 @@ def send(self, msg): msg = json.dumps(msg).encode("utf-8") serialization = b"j" else: - raise ValueError("Unsupported serialization type set: %s" % serialization) + raise ValueError( + "Unsupported serialization type set: %s" % self.serialization + ) if self.verbose: print("Sending message with serialization %s" % serialization) @@ -162,6 +164,19 @@ def close(self): async def get_message_from_reader(reader: asyncio.StreamReader) -> Any: + """ + Reads a message from the StreamReader + :exception IncompleteReadError if the underlying socket is closed + + To safely use this method, you should guard against the exception + like this: + ``` + try: + record: Tuple[str, Any] = await get_message_from_reader(reader) + except IncompleteReadError as e: + print("The underlying socket closed", repr(e)) + ``` + """ msg = await reader.readexactly(5) msglen, serialization = struct.unpack(">Lc", msg) msg = await reader.readexactly(msglen) diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index ee66a3bfa..8903b0e6a 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -5,7 +5,7 @@ import random import socket import time -from asyncio import Task +from asyncio import IncompleteReadError, Task from collections import defaultdict from typing import Any, DefaultDict, Dict, List, NoReturn, Optional, Tuple @@ -103,15 +103,14 @@ async def handler( """Created for every new connection to the Server""" self.logger.debug("Initializing new handler") while True: - if reader.at_eof(): - # the socket is closed - self.logger.debug( + try: + record: Tuple[str, Any] = await get_message_from_reader(reader) + except IncompleteReadError: + self.logger.info( "Terminating handler, because the underlying socket closed", + exc_info=True, ) break - - record: Tuple[str, Any] = await get_message_from_reader(reader) - if len(record) != 2: self.logger.error("Query is not the correct length %s", repr(record)) continue From 2dd339cfa0ae1ad25e5772a88b23ef4a4608e34d Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 22 Feb 2021 14:20:08 +0100 Subject: [PATCH 135/139] Fixed TaskManager.__exit__ --- demo.py | 44 ++++++++++++--------------- openwpm/storage/storage_controller.py | 3 +- openwpm/task_manager.py | 11 +++++-- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/demo.py b/demo.py index a7e3902c1..4267faf43 100644 --- a/demo.py +++ b/demo.py @@ -47,36 +47,32 @@ # Commands time out by default after 60 seconds -manager = TaskManager( +with TaskManager( manager_params, browser_params, SQLiteStorageProvider(Path("./datadir/crawl-data.sqlite")), None, -) +) as manager: + # Visits the sites + for index, site in enumerate(sites): -# Visits the sites -for index, site in enumerate(sites): + def callback(success: bool, val: str = site) -> None: + print( + f"CommandSequence for {val} ran {'successfully' if success else 'unsuccessfully'}" + ) - def callback(success: bool, val: str = site) -> None: - print( - f"CommandSequence for {val} ran {'successfully' if success else 'unsuccessfully'}" + # Parallelize sites over all number of browsers set above. + command_sequence = CommandSequence( + site, + site_rank=index, + reset=True, + callback=callback, ) - # Parallelize sites over all number of browsers set above. - command_sequence = CommandSequence( - site, - site_rank=index, - reset=True, - callback=callback, - ) + # Start by visiting the page + command_sequence.append_command(GetCommand(url=site, sleep=3), timeout=60) + # Have a look at custom_command.py to see how to implement your own command + command_sequence.append_command(LinkCountingCommand()) - # Start by visiting the page - command_sequence.append_command(GetCommand(url=site, sleep=3), timeout=60) - # Have a look at custom_command.py to see how to implement your own command - command_sequence.append_command(LinkCountingCommand()) - - # Run commands across the three browsers (simple parallelization) - manager.execute_command_sequence(command_sequence) - -# Shuts down the browsers and waits for the data to finish logging -manager.close() + # Run commands across the three browsers (simple parallelization) + manager.execute_command_sequence(command_sequence) diff --git a/openwpm/storage/storage_controller.py b/openwpm/storage/storage_controller.py index 8903b0e6a..28b441f6e 100644 --- a/openwpm/storage/storage_controller.py +++ b/openwpm/storage/storage_controller.py @@ -107,8 +107,7 @@ async def handler( record: Tuple[str, Any] = await get_message_from_reader(reader) except IncompleteReadError: self.logger.info( - "Terminating handler, because the underlying socket closed", - exc_info=True, + "Terminating handler, because the underlying socket closed" ) break if len(record) != 2: diff --git a/openwpm/task_manager.py b/openwpm/task_manager.py index 2404e1b78..93d05245a 100644 --- a/openwpm/task_manager.py +++ b/openwpm/task_manager.py @@ -2,11 +2,13 @@ import logging import os import pickle +import sys import threading import time import traceback from queue import Empty as EmptyQueue -from typing import Any, Dict, List, Optional, Set, Tuple +from types import TracebackType +from typing import Any, Dict, List, Optional, Set, Tuple, Type import psutil import tblib @@ -172,7 +174,12 @@ def __enter__(self): """ return self - def __exit__(self) -> None: + def __exit__( + self, + exc_type: Type[BaseException], + exc_val: BaseException, + exc_tb: TracebackType, + ) -> None: """ Execute shutdown procedure for TaskManager """ From a555c14e8201b769565ef604122432fa939cf652 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 22 Feb 2021 14:30:42 +0100 Subject: [PATCH 136/139] Moved validation code into config.py --- openwpm/config.py | 3 +++ openwpm/task_manager.py | 9 +++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/openwpm/config.py b/openwpm/config.py index 087474202..d8f4a4777 100644 --- a/openwpm/config.py +++ b/openwpm/config.py @@ -266,6 +266,9 @@ def validate_manager_params(manager_params: ManagerParams) -> None: def validate_crawl_configs( manager_params: ManagerParams, browser_params: List[BrowserParams] ) -> None: + validate_manager_params(manager_params) + for bp in browser_params: + validate_browser_params(bp) if len(browser_params) != manager_params.num_browsers: raise ConfigError( diff --git a/openwpm/task_manager.py b/openwpm/task_manager.py index 93d05245a..7dd362349 100644 --- a/openwpm/task_manager.py +++ b/openwpm/task_manager.py @@ -78,9 +78,6 @@ def __init__( Keyword arguments to pass to MPLogger on initialization. """ - validate_manager_params(manager_params_temp) - for bp in browser_params_temp: - validate_browser_params(bp) validate_crawl_configs(manager_params_temp, browser_params_temp) manager_params = ManagerParamsInternal.from_dict(manager_params_temp.to_dict()) browser_params = [ @@ -176,9 +173,9 @@ def __enter__(self): def __exit__( self, - exc_type: Type[BaseException], - exc_val: BaseException, - exc_tb: TracebackType, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], ) -> None: """ Execute shutdown procedure for TaskManager From 4f6aed128a18d9a51c1e386cec6e5096c25888b7 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 22 Feb 2021 15:38:11 +0100 Subject: [PATCH 137/139] Removed comment --- openwpm/task_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openwpm/task_manager.py b/openwpm/task_manager.py index 7dd362349..e61d98f04 100644 --- a/openwpm/task_manager.py +++ b/openwpm/task_manager.py @@ -609,7 +609,6 @@ def execute_command_sequence( """ # Block if the storage controller has too many unfinished records - # TODO create tests that these numbers are still meaningful agg_queue_size = self.storage_controller_handle.get_most_recent_status() if agg_queue_size >= STORAGE_CONTROLLER_JOB_LIMIT: while agg_queue_size >= STORAGE_CONTROLLER_JOB_LIMIT: From eb08c13514ae60fb1d1d22bbdbd8cf5ca4bfc4e9 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 22 Feb 2021 15:39:34 +0100 Subject: [PATCH 138/139] Removed comment --- test/storage/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/storage/__init__.py b/test/storage/__init__.py index 374c8f7b5..e69de29bb 100644 --- a/test/storage/__init__.py +++ b/test/storage/__init__.py @@ -1,3 +0,0 @@ -"""TODO: Write tests for a correctly set interrupt status - -""" From 4820b2cdd819869ed5ae0f014aa035a29f01b2f3 Mon Sep 17 00:00:00 2001 From: vringar Date: Mon, 22 Feb 2021 15:53:39 +0100 Subject: [PATCH 139/139] Removed comment --- openwpm/storage/parquet_schema.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/openwpm/storage/parquet_schema.py b/openwpm/storage/parquet_schema.py index 7c11c8302..cf83f4406 100644 --- a/openwpm/storage/parquet_schema.py +++ b/openwpm/storage/parquet_schema.py @@ -16,7 +16,6 @@ pa.field("openwpm_version", pa.string(), nullable=False), pa.field("browser_version", pa.string(), nullable=False), pa.field("instance_id", pa.uint32(), nullable=False), - # Omitting start_time as I couldn't figure out how to implement it ] PQ_SCHEMAS["task"] = pa.schema(fields) @@ -25,7 +24,6 @@ pa.field("task_id", pa.int64(), nullable=False), pa.field("browser_params", pa.string(), nullable=False), pa.field("instance_id", pa.uint32(), nullable=False), - # Omitting start_time as I couldn't figure out how to implement it ] PQ_SCHEMAS["crawl"] = pa.schema(fields)