From 4779d3a1618d3f15ec8ac8d74a4ad6012164a79f Mon Sep 17 00:00:00 2001 From: John Sirois Date: Mon, 13 Apr 2020 20:56:50 -0700 Subject: [PATCH] Utilities to create subsystem instances. (#9528) Introduce utility functions to create GoalSubsystem and Subsystem instances for testing. Fixes #9141 --- src/python/pants/engine/goal_test.py | 25 +++---- src/python/pants/rules/core/fmt_test.py | 19 +++--- src/python/pants/rules/core/lint_test.py | 13 +--- .../pants/rules/core/list_backends_test.py | 18 ++--- .../rules/core/list_target_types_test.py | 28 ++++---- .../pants/rules/core/list_targets_test.py | 22 ++---- src/python/pants/rules/core/run_test.py | 21 +++--- src/python/pants/rules/core/test_test.py | 12 +--- src/python/pants/testutil/engine/util.py | 67 ++++++++++++++++++- 9 files changed, 127 insertions(+), 98 deletions(-) diff --git a/src/python/pants/engine/goal_test.py b/src/python/pants/engine/goal_test.py index c0115e34878..a1fa54109b0 100644 --- a/src/python/pants/engine/goal_test.py +++ b/src/python/pants/engine/goal_test.py @@ -1,19 +1,12 @@ # Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from unittest.mock import Mock - -import pytest - from pants.engine.console import Console from pants.engine.goal import Goal, GoalSubsystem, LineOriented from pants.engine.rules import goal_rule -from pants.testutil.engine.util import MockConsole, run_rule +from pants.testutil.engine.util import MockConsole, create_goal_subsystem, run_rule -@pytest.mark.skip( - reason="Figure out how to create a GoalSubsystem for tests. We can't call global_instance()" -) def test_line_oriented_goal() -> None: class OutputtingGoalOptions(LineOriented, GoalSubsystem): name = "dummy" @@ -29,11 +22,13 @@ def output_rule(console: Console, options: OutputtingGoalOptions) -> OutputtingG print_stdout("line oriented") return OutputtingGoal(0) - mock_console = MockConsole() - # TODO: how should we mock `GoalSubsystem`s passed to `run_rule`? - mock_options = Mock() - mock_options.output = OutputtingGoalOptions.output - mock_options.line_oriented = OutputtingGoalOptions.line_oriented - result: OutputtingGoal = run_rule(output_rule, rule_args=[mock_console, mock_options]) + console = MockConsole() + result: OutputtingGoal = run_rule( + output_rule, + rule_args=[ + console, + create_goal_subsystem(OutputtingGoalOptions, sep="\\n", output_file=None), + ], + ) assert result.exit_code == 0 - assert mock_console.stdout.getvalue() == "output...line oriented" + assert console.stdout.getvalue() == "output...line oriented\n" diff --git a/src/python/pants/rules/core/fmt_test.py b/src/python/pants/rules/core/fmt_test.py index 849574092e0..94f75645c43 100644 --- a/src/python/pants/rules/core/fmt_test.py +++ b/src/python/pants/rules/core/fmt_test.py @@ -27,18 +27,19 @@ TargetAdaptorWithOrigin, ) from pants.engine.rules import UnionMembership -from pants.rules.core.fmt import Fmt, FmtResult, LanguageFmtResults, LanguageFormatters, fmt -from pants.testutil.engine.util import MockConsole, MockGet, run_rule +from pants.rules.core.fmt import ( + Fmt, + FmtOptions, + FmtResult, + LanguageFmtResults, + LanguageFormatters, + fmt, +) +from pants.testutil.engine.util import MockConsole, MockGet, create_goal_subsystem, run_rule from pants.testutil.test_base import TestBase from pants.util.ordered_set import OrderedSet -# TODO(#9141): replace this with a proper util to create `GoalSubsystem`s -class MockOptions: - def __init__(self, **values): - self.values = Mock(**values) - - class MockLanguageFormatters(LanguageFormatters, metaclass=ABCMeta): @staticmethod @abstractmethod @@ -132,7 +133,7 @@ def run_fmt_rule( rule_args=[ console, HydratedTargetsWithOrigins(targets), - MockOptions(per_target_caching=per_target_caching), + create_goal_subsystem(FmtOptions, per_target_caching=per_target_caching), Workspace(self.scheduler), union_membership, ], diff --git a/src/python/pants/rules/core/lint_test.py b/src/python/pants/rules/core/lint_test.py index 9f91b8c73d0..c7ba4d0ce82 100644 --- a/src/python/pants/rules/core/lint_test.py +++ b/src/python/pants/rules/core/lint_test.py @@ -3,25 +3,18 @@ from abc import ABCMeta, abstractmethod from typing import Iterable, List, Tuple, Type -from unittest.mock import Mock from pants.build_graph.address import Address from pants.engine.legacy.graph import HydratedTargetsWithOrigins, HydratedTargetWithOrigin from pants.engine.legacy.structs import TargetAdaptorWithOrigin from pants.engine.rules import UnionMembership from pants.rules.core.fmt_test import FmtTest -from pants.rules.core.lint import Lint, Linter, LintResult, lint -from pants.testutil.engine.util import MockConsole, MockGet, run_rule +from pants.rules.core.lint import Lint, Linter, LintOptions, LintResult, lint +from pants.testutil.engine.util import MockConsole, MockGet, create_goal_subsystem, run_rule from pants.testutil.test_base import TestBase from pants.util.ordered_set import OrderedSet -# TODO(#9141): replace this with a proper util to create `GoalSubsystem`s -class MockOptions: - def __init__(self, **values): - self.values = Mock(**values) - - class MockLinter(Linter, metaclass=ABCMeta): @staticmethod def is_valid_target(_: TargetAdaptorWithOrigin) -> bool: @@ -107,7 +100,7 @@ def run_lint_rule( rule_args=[ console, HydratedTargetsWithOrigins(targets), - MockOptions(per_target_caching=per_target_caching), + create_goal_subsystem(LintOptions, per_target_caching=per_target_caching), union_membership, ], mock_gets=[ diff --git a/src/python/pants/rules/core/list_backends_test.py b/src/python/pants/rules/core/list_backends_test.py index 44e5dfd1ca5..668f99e13e4 100644 --- a/src/python/pants/rules/core/list_backends_test.py +++ b/src/python/pants/rules/core/list_backends_test.py @@ -1,24 +1,20 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from contextlib import contextmanager from textwrap import dedent from pants.engine.fs import EMPTY_SNAPSHOT, Digest, FileContent, FilesContent, PathGlobs, Snapshot from pants.option.global_options import GlobalOptions -from pants.rules.core.list_backends import hackily_get_module_docstring, list_backends +from pants.rules.core.list_backends import ( + BackendsOptions, + hackily_get_module_docstring, + list_backends, +) from pants.source.source_root import SourceRootConfig -from pants.testutil.engine.util import MockConsole, MockGet, run_rule +from pants.testutil.engine.util import MockConsole, MockGet, create_goal_subsystem, run_rule from pants.testutil.subsystem.util import global_subsystem_instance -# TODO(#9141): replace this with a proper util to create `GoalSubsystem`s -class MockOptions: - @contextmanager - def line_oriented(self, console: MockConsole): - yield lambda msg: console.print_stdout(msg) - - def test_hackily_get_module_docstring() -> None: assert ( hackily_get_module_docstring( @@ -156,7 +152,7 @@ def rules(): run_rule( list_backends, rule_args=[ - MockOptions(), + create_goal_subsystem(BackendsOptions, sep="\\n", output_file=None), global_subsystem_instance(SourceRootConfig), global_subsystem_instance(GlobalOptions), console, diff --git a/src/python/pants/rules/core/list_target_types_test.py b/src/python/pants/rules/core/list_target_types_test.py index e259b144dc6..dfac1f6b51c 100644 --- a/src/python/pants/rules/core/list_target_types_test.py +++ b/src/python/pants/rules/core/list_target_types_test.py @@ -1,29 +1,23 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from contextlib import contextmanager from enum import Enum from textwrap import dedent from typing import Optional, cast -from unittest.mock import Mock from pants.engine.rules import UnionMembership from pants.engine.target import BoolField, IntField, RegisteredTargetTypes, StringField, Target -from pants.rules.core.list_target_types import list_target_types -from pants.testutil.engine.util import MockConsole, run_rule +from pants.option.global_options import GlobalOptions +from pants.rules.core.list_target_types import TargetTypesOptions, list_target_types +from pants.testutil.engine.util import ( + MockConsole, + create_goal_subsystem, + create_subsystem, + run_rule, +) from pants.util.ordered_set import OrderedSet -# TODO(#9141): replace this with a proper util to create `GoalSubsystem`s -class MockOptions: - def __init__(self, **values): - self.values = Mock(**values) - - @contextmanager - def line_oriented(self, console: MockConsole): - yield lambda msg: console.print_stdout(msg) - - # Note no docstring. class FortranVersion(StringField): alias = "fortran_version" @@ -90,8 +84,10 @@ def run_goal( rule_args=[ RegisteredTargetTypes.create([FortranBinary, FortranLibrary, FortranTests]), union_membership or UnionMembership({}), - MockOptions(details=details_target), - Mock(), + create_goal_subsystem( + TargetTypesOptions, sep="\\n", output_file=None, details=details_target + ), + create_subsystem(GlobalOptions, v1=False), console, ], ) diff --git a/src/python/pants/rules/core/list_targets_test.py b/src/python/pants/rules/core/list_targets_test.py index 14461f0545f..af725655e2c 100644 --- a/src/python/pants/rules/core/list_targets_test.py +++ b/src/python/pants/rules/core/list_targets_test.py @@ -1,29 +1,16 @@ # Copyright 2020 Pants project contributors (see CONTRIBUTORS.md). # Licensed under the Apache License, Version 2.0 (see LICENSE). -from contextlib import contextmanager from textwrap import dedent from typing import List, Optional, Tuple, cast -from unittest.mock import Mock from pants.backend.jvm.artifact import Artifact from pants.backend.jvm.repository import Repository from pants.build_graph.address import Address from pants.engine.addressable import Addresses from pants.engine.target import DescriptionField, ProvidesField, Target, Targets -from pants.rules.core.list_targets import list_targets -from pants.testutil.engine.util import MockConsole, MockGet, run_rule - - -# TODO(#9141): replace this with a proper util to create `GoalSubsystem`s -class MockOptions: - def __init__(self, **values): - self.values = Mock(**values) - self.name = "list" - - @contextmanager - def line_oriented(self, console: MockConsole): - yield lambda msg: console.print_stdout(msg) +from pants.rules.core.list_targets import ListOptions, list_targets +from pants.testutil.engine.util import MockConsole, MockGet, create_goal_subsystem, run_rule class MockTarget(Target): @@ -43,7 +30,10 @@ def run_goal( list_targets, rule_args=[ Addresses(tgt.address for tgt in targets), - MockOptions( + create_goal_subsystem( + ListOptions, + sep="\\n", + output_file=None, documented=show_documented, provides=show_provides, provides_columns=provides_columns or "address,artifact_id", diff --git a/src/python/pants/rules/core/run_test.py b/src/python/pants/rules/core/run_test.py index 63a19db1535..a5208b8b8a8 100644 --- a/src/python/pants/rules/core/run_test.py +++ b/src/python/pants/rules/core/run_test.py @@ -2,7 +2,6 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from typing import cast -from unittest.mock import Mock from pants.base.build_root import BuildRoot from pants.build_graph.address import Address @@ -11,17 +10,17 @@ from pants.engine.interactive_runner import InteractiveProcessRequest, InteractiveRunner from pants.option.global_options import GlobalOptions from pants.rules.core.binary import CreatedBinary -from pants.rules.core.run import Run, run -from pants.testutil.engine.util import MockConsole, MockGet, run_rule +from pants.rules.core.run import Run, RunOptions, run +from pants.testutil.engine.util import ( + MockConsole, + MockGet, + create_goal_subsystem, + create_subsystem, + run_rule, +) from pants.testutil.test_base import TestBase -# TODO(#9141): replace this with a proper util to create `GoalSubsystem`s -class MockOptions: - def __init__(self, **values): - self.values = Mock(**values) - - class RunTest(TestBase): def create_mock_binary(self, program_text: bytes) -> CreatedBinary: input_files_content = InputFilesContent( @@ -44,8 +43,8 @@ def single_target_run( interactive_runner, BuildRoot(), Addresses([Address.parse(address_spec)]), - MockOptions(args=[]), - GlobalOptions.global_instance(), + create_goal_subsystem(RunOptions, args=[]), + create_subsystem(GlobalOptions, pants_workdir=self.pants_workdir), ], mock_gets=[ MockGet( diff --git a/src/python/pants/rules/core/test_test.py b/src/python/pants/rules/core/test_test.py index 049dd8661b6..91816d91e66 100644 --- a/src/python/pants/rules/core/test_test.py +++ b/src/python/pants/rules/core/test_test.py @@ -5,7 +5,6 @@ from pathlib import PurePath from textwrap import dedent from typing import List, Optional, Tuple, Type, cast -from unittest.mock import Mock import pytest @@ -40,21 +39,16 @@ Test, TestConfiguration, TestDebugRequest, + TestOptions, TestResult, WrappedTestConfiguration, run_tests, ) -from pants.testutil.engine.util import MockConsole, MockGet, run_rule +from pants.testutil.engine.util import MockConsole, MockGet, create_goal_subsystem, run_rule from pants.testutil.test_base import TestBase from pants.util.ordered_set import OrderedSet -# TODO(#9141): replace this with a proper util to create `GoalSubsystem`s -class MockOptions: - def __init__(self, **values): - self.values = Mock(**values) - - class MockTarget(Target): alias = "mock_target" core_fields = (Sources,) @@ -157,7 +151,7 @@ def run_test_rule( include_sources: bool = True, ) -> Tuple[int, str]: console = MockConsole(use_colors=False) - options = MockOptions(debug=debug, run_coverage=False) + options = create_goal_subsystem(TestOptions, debug=debug, run_coverage=False) interactive_runner = InteractiveRunner(self.scheduler) workspace = Workspace(self.scheduler) union_membership = UnionMembership({TestConfiguration: OrderedSet([config])}) diff --git a/src/python/pants/testutil/engine/util.py b/src/python/pants/testutil/engine/util.py index 66f2539add9..891ae3dfc0b 100644 --- a/src/python/pants/testutil/engine/util.py +++ b/src/python/pants/testutil/engine/util.py @@ -6,12 +6,25 @@ from dataclasses import dataclass from io import StringIO from types import CoroutineType, GeneratorType -from typing import Any, Callable, List, Optional, Sequence, Tuple, Type, get_type_hints +from typing import ( + Any, + Callable, + List, + Optional, + Sequence, + Tuple, + Type, + TypeVar, + Union, + cast, + get_type_hints, +) from colors import blue, cyan, green, magenta, red from pants.base.file_system_project_tree import FileSystemProjectTree from pants.engine.addressable import addressable_sequence +from pants.engine.goal import GoalSubsystem from pants.engine.native import Native from pants.engine.parser import SymbolTable from pants.engine.rules import UnionMembership @@ -19,6 +32,9 @@ from pants.engine.selectors import Get from pants.engine.struct import Struct from pants.option.global_options import DEFAULT_EXECUTION_OPTIONS +from pants.option.option_value_container import OptionValueContainer +from pants.option.ranked_value import Rank, RankedValue, Value +from pants.subsystem.subsystem import Subsystem from pants.util.objects import SubclassesOf @@ -31,6 +47,55 @@ class MockGet: mock: Callable[[Any], Any] +def _create_scoped_options( + default_rank: Rank, **options: Union[RankedValue, Value] +) -> OptionValueContainer: + scoped_options = OptionValueContainer() + for key, value in options.items(): + if not isinstance(value, RankedValue): + value = RankedValue(default_rank, value) + setattr(scoped_options, key, value) + return scoped_options + + +GS = TypeVar("GS", bound=GoalSubsystem) + + +def create_goal_subsystem( + goal_subsystem_type: Type[GS], + default_rank: Rank = Rank.NONE, + **options: Union[RankedValue, Value], +) -> GS: + """Creates a new goal subsystem instance populated with the given option values. + + :param goal_subsystem_type: The `GoalSubsystem` type to create. + :param default_rank: The rank to assign any raw option values passed. + :param options: The option values to populate the new goal subsystem instance with. + """ + return goal_subsystem_type( + scope=goal_subsystem_type.name, + scoped_options=_create_scoped_options(default_rank, **options), + ) + + +SS = TypeVar("SS", bound=Subsystem) + + +def create_subsystem( + subsystem_type: Type[SS], default_rank: Rank = Rank.NONE, **options: Union[RankedValue, Value], +) -> SS: + """Creates a new subsystem instance populated with the given option values. + + :param subsystem_type: The `Subsystem` type to create. + :param default_rank: The rank to assign any raw option values passed. + :param options: The option values to populate the new subsystem instance with. + """ + options_scope = cast(str, subsystem_type.options_scope) + return subsystem_type( + scope=options_scope, scoped_options=_create_scoped_options(default_rank, **options), + ) + + def run_rule( rule, *,