diff --git a/src/python/pants/backend/codegen/protobuf/target_types.py b/src/python/pants/backend/codegen/protobuf/target_types.py index 1f8c032227c..0c671dc96db 100644 --- a/src/python/pants/backend/codegen/protobuf/target_types.py +++ b/src/python/pants/backend/codegen/protobuf/target_types.py @@ -2,7 +2,8 @@ # Licensed under the Apache License, Version 2.0 (see LICENSE). from pants.backend.codegen.protobuf.protoc import Protoc -from pants.engine.rules import Get, collect_rules, rule +from pants.engine.fs import PathGlobs, Paths +from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import ( COMMON_TARGET_FIELDS, AllTargets, @@ -11,6 +12,7 @@ GeneratedTargets, GenerateTargetsRequest, MultipleSourcesField, + OverridesField, SingleSourceField, SourcesPaths, SourcesPathsRequest, @@ -19,6 +21,7 @@ generate_file_level_targets, ) from pants.engine.unions import UnionMembership, UnionRule +from pants.option.global_options import FilesNotFoundBehavior from pants.util.docutil import doc_url from pants.util.logging import LogLevel @@ -77,6 +80,29 @@ class ProtobufSourcesGeneratingSourcesField(MultipleSourcesField): expected_file_extensions = (".proto",) +class ProtobufSourcesOverridesField(OverridesField): + help = ( + "Override the field values for generated `protobuf_source` targets.\n\n" + "Expects a dictionary of relative file paths and globs to a dictionary for the " + "overrides. You may either use a string for a single path / glob, " + "or a string tuple for multiple paths / globs. Each override is a dictionary of " + "field names to the overridden value.\n\n" + "For example:\n\n" + " overrides={\n" + ' "foo.proto": {"grpc": True]},\n' + ' "bar.proto": {"description": "our user model"]},\n' + ' ("foo.proto", "bar.proto"): {"tags": ["overridden"]},\n' + " }\n\n" + "File paths and globs are relative to the BUILD file's directory. Every overridden file is " + "validated to belong to this target's `sources` field.\n\n" + "If you'd like to override a field's value for every `protobuf_source` target generated by " + "this target, change the field directly on this target rather than using the " + "`overrides` field.\n\n" + "You can specify the same file name in multiple keys, so long as you don't override the " + "same field more than one time for the file." + ) + + class ProtobufSourcesGeneratorTarget(Target): alias = "protobuf_sources" core_fields = ( @@ -84,6 +110,7 @@ class ProtobufSourcesGeneratorTarget(Target): ProtobufDependenciesField, ProtobufSourcesGeneratingSourcesField, ProtobufGrpcToggleField, + ProtobufSourcesOverridesField, ) help = "Generate a `protobuf_source` target for each file in the `sources` field." @@ -98,18 +125,32 @@ class GenerateTargetsFromProtobufSources(GenerateTargetsRequest): @rule async def generate_targets_from_protobuf_sources( request: GenerateTargetsFromProtobufSources, + files_not_found_behavior: FilesNotFoundBehavior, protoc: Protoc, union_membership: UnionMembership, ) -> GeneratedTargets: - paths = await Get( + sources_paths = await Get( SourcesPaths, SourcesPathsRequest(request.generator[ProtobufSourcesGeneratingSourcesField]) ) + + all_overrides = {} + overrides_field = request.generator[OverridesField] + if overrides_field.value: + _all_override_paths = await MultiGet( + Get(Paths, PathGlobs, path_globs) + for path_globs in overrides_field.to_path_globs(files_not_found_behavior) + ) + all_overrides = overrides_field.flatten_paths( + dict(zip(_all_override_paths, overrides_field.value.values())) + ) + return generate_file_level_targets( ProtobufSourceTarget, request.generator, - paths.files, + sources_paths.files, union_membership, add_dependencies_on_all_siblings=not protoc.dependency_inference, + overrides=all_overrides, ) diff --git a/src/python/pants/backend/codegen/protobuf/target_types_test.py b/src/python/pants/backend/codegen/protobuf/target_types_test.py new file mode 100644 index 00000000000..bd53bf0a0c6 --- /dev/null +++ b/src/python/pants/backend/codegen/protobuf/target_types_test.py @@ -0,0 +1,64 @@ +# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import annotations + +import os +from textwrap import dedent + +from pants.backend.codegen.protobuf import target_types +from pants.backend.codegen.protobuf.target_types import ( + GenerateTargetsFromProtobufSources, + ProtobufSourcesGeneratorTarget, + ProtobufSourceTarget, +) +from pants.engine.addresses import Address +from pants.engine.target import GeneratedTargets, SingleSourceField, Tags +from pants.testutil.rule_runner import QueryRule, RuleRunner + + +def test_generate_source_and_test_targets() -> None: + rule_runner = RuleRunner( + rules=[ + *target_types.rules(), + QueryRule(GeneratedTargets, [GenerateTargetsFromProtobufSources]), + ], + target_types=[ProtobufSourcesGeneratorTarget], + ) + rule_runner.write_files( + { + "src/proto/BUILD": dedent( + """\ + protobuf_sources( + name='lib', + sources=['**/*.proto'], + overrides={'f1.proto': {'tags': ['overridden']}}, + ) + """ + ), + "src/proto/f1.proto": "", + "src/proto/f2.proto": "", + "src/proto/subdir/f.proto": "", + } + ) + + generator = rule_runner.get_target(Address("src/proto", target_name="lib")) + + def gen_tgt(rel_fp: str, tags: list[str] | None = None) -> ProtobufSourceTarget: + return ProtobufSourceTarget( + {SingleSourceField.alias: rel_fp, Tags.alias: tags}, + Address("src/proto", target_name="lib", relative_file_path=rel_fp), + residence_dir=os.path.dirname(os.path.join("src/proto", rel_fp)), + ) + + generated = rule_runner.request( + GeneratedTargets, [GenerateTargetsFromProtobufSources(generator)] + ) + assert generated == GeneratedTargets( + generator, + { + gen_tgt("f1.proto", tags=["overridden"]), + gen_tgt("f2.proto"), + gen_tgt("subdir/f.proto"), + }, + ) diff --git a/src/python/pants/backend/shell/target_types.py b/src/python/pants/backend/shell/target_types.py index 951688f9722..be094e0685e 100644 --- a/src/python/pants/backend/shell/target_types.py +++ b/src/python/pants/backend/shell/target_types.py @@ -11,8 +11,9 @@ from pants.backend.shell.shell_setup import ShellSetup from pants.core.goals.test import RuntimePackageDependenciesField from pants.engine.addresses import Address +from pants.engine.fs import PathGlobs, Paths from pants.engine.process import BinaryPathTest -from pants.engine.rules import Get, collect_rules, rule +from pants.engine.rules import Get, MultiGet, collect_rules, rule from pants.engine.target import ( COMMON_TARGET_FIELDS, BoolField, @@ -22,6 +23,7 @@ IntField, InvalidFieldException, MultipleSourcesField, + OverridesField, SingleSourceField, SourcesPaths, SourcesPathsRequest, @@ -31,6 +33,7 @@ generate_file_level_targets, ) from pants.engine.unions import UnionMembership, UnionRule +from pants.option.global_options import FilesNotFoundBehavior from pants.util.enums import match @@ -154,6 +157,29 @@ class Shunit2TestsGeneratorSourcesField(ShellGeneratingSourcesBases): default = ("*_test.sh", "test_*.sh", "tests.sh") +class Shunit2TestsOverrideField(OverridesField): + help = ( + "Override the field values for generated `shunit2_test` targets.\n\n" + "Expects a dictionary of relative file paths and globs to a dictionary for the " + "overrides. You may either use a string for a single path / glob, " + "or a string tuple for multiple paths / globs. Each override is a dictionary of " + "field names to the overridden value.\n\n" + "For example:\n\n" + " overrides={\n" + ' "foo_test.sh": {"timeout": 120]},\n' + ' "bar_test.sh": {"timeout": 200]},\n' + ' ("foo_test.sh", "bar_test.sh"): {"tags": ["slow_tests"]},\n' + " }\n\n" + "File paths and globs are relative to the BUILD file's directory. Every overridden file is " + "validated to belong to this target's `sources` field.\n\n" + "If you'd like to override a field's value for every `shunit2_test` target generated by " + "this target, change the field directly on this target rather than using the " + "`overrides` field.\n\n" + "You can specify the same file name in multiple keys, so long as you don't override the " + "same field more than one time for the file." + ) + + class Shunit2TestsGeneratorTarget(Target): alias = "shunit2_tests" core_fields = ( @@ -163,6 +189,7 @@ class Shunit2TestsGeneratorTarget(Target): Shunit2TestTimeoutField, Shunit2ShellField, RuntimePackageDependenciesField, + Shunit2TestsOverrideField, ) help = "Generate a `shunit2_test` target for each file in the `sources` field." @@ -174,18 +201,32 @@ class GenerateTargetsFromShunit2Tests(GenerateTargetsRequest): @rule async def generate_targets_from_shunit2_tests( request: GenerateTargetsFromShunit2Tests, + files_not_found_behavior: FilesNotFoundBehavior, shell_setup: ShellSetup, union_membership: UnionMembership, ) -> GeneratedTargets: - paths = await Get( + sources_paths = await Get( SourcesPaths, SourcesPathsRequest(request.generator[Shunit2TestsGeneratorSourcesField]) ) + + all_overrides = {} + overrides_field = request.generator[OverridesField] + if overrides_field.value: + _all_override_paths = await MultiGet( + Get(Paths, PathGlobs, path_globs) + for path_globs in overrides_field.to_path_globs(files_not_found_behavior) + ) + all_overrides = overrides_field.flatten_paths( + dict(zip(_all_override_paths, overrides_field.value.values())) + ) + return generate_file_level_targets( Shunit2TestTarget, request.generator, - paths.files, + sources_paths.files, union_membership, add_dependencies_on_all_siblings=not shell_setup.dependency_inference, + overrides=all_overrides, ) @@ -204,9 +245,37 @@ class ShellSourcesGeneratingSourcesField(ShellGeneratingSourcesBases): default = ("*.sh",) + tuple(f"!{pat}" for pat in Shunit2TestsGeneratorSourcesField.default) +class ShellSourcesOverridesField(OverridesField): + help = ( + "Override the field values for generated `shell_source` targets.\n\n" + "Expects a dictionary of relative file paths and globs to a dictionary for the " + "overrides. You may either use a string for a single path / glob, " + "or a string tuple for multiple paths / globs. Each override is a dictionary of " + "field names to the overridden value.\n\n" + "For example:\n\n" + " overrides={\n" + ' "foo.sh": {"skip_shellcheck": True]},\n' + ' "bar.sh": {"skip_shfmt": True]},\n' + ' ("foo.sh", "bar.sh"): {"tags": ["linter_disabled"]},\n' + " }\n\n" + "File paths and globs are relative to the BUILD file's directory. Every overridden file is " + "validated to belong to this target's `sources` field.\n\n" + "If you'd like to override a field's value for every `shell_source` target generated by " + "this target, change the field directly on this target rather than using the " + "`overrides` field.\n\n" + "You can specify the same file name in multiple keys, so long as you don't override the " + "same field more than one time for the file." + ) + + class ShellSourcesGeneratorTarget(Target): alias = "shell_sources" - core_fields = (*COMMON_TARGET_FIELDS, Dependencies, ShellSourcesGeneratingSourcesField) + core_fields = ( + *COMMON_TARGET_FIELDS, + Dependencies, + ShellSourcesGeneratingSourcesField, + ShellSourcesOverridesField, + ) help = "Generate a `shell_source` target for each file in the `sources` field." deprecated_alias = "shell_library" @@ -220,18 +289,32 @@ class GenerateTargetsFromShellSources(GenerateTargetsRequest): @rule async def generate_targets_from_shell_sources( request: GenerateTargetsFromShellSources, + files_not_found_behavior: FilesNotFoundBehavior, shell_setup: ShellSetup, union_membership: UnionMembership, ) -> GeneratedTargets: - paths = await Get( + sources_paths = await Get( SourcesPaths, SourcesPathsRequest(request.generator[ShellSourcesGeneratingSourcesField]) ) + + all_overrides = {} + overrides_field = request.generator[OverridesField] + if overrides_field.value: + _all_override_paths = await MultiGet( + Get(Paths, PathGlobs, path_globs) + for path_globs in overrides_field.to_path_globs(files_not_found_behavior) + ) + all_overrides = overrides_field.flatten_paths( + dict(zip(_all_override_paths, overrides_field.value.values())) + ) + return generate_file_level_targets( ShellSourceTarget, request.generator, - paths.files, + sources_paths.files, union_membership, add_dependencies_on_all_siblings=not shell_setup.dependency_inference, + overrides=all_overrides, ) diff --git a/src/python/pants/backend/shell/target_types_test.py b/src/python/pants/backend/shell/target_types_test.py index 9d0b930e6b6..3e8e966057f 100644 --- a/src/python/pants/backend/shell/target_types_test.py +++ b/src/python/pants/backend/shell/target_types_test.py @@ -3,9 +3,24 @@ from __future__ import annotations +import os +from textwrap import dedent + import pytest -from pants.backend.shell.target_types import Shunit2Shell +from pants.backend.shell import target_types +from pants.backend.shell.target_types import ( + GenerateTargetsFromShellSources, + GenerateTargetsFromShunit2Tests, + ShellSourcesGeneratorTarget, + ShellSourceTarget, + Shunit2Shell, + Shunit2TestsGeneratorTarget, + Shunit2TestTarget, +) +from pants.engine.addresses import Address +from pants.engine.target import GeneratedTargets, SingleSourceField, Tags +from pants.testutil.rule_runner import QueryRule, RuleRunner @pytest.mark.parametrize( @@ -48,3 +63,80 @@ def test_shunit2_shell_parse_shebang(content: bytes, expected: Shunit2Shell | No assert result is None else: assert result == expected + + +def test_generate_source_and_test_targets() -> None: + rule_runner = RuleRunner( + rules=[ + *target_types.rules(), + QueryRule(GeneratedTargets, [GenerateTargetsFromShunit2Tests]), + QueryRule(GeneratedTargets, [GenerateTargetsFromShellSources]), + ], + target_types=[Shunit2TestsGeneratorTarget, ShellSourcesGeneratorTarget], + ) + rule_runner.write_files( + { + "src/sh/BUILD": dedent( + """\ + shell_sources( + name='lib', + sources=['**/*.sh', '!**/*_test.sh'], + overrides={'f1.sh': {'tags': ['overridden']}}, + ) + + shunit2_tests( + name='tests', + sources=['**/*_test.sh'], + overrides={'f1_test.sh': {'tags': ['overridden']}}, + ) + """ + ), + "src/sh/f1.sh": "", + "src/sh/f1_test.sh": "", + "src/sh/f2.sh": "", + "src/sh/f2_test.sh": "", + "src/sh/subdir/f.sh": "", + "src/sh/subdir/f_test.sh": "", + } + ) + + sources_generator = rule_runner.get_target(Address("src/sh", target_name="lib")) + tests_generator = rule_runner.get_target(Address("src/sh", target_name="tests")) + + def gen_source_tgt(rel_fp: str, tags: list[str] | None = None) -> ShellSourceTarget: + return ShellSourceTarget( + {SingleSourceField.alias: rel_fp, Tags.alias: tags}, + Address("src/sh", target_name="lib", relative_file_path=rel_fp), + residence_dir=os.path.dirname(os.path.join("src/sh", rel_fp)), + ) + + def gen_test_tgt(rel_fp: str, tags: list[str] | None = None) -> Shunit2TestTarget: + return Shunit2TestTarget( + {SingleSourceField.alias: rel_fp, Tags.alias: tags}, + Address("src/sh", target_name="tests", relative_file_path=rel_fp), + residence_dir=os.path.dirname(os.path.join("src/sh", rel_fp)), + ) + + sources_generated = rule_runner.request( + GeneratedTargets, [GenerateTargetsFromShellSources(sources_generator)] + ) + tests_generated = rule_runner.request( + GeneratedTargets, [GenerateTargetsFromShunit2Tests(tests_generator)] + ) + + assert sources_generated == GeneratedTargets( + sources_generator, + { + gen_source_tgt("f1.sh", tags=["overridden"]), + gen_source_tgt("f2.sh"), + gen_source_tgt("subdir/f.sh"), + }, + ) + assert tests_generated == GeneratedTargets( + tests_generator, + { + gen_test_tgt("f1_test.sh", tags=["overridden"]), + gen_test_tgt("f2_test.sh"), + gen_test_tgt("subdir/f_test.sh"), + }, + )