Add overrides field to shell_sources, shunit2_sources, and `pro…

…tobuf_sources` (#13298)

See #13270.

This does not yet add the field to `java_sources` and `junit_tests`, tracked by #13297.

I'm not sure if we should add this mechanism to `files` and `resources`? I'm having a hard time thinking of when you would want to override the metadata for a single file. But, we might want to do it for the sake of consistency?

This also does not hook up `python_requirements` and `poetry_requirements` because those use the old CAOF macro system, rather than the Target API.

[ci skip-rust]

src/python/pants/backend/codegen/protobuf/target_types.py

@@ -2,7 +2,8 @@
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
 from pants.backend.codegen.protobuf.protoc import Protoc
-from pants.engine.rules import Get, collect_rules, rule
+from pants.engine.fs import PathGlobs, Paths
+from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.engine.target import (
     COMMON_TARGET_FIELDS,
     AllTargets,
@@ -11,6 +12,7 @@
     GeneratedTargets,
     GenerateTargetsRequest,
     MultipleSourcesField,
+    OverridesField,
     SingleSourceField,
     SourcesPaths,
     SourcesPathsRequest,
@@ -19,6 +21,7 @@
     generate_file_level_targets,
 )
 from pants.engine.unions import UnionMembership, UnionRule
+from pants.option.global_options import FilesNotFoundBehavior
 from pants.util.docutil import doc_url
 from pants.util.logging import LogLevel
 
@@ -77,13 +80,37 @@ class ProtobufSourcesGeneratingSourcesField(MultipleSourcesField):
     expected_file_extensions = (".proto",)
 
 
+class ProtobufSourcesOverridesField(OverridesField):
+    help = (
+        "Override the field values for generated `protobuf_source` targets.\n\n"
+        "Expects a dictionary of relative file paths and globs to a dictionary for the "
+        "overrides. You may either use a string for a single path / glob, "
+        "or a string tuple for multiple paths / globs. Each override is a dictionary of "
+        "field names to the overridden value.\n\n"
+        "For example:\n\n"
+        "  overrides={\n"
+        '    "foo.proto": {"grpc": True]},\n'
+        '    "bar.proto": {"description": "our user model"]},\n'
+        '    ("foo.proto", "bar.proto"): {"tags": ["overridden"]},\n'
+        "  }\n\n"
+        "File paths and globs are relative to the BUILD file's directory. Every overridden file is "
+        "validated to belong to this target's `sources` field.\n\n"
+        "If you'd like to override a field's value for every `protobuf_source` target generated by "
+        "this target, change the field directly on this target rather than using the "
+        "`overrides` field.\n\n"
+        "You can specify the same file name in multiple keys, so long as you don't override the "
+        "same field more than one time for the file."
+    )
+
+
 class ProtobufSourcesGeneratorTarget(Target):
     alias = "protobuf_sources"
     core_fields = (
         *COMMON_TARGET_FIELDS,
         ProtobufDependenciesField,
         ProtobufSourcesGeneratingSourcesField,
         ProtobufGrpcToggleField,
+        ProtobufSourcesOverridesField,
     )
     help = "Generate a `protobuf_source` target for each file in the `sources` field."
 
@@ -98,18 +125,32 @@ class GenerateTargetsFromProtobufSources(GenerateTargetsRequest):
 @rule
 async def generate_targets_from_protobuf_sources(
     request: GenerateTargetsFromProtobufSources,
+    files_not_found_behavior: FilesNotFoundBehavior,
     protoc: Protoc,
     union_membership: UnionMembership,
 ) -> GeneratedTargets:
-    paths = await Get(
+    sources_paths = await Get(
         SourcesPaths, SourcesPathsRequest(request.generator[ProtobufSourcesGeneratingSourcesField])
     )
+
+    all_overrides = {}
+    overrides_field = request.generator[OverridesField]
+    if overrides_field.value:
+        _all_override_paths = await MultiGet(
+            Get(Paths, PathGlobs, path_globs)
+            for path_globs in overrides_field.to_path_globs(files_not_found_behavior)
+        )
+        all_overrides = overrides_field.flatten_paths(
+            dict(zip(_all_override_paths, overrides_field.value.values()))
+        )
+
     return generate_file_level_targets(
         ProtobufSourceTarget,
         request.generator,
-        paths.files,
+        sources_paths.files,
         union_membership,
         add_dependencies_on_all_siblings=not protoc.dependency_inference,
+        overrides=all_overrides,
     )
 
 

src/python/pants/backend/codegen/protobuf/target_types_test.py

@@ -0,0 +1,64 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import annotations
+
+import os
+from textwrap import dedent
+
+from pants.backend.codegen.protobuf import target_types
+from pants.backend.codegen.protobuf.target_types import (
+    GenerateTargetsFromProtobufSources,
+    ProtobufSourcesGeneratorTarget,
+    ProtobufSourceTarget,
+)
+from pants.engine.addresses import Address
+from pants.engine.target import GeneratedTargets, SingleSourceField, Tags
+from pants.testutil.rule_runner import QueryRule, RuleRunner
+
+
+def test_generate_source_and_test_targets() -> None:
+    rule_runner = RuleRunner(
+        rules=[
+            *target_types.rules(),
+            QueryRule(GeneratedTargets, [GenerateTargetsFromProtobufSources]),
+        ],
+        target_types=[ProtobufSourcesGeneratorTarget],
+    )
+    rule_runner.write_files(
+        {
+            "src/proto/BUILD": dedent(
+                """\
+                protobuf_sources(
+                    name='lib',
+                    sources=['**/*.proto'],
+                    overrides={'f1.proto': {'tags': ['overridden']}},
+                )
+                """
+            ),
+            "src/proto/f1.proto": "",
+            "src/proto/f2.proto": "",
+            "src/proto/subdir/f.proto": "",
+        }
+    )
+
+    generator = rule_runner.get_target(Address("src/proto", target_name="lib"))
+
+    def gen_tgt(rel_fp: str, tags: list[str] | None = None) -> ProtobufSourceTarget:
+        return ProtobufSourceTarget(
+            {SingleSourceField.alias: rel_fp, Tags.alias: tags},
+            Address("src/proto", target_name="lib", relative_file_path=rel_fp),
+            residence_dir=os.path.dirname(os.path.join("src/proto", rel_fp)),
+        )
+
+    generated = rule_runner.request(
+        GeneratedTargets, [GenerateTargetsFromProtobufSources(generator)]
+    )
+    assert generated == GeneratedTargets(
+        generator,
+        {
+            gen_tgt("f1.proto", tags=["overridden"]),
+            gen_tgt("f2.proto"),
+            gen_tgt("subdir/f.proto"),
+        },
+    )

src/python/pants/backend/shell/target_types.py

@@ -11,8 +11,9 @@
 from pants.backend.shell.shell_setup import ShellSetup
 from pants.core.goals.test import RuntimePackageDependenciesField
 from pants.engine.addresses import Address
+from pants.engine.fs import PathGlobs, Paths
 from pants.engine.process import BinaryPathTest
-from pants.engine.rules import Get, collect_rules, rule
+from pants.engine.rules import Get, MultiGet, collect_rules, rule
 from pants.engine.target import (
     COMMON_TARGET_FIELDS,
     BoolField,
@@ -22,6 +23,7 @@
     IntField,
     InvalidFieldException,
     MultipleSourcesField,
+    OverridesField,
     SingleSourceField,
     SourcesPaths,
     SourcesPathsRequest,
@@ -31,6 +33,7 @@
     generate_file_level_targets,
 )
 from pants.engine.unions import UnionMembership, UnionRule
+from pants.option.global_options import FilesNotFoundBehavior
 from pants.util.enums import match
 
 
@@ -154,6 +157,29 @@ class Shunit2TestsGeneratorSourcesField(ShellGeneratingSourcesBases):
     default = ("*_test.sh", "test_*.sh", "tests.sh")
 
 
+class Shunit2TestsOverrideField(OverridesField):
+    help = (
+        "Override the field values for generated `shunit2_test` targets.\n\n"
+        "Expects a dictionary of relative file paths and globs to a dictionary for the "
+        "overrides. You may either use a string for a single path / glob, "
+        "or a string tuple for multiple paths / globs. Each override is a dictionary of "
+        "field names to the overridden value.\n\n"
+        "For example:\n\n"
+        "  overrides={\n"
+        '    "foo_test.sh": {"timeout": 120]},\n'
+        '    "bar_test.sh": {"timeout": 200]},\n'
+        '    ("foo_test.sh", "bar_test.sh"): {"tags": ["slow_tests"]},\n'
+        "  }\n\n"
+        "File paths and globs are relative to the BUILD file's directory. Every overridden file is "
+        "validated to belong to this target's `sources` field.\n\n"
+        "If you'd like to override a field's value for every `shunit2_test` target generated by "
+        "this target, change the field directly on this target rather than using the "
+        "`overrides` field.\n\n"
+        "You can specify the same file name in multiple keys, so long as you don't override the "
+        "same field more than one time for the file."
+    )
+
+
 class Shunit2TestsGeneratorTarget(Target):
     alias = "shunit2_tests"
     core_fields = (
@@ -163,6 +189,7 @@ class Shunit2TestsGeneratorTarget(Target):
         Shunit2TestTimeoutField,
         Shunit2ShellField,
         RuntimePackageDependenciesField,
+        Shunit2TestsOverrideField,
     )
     help = "Generate a `shunit2_test` target for each file in the `sources` field."
 
@@ -174,18 +201,32 @@ class GenerateTargetsFromShunit2Tests(GenerateTargetsRequest):
 @rule
 async def generate_targets_from_shunit2_tests(
     request: GenerateTargetsFromShunit2Tests,
+    files_not_found_behavior: FilesNotFoundBehavior,
     shell_setup: ShellSetup,
     union_membership: UnionMembership,
 ) -> GeneratedTargets:
-    paths = await Get(
+    sources_paths = await Get(
         SourcesPaths, SourcesPathsRequest(request.generator[Shunit2TestsGeneratorSourcesField])
     )
+
+    all_overrides = {}
+    overrides_field = request.generator[OverridesField]
+    if overrides_field.value:
+        _all_override_paths = await MultiGet(
+            Get(Paths, PathGlobs, path_globs)
+            for path_globs in overrides_field.to_path_globs(files_not_found_behavior)
+        )
+        all_overrides = overrides_field.flatten_paths(
+            dict(zip(_all_override_paths, overrides_field.value.values()))
+        )
+
     return generate_file_level_targets(
         Shunit2TestTarget,
         request.generator,
-        paths.files,
+        sources_paths.files,
         union_membership,
         add_dependencies_on_all_siblings=not shell_setup.dependency_inference,
+        overrides=all_overrides,
     )
 
 
@@ -204,9 +245,37 @@ class ShellSourcesGeneratingSourcesField(ShellGeneratingSourcesBases):
     default = ("*.sh",) + tuple(f"!{pat}" for pat in Shunit2TestsGeneratorSourcesField.default)
 
 
+class ShellSourcesOverridesField(OverridesField):
+    help = (
+        "Override the field values for generated `shell_source` targets.\n\n"
+        "Expects a dictionary of relative file paths and globs to a dictionary for the "
+        "overrides. You may either use a string for a single path / glob, "
+        "or a string tuple for multiple paths / globs. Each override is a dictionary of "
+        "field names to the overridden value.\n\n"
+        "For example:\n\n"
+        "  overrides={\n"
+        '    "foo.sh": {"skip_shellcheck": True]},\n'
+        '    "bar.sh": {"skip_shfmt": True]},\n'
+        '    ("foo.sh", "bar.sh"): {"tags": ["linter_disabled"]},\n'
+        "  }\n\n"
+        "File paths and globs are relative to the BUILD file's directory. Every overridden file is "
+        "validated to belong to this target's `sources` field.\n\n"
+        "If you'd like to override a field's value for every `shell_source` target generated by "
+        "this target, change the field directly on this target rather than using the "
+        "`overrides` field.\n\n"
+        "You can specify the same file name in multiple keys, so long as you don't override the "
+        "same field more than one time for the file."
+    )
+
+
 class ShellSourcesGeneratorTarget(Target):
     alias = "shell_sources"
-    core_fields = (*COMMON_TARGET_FIELDS, Dependencies, ShellSourcesGeneratingSourcesField)
+    core_fields = (
+        *COMMON_TARGET_FIELDS,
+        Dependencies,
+        ShellSourcesGeneratingSourcesField,
+        ShellSourcesOverridesField,
+    )
     help = "Generate a `shell_source` target for each file in the `sources` field."
 
     deprecated_alias = "shell_library"
@@ -220,18 +289,32 @@ class GenerateTargetsFromShellSources(GenerateTargetsRequest):
 @rule
 async def generate_targets_from_shell_sources(
     request: GenerateTargetsFromShellSources,
+    files_not_found_behavior: FilesNotFoundBehavior,
     shell_setup: ShellSetup,
     union_membership: UnionMembership,
 ) -> GeneratedTargets:
-    paths = await Get(
+    sources_paths = await Get(
         SourcesPaths, SourcesPathsRequest(request.generator[ShellSourcesGeneratingSourcesField])
     )
+
+    all_overrides = {}
+    overrides_field = request.generator[OverridesField]
+    if overrides_field.value:
+        _all_override_paths = await MultiGet(
+            Get(Paths, PathGlobs, path_globs)
+            for path_globs in overrides_field.to_path_globs(files_not_found_behavior)
+        )
+        all_overrides = overrides_field.flatten_paths(
+            dict(zip(_all_override_paths, overrides_field.value.values()))
+        )
+
     return generate_file_level_targets(
         ShellSourceTarget,
         request.generator,
-        paths.files,
+        sources_paths.files,
         union_membership,
         add_dependencies_on_all_siblings=not shell_setup.dependency_inference,
+        overrides=all_overrides,
     )