diff --git a/pkg/BUILD.bazel b/pkg/BUILD.bazel index f30c8367cfe9f..a2663dc2b0430 100644 --- a/pkg/BUILD.bazel +++ b/pkg/BUILD.bazel @@ -305,9 +305,9 @@ gen_file_lists( out_stem = "src_file_lists", src_libs = { # source rule: name in generated file - "//:protobuf": "libprotobuf", - "//src/google/protobuf/compiler:protoc_lib": "libprotoc", - "//:protobuf_lite": "libprotobuf_lite", + ":protobuf": "libprotobuf", + ":protoc": "libprotoc", + ":protobuf_lite": "libprotobuf_lite", }, ) @@ -343,8 +343,8 @@ cc_dist_library( }), tags = ["manual"], deps = [ - "//:protobuf_lite", "//src/google/protobuf:arena", + "//src/google/protobuf:protobuf_lite", "//src/google/protobuf/io", "//src/google/protobuf/io:io_win32", "//src/google/protobuf/stubs:lite", @@ -362,8 +362,6 @@ cc_dist_library( }), tags = ["manual"], deps = [ - "//:protobuf", - "//:protobuf_lite", "//src/google/protobuf:arena", "//src/google/protobuf/compiler:importer", "//src/google/protobuf/io", @@ -371,6 +369,8 @@ cc_dist_library( "//src/google/protobuf/io:io_win32", "//src/google/protobuf/io:printer", "//src/google/protobuf/io:tokenizer", + "//src/google/protobuf:protobuf", + "//src/google/protobuf:protobuf_lite", "//src/google/protobuf/stubs", "//src/google/protobuf/stubs:lite", "//src/google/protobuf/util:delimited_message_util", @@ -382,6 +382,22 @@ cc_dist_library( ], ) +cc_dist_library( + name = "protoc", + tags = ["manual"], + deps = [ + "//src/google/protobuf/compiler:code_generator", + "//src/google/protobuf/compiler:command_line_interface", + "//src/google/protobuf/compiler/cpp", + "//src/google/protobuf/compiler/csharp", + "//src/google/protobuf/compiler/java", + "//src/google/protobuf/compiler/objectivec", + "//src/google/protobuf/compiler/php", + "//src/google/protobuf/compiler/python", + "//src/google/protobuf/compiler/ruby", + ], +) + ################################################################################ # Distribution sources ################################################################################ diff --git a/pkg/build_systems.bzl b/pkg/build_systems.bzl index 2d8cdd70bee1d..40bdaad659db1 100644 --- a/pkg/build_systems.bzl +++ b/pkg/build_systems.bzl @@ -1,6 +1,7 @@ # Starlark utilities for working with other build systems load("@rules_pkg//:providers.bzl", "PackageFilegroupInfo", "PackageFilesInfo") +load(":cc_dist_library.bzl", "CcFileList") ################################################################################ # Macro to create CMake and Automake source lists. @@ -31,21 +32,6 @@ def gen_file_lists(name, out_stem, **kwargs): # Aspect that extracts srcs, hdrs, etc. ################################################################################ -CcFileList = provider( - doc = "List of files to be built into a library.", - fields = { - # As a rule of thumb, `hdrs` and `textual_hdrs` are the files that - # would be installed along with a prebuilt library. - "hdrs": "public header files, including those used by generated code", - "textual_hdrs": "files which are included but are not self-contained", - - # The `internal_hdrs` are header files which appear in `srcs`. - # These are only used when compiling the library. - "internal_hdrs": "internal header files (only used to build .cc files)", - "srcs": "source files", - }, -) - ProtoFileList = provider( doc = "List of proto files and generated code to be built into a library.", fields = { @@ -65,56 +51,11 @@ def _flatten_target_files(targets): files.append(tfile) return files -def _combine_cc_file_lists(file_lists): - hdrs = {} - textual_hdrs = {} - internal_hdrs = {} - srcs = {} - for file_list in file_lists: - hdrs.update({f: 1 for f in file_list.hdrs}) - textual_hdrs.update({f: 1 for f in file_list.textual_hdrs}) - internal_hdrs.update({f: 1 for f in file_list.internal_hdrs}) - srcs.update({f: 1 for f in file_list.srcs}) - return CcFileList( - hdrs = sorted(hdrs.keys()), - textual_hdrs = sorted(textual_hdrs.keys()), - internal_hdrs = sorted(internal_hdrs.keys()), - srcs = sorted(srcs.keys()), - ) - def _file_list_aspect_impl(target, ctx): # We're going to reach directly into the attrs on the traversed rule. rule_attr = ctx.rule.attr providers = [] - # Extract sources from a `cc_library` (or similar): - if CcInfo in target: - # CcInfo is a proxy for what we expect this rule to look like. - # However, some deps may expose `CcInfo` without having `srcs`, - # `hdrs`, etc., so we use `getattr` to handle that gracefully. - - internal_hdrs = [] - srcs = [] - - # Filter `srcs` so it only contains source files. Headers will go - # into `internal_headers`. - for src in _flatten_target_files(getattr(rule_attr, "srcs", [])): - if src.extension.lower() in ["c", "cc", "cpp", "cxx"]: - srcs.append(src) - else: - internal_hdrs.append(src) - - providers.append(CcFileList( - hdrs = _flatten_target_files(getattr(rule_attr, "hdrs", [])), - textual_hdrs = _flatten_target_files(getattr( - rule_attr, - "textual_hdrs", - [], - )), - internal_hdrs = internal_hdrs, - srcs = srcs, - )) - # Extract sources from a `proto_library`: if ProtoInfo in target: proto_srcs = [] @@ -197,19 +138,28 @@ def _create_file_list_impl(ctx, fragment_generator): for srcrule, libname in ctx.attr.src_libs.items(): if CcFileList in srcrule: cc_file_list = srcrule[CcFileList] + + # Turn depsets of files into sorted lists. + srcs = sorted(cc_file_list.srcs.to_list()) + hdrs = sorted( + depset(transitive = [ + cc_file_list.textual_hdrs, + cc_file_list.hdrs, + ]).to_list(), + ) + fragments.extend([ fragment_generator( srcrule.label, libname + "_srcs", ctx.attr.source_prefix, - [f.short_path for f in cc_file_list.srcs], + [f.short_path for f in srcs], ), fragment_generator( srcrule.label, libname + "_hdrs", ctx.attr.source_prefix, - [f.short_path for f in (cc_file_list.hdrs + - cc_file_list.textual_hdrs)], + [f.short_path for f in hdrs], ), ]) @@ -247,7 +197,7 @@ def _create_file_list_impl(ctx, fragment_generator): # keys are the destination: files.update(srcrule[PackageFilesInfo].dest_src_map) - if files == {} and DefaultInfo in srcrule and CcInfo not in srcrule: + if files == {} and DefaultInfo in srcrule and CcFileList not in srcrule: # This could be an individual file or filegroup. # We explicitly ignore rules with CcInfo, since their # output artifacts are libraries or binaries. diff --git a/pkg/cc_dist_library.bzl b/pkg/cc_dist_library.bzl index d48e8bed8bc48..383979e6cd27d 100644 --- a/pkg/cc_dist_library.bzl +++ b/pkg/cc_dist_library.bzl @@ -3,12 +3,32 @@ load("@rules_cc//cc:action_names.bzl", cc_action_names = "ACTION_NAMES") load("@rules_cc//cc:find_cc_toolchain.bzl", "find_cc_toolchain") +################################################################################ +# Archive/linking support +################################################################################ + +def _collect_linker_input_objects(dep_label, cc_info, objs, pic_objs): + """Accumulate .o and .pic.o files into `objs` and `pic_objs`.""" + link_ctx = cc_info.linking_context + if link_ctx == None: + return + + linker_inputs = link_ctx.linker_inputs.to_list() + for link_input in linker_inputs: + if link_input.owner != dep_label: + # This is a transitive dep: skip it. + continue + + for lib in link_input.libraries: + objs.extend(lib.objects or []) + pic_objs.extend(lib.pic_objects or []) + # Creates an action to build the `output_file` static library (archive) # using `object_files`. def _create_archive_action( ctx, feature_configuration, - cc_toolchain, + cc_toolchain_info, output_file, object_files): # Based on Bazel's src/main/starlark/builtins_bzl/common/cc/cc_import.bzl: @@ -16,7 +36,7 @@ def _create_archive_action( # Build the command line and add args for all of the input files: archiver_variables = cc_common.create_link_variables( feature_configuration = feature_configuration, - cc_toolchain = cc_toolchain, + cc_toolchain = cc_toolchain_info, output_file = output_file.path, is_using_linker = False, ) @@ -48,7 +68,7 @@ def _create_archive_action( inputs = depset( direct = object_files, transitive = [ - cc_toolchain.all_files, + cc_toolchain_info.all_files, ], ), use_default_shell_env = False, @@ -56,96 +76,242 @@ def _create_archive_action( mnemonic = "CppArchiveDist", ) -# Implementation for cc_dist_library rule. -def _cc_dist_library_impl(ctx): - cc_toolchain_info = find_cc_toolchain(ctx) - if cc_toolchain_info.ar_executable == None: - return [] - - feature_configuration = cc_common.configure_features( - ctx = ctx, +def _create_dso_link_action( + ctx, + feature_configuration, + cc_toolchain_info, + object_files, + pic_object_files): + compilation_outputs = cc_common.create_compilation_outputs( + objects = depset(object_files), + pic_objects = depset(pic_object_files), + ) + link_output = cc_common.link( + actions = ctx.actions, + feature_configuration = feature_configuration, cc_toolchain = cc_toolchain_info, + compilation_outputs = compilation_outputs, + name = ctx.label.name, + output_type = "dynamic_library", + user_link_flags = ctx.attr.linkopts, ) + library_to_link = link_output.library_to_link + + outputs = [] + + # Note: library_to_link.dynamic_library and interface_library are often + # symlinks in the solib directory. For DefaultInfo, prefer reporting + # the resolved artifact paths. + if library_to_link.resolved_symlink_dynamic_library != None: + outputs.append(library_to_link.resolved_symlink_dynamic_library) + elif library_to_link.dynamic_library != None: + outputs.append(library_to_link.dynamic_library) + + if library_to_link.resolved_symlink_interface_library != None: + outputs.append(library_to_link.resolved_symlink_interface_library) + elif library_to_link.interface_library != None: + outputs.append(library_to_link.interface_library) + + return outputs + +################################################################################ +# Source file/header support +################################################################################ - # Collect the set of object files from the immediate deps. +CcFileList = provider( + doc = "List of files to be built into a library.", + fields = { + # As a rule of thumb, `hdrs` and `textual_hdrs` are the files that + # would be installed along with a prebuilt library. + "hdrs": "public header files, including those used by generated code", + "textual_hdrs": "files which are included but are not self-contained", + + # The `internal_hdrs` are header files which appear in `srcs`. + # These are only used when compiling the library. + "internal_hdrs": "internal header files (only used to build .cc files)", + "srcs": "source files", + }, +) + +def _flatten_target_files(targets): + return depset(transitive = [target.files for target in targets]) + + files = [] + for target in targets: + files.extend(target.files.to_list()) + return files + +def _cc_file_list_aspect_impl(target, ctx): + # Extract sources from a `cc_library` (or similar): + if CcInfo not in target: + return [] + + # We're going to reach directly into the attrs on the traversed rule. + rule_attr = ctx.rule.attr + + # CcInfo is a proxy for what we expect this rule to look like. + # However, some deps may expose `CcInfo` without having `srcs`, + # `hdrs`, etc., so we use `getattr` to handle that gracefully. + + internal_hdrs = [] + srcs = [] + + # Filter `srcs` so it only contains source files. Headers will go + # into `internal_headers`. + for src in _flatten_target_files(getattr(rule_attr, "srcs", [])).to_list(): + if src.extension.lower() in ["c", "cc", "cpp", "cxx"]: + srcs.append(src) + else: + internal_hdrs.append(src) + + return [CcFileList( + hdrs = _flatten_target_files(getattr(rule_attr, "hdrs", depset())), + textual_hdrs = _flatten_target_files(getattr( + rule_attr, + "textual_hdrs", + depset(), + )), + internal_hdrs = depset(internal_hdrs), + srcs = depset(srcs), + )] + +cc_file_list_aspect = aspect( + doc = """ +Aspect to provide the list of sources and headers from a rule. + +Output is CcFileList. Example: + + cc_library( + name = "foo", + srcs = [ + "foo.cc", + "foo_internal.h", + ], + hdrs = ["foo.h"], + textual_hdrs = ["foo_inl.inc"], + ) + # produces: + # CcFileList( + # hdrs = depset([File("foo.h")]), + # textual_hdrs = depset([File("foo_inl.inc")]), + # internal_hdrs = depset([File("foo_internal.h")]), + # srcs = depset([File("foo.cc")]), + # ) +""", + implementation = _cc_file_list_aspect_impl, +) + +################################################################################ +# Rule impl +################################################################################ + +def _collect_inputs(deps): + """Collects files from a list of immediate deps. + + This rule collects source files and linker inputs for C++ deps. Only + these immediate deps are considered, not transitive deps. + + The return value is a struct with object files (linker inputs), + partitioned by PIC and non-pic, and the rules' source and header files: + + struct( + objects = ..., # non-PIC object files + pic_objects = ..., # PIC objects + cc_file_list = ..., # a CcFileList + ) + + Args: + deps: Iterable of immediate deps. These will be treated as the "inputs," + but not the transitive deps. + + Returns: + A struct with linker inputs, source files, and header files. + """ objs = [] pic_objs = [] - for dep in ctx.attr.deps: - if CcInfo not in dep: - continue - link_ctx = dep[CcInfo].linking_context - if link_ctx == None: - continue + # The returned CcFileList will contain depsets of the deps' file lists. + # These lists hold `depset()`s from each of `deps`. + srcs = [] + hdrs = [] + internal_hdrs = [] + textual_hdrs = [] - linker_inputs = link_ctx.linker_inputs.to_list() - for link_input in linker_inputs: - if link_input.owner != dep.label: - # This is a transitive dep: skip it. - continue + for dep in deps: + if CcInfo in dep: + _collect_linker_input_objects( + dep.label, + dep[CcInfo], + objs, + pic_objs, + ) - for lib in link_input.libraries: - objs.extend(lib.objects or []) - pic_objs.extend(lib.pic_objects or []) + if CcFileList in dep: + cfl = dep[CcFileList] + srcs.append(cfl.srcs) + hdrs.append(cfl.hdrs) + internal_hdrs.append(cfl.internal_hdrs) + textual_hdrs.append(cfl.textual_hdrs) + + return struct( + objects = objs, + pic_objects = pic_objs, + cc_file_list = CcFileList( + srcs = depset(transitive = srcs), + hdrs = depset(transitive = hdrs), + internal_hdrs = depset(transitive = internal_hdrs), + textual_hdrs = depset(transitive = textual_hdrs), + ), + ) + +# Implementation for cc_dist_library rule. +def _cc_dist_library_impl(ctx): + cc_toolchain_info = find_cc_toolchain(ctx) + + feature_configuration = cc_common.configure_features( + ctx = ctx, + cc_toolchain = cc_toolchain_info, + ) + + inputs = _collect_inputs(ctx.attr.deps) # For static libraries, build separately with and without pic. stemname = "lib" + ctx.label.name outputs = [] - if len(objs) > 0: + if len(inputs.objects) > 0: archive_out = ctx.actions.declare_file(stemname + ".a") _create_archive_action( ctx, feature_configuration, cc_toolchain_info, archive_out, - objs, + inputs.objects, ) outputs.append(archive_out) - if len(pic_objs) > 0: + if len(inputs.pic_objects) > 0: pic_archive_out = ctx.actions.declare_file(stemname + ".pic.a") _create_archive_action( ctx, feature_configuration, cc_toolchain_info, pic_archive_out, - pic_objs, + inputs.pic_objects, ) outputs.append(pic_archive_out) # For dynamic libraries, use the `cc_common.link` command to ensure # everything gets built correctly according to toolchain definitions. - - compilation_outputs = cc_common.create_compilation_outputs( - objects = depset(objs), - pic_objects = depset(pic_objs), - ) - link_output = cc_common.link( - actions = ctx.actions, - feature_configuration = feature_configuration, - cc_toolchain = cc_toolchain_info, - compilation_outputs = compilation_outputs, - name = ctx.label.name, - output_type = "dynamic_library", - user_link_flags = ctx.attr.linkopts, - ) - library_to_link = link_output.library_to_link - - # Note: library_to_link.dynamic_library and interface_library are often - # symlinks in the solib directory. For DefaultInfo, prefer reporting - # the resolved artifact paths. - if library_to_link.resolved_symlink_dynamic_library != None: - outputs.append(library_to_link.resolved_symlink_dynamic_library) - elif library_to_link.dynamic_library != None: - outputs.append(library_to_link.dynamic_library) - - if library_to_link.resolved_symlink_interface_library != None: - outputs.append(library_to_link.resolved_symlink_interface_library) - elif library_to_link.interface_library != None: - outputs.append(library_to_link.interface_library) + outputs.extend(_create_dso_link_action( + ctx, + feature_configuration, + cc_toolchain_info, + inputs.objects, + inputs.pic_objects, + )) # We could expose the libraries for use from cc rules: # @@ -169,6 +335,7 @@ def _cc_dist_library_impl(ctx): return [ DefaultInfo(files = depset(outputs)), + inputs.cc_file_list, ] cc_dist_library = rule( @@ -214,6 +381,7 @@ Example: "Only these targets' compilation outputs will be " + "included (i.e., the transitive dependencies are not " + "included in the output)."), + aspects = [cc_file_list_aspect], ), "linkopts": attr.string_list( doc = ("Add these flags to the C++ linker command when creating " + diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index ac5da90495499..a8736c337b9d3 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel @@ -160,6 +160,7 @@ cc_library( linkopts = LINK_OPTS, visibility = [ "//:__pkg__", + "//pkg:__pkg__", "//src/google/protobuf:__subpackages__", ], # In Bazel 6.0+, these will be `interface_deps`: @@ -209,6 +210,7 @@ cc_library( linkopts = LINK_OPTS, visibility = [ "//:__pkg__", + "//pkg:__pkg__", "//src/google/protobuf:__subpackages__", ], deps = [ diff --git a/src/google/protobuf/compiler/cpp/BUILD.bazel b/src/google/protobuf/compiler/cpp/BUILD.bazel index 37e31e8037b2d..b14259495c365 100644 --- a/src/google/protobuf/compiler/cpp/BUILD.bazel +++ b/src/google/protobuf/compiler/cpp/BUILD.bazel @@ -48,7 +48,10 @@ cc_library( ], copts = COPTS, include_prefix = "google/protobuf/compiler/cpp", - visibility = ["//src/google/protobuf/compiler:__pkg__"], + visibility = [ + "//pkg:__pkg__", + "//src/google/protobuf/compiler:__pkg__", + ], deps = [ "//:protobuf", "//src/google/protobuf/compiler:code_generator", diff --git a/src/google/protobuf/compiler/csharp/BUILD.bazel b/src/google/protobuf/compiler/csharp/BUILD.bazel index 1271b94ff0980..819c9dec27088 100644 --- a/src/google/protobuf/compiler/csharp/BUILD.bazel +++ b/src/google/protobuf/compiler/csharp/BUILD.bazel @@ -51,7 +51,10 @@ cc_library( "//conditions:default": ["-Wno-overloaded-virtual"], }), include_prefix = "google/protobuf/compiler/csharp", - visibility = ["//src/google/protobuf/compiler:__pkg__"], + visibility = [ + "//pkg:__pkg__", + "//src/google/protobuf/compiler:__pkg__", + ], deps = [ "//:protobuf", "//src/google/protobuf/compiler:code_generator", @@ -62,11 +65,11 @@ cc_test( name = "bootstrap_unittest", srcs = ["csharp_bootstrap_unittest.cc"], data = [ - "//src/google/protobuf:descriptor_proto_srcs", "//:well_known_type_protos", "//conformance:all_files", "//conformance:conformance_proto", "//csharp:wkt_cs_srcs", + "//src/google/protobuf:descriptor_proto_srcs", "//src/google/protobuf:testdata", ], deps = [ diff --git a/src/google/protobuf/compiler/java/BUILD.bazel b/src/google/protobuf/compiler/java/BUILD.bazel index f035f32de9ca3..66a809ec2e639 100644 --- a/src/google/protobuf/compiler/java/BUILD.bazel +++ b/src/google/protobuf/compiler/java/BUILD.bazel @@ -74,7 +74,10 @@ cc_library( ], copts = COPTS, include_prefix = "google/protobuf/compiler/java", - visibility = ["//src/google/protobuf/compiler:__pkg__"], + visibility = [ + "//pkg:__pkg__", + "//src/google/protobuf/compiler:__pkg__", + ], deps = [ "//:protobuf", "//src/google/protobuf/compiler:code_generator", @@ -85,9 +88,9 @@ cc_test( name = "doc_comment_unittest", srcs = ["doc_comment_unittest.cc"], data = [ - "//src/google/protobuf:descriptor_proto_srcs", "//:well_known_type_protos", "//conformance:conformance_proto", + "//src/google/protobuf:descriptor_proto_srcs", ], deps = [ ":java", diff --git a/src/google/protobuf/compiler/objectivec/BUILD.bazel b/src/google/protobuf/compiler/objectivec/BUILD.bazel index 07153907543a7..634dda2483121 100644 --- a/src/google/protobuf/compiler/objectivec/BUILD.bazel +++ b/src/google/protobuf/compiler/objectivec/BUILD.bazel @@ -39,7 +39,10 @@ cc_library( ], copts = COPTS, include_prefix = "google/protobuf/compiler/objectivec", - visibility = ["//src/google/protobuf/compiler:__pkg__"], + visibility = [ + "//pkg:__pkg__", + "//src/google/protobuf/compiler:__pkg__", + ], deps = [ "//:protobuf", "//src/google/protobuf/compiler:code_generator", diff --git a/src/google/protobuf/compiler/php/BUILD.bazel b/src/google/protobuf/compiler/php/BUILD.bazel index 969d2df849f6f..82ba185fd95a3 100644 --- a/src/google/protobuf/compiler/php/BUILD.bazel +++ b/src/google/protobuf/compiler/php/BUILD.bazel @@ -12,7 +12,10 @@ cc_library( hdrs = ["php_generator.h"], copts = COPTS, include_prefix = "google/protobuf/compiler/php", - visibility = ["//src/google/protobuf/compiler:__pkg__"], + visibility = [ + "//pkg:__pkg__", + "//src/google/protobuf/compiler:__pkg__", + ], deps = [ "//:protobuf", "//src/google/protobuf/compiler:code_generator", diff --git a/src/google/protobuf/compiler/python/BUILD.bazel b/src/google/protobuf/compiler/python/BUILD.bazel index 3fe6807d1d06c..1924a9ac5bebe 100644 --- a/src/google/protobuf/compiler/python/BUILD.bazel +++ b/src/google/protobuf/compiler/python/BUILD.bazel @@ -20,7 +20,10 @@ cc_library( ], copts = COPTS, include_prefix = "google/protobuf/compiler/python", - visibility = ["//src/google/protobuf/compiler:__pkg__"], + visibility = [ + "//pkg:__pkg__", + "//src/google/protobuf/compiler:__pkg__", + ], deps = [ "//:protobuf", "//src/google/protobuf/compiler:code_generator", diff --git a/src/google/protobuf/compiler/ruby/BUILD.bazel b/src/google/protobuf/compiler/ruby/BUILD.bazel index 3a3469eb0d0a1..455865322d307 100644 --- a/src/google/protobuf/compiler/ruby/BUILD.bazel +++ b/src/google/protobuf/compiler/ruby/BUILD.bazel @@ -12,7 +12,10 @@ cc_library( hdrs = ["ruby_generator.h"], copts = COPTS, include_prefix = "google/protobuf/compiler/ruby", - visibility = ["//src/google/protobuf/compiler:__pkg__"], + visibility = [ + "//pkg:__pkg__", + "//src/google/protobuf/compiler:__pkg__", + ], deps = [ "//:protobuf", "//src/google/protobuf/compiler:code_generator",