Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[6.2.0]Add native.module_{name,version} #17873

Closed
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -128,17 +128,31 @@ private NameConflictException(String message) {
private RootedPath filename;

/**
* The directory in which this package's BUILD file resides. All InputFile
* members of the packages are located relative to this directory.
* The directory in which this package's BUILD file resides. All InputFile members of the packages
* are located relative to this directory.
*/
private Path packageDirectory;

/**
* The name of the workspace this package is in. Used as a prefix for the runfiles directory.
* This can be set in the WORKSPACE file. This must be a valid target name.
* The name of the workspace this package is in. Used as a prefix for the runfiles directory. This
* can be set in the WORKSPACE file. This must be a valid target name.
*/
private String workspaceName;

/**
* The name of the Bzlmod module associated with the repo this package is in. If this package is
* not from a Bzlmod repo, this is empty. For repos generated by module extensions, this is the
* name of the module hosting the extension.
*/
private final Optional<String> associatedModuleName;

/**
* The version of the Bzlmod module associated with the repo this package is in. If this package
* is not from a Bzlmod repo, this is empty. For repos generated by module extensions, this is the
* version of the module hosting the extension.
*/
private final Optional<String> associatedModuleVersion;

/**
* The root of the source tree in which this package was found. It is an invariant that {@code
* sourceRoot.getRelative(packageId.getSourceRoot()).equals(packageDirectory)}. Returns {@link
Expand All @@ -147,8 +161,8 @@ private NameConflictException(String message) {
private Optional<Root> sourceRoot;

/**
* The "Make" environment of this package, containing package-local
* definitions of "Make" variables.
* The "Make" environment of this package, containing package-local definitions of "Make"
* variables.
*/
private ImmutableMap<String, String> makeEnv;

Expand Down Expand Up @@ -293,9 +307,15 @@ public ImmutableMap<String, Module> getLoads() {
* <p>{@code name} <b>MUST</b> be a suffix of {@code filename.getParentDirectory())}.
*/
private Package(
PackageIdentifier packageId, String workspaceName, boolean succinctTargetNotFoundErrors) {
PackageIdentifier packageId,
String workspaceName,
Optional<String> associatedModuleName,
Optional<String> associatedModuleVersion,
boolean succinctTargetNotFoundErrors) {
this.packageIdentifier = packageId;
this.workspaceName = workspaceName;
this.associatedModuleName = associatedModuleName;
this.associatedModuleVersion = associatedModuleVersion;
this.succinctTargetNotFoundErrors = succinctTargetNotFoundErrors;
}

Expand Down Expand Up @@ -887,6 +907,8 @@ public static Builder newExternalPackageBuilder(
helper,
LabelConstants.EXTERNAL_PACKAGE_IDENTIFIER,
workspaceName,
/* associatedModuleName= */ Optional.empty(),
/* associatedModuleVersion= */ Optional.empty(),
starlarkSemantics.getBool(BuildLanguageOptions.INCOMPATIBLE_NO_IMPLICIT_FILE_EXPORT),
mainRepoMapping,
mainRepoMapping)
Expand All @@ -902,6 +924,8 @@ public static Builder newExternalPackageBuilderForBzlmod(
DefaultPackageSettings.INSTANCE,
basePackageId,
DUMMY_WORKSPACE_NAME_FOR_BZLMOD_PACKAGES,
/* associatedModuleName= */ Optional.empty(),
/* associatedModuleVersion= */ Optional.empty(),
starlarkSemantics.getBool(BuildLanguageOptions.INCOMPATIBLE_NO_IMPLICIT_FILE_EXPORT),
repoMapping,
// This mapping is *not* the main repository's mapping, but since it is only used to
Expand Down Expand Up @@ -1115,10 +1139,18 @@ public T intern(T sample) {
PackageSettings packageSettings,
PackageIdentifier id,
String workspaceName,
Optional<String> associatedModuleName,
Optional<String> associatedModuleVersion,
boolean noImplicitFileExport,
RepositoryMapping repositoryMapping,
RepositoryMapping mainRepositoryMapping) {
this.pkg = new Package(id, workspaceName, packageSettings.succinctTargetNotFoundErrors());
this.pkg =
new Package(
id,
workspaceName,
associatedModuleName,
associatedModuleVersion,
packageSettings.succinctTargetNotFoundErrors());
this.noImplicitFileExport = noImplicitFileExport;
this.repositoryMapping = repositoryMapping;
this.mainRepositoryMapping = mainRepositoryMapping;
Expand All @@ -1144,6 +1176,24 @@ String getPackageWorkspaceName() {
return pkg.getWorkspaceName();
}

/**
* Returns the name of the Bzlmod module associated with the repo this package is in. If this
* package is not from a Bzlmod repo, this is empty. For repos generated by module extensions,
* this is the name of the module hosting the extension.
*/
Optional<String> getAssociatedModuleName() {
return pkg.associatedModuleName;
}

/**
* Returns the version of the Bzlmod module associated with the repo this package is in. If this
* package is not from a Bzlmod repo, this is empty. For repos generated by module extensions,
* this is the version of the module hosting the extension.
*/
Optional<String> getAssociatedModuleVersion() {
return pkg.associatedModuleVersion;
}

/**
* Updates the externalPackageRepositoryMappings entry for {@code repoWithin}. Adds new entry
* from {@code localName} to {@code mappedName} in {@code repoWithin}'s map.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.OptionalLong;
import java.util.Set;
import java.util.concurrent.ForkJoinPool;
Expand Down Expand Up @@ -464,13 +465,17 @@ public Package.Builder newExternalPackageBuilder(
public Package.Builder newPackageBuilder(
PackageIdentifier packageId,
String workspaceName,
Optional<String> associatedModuleName,
Optional<String> associatedModuleVersion,
StarlarkSemantics starlarkSemantics,
RepositoryMapping repositoryMapping,
RepositoryMapping mainRepositoryMapping) {
return new Package.Builder(
packageSettings,
packageId,
workspaceName,
associatedModuleName,
associatedModuleVersion,
starlarkSemantics.getBool(BuildLanguageOptions.INCOMPATIBLE_NO_IMPLICIT_FILE_EXPORT),
repositoryMapping,
mainRepositoryMapping);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -635,6 +635,20 @@ public Label packageRelativeLabel(Object input, StarlarkThread thread) throws Ev
}
}

@Override
@Nullable
public String moduleName(StarlarkThread thread) throws EvalException {
BazelStarlarkContext.from(thread).checkLoadingPhase("native.module_name");
return PackageFactory.getContext(thread).getBuilder().getAssociatedModuleName().orElse(null);
}

@Override
@Nullable
public String moduleVersion(StarlarkThread thread) throws EvalException {
BazelStarlarkContext.from(thread).checkLoadingPhase("native.module_version");
return PackageFactory.getContext(thread).getBuilder().getAssociatedModuleVersion().orElse(null);
}

private static Dict<String, Object> getRuleDict(Rule rule, Mutability mu) throws EvalException {
Dict.Builder<String, Object> values = Dict.builder();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2215,8 +2215,8 @@ java_library(
srcs = ["RepositoryMappingValue.java"],
deps = [
":sky_functions",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:common",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/concurrent",
"//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec",
"//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code",
"//src/main/java/com/google/devtools/build/lib/util:exit_code",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1348,6 +1348,8 @@ private LoadedPackage loadPackage(
.newPackageBuilder(
packageId,
workspaceName,
repositoryMappingValue.getAssociatedModuleName(),
repositoryMappingValue.getAssociatedModuleVersion(),
starlarkBuiltinsValue.starlarkSemantics,
repositoryMapping,
mainRepositoryMappingValue.getRepositoryMapping())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,12 @@
package com.google.devtools.build.lib.skyframe;

import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionValue;
import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphValue;
import com.google.devtools.build.lib.bazel.bzlmod.Module;
import com.google.devtools.build.lib.bazel.bzlmod.ModuleExtensionId;
import com.google.devtools.build.lib.bazel.bzlmod.ModuleKey;
import com.google.devtools.build.lib.bazel.bzlmod.SingleExtensionEvalValue;
import com.google.devtools.build.lib.bazel.bzlmod.Version;
import com.google.devtools.build.lib.cmdline.LabelConstants;
import com.google.devtools.build.lib.cmdline.RepositoryMapping;
import com.google.devtools.build.lib.cmdline.RepositoryName;
Expand Down Expand Up @@ -61,19 +63,21 @@ public SkyValue compute(SkyKey skyKey, Environment env)
return null;
}
// We need to make sure that @_builtins maps to @_builtins too.
return RepositoryMappingValue.withMapping(
return RepositoryMappingValue.createForBzlmodRepo(
RepositoryMapping.create(
ImmutableMap.of(
StarlarkBuiltinsValue.BUILTINS_NAME,
StarlarkBuiltinsValue.BUILTINS_REPO,
// TODO(wyv): Google internal tests that have blzmod enabled fail because
// TODO(wyv): Google internal tests that have Bzlmod enabled fail because
// they try to access cpp tools targets in the main repo from inside the
// @_builtin repo. This is just a workaround and needs a proper way to
// inject this mapping for google internal tests only.
"",
RepositoryName.MAIN),
StarlarkBuiltinsValue.BUILTINS_REPO)
.withAdditionalMappings(bazelToolsMapping.getRepositoryMapping()));
.withAdditionalMappings(bazelToolsMapping.getRepositoryMapping()),
"bazel_tools",
Version.EMPTY);
}

BazelModuleResolutionValue bazelModuleResolutionValue =
Expand Down Expand Up @@ -101,21 +105,20 @@ public SkyValue compute(SkyKey skyKey, Environment env)
.collect(
Collectors.toMap(
Entry::getKey, entry -> RepositoryName.createUnvalidated(entry.getKey())));
return RepositoryMappingValue.withMapping(
computeForBazelModuleRepo(repositoryName, bazelModuleResolutionValue)
.get()
// For the transitional period, we need to map the workspace name to the main repo.
.withAdditionalMappings(
ImmutableMap.of(
externalPackageValue.getPackage().getWorkspaceName(), RepositoryName.MAIN))
.withAdditionalMappings(additionalMappings));
return computeForBazelModuleRepo(repositoryName, bazelDepGraphValue)
.get()
// For the transitional period, we need to map the workspace name to the main repo.
.withAdditionalMappings(
ImmutableMap.of(
externalPackageValue.getPackage().getWorkspaceName(), RepositoryName.MAIN))
.withAdditionalMappings(additionalMappings);
}

// Try and see if this is a repo generated from a Bazel module.
Optional<RepositoryMapping> mapping =
computeForBazelModuleRepo(repositoryName, bazelModuleResolutionValue);
if (mapping.isPresent()) {
return RepositoryMappingValue.withMapping(mapping.get());
Optional<RepositoryMappingValue> mappingValue =
computeForBazelModuleRepo(repositoryName, bazelDepGraphValue);
if (mappingValue.isPresent()) {
return mappingValue.get();
}

// Now try and see if this is a repo generated from a module extension.
Expand All @@ -129,12 +132,8 @@ public SkyValue compute(SkyKey skyKey, Environment env)
if (extensionEvalValue == null) {
return null;
}
return RepositoryMappingValue.withMapping(
computeForModuleExtensionRepo(
repositoryName,
moduleExtensionId.get(),
extensionEvalValue,
bazelModuleResolutionValue));
return computeForModuleExtensionRepo(
repositoryName, moduleExtensionId.get(), extensionEvalValue, bazelDepGraphValue);
}
}

Expand All @@ -157,32 +156,33 @@ public SkyValue compute(SkyKey skyKey, Environment env)
}

/**
* Calculate repo mappings for a repo generated from a Bazel module. Such a repo can see all its
* Calculates repo mappings for a repo generated from a Bazel module. Such a repo can see all its
* {@code bazel_dep}s, as well as any repos generated by an extension it has a {@code use_repo}
* clause for.
*
* @return the repo mappings for the repo if it's generated from a Bazel module, otherwise return
* Optional.empty().
*/
private Optional<RepositoryMapping> computeForBazelModuleRepo(
RepositoryName repositoryName, BazelModuleResolutionValue bazelModuleResolutionValue) {
ModuleKey moduleKey =
bazelModuleResolutionValue.getCanonicalRepoNameLookup().get(repositoryName);
private Optional<RepositoryMappingValue> computeForBazelModuleRepo(
RepositoryName repositoryName, BazelDepGraphValue bazelDepGraphValue) {
ModuleKey moduleKey = bazelDepGraphValue.getCanonicalRepoNameLookup().get(repositoryName);
if (moduleKey == null) {
return Optional.empty();
}
return Optional.of(bazelModuleResolutionValue.getFullRepoMapping(moduleKey));
Module module = bazelDepGraphValue.getDepGraph().get(moduleKey);
return Optional.of(
RepositoryMappingValue.createForBzlmodRepo(
bazelDepGraphValue.getFullRepoMapping(moduleKey),
module.getName(),
module.getVersion()));
}

/**
* Calculate repo mappings for a repo generated from a module extension. Such a repo can see all
* Calculates repo mappings for a repo generated from a module extension. Such a repo can see all
* repos generated by the same module extension, as well as all repos that the Bazel module
* hosting the extension can see (see above).
*
* @return the repo mappings for the repo if it's generated from a module extension, otherwise
* return Optional.empty().
*/
private RepositoryMapping computeForModuleExtensionRepo(
private RepositoryMappingValue computeForModuleExtensionRepo(
RepositoryName repositoryName,
ModuleExtensionId extensionId,
SingleExtensionEvalValue extensionEvalValue,
Expand All @@ -194,16 +194,24 @@ private RepositoryMapping computeForModuleExtensionRepo(
bazelModuleResolutionValue
.getCanonicalRepoNameLookup()
.get(extensionId.getBzlFileLabel().getRepository());
Module module = bazelDepGraphValue.getDepGraph().get(moduleKey);
// NOTE(wyv): This means that if "foo" has a bazel_dep with the repo name "bar", and the
// extension generates an internal repo name "bar", then within a repo generated by the
// extension, "bar" will refer to the latter. We should explore a way to differentiate between
// the two to avoid any surprises.
return RepositoryMapping.create(
extensionEvalValue.getCanonicalRepoNameToInternalNames().inverse(), repositoryName)
.withAdditionalMappings(bazelModuleResolutionValue.getFullRepoMapping(moduleKey));
return RepositoryMappingValue.createForBzlmodRepo(
RepositoryMapping.create(
extensionEvalValue.getCanonicalRepoNameToInternalNames().inverse(), repositoryName)
.withAdditionalMappings(bazelDepGraphValue.getFullRepoMapping(moduleKey)),
module.getName(),
module.getVersion());
}

private SkyValue computeFromWorkspace(
/**
* Calculate repo mappings for a repo generated from WORKSPACE. Such a repo is not subject to
* strict deps, and can additionally see all repos that the root module can see.
*/
private RepositoryMappingValue computeFromWorkspace(
RepositoryName repositoryName,
PackageValue externalPackageValue,
@Nullable RepositoryMapping rootModuleRepoMapping)
Expand All @@ -217,12 +225,13 @@ private SkyValue computeFromWorkspace(
externalPackage.getRepositoryMapping(repositoryName));
if (rootModuleRepoMapping == null) {
// This means Bzlmod is disabled.
return RepositoryMappingValue.withMapping(workspaceMapping);
return RepositoryMappingValue.createForWorkspaceRepo(workspaceMapping);
}
// If Bzlmod is in play, we need to make sure that WORKSPACE repos see all repos that root
// If Bzlmod is in play, we need to make sure that WORKSPACE repos see all repos that the root
// module can see, taking care to compose the existing WORKSPACE mapping with the main repo
// mapping from Bzlmod.
return RepositoryMappingValue.withMapping(workspaceMapping.composeWith(rootModuleRepoMapping));
return RepositoryMappingValue.createForWorkspaceRepo(
workspaceMapping.composeWith(rootModuleRepoMapping));
}

private static Optional<ModuleExtensionId> maybeGetModuleExtensionForRepo(
Expand Down
Loading