Skip to content

Commit

Permalink
feat: support for nested node_modules in linker
Browse files Browse the repository at this point in the history 
This PR adds support for nested node_modules folders in the linker, which allows targets to depend on npm packages from multiple yarn_install & npm_install repository rules (max one per directory in the workspace).

This is an opt-in feature which is enabled by specifying package_path in yarn_install or npm_install for nested package.json files. Setting package_path informs the linker to link the external npm repository to a node_modules folder in the package_path sub-directory specified.

For example, given two yarn_install rules:

```
yarn_install(
    name = "npm",
    package_json = "//:package.json",
    yarn_lock = "//:yarn.lock",
)

yarn_install(
    name = "npm_subdir",
    package_json = "//subdir:package.json",
    package_path = "subdir",
    yarn_lock = "//subdir:yarn.lock",
)
```

a target may depend on npm packages from both,

```
jasmine_node_test(
    name = "test",
    srcs = ["test.js"],
    deps = [
        "@npm//foo",
        "@npm_subdir//bar",
    ],
)
```

and the linker will link multiple 3rd party node_modules folders,

```
/node_modules => @npm//:node_modules
/subdir/node_modules => @npm_subdir/:node_modules
```

making the 3rd party npm deps foo & bar available at

```
/node_modules/foo
/subdir/node_modules/bar
```

This feature is opt-in as package_path currently defaults to "". In a future major release (4.0.0), package_path will default to the directory the package.json file is in, which will turn it on by default.

This feature is an alternative to yarn workspaces & npm workspaces, which also lay out node_modules in multiple directories from multiple package.json files.

To date, targets have been limited to depending on and resolving form a single yarn_install or npm_install workspace which was always linked to a node_modules folder at the root of the workspace. With this change, the linker is able to link to both the root of the workspace as well as sub-directories in the workspace so targets may now resolve 3rd party npm deps from multiple package.json files which correspond to multiple node_modules folders in the tree.

1st party deps are still always linked to the root node_modules folder as they were before this change.

NB: depending on multiple npm workspaces will not be supported by ts_library as its resolution logic is limited to a single node_modules root

NB: pre-linker node require patches (deprecated as of 3.0.0) will only include root node_modules; no nested node_modules shall included in require patch resolution
  • Loading branch information
@gregmagolan
Greg Magolan authored and gregmagolan committed Feb 8, 2021
1 parent f557f9d commit 2c2cc6e
Show file tree
Hide file tree
Showing 49 changed files with 974 additions and 701 deletions.
16 changes: 16 additions & 0 deletions WORKSPACE
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ workspace(
# cypress_deps must be a managed directory to ensure it is downloaded before cypress_repository is run.
"@cypress_deps": ["packages/cypress/test/node_modules"],
"@npm": ["node_modules"],
"@npm_internal_linker_test_multi_linker": ["internal/linker/test/multi_linker/node_modules"],
"@npm_node_patches": ["packages/node-patches/node_modules"],
},
)
Expand Down Expand Up @@ -57,6 +58,21 @@ yarn_install(
yarn_lock = "//:yarn.lock",
)

yarn_install(
name = "npm_internal_linker_test_multi_linker",
package_json = "//internal/linker/test/multi_linker:package.json",
package_path = "internal/linker/test/multi_linker",
yarn_lock = "//internal/linker/test/multi_linker:yarn.lock",
)

yarn_install(
name = "onepa_npm_deps",
package_json = "//internal/linker/test/multi_linker/onepa:package.json",
package_path = "internal/linker/test/multi_linker/onepa",
symlink_node_modules = False,
yarn_lock = "//internal/linker/test/multi_linker/onepa:yarn.lock",
)

npm_install(
name = "npm_node_patches",
package_json = "//packages/node-patches:package.json",
Expand Down
5 changes: 2 additions & 3 deletions internal/bazel_integration_test/test_runner.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -358,11 +358,10 @@ for (const bazelCommand of config.bazelCommands) {
'BASH_FUNC_is_absolute%%',
'BASH_FUNC_rlocation%%',
'BASH_FUNC_runfiles_export_envvars%%',
'BAZEL_NODE_MODULES_ROOT',
'BAZEL_NODE_MODULES_ROOTS',
'BAZEL_NODE_PATCH_REQUIRE',
'BAZEL_NODE_RUNFILES_HELPER',
'BAZEL_PATCH_GUARDS',
'BAZEL_PATCH_ROOT',
'BAZEL_PATCH_ROOTS',
'BAZEL_TARGET',
'BAZEL_WORKSPACE',
'BAZELISK_SKIP_WRAPPER',
Expand Down
9 changes: 8 additions & 1 deletion internal/js_library/js_library.bzl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ _ATTRS = {
),
"deps": attr.label_list(),
"external_npm_package": attr.bool(
doc = """Indictates that this js_library target is one or more external npm packages in node_modules.
doc = """Internal use only. Indictates that this js_library target is one or more external npm packages in node_modules.
This is used by the yarn_install & npm_install repository rules for npm dependencies installed by
yarn & npm. When true, js_library will provide ExternalNpmPackageInfo.
Expand Down Expand Up @@ -73,6 +73,12 @@ _ATTRS = {
See `examples/user_managed_deps` for a working example of user-managed npm dependencies.""",
default = False,
),
"external_npm_package_path": attr.string(
doc = """Internal use only. The local workspace path that the linker should link these node_modules to.
Used only when external_npm_package is True. If empty, the linker will link these node_modules at the root.""",
default = "",
),
"is_windows": attr.bool(
doc = "Internal use only. Automatically set by macro",
mandatory = True,
Expand Down Expand Up @@ -230,6 +236,7 @@ def _impl(ctx):
direct_sources = depset(transitive = direct_sources_depsets),
sources = depset(transitive = npm_sources_depsets),
workspace = workspace_name,
path = ctx.attr.external_npm_package_path,
))

# Don't provide DeclarationInfo if there are no typings to provide.
Expand Down
231 changes: 138 additions & 93 deletions internal/linker/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ function mkdirp(p) {
return __awaiter(this, void 0, void 0, function* () {
if (p && !(yield exists(p))) {
yield mkdirp(path.dirname(p));
log_verbose(`mkdir( ${p} )`);
log_verbose(`creating directory ${p} in ${process.cwd()}`);
try {
yield fs.promises.mkdir(p);
}
Expand Down Expand Up @@ -98,7 +98,10 @@ function deleteDirectory(p) {
}
function symlink(target, p) {
return __awaiter(this, void 0, void 0, function* () {
log_verbose(`symlink( ${p} -> ${target} )`);
if (!path.isAbsolute(target)) {
target = path.resolve(process.cwd(), target);
}
log_verbose(`creating symlink ${p} -> ${target}`);
try {
yield fs.promises.symlink(target, p, 'junction');
return true;
Expand All @@ -117,33 +120,24 @@ function symlink(target, p) {
}
});
}
function resolveRoot(root, startCwd, isExecroot, runfiles) {
function resolveExternalWorkspacePath(workspace, startCwd, isExecroot, execroot, runfiles) {
return __awaiter(this, void 0, void 0, function* () {
if (isExecroot) {
return root ? `${startCwd}/external/${root}` : `${startCwd}/node_modules`;
}
const match = startCwd.match(BAZEL_OUT_REGEX);
if (!match) {
if (!root) {
return `${startCwd}/node_modules`;
}
return path.resolve(`${startCwd}/../${root}`);
return `${execroot}/external/${workspace}`;
}
const symlinkRoot = startCwd.slice(0, match.index);
process.chdir(symlinkRoot);
if (!root) {
return `${symlinkRoot}/node_modules`;
if (!execroot) {
return path.resolve(`${startCwd}/../${workspace}`);
}
const fromManifest = runfiles.lookupDirectory(root);
const fromManifest = runfiles.lookupDirectory(workspace);
if (fromManifest) {
return fromManifest;
}
else {
const maybe = path.resolve(`${symlinkRoot}/external/${root}`);
if (fs.existsSync(maybe)) {
const maybe = path.resolve(`${execroot}/external/${workspace}`);
if (yield exists(maybe)) {
return maybe;
}
return path.resolve(`${startCwd}/../${root}`);
return path.resolve(`${startCwd}/../${workspace}`);
}
});
}
Expand Down Expand Up @@ -211,7 +205,7 @@ class Runfiles {
if (result) {
return result;
}
const e = new Error(`could not resolve modulePath ${modulePath}`);
const e = new Error(`could not resolve module ${modulePath}`);
e.code = 'MODULE_NOT_FOUND';
throw e;
}
Expand Down Expand Up @@ -272,6 +266,9 @@ function exists(p) {
});
}
function existsSync(p) {
if (!p) {
return false;
}
try {
fs.lstatSync(p);
return true;
Expand Down Expand Up @@ -328,19 +325,6 @@ function liftElement(element) {
}
return element;
}
function toParentLink(link) {
return [link[0], path.dirname(link[1])];
}
function allElementsAlign(name, elements) {
if (!elements[0].link) {
return false;
}
const parentLink = toParentLink(elements[0].link);
if (!elements.every(e => !!e.link && isDirectChildLink(parentLink, e.link))) {
return false;
}
return !!elements[0].link && allElementsAlignUnder(name, parentLink, elements);
}
function allElementsAlignUnder(parentName, parentLink, elements) {
for (const { name, link, children } of elements) {
if (!link || children) {
Expand All @@ -361,16 +345,10 @@ function allElementsAlignUnder(parentName, parentLink, elements) {
function isDirectChildPath(parent, child) {
return parent === path.dirname(child);
}
function isDirectChildLink([parentRel, parentPath], [childRel, childPath]) {
if (parentRel !== childRel) {
return false;
}
if (!isDirectChildPath(parentPath, childPath)) {
return false;
}
return true;
function isDirectChildLink(parentLink, childLink) {
return parentLink === path.dirname(childLink);
}
function isNameLinkPathTopAligned(namePath, [, linkPath]) {
function isNameLinkPathTopAligned(namePath, linkPath) {
return path.basename(namePath) === path.basename(linkPath);
}
function visitDirectoryPreserveLinks(dirPath, visit) {
Expand All @@ -390,28 +368,96 @@ function visitDirectoryPreserveLinks(dirPath, visit) {
}
});
}
function findExecroot(startCwd) {
if (existsSync(`${startCwd}/bazel-out`)) {
return startCwd;
}
const bazelOutMatch = startCwd.match(BAZEL_OUT_REGEX);
return bazelOutMatch ? startCwd.slice(0, bazelOutMatch.index) : undefined;
}
function main(args, runfiles) {
return __awaiter(this, void 0, void 0, function* () {
if (!args || args.length < 1)
throw new Error('requires one argument: modulesManifest path');
const [modulesManifest] = args;
let { bin, root, modules, workspace } = JSON.parse(fs.readFileSync(modulesManifest));
log_verbose('manifest file:', modulesManifest);
let { workspace, bin, roots, modules } = JSON.parse(fs.readFileSync(modulesManifest));
modules = modules || {};
log_verbose('manifest file', modulesManifest);
log_verbose('manifest contents', JSON.stringify({ workspace, bin, root, modules }, null, 2));
log_verbose('manifest contents:', JSON.stringify({ workspace, bin, roots, modules }, null, 2));
const startCwd = process.cwd().replace(/\\/g, '/');
log_verbose('startCwd', startCwd);
const isExecroot = existsSync(`${startCwd}/bazel-out`);
log_verbose('isExecroot', isExecroot.toString());
const rootDir = yield resolveRoot(root, startCwd, isExecroot, runfiles);
log_verbose('resolved node_modules root', root, 'to', rootDir);
log_verbose('cwd', process.cwd());
if (!(yield exists(rootDir))) {
log_verbose('no third-party packages; mkdir node_modules at', root);
yield mkdirp(rootDir);
}
yield symlink(rootDir, 'node_modules');
process.chdir(rootDir);
log_verbose('startCwd:', startCwd);
const execroot = findExecroot(startCwd);
log_verbose('execroot:', execroot ? execroot : 'not found');
const isExecroot = startCwd == execroot;
log_verbose('isExecroot:', isExecroot.toString());
if (!isExecroot && execroot) {
process.chdir(execroot);
log_verbose('changed directory to execroot', execroot);
}
function symlinkWithUnlink(target, p, stats = null) {
return __awaiter(this, void 0, void 0, function* () {
if (!path.isAbsolute(target)) {
target = path.resolve(process.cwd(), target);
}
if (stats === null) {
stats = yield gracefulLstat(p);
}
if (runfiles.manifest && execroot && stats !== null && stats.isSymbolicLink()) {
const symlinkPath = fs.readlinkSync(p).replace(/\\/g, '/');
if (path.relative(symlinkPath, target) != '' &&
!path.relative(execroot, symlinkPath).startsWith('..')) {
log_verbose(`Out-of-date symlink for ${p} to ${symlinkPath} detected. Target should be ${target}. Unlinking.`);
yield unlink(p);
}
}
return symlink(target, p);
});
}
for (const packagePath of Object.keys(roots)) {
const workspace = roots[packagePath];
const workspacePath = yield resolveExternalWorkspacePath(workspace, startCwd, isExecroot, execroot, runfiles);
log_verbose(`resolved ${workspace} workspace path to ${workspacePath}`);
const workspaceNodeModules = `${workspacePath}/node_modules`;
if (packagePath) {
if (yield exists(workspaceNodeModules)) {
let resolvedPackagePath;
if (yield exists(packagePath)) {
yield symlinkWithUnlink(workspaceNodeModules, `${packagePath}/node_modules`);
resolvedPackagePath = packagePath;
}
if (!isExecroot) {
const runfilesPackagePath = `${startCwd}/${packagePath}`;
if (yield exists(runfilesPackagePath)) {
if (resolvedPackagePath) {
yield symlinkWithUnlink(`${resolvedPackagePath}/node_modules`, `${runfilesPackagePath}/node_modules`);
}
else {
yield symlinkWithUnlink(workspaceNodeModules, `${runfilesPackagePath}/node_modules`);
}
resolvedPackagePath = runfilesPackagePath;
}
}
const packagePathBin = `${bin}/${packagePath}`;
if (resolvedPackagePath && (yield exists(packagePathBin))) {
yield symlinkWithUnlink(`${resolvedPackagePath}/node_modules`, `${packagePathBin}/node_modules`);
}
}
}
else {
if (yield exists(workspaceNodeModules)) {
yield symlinkWithUnlink(workspaceNodeModules, `node_modules`);
}
else {
log_verbose('no root npm workspace node_modules folder to link to; creating node_modules directory in', process.cwd());
yield mkdirp('node_modules');
}
}
}
if (!roots || !roots['']) {
log_verbose('no root npm workspace; creating node_modules directory in ', process.cwd());
yield mkdirp('node_modules');
}
process.chdir('node_modules');
function isLeftoverDirectoryFromLinker(stats, modulePath) {
return __awaiter(this, void 0, void 0, function* () {
if (runfiles.manifest === undefined) {
Expand Down Expand Up @@ -451,44 +497,43 @@ function main(args, runfiles) {
return __awaiter(this, void 0, void 0, function* () {
yield mkdirp(path.dirname(m.name));
if (m.link) {
const [root, modulePath] = m.link;
const modulePath = m.link;
let target;
switch (root) {
case 'execroot':
if (isExecroot) {
target = `${startCwd}/${modulePath}`;
break;
}
case 'runfiles':
let runfilesPath = modulePath;
if (runfilesPath.startsWith(`${bin}/`)) {
runfilesPath = runfilesPath.slice(bin.length + 1);
}
else if (runfilesPath === bin) {
runfilesPath = '';
}
const externalPrefix = 'external/';
if (runfilesPath.startsWith(externalPrefix)) {
runfilesPath = runfilesPath.slice(externalPrefix.length);
}
else {
runfilesPath = `${workspace}/${runfilesPath}`;
}
try {
target = runfiles.resolve(runfilesPath);
if (runfiles.manifest && root == 'execroot' && modulePath.startsWith(`${bin}/`)) {
if (!target.includes(`/${bin}/`)) {
const e = new Error(`could not resolve modulePath ${modulePath}`);
e.code = 'MODULE_NOT_FOUND';
throw e;
}
if (isExecroot) {
target = `${startCwd}/${modulePath}`;
}
if (!isExecroot || !existsSync(target)) {
let runfilesPath = modulePath;
if (runfilesPath.startsWith(`${bin}/`)) {
runfilesPath = runfilesPath.slice(bin.length + 1);
}
else if (runfilesPath === bin) {
runfilesPath = '';
}
const externalPrefix = 'external/';
if (runfilesPath.startsWith(externalPrefix)) {
runfilesPath = runfilesPath.slice(externalPrefix.length);
}
else {
runfilesPath = `${workspace}/${runfilesPath}`;
}
try {
target = runfiles.resolve(runfilesPath);
if (runfiles.manifest && modulePath.startsWith(`${bin}/`)) {
if (!target.match(BAZEL_OUT_REGEX)) {
const e = new Error(`could not resolve module ${runfilesPath} in output tree`);
e.code = 'MODULE_NOT_FOUND';
throw e;
}
}
catch (err) {
target = undefined;
log_verbose(`runfiles resolve failed for module '${m.name}': ${err.message}`);
}
break;
}
catch (err) {
target = undefined;
log_verbose(`runfiles resolve failed for module '${m.name}': ${err.message}`);
}
}
if (target && !path.isAbsolute(target)) {
target = path.resolve(process.cwd(), target);
}
const stats = yield gracefulLstat(m.name);
const isLeftOver = (stats !== null && (yield isLeftoverDirectoryFromLinker(stats, m.name)));
Expand All @@ -497,7 +542,7 @@ function main(args, runfiles) {
yield createSymlinkAndPreserveContents(stats, m.name, target);
}
else {
yield symlink(target, m.name);
yield symlinkWithUnlink(target, m.name, stats);
}
}
else {
Expand Down
Loading

0 comments on commit 2c2cc6e

Please sign in to comment.