diff --git a/CHANGES.md b/CHANGES.md index 11f3f9da3b..87af50c159 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,10 +13,12 @@ We adhere to the [keepachangelog](https://keepachangelog.com/en/1.0.0/) format ( ### Added * `gradlew equoIde` opens a repeatable clean Spotless dev environment. ([#1523](https://github.com/diffplug/spotless/pull/1523)) * `cleanthat` added `includeDraft` option, to include draft mutators from composite mutators. ([#1574](https://github.com/diffplug/spotless/pull/1574)) +* `npm`-based formatters now support caching of `node_modules` directory ([#1590](https://github.com/diffplug/spotless/pull/1590)) ### Fixed * `JacksonJsonFormatterFunc` handles json files with an Array as root. ([#1585](https://github.com/diffplug/spotless/pull/1585)) ### Changes * Bump default `cleanthat` version to latest `2.1` -> `2.6` ([#1569](https://github.com/diffplug/spotless/pull/1569) and [#1574](https://github.com/diffplug/spotless/pull/1574)) +* Reduce logging-noise created by `npm`-based formatters ([#1590](https://github.com/diffplug/spotless/pull/1590) fixes [#1582](https://github.com/diffplug/spotless/issues/1582)) ## [2.35.0] - 2023-02-10 ### Added diff --git a/lib/build.gradle b/lib/build.gradle index f6e0446f73..9a6e7913d5 100644 --- a/lib/build.gradle +++ b/lib/build.gradle @@ -56,6 +56,8 @@ tasks.named("check").configure { dependencies { compileOnly 'org.slf4j:slf4j-api:2.0.0' + testCommonImplementation 'org.slf4j:slf4j-api:2.0.0' + // zero runtime reqs is a hard requirements for spotless-lib // if you need a dep, put it in lib-extra testCommonImplementation "org.junit.jupiter:junit-jupiter:$VER_JUNIT" diff --git a/lib/src/main/java/com/diffplug/spotless/npm/EslintFormatterStep.java b/lib/src/main/java/com/diffplug/spotless/npm/EslintFormatterStep.java index b262bb4b98..a4480fe7fc 100644 --- a/lib/src/main/java/com/diffplug/spotless/npm/EslintFormatterStep.java +++ b/lib/src/main/java/com/diffplug/spotless/npm/EslintFormatterStep.java @@ -15,7 +15,6 @@ */ package com.diffplug.spotless.npm; -import static com.diffplug.spotless.LazyArgLogger.lazy; import static java.util.Objects.requireNonNull; import java.io.File; @@ -71,13 +70,13 @@ public static Map defaultDevDependenciesWithEslint(String versio return Collections.singletonMap("eslint", version); } - public static FormatterStep create(Map devDependencies, Provisioner provisioner, File projectDir, File buildDir, NpmPathResolver npmPathResolver, EslintConfig eslintConfig) { + public static FormatterStep create(Map devDependencies, Provisioner provisioner, File projectDir, File buildDir, File cacheDir, NpmPathResolver npmPathResolver, EslintConfig eslintConfig) { requireNonNull(devDependencies); requireNonNull(provisioner); requireNonNull(projectDir); requireNonNull(buildDir); return FormatterStep.createLazy(NAME, - () -> new State(NAME, devDependencies, projectDir, buildDir, npmPathResolver, eslintConfig), + () -> new State(NAME, devDependencies, projectDir, buildDir, cacheDir, npmPathResolver, eslintConfig), State::createFormatterFunc); } @@ -89,13 +88,12 @@ private static class State extends NpmFormatterStepStateBase implements Serializ @SuppressFBWarnings("SE_TRANSIENT_FIELD_NOT_RESTORED") private transient EslintConfig eslintConfigInUse; - State(String stepName, Map devDependencies, File projectDir, File buildDir, NpmPathResolver npmPathResolver, EslintConfig eslintConfig) throws IOException { + State(String stepName, Map devDependencies, File projectDir, File buildDir, File cacheDir, NpmPathResolver npmPathResolver, EslintConfig eslintConfig) throws IOException { super(stepName, new NpmConfig( replaceDevDependencies( NpmResourceHelper.readUtf8StringFromClasspath(EslintFormatterStep.class, "/com/diffplug/spotless/npm/eslint-package.json"), new TreeMap<>(devDependencies)), - "eslint", NpmResourceHelper.readUtf8StringFromClasspath(EslintFormatterStep.class, "/com/diffplug/spotless/npm/common-serve.js", "/com/diffplug/spotless/npm/eslint-serve.js"), @@ -103,6 +101,7 @@ private static class State extends NpmFormatterStepStateBase implements Serializ new NpmFormatterStepLocations( projectDir, buildDir, + cacheDir, npmPathResolver::resolveNpmExecutable, npmPathResolver::resolveNodeExecutable)); this.origEslintConfig = requireNonNull(eslintConfig.verify()); @@ -116,7 +115,7 @@ protected void prepareNodeServerLayout() throws IOException { // If any config files are provided, we need to make sure they are at the same location as the node modules // as eslint will try to resolve plugin/config names relatively to the config file location and some // eslint configs contain relative paths to additional config files (such as tsconfig.json e.g.) - logger.info("Copying config file <{}> to <{}> and using the copy", origEslintConfig.getEslintConfigPath(), nodeServerLayout.nodeModulesDir()); + logger.debug("Copying config file <{}> to <{}> and using the copy", origEslintConfig.getEslintConfigPath(), nodeServerLayout.nodeModulesDir()); File configFileCopy = NpmResourceHelper.copyFileToDir(origEslintConfig.getEslintConfigPath(), nodeServerLayout.nodeModulesDir()); this.eslintConfigInUse = this.origEslintConfig.withEslintConfigPath(configFileCopy).verify(); } @@ -162,8 +161,6 @@ public EslintFilePathPassingFormatterFunc(File projectDir, File nodeModulesDir, @Override public String applyWithFile(String unix, File file) throws Exception { - logger.info("formatting String '{}[...]' in file '{}'", lazy(() -> unix.substring(0, Math.min(50, unix.length()))), file); - Map eslintCallOptions = new HashMap<>(); setConfigToCallOptions(eslintCallOptions); setFilePathToCallOptions(eslintCallOptions, file); diff --git a/lib/src/main/java/com/diffplug/spotless/npm/NodeApp.java b/lib/src/main/java/com/diffplug/spotless/npm/NodeApp.java new file mode 100644 index 0000000000..edac59bbfd --- /dev/null +++ b/lib/src/main/java/com/diffplug/spotless/npm/NodeApp.java @@ -0,0 +1,88 @@ +/* + * Copyright 2023 DiffPlug + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.diffplug.spotless.npm; + +import java.util.Objects; + +import javax.annotation.Nonnull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class NodeApp { + + private static final Logger logger = LoggerFactory.getLogger(NodeApp.class); + + private static final TimedLogger timedLogger = TimedLogger.forLogger(logger); + + @Nonnull + protected final NodeServerLayout nodeServerLayout; + + @Nonnull + protected final NpmConfig npmConfig; + + @Nonnull + protected final NpmProcessFactory npmProcessFactory; + + @Nonnull + protected final NpmFormatterStepLocations formatterStepLocations; + + public NodeApp(@Nonnull NodeServerLayout nodeServerLayout, @Nonnull NpmConfig npmConfig, @Nonnull NpmFormatterStepLocations formatterStepLocations) { + this.nodeServerLayout = Objects.requireNonNull(nodeServerLayout); + this.npmConfig = Objects.requireNonNull(npmConfig); + this.npmProcessFactory = processFactory(formatterStepLocations); + this.formatterStepLocations = Objects.requireNonNull(formatterStepLocations); + } + + private static NpmProcessFactory processFactory(NpmFormatterStepLocations formatterStepLocations) { + if (formatterStepLocations.cacheDir() != null) { + logger.info("Caching npm install results in {}.", formatterStepLocations.cacheDir()); + return NodeModulesCachingNpmProcessFactory.create(formatterStepLocations.cacheDir()); + } + logger.debug("Not caching npm install results."); + return StandardNpmProcessFactory.INSTANCE; + } + + boolean needsNpmInstall() { + return !this.nodeServerLayout.isNodeModulesPrepared(); + } + + boolean needsPrepareNodeAppLayout() { + return !this.nodeServerLayout.isLayoutPrepared(); + } + + void prepareNodeAppLayout() { + timedLogger.withInfo("Preparing {} for npm step {}.", this.nodeServerLayout, getClass().getName()).run(() -> { + NpmResourceHelper.assertDirectoryExists(nodeServerLayout.nodeModulesDir()); + NpmResourceHelper.writeUtf8StringToFile(nodeServerLayout.packageJsonFile(), this.npmConfig.getPackageJsonContent()); + if (this.npmConfig.getServeScriptContent() != null) { + NpmResourceHelper.writeUtf8StringToFile(nodeServerLayout.serveJsFile(), this.npmConfig.getServeScriptContent()); + } else { + NpmResourceHelper.deleteFileIfExists(nodeServerLayout.serveJsFile()); + } + if (this.npmConfig.getNpmrcContent() != null) { + NpmResourceHelper.writeUtf8StringToFile(nodeServerLayout.npmrcFile(), this.npmConfig.getNpmrcContent()); + } else { + NpmResourceHelper.deleteFileIfExists(nodeServerLayout.npmrcFile()); + } + }); + } + + void npmInstall() { + timedLogger.withInfo("Installing npm dependencies for {} with {}.", this.nodeServerLayout, this.npmProcessFactory.describe()) + .run(() -> npmProcessFactory.createNpmInstallProcess(nodeServerLayout, formatterStepLocations).waitFor()); + } +} diff --git a/lib/src/main/java/com/diffplug/spotless/npm/NodeModulesCachingNpmProcessFactory.java b/lib/src/main/java/com/diffplug/spotless/npm/NodeModulesCachingNpmProcessFactory.java new file mode 100644 index 0000000000..0086064194 --- /dev/null +++ b/lib/src/main/java/com/diffplug/spotless/npm/NodeModulesCachingNpmProcessFactory.java @@ -0,0 +1,118 @@ +/* + * Copyright 2023 DiffPlug + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.diffplug.spotless.npm; + +import java.io.File; +import java.util.List; +import java.util.Objects; + +import javax.annotation.Nonnull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.diffplug.spotless.ProcessRunner.Result; + +public class NodeModulesCachingNpmProcessFactory implements NpmProcessFactory { + + private static final Logger logger = LoggerFactory.getLogger(NodeModulesCachingNpmProcessFactory.class); + + private static final TimedLogger timedLogger = TimedLogger.forLogger(logger); + + private final File cacheDir; + + private final ShadowCopy shadowCopy; + + private NodeModulesCachingNpmProcessFactory(@Nonnull File cacheDir) { + this.cacheDir = Objects.requireNonNull(cacheDir); + assertDir(cacheDir); + this.shadowCopy = new ShadowCopy(cacheDir); + } + + private void assertDir(File cacheDir) { + if (cacheDir.exists() && !cacheDir.isDirectory()) { + throw new IllegalArgumentException("Cache dir must be a directory"); + } + if (!cacheDir.exists()) { + if (!cacheDir.mkdirs()) { + throw new IllegalArgumentException("Cache dir could not be created."); + } + } + } + + public static NodeModulesCachingNpmProcessFactory create(@Nonnull File cacheDir) { + return new NodeModulesCachingNpmProcessFactory(cacheDir); + } + + @Override + public NpmProcess createNpmInstallProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) { + NpmProcess actualNpmInstallProcess = StandardNpmProcessFactory.INSTANCE.createNpmInstallProcess(nodeServerLayout, formatterStepLocations); + return new CachingNmpInstall(actualNpmInstallProcess, nodeServerLayout); + } + + @Override + public NpmLongRunningProcess createNpmServeProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) { + return StandardNpmProcessFactory.INSTANCE.createNpmServeProcess(nodeServerLayout, formatterStepLocations); + } + + private class CachingNmpInstall implements NpmProcess { + + private final NpmProcess actualNpmInstallProcess; + private final NodeServerLayout nodeServerLayout; + + public CachingNmpInstall(NpmProcess actualNpmInstallProcess, NodeServerLayout nodeServerLayout) { + this.actualNpmInstallProcess = actualNpmInstallProcess; + this.nodeServerLayout = nodeServerLayout; + } + + @Override + public Result waitFor() { + String entryName = entryName(); + if (shadowCopy.entryExists(entryName, NodeServerLayout.NODE_MODULES)) { + timedLogger.withInfo("Using cached node_modules for {} from {}", entryName, cacheDir) + .run(() -> shadowCopy.copyEntryInto(entryName(), NodeServerLayout.NODE_MODULES, nodeServerLayout.nodeModulesDir())); + return new CachedResult(); + } else { + Result result = timedLogger.withInfo("calling actual npm install {}", actualNpmInstallProcess.describe()) + .call(actualNpmInstallProcess::waitFor); + assert result.exitCode() == 0; + storeShadowCopy(entryName); + return result; + } + } + + private void storeShadowCopy(String entryName) { + timedLogger.withInfo("Caching node_modules for {} in {}", entryName, cacheDir) + .run(() -> shadowCopy.addEntry(entryName(), new File(nodeServerLayout.nodeModulesDir(), NodeServerLayout.NODE_MODULES))); + } + + private String entryName() { + return nodeServerLayout.nodeModulesDir().getName(); + } + + @Override + public String describe() { + return String.format("Wrapper around [%s] to cache node_modules in [%s]", actualNpmInstallProcess.describe(), cacheDir.getAbsolutePath()); + } + } + + private class CachedResult extends Result { + + public CachedResult() { + super(List.of("(from cache dir " + cacheDir + ")"), 0, new byte[0], new byte[0]); + } + } +} diff --git a/lib/src/main/java/com/diffplug/spotless/npm/NodeServeApp.java b/lib/src/main/java/com/diffplug/spotless/npm/NodeServeApp.java new file mode 100644 index 0000000000..c9311a5589 --- /dev/null +++ b/lib/src/main/java/com/diffplug/spotless/npm/NodeServeApp.java @@ -0,0 +1,40 @@ +/* + * Copyright 2023 DiffPlug + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.diffplug.spotless.npm; + +import javax.annotation.Nonnull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.diffplug.spotless.ProcessRunner; + +public class NodeServeApp extends NodeApp { + + private static final Logger logger = LoggerFactory.getLogger(NodeApp.class); + + private static final TimedLogger timedLogger = TimedLogger.forLogger(logger); + + public NodeServeApp(@Nonnull NodeServerLayout nodeServerLayout, @Nonnull NpmConfig npmConfig, @Nonnull NpmFormatterStepLocations formatterStepLocations) { + super(nodeServerLayout, npmConfig, formatterStepLocations); + } + + ProcessRunner.LongRunningProcess startNpmServeProcess() { + return timedLogger.withInfo("Starting npm based server in {} with {}.", this.nodeServerLayout.nodeModulesDir(), this.npmProcessFactory.describe()) + .call(() -> npmProcessFactory.createNpmServeProcess(nodeServerLayout, formatterStepLocations).start()); + } + +} diff --git a/lib/src/main/java/com/diffplug/spotless/npm/NodeServerLayout.java b/lib/src/main/java/com/diffplug/spotless/npm/NodeServerLayout.java index 8b39c5e4ab..850ea4eb6b 100644 --- a/lib/src/main/java/com/diffplug/spotless/npm/NodeServerLayout.java +++ b/lib/src/main/java/com/diffplug/spotless/npm/NodeServerLayout.java @@ -27,6 +27,7 @@ class NodeServerLayout { private static final Pattern PACKAGE_JSON_NAME_PATTERN = Pattern.compile("\"name\"\\s*:\\s*\"([^\"]+)\""); + static final String NODE_MODULES = "node_modules"; private final File nodeModulesDir; private final File packageJsonFile; @@ -55,7 +56,6 @@ private static String nodeModulesDirName(String packageJsonContent) { } File nodeModulesDir() { - return nodeModulesDir; } @@ -89,7 +89,7 @@ public boolean isLayoutPrepared() { } public boolean isNodeModulesPrepared() { - Path nodeModulesInstallDirPath = new File(nodeModulesDir(), "node_modules").toPath(); + Path nodeModulesInstallDirPath = new File(nodeModulesDir(), NODE_MODULES).toPath(); if (!Files.isDirectory(nodeModulesInstallDirPath)) { return false; } diff --git a/lib/src/main/java/com/diffplug/spotless/npm/NpmConfig.java b/lib/src/main/java/com/diffplug/spotless/npm/NpmConfig.java index 1492fe7a99..863be41193 100644 --- a/lib/src/main/java/com/diffplug/spotless/npm/NpmConfig.java +++ b/lib/src/main/java/com/diffplug/spotless/npm/NpmConfig.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2020 DiffPlug + * Copyright 2016-2023 DiffPlug * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ package com.diffplug.spotless.npm; import java.io.Serializable; +import java.util.Objects; import javax.annotation.Nonnull; @@ -23,30 +24,24 @@ class NpmConfig implements Serializable { private static final long serialVersionUID = 684264546497914877L; + @Nonnull private final String packageJsonContent; - private final String npmModule; - private final String serveScriptContent; private final String npmrcContent; - public NpmConfig(String packageJsonContent, String npmModule, String serveScriptContent, String npmrcContent) { - this.packageJsonContent = packageJsonContent; - this.npmModule = npmModule; + public NpmConfig(@Nonnull String packageJsonContent, String serveScriptContent, String npmrcContent) { + this.packageJsonContent = Objects.requireNonNull(packageJsonContent); this.serveScriptContent = serveScriptContent; this.npmrcContent = npmrcContent; } + @Nonnull public String getPackageJsonContent() { return packageJsonContent; } - public String getNpmModule() { - return npmModule; - } - - @Nonnull public String getServeScriptContent() { return serveScriptContent; } diff --git a/lib/src/main/java/com/diffplug/spotless/npm/NpmFormatterStepLocations.java b/lib/src/main/java/com/diffplug/spotless/npm/NpmFormatterStepLocations.java index 0e99e1afab..67763e59ec 100644 --- a/lib/src/main/java/com/diffplug/spotless/npm/NpmFormatterStepLocations.java +++ b/lib/src/main/java/com/diffplug/spotless/npm/NpmFormatterStepLocations.java @@ -21,6 +21,8 @@ import java.io.Serializable; import java.util.function.Supplier; +import javax.annotation.Nonnull; + import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; class NpmFormatterStepLocations implements Serializable { @@ -32,15 +34,19 @@ class NpmFormatterStepLocations implements Serializable { @SuppressFBWarnings("SE_TRANSIENT_FIELD_NOT_RESTORED") private final transient File buildDir; + @SuppressFBWarnings("SE_TRANSIENT_FIELD_NOT_RESTORED") + private final transient File cacheDir; + @SuppressFBWarnings("SE_TRANSIENT_FIELD_NOT_RESTORED") private final transient Supplier npmExecutable; @SuppressFBWarnings("SE_TRANSIENT_FIELD_NOT_RESTORED") private final transient Supplier nodeExecutable; - public NpmFormatterStepLocations(File projectDir, File buildDir, Supplier npmExecutable, Supplier nodeExecutable) { + public NpmFormatterStepLocations(@Nonnull File projectDir, @Nonnull File buildDir, File cacheDir, @Nonnull Supplier npmExecutable, @Nonnull Supplier nodeExecutable) { this.projectDir = requireNonNull(projectDir); this.buildDir = requireNonNull(buildDir); + this.cacheDir = cacheDir; this.npmExecutable = requireNonNull(npmExecutable); this.nodeExecutable = requireNonNull(nodeExecutable); } @@ -53,6 +59,10 @@ public File buildDir() { return buildDir; } + public File cacheDir() { + return cacheDir; + } + public File npmExecutable() { return npmExecutable.get(); } diff --git a/lib/src/main/java/com/diffplug/spotless/npm/NpmFormatterStepStateBase.java b/lib/src/main/java/com/diffplug/spotless/npm/NpmFormatterStepStateBase.java index f3f8a80fe8..3f262c813c 100644 --- a/lib/src/main/java/com/diffplug/spotless/npm/NpmFormatterStepStateBase.java +++ b/lib/src/main/java/com/diffplug/spotless/npm/NpmFormatterStepStateBase.java @@ -41,6 +41,8 @@ abstract class NpmFormatterStepStateBase implements Serializable { private static final Logger logger = LoggerFactory.getLogger(NpmFormatterStepStateBase.class); + private static final TimedLogger timedLogger = TimedLogger.forLogger(logger); + private static final long serialVersionUID = 1460749955865959948L; @SuppressFBWarnings("SE_TRANSIENT_FIELD_NOT_RESTORED") @@ -52,39 +54,22 @@ abstract class NpmFormatterStepStateBase implements Serializable { private final String stepName; + private final transient NodeServeApp nodeServeApp; + protected NpmFormatterStepStateBase(String stepName, NpmConfig npmConfig, NpmFormatterStepLocations locations) throws IOException { this.stepName = requireNonNull(stepName); this.npmConfig = requireNonNull(npmConfig); this.locations = locations; this.nodeServerLayout = new NodeServerLayout(locations.buildDir(), npmConfig.getPackageJsonContent()); + this.nodeServeApp = new NodeServeApp(nodeServerLayout, npmConfig, locations); } protected void prepareNodeServerLayout() throws IOException { - final long started = System.currentTimeMillis(); - // maybe introduce trace logger? - logger.info("Preparing {} for npm step {}.", this.nodeServerLayout, getClass().getName()); - NpmResourceHelper.assertDirectoryExists(nodeServerLayout.nodeModulesDir()); - NpmResourceHelper.writeUtf8StringToFile(nodeServerLayout.packageJsonFile(), - this.npmConfig.getPackageJsonContent()); - NpmResourceHelper - .writeUtf8StringToFile(nodeServerLayout.serveJsFile(), this.npmConfig.getServeScriptContent()); - if (this.npmConfig.getNpmrcContent() != null) { - NpmResourceHelper.writeUtf8StringToFile(nodeServerLayout.npmrcFile(), this.npmConfig.getNpmrcContent()); - } else { - NpmResourceHelper.deleteFileIfExists(nodeServerLayout.npmrcFile()); - } - logger.info("Prepared {} for npm step {} in {} ms.", this.nodeServerLayout, getClass().getName(), System.currentTimeMillis() - started); + nodeServeApp.prepareNodeAppLayout(); } protected void prepareNodeServer() throws IOException { - final long started = System.currentTimeMillis(); - logger.info("running npm install in {} for npm step {}", this.nodeServerLayout.nodeModulesDir(), getClass().getName()); - runNpmInstall(nodeServerLayout.nodeModulesDir()); - logger.info("npm install finished in {} ms in {} for npm step {}", System.currentTimeMillis() - started, this.nodeServerLayout.nodeModulesDir(), getClass().getName()); - } - - private void runNpmInstall(File npmProjectDir) throws IOException { - new NpmProcess(npmProjectDir, this.locations.npmExecutable(), this.locations.nodeExecutable()).install(); + nodeServeApp.npmInstall(); } protected void assertNodeServerDirReady() throws IOException { @@ -99,11 +84,11 @@ protected void assertNodeServerDirReady() throws IOException { } protected boolean needsPrepareNodeServer() { - return !this.nodeServerLayout.isNodeModulesPrepared(); + return nodeServeApp.needsNpmInstall(); } protected boolean needsPrepareNodeServerLayout() { - return !this.nodeServerLayout.isLayoutPrepared(); + return nodeServeApp.needsPrepareNodeAppLayout(); } protected ServerProcessInfo npmRunServer() throws ServerStartException, IOException { @@ -115,7 +100,7 @@ protected ServerProcessInfo npmRunServer() throws ServerStartException, IOExcept final File serverPortFile = new File(this.nodeServerLayout.nodeModulesDir(), "server.port"); NpmResourceHelper.deleteFileIfExists(serverPortFile); // start the http server in node - server = new NpmProcess(this.nodeServerLayout.nodeModulesDir(), this.locations.npmExecutable(), this.locations.nodeExecutable()).start(); + server = nodeServeApp.startNpmServeProcess(); // await the readiness of the http server - wait for at most 60 seconds try { @@ -206,7 +191,7 @@ protected static class ServerStartException extends RuntimeException { private static final long serialVersionUID = -8803977379866483002L; public ServerStartException(String message, Throwable cause) { - super(cause); + super(message, cause); } } } diff --git a/lib/src/main/java/com/diffplug/spotless/npm/NpmLongRunningProcess.java b/lib/src/main/java/com/diffplug/spotless/npm/NpmLongRunningProcess.java new file mode 100644 index 0000000000..f5bece3e06 --- /dev/null +++ b/lib/src/main/java/com/diffplug/spotless/npm/NpmLongRunningProcess.java @@ -0,0 +1,26 @@ +/* + * Copyright 2023 DiffPlug + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.diffplug.spotless.npm; + +import com.diffplug.spotless.ProcessRunner.LongRunningProcess; + +interface NpmLongRunningProcess { + + String describe(); + + LongRunningProcess start(); + +} diff --git a/lib/src/main/java/com/diffplug/spotless/npm/NpmProcess.java b/lib/src/main/java/com/diffplug/spotless/npm/NpmProcess.java index 6384900d82..473057a769 100644 --- a/lib/src/main/java/com/diffplug/spotless/npm/NpmProcess.java +++ b/lib/src/main/java/com/diffplug/spotless/npm/NpmProcess.java @@ -1,5 +1,5 @@ /* - * Copyright 2016-2023 DiffPlug + * Copyright 2023 DiffPlug * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,96 +15,12 @@ */ package com.diffplug.spotless.npm; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; +import com.diffplug.spotless.ProcessRunner.Result; -import com.diffplug.spotless.ProcessRunner; -import com.diffplug.spotless.ProcessRunner.LongRunningProcess; +interface NpmProcess { -class NpmProcess { + String describe(); - private final File workingDir; + Result waitFor(); - private final File npmExecutable; - - private final File nodeExecutable; - - private final ProcessRunner processRunner; - - NpmProcess(File workingDir, File npmExecutable, File nodeExecutable) { - this.workingDir = workingDir; - this.npmExecutable = npmExecutable; - this.nodeExecutable = nodeExecutable; - processRunner = ProcessRunner.usingRingBuffersOfCapacity(100 * 1024); // 100kB - } - - void install() { - npmAwait("install", - "--no-audit", - "--no-package-lock", - "--no-fund", - "--prefer-offline"); - } - - LongRunningProcess start() { - // adding --scripts-prepend-node-path=true due to https://github.com/diffplug/spotless/issues/619#issuecomment-648018679 - return npm("start", "--scripts-prepend-node-path=true"); - } - - private void npmAwait(String... args) { - try (LongRunningProcess npmProcess = npm(args)) { - if (npmProcess.waitFor() != 0) { - throw new NpmProcessException("Running npm command '" + commandLine(args) + "' failed with exit code: " + npmProcess.exitValue() + "\n\n" + npmProcess.result()); - } - } catch (InterruptedException e) { - throw new NpmProcessException("Running npm command '" + commandLine(args) + "' was interrupted.", e); - } catch (ExecutionException e) { - throw new NpmProcessException("Running npm command '" + commandLine(args) + "' failed.", e); - } - } - - private LongRunningProcess npm(String... args) { - List processCommand = processCommand(args); - try { - return processRunner.start(this.workingDir, environmentVariables(), null, true, processCommand); - } catch (IOException e) { - throw new NpmProcessException("Failed to launch npm command '" + commandLine(args) + "'.", e); - } - } - - private List processCommand(String... args) { - List command = new ArrayList<>(args.length + 1); - command.add(this.npmExecutable.getAbsolutePath()); - command.addAll(Arrays.asList(args)); - return command; - } - - private Map environmentVariables() { - Map environmentVariables = new HashMap<>(); - environmentVariables.put("PATH", this.nodeExecutable.getParentFile().getAbsolutePath() + File.pathSeparator + System.getenv("PATH")); - return environmentVariables; - } - - private String commandLine(String... args) { - return "npm " + Arrays.stream(args).collect(Collectors.joining(" ")); - } - - static class NpmProcessException extends RuntimeException { - private static final long serialVersionUID = 6424331316676759525L; - - public NpmProcessException(String message) { - super(message); - } - - public NpmProcessException(String message, Throwable cause) { - super(message, cause); - } - } } diff --git a/lib/src/main/java/com/diffplug/spotless/npm/NpmProcessException.java b/lib/src/main/java/com/diffplug/spotless/npm/NpmProcessException.java new file mode 100644 index 0000000000..bff621df07 --- /dev/null +++ b/lib/src/main/java/com/diffplug/spotless/npm/NpmProcessException.java @@ -0,0 +1,28 @@ +/* + * Copyright 2023 DiffPlug + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.diffplug.spotless.npm; + +public class NpmProcessException extends RuntimeException { + private static final long serialVersionUID = 6424331316676759525L; + + public NpmProcessException(String message) { + super(message); + } + + public NpmProcessException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/lib/src/main/java/com/diffplug/spotless/npm/NpmProcessFactory.java b/lib/src/main/java/com/diffplug/spotless/npm/NpmProcessFactory.java new file mode 100644 index 0000000000..41543a2099 --- /dev/null +++ b/lib/src/main/java/com/diffplug/spotless/npm/NpmProcessFactory.java @@ -0,0 +1,27 @@ +/* + * Copyright 2023 DiffPlug + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.diffplug.spotless.npm; + +public interface NpmProcessFactory { + NpmProcess createNpmInstallProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations); + + NpmLongRunningProcess createNpmServeProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations); + + default String describe() { + return getClass().getSimpleName(); + } + +} diff --git a/lib/src/main/java/com/diffplug/spotless/npm/NpmResourceHelper.java b/lib/src/main/java/com/diffplug/spotless/npm/NpmResourceHelper.java index aa66c54fcf..7a28685de0 100644 --- a/lib/src/main/java/com/diffplug/spotless/npm/NpmResourceHelper.java +++ b/lib/src/main/java/com/diffplug/spotless/npm/NpmResourceHelper.java @@ -103,6 +103,18 @@ static void awaitReadableFile(File file, Duration maxWaitTime) throws TimeoutExc if ((System.currentTimeMillis() - startedAt) > maxWaitTime.toMillis()) { throw new TimeoutException("The file did not appear within " + maxWaitTime); } + ThrowingEx.run(() -> Thread.sleep(100)); + } + } + + static void awaitFileDeleted(File file, Duration maxWaitTime) throws TimeoutException { + final long startedAt = System.currentTimeMillis(); + while (file.exists()) { + // wait for at most maxWaitTime + if ((System.currentTimeMillis() - startedAt) > maxWaitTime.toMillis()) { + throw new TimeoutException("The file did not disappear within " + maxWaitTime); + } + ThrowingEx.run(() -> Thread.sleep(100)); } } diff --git a/lib/src/main/java/com/diffplug/spotless/npm/PrettierFormatterStep.java b/lib/src/main/java/com/diffplug/spotless/npm/PrettierFormatterStep.java index 05c61f9bdf..11a6230e4c 100644 --- a/lib/src/main/java/com/diffplug/spotless/npm/PrettierFormatterStep.java +++ b/lib/src/main/java/com/diffplug/spotless/npm/PrettierFormatterStep.java @@ -15,7 +15,6 @@ */ package com.diffplug.spotless.npm; -import static com.diffplug.spotless.LazyArgLogger.lazy; import static java.util.Objects.requireNonNull; import java.io.File; @@ -50,12 +49,12 @@ public static final Map defaultDevDependenciesWithPrettier(Strin return Collections.singletonMap("prettier", version); } - public static FormatterStep create(Map devDependencies, Provisioner provisioner, File projectDir, File buildDir, NpmPathResolver npmPathResolver, PrettierConfig prettierConfig) { + public static FormatterStep create(Map devDependencies, Provisioner provisioner, File projectDir, File buildDir, File cacheDir, NpmPathResolver npmPathResolver, PrettierConfig prettierConfig) { requireNonNull(devDependencies); requireNonNull(provisioner); requireNonNull(buildDir); return FormatterStep.createLazy(NAME, - () -> new State(NAME, devDependencies, projectDir, buildDir, npmPathResolver, prettierConfig), + () -> new State(NAME, devDependencies, projectDir, buildDir, cacheDir, npmPathResolver, prettierConfig), State::createFormatterFunc); } @@ -64,13 +63,12 @@ private static class State extends NpmFormatterStepStateBase implements Serializ private static final long serialVersionUID = -539537027004745812L; private final PrettierConfig prettierConfig; - State(String stepName, Map devDependencies, File projectDir, File buildDir, NpmPathResolver npmPathResolver, PrettierConfig prettierConfig) throws IOException { + State(String stepName, Map devDependencies, File projectDir, File buildDir, File cacheDir, NpmPathResolver npmPathResolver, PrettierConfig prettierConfig) throws IOException { super(stepName, new NpmConfig( replaceDevDependencies( NpmResourceHelper.readUtf8StringFromClasspath(PrettierFormatterStep.class, "/com/diffplug/spotless/npm/prettier-package.json"), new TreeMap<>(devDependencies)), - "prettier", NpmResourceHelper.readUtf8StringFromClasspath(PrettierFormatterStep.class, "/com/diffplug/spotless/npm/common-serve.js", "/com/diffplug/spotless/npm/prettier-serve.js"), @@ -78,6 +76,7 @@ private static class State extends NpmFormatterStepStateBase implements Serializ new NpmFormatterStepLocations( projectDir, buildDir, + cacheDir, npmPathResolver::resolveNpmExecutable, npmPathResolver::resolveNodeExecutable)); this.prettierConfig = requireNonNull(prettierConfig); @@ -120,8 +119,6 @@ public PrettierFilePathPassingFormatterFunc(String prettierConfigOptions, Pretti @Override public String applyWithFile(String unix, File file) throws Exception { - logger.info("formatting String '{}[...]' in file '{}'", lazy(() -> unix.substring(0, Math.min(50, unix.length()))), file); - final String prettierConfigOptionsWithFilepath = assertFilepathInConfigOptions(file); try { return restService.format(unix, prettierConfigOptionsWithFilepath); diff --git a/lib/src/main/java/com/diffplug/spotless/npm/ShadowCopy.java b/lib/src/main/java/com/diffplug/spotless/npm/ShadowCopy.java new file mode 100644 index 0000000000..044b5d70ea --- /dev/null +++ b/lib/src/main/java/com/diffplug/spotless/npm/ShadowCopy.java @@ -0,0 +1,175 @@ +/* + * Copyright 2023 DiffPlug + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.diffplug.spotless.npm; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystemException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.time.Duration; +import java.util.concurrent.TimeoutException; + +import javax.annotation.Nonnull; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.diffplug.spotless.ThrowingEx; + +class ShadowCopy { + + private static final Logger logger = LoggerFactory.getLogger(ShadowCopy.class); + + private final File shadowCopyRoot; + + public ShadowCopy(@Nonnull File shadowCopyRoot) { + this.shadowCopyRoot = shadowCopyRoot; + if (!shadowCopyRoot.isDirectory()) { + throw new IllegalArgumentException("Shadow copy root must be a directory: " + shadowCopyRoot); + } + } + + public void addEntry(String key, File orig) { + // prevent concurrent adding of entry with same key + if (!reserveSubFolder(key)) { + logger.debug("Shadow copy entry already in progress: {}. Awaiting finalization.", key); + try { + NpmResourceHelper.awaitFileDeleted(markerFilePath(key).toFile(), Duration.ofSeconds(120)); + } catch (TimeoutException e) { + throw new RuntimeException(e); + } + } + try { + storeEntry(key, orig); + } finally { + cleanupReservation(key); + } + } + + public File getEntry(String key, String fileName) { + return entry(key, fileName); + } + + private void storeEntry(String key, File orig) { + File target = entry(key, orig.getName()); + if (target.exists()) { + logger.debug("Shadow copy entry already exists: {}", key); + // delete directory "target" recursively + // https://stackoverflow.com/questions/3775694/deleting-folder-from-java + ThrowingEx.run(() -> Files.walkFileTree(target.toPath(), new DeleteDirectoryRecursively())); + } + // copy directory "orig" to "target" using hard links if possible or a plain copy otherwise + ThrowingEx.run(() -> Files.walkFileTree(orig.toPath(), new CopyDirectoryRecursively(target, orig))); + } + + private void cleanupReservation(String key) { + ThrowingEx.run(() -> Files.delete(markerFilePath(key))); + } + + private Path markerFilePath(String key) { + return Paths.get(shadowCopyRoot.getAbsolutePath(), key + ".marker"); + } + + private File entry(String key, String origName) { + return Paths.get(shadowCopyRoot.getAbsolutePath(), key, origName).toFile(); + } + + private boolean reserveSubFolder(String key) { + // put a marker file named "key".marker in "shadowCopyRoot" to make sure no other process is using it or return false if it already exists + try { + Files.createFile(Paths.get(shadowCopyRoot.getAbsolutePath(), key + ".marker")); + return true; + } catch (FileAlreadyExistsException e) { + return false; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public File copyEntryInto(String key, String origName, File targetParentFolder) { + File target = Paths.get(targetParentFolder.getAbsolutePath(), origName).toFile(); + if (target.exists()) { + logger.warn("Shadow copy destination already exists, deleting! {}: {}", key, target); + ThrowingEx.run(() -> Files.walkFileTree(target.toPath(), new DeleteDirectoryRecursively())); + } + // copy directory "orig" to "target" using hard links if possible or a plain copy otherwise + ThrowingEx.run(() -> Files.walkFileTree(entry(key, origName).toPath(), new CopyDirectoryRecursively(target, entry(key, origName)))); + return target; + } + + public boolean entryExists(String key, String origName) { + return entry(key, origName).exists(); + } + + private static class CopyDirectoryRecursively extends SimpleFileVisitor { + private final File target; + private final File orig; + + private boolean tryHardLink = true; + + public CopyDirectoryRecursively(File target, File orig) { + this.target = target; + this.orig = orig; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + // create directory on target + Files.createDirectories(target.toPath().resolve(orig.toPath().relativize(dir))); + return super.preVisitDirectory(dir, attrs); + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + // first try to hardlink, if that fails, copy + if (tryHardLink) { + try { + Files.createLink(target.toPath().resolve(orig.toPath().relativize(file)), file); + return super.visitFile(file, attrs); + } catch (UnsupportedOperationException | SecurityException | FileSystemException e) { + logger.debug("Shadow copy entry does not support hard links: {}. Switching to 'copy'.", file, e); + tryHardLink = false; // remember that hard links are not supported + } catch (IOException e) { + logger.debug("Shadow copy entry failed to create hard link: {}. Switching to 'copy'.", file, e); + tryHardLink = false; // remember that hard links are not supported + } + } + // copy file to target + Files.copy(file, target.toPath().resolve(orig.toPath().relativize(file))); + return super.visitFile(file, attrs); + } + } + + private static class DeleteDirectoryRecursively extends SimpleFileVisitor { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return super.visitFile(file, attrs); + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + Files.delete(dir); + return super.postVisitDirectory(dir, exc); + } + } +} diff --git a/lib/src/main/java/com/diffplug/spotless/npm/StandardNpmProcessFactory.java b/lib/src/main/java/com/diffplug/spotless/npm/StandardNpmProcessFactory.java new file mode 100644 index 0000000000..2c82768da2 --- /dev/null +++ b/lib/src/main/java/com/diffplug/spotless/npm/StandardNpmProcessFactory.java @@ -0,0 +1,141 @@ +/* + * Copyright 2023 DiffPlug + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.diffplug.spotless.npm; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutionException; + +import com.diffplug.spotless.ProcessRunner; + +public class StandardNpmProcessFactory implements NpmProcessFactory { + + public static final StandardNpmProcessFactory INSTANCE = new StandardNpmProcessFactory(); + + private StandardNpmProcessFactory() { + // only one instance neeeded + } + + @Override + public NpmProcess createNpmInstallProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) { + return new NpmInstall(nodeServerLayout.nodeModulesDir(), formatterStepLocations); + } + + @Override + public NpmLongRunningProcess createNpmServeProcess(NodeServerLayout nodeServerLayout, NpmFormatterStepLocations formatterStepLocations) { + return new NpmServe(nodeServerLayout.nodeModulesDir(), formatterStepLocations); + } + + private static abstract class AbstractStandardNpmProcess { + protected final ProcessRunner processRunner = ProcessRunner.usingRingBuffersOfCapacity(100 * 1024); // 100kB + + protected final File workingDir; + protected final NpmFormatterStepLocations formatterStepLocations; + + public AbstractStandardNpmProcess(File workingDir, NpmFormatterStepLocations formatterStepLocations) { + this.formatterStepLocations = formatterStepLocations; + this.workingDir = workingDir; + } + + protected String npmExecutable() { + return formatterStepLocations.npmExecutable().getAbsolutePath(); + } + + protected abstract List commandLine(); + + protected Map environmentVariables() { + return Map.of( + "PATH", formatterStepLocations.nodeExecutable().getParentFile().getAbsolutePath() + File.pathSeparator + System.getenv("PATH")); + } + + protected ProcessRunner.LongRunningProcess doStart() { + try { + return processRunner.start(workingDir, environmentVariables(), null, true, commandLine()); + } catch (IOException e) { + throw new NpmProcessException("Failed to launch npm command '" + describe() + "'.", e); + } + } + + protected abstract String describe(); + + public String doDescribe() { + return String.format("%s in %s [%s]", getClass().getSimpleName(), workingDir, String.join(" ", commandLine())); + } + } + + private static class NpmInstall extends AbstractStandardNpmProcess implements NpmProcess { + + public NpmInstall(File workingDir, NpmFormatterStepLocations formatterStepLocations) { + super(workingDir, formatterStepLocations); + } + + @Override + protected List commandLine() { + return List.of( + npmExecutable(), + "install", + "--no-audit", + "--no-fund", + "--prefer-offline"); + } + + @Override + public String describe() { + return doDescribe(); + } + + @Override + public ProcessRunner.Result waitFor() { + try (ProcessRunner.LongRunningProcess npmProcess = doStart()) { + if (npmProcess.waitFor() != 0) { + throw new NpmProcessException("Running npm command '" + describe() + "' failed with exit code: " + npmProcess.exitValue() + "\n\n" + npmProcess.result()); + } + return npmProcess.result(); + } catch (InterruptedException e) { + throw new NpmProcessException("Running npm command '" + describe() + "' was interrupted.", e); + } catch (ExecutionException e) { + throw new NpmProcessException("Running npm command '" + describe() + "' failed.", e); + } + } + } + + private static class NpmServe extends AbstractStandardNpmProcess implements NpmLongRunningProcess { + + public NpmServe(File workingDir, NpmFormatterStepLocations formatterStepLocations) { + super(workingDir, formatterStepLocations); + } + + @Override + protected List commandLine() { + return List.of( + npmExecutable(), + "start", + "--scripts-prepend-node-path=true"); + } + + @Override + public String describe() { + return doDescribe(); + } + + @Override + public ProcessRunner.LongRunningProcess start() { + return doStart(); + } + } +} diff --git a/lib/src/main/java/com/diffplug/spotless/npm/TimedLogger.java b/lib/src/main/java/com/diffplug/spotless/npm/TimedLogger.java new file mode 100644 index 0000000000..537234fcee --- /dev/null +++ b/lib/src/main/java/com/diffplug/spotless/npm/TimedLogger.java @@ -0,0 +1,228 @@ +/* + * Copyright 2023 DiffPlug + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.diffplug.spotless.npm; + +import static com.diffplug.spotless.LazyArgLogger.lazy; + +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; + +import javax.annotation.Nonnull; + +import org.slf4j.Logger; + +import com.diffplug.spotless.ThrowingEx; + +/** + * A logger that logs the time it took to execute a block of code. + */ +class TimedLogger { + + public static final String MESSAGE_PREFIX_BEGIN = "[BEGIN] "; + + public static final String MESSAGE_PREFIX_END = "[END] "; + + public static final String MESSAGE_SUFFIX_TOOK = " (took {})"; + + private final Logger logger; + private final Ticker ticker; + + private TimedLogger(@Nonnull Logger logger, Ticker ticker) { + this.logger = Objects.requireNonNull(logger); + this.ticker = ticker; + } + + public static TimedLogger forLogger(@Nonnull Logger logger) { + return forLogger(logger, Ticker.systemTicker()); + } + + public static TimedLogger forLogger(@Nonnull Logger logger, Ticker ticker) { + return new TimedLogger(logger, ticker); + } + + public TimedExec withInfo(@Nonnull String message, Object... args) { + return new TimedExec(logger::isInfoEnabled, logger::info, ticker, message, args); + } + + public TimedExec withDebug(@Nonnull String message, Object... args) { + return new TimedExec(logger::isDebugEnabled, logger::debug, ticker, message, args); + } + + public TimedExec withTrace(@Nonnull String message, Object... args) { + return new TimedExec(logger::isTraceEnabled, logger::trace, ticker, message, args); + } + + public TimedExec withWarn(@Nonnull String message, Object... args) { + return new TimedExec(logger::isWarnEnabled, logger::warn, ticker, message, args); + } + + public TimedExec withError(@Nonnull String message, Object... args) { + return new TimedExec(logger::isErrorEnabled, logger::error, ticker, message, args); + } + + public static class Timed implements AutoCloseable { + + @Nonnull + private final String msg; + + @Nonnull + private final List params; + @Nonnull + private final LogToLevelMethod delegatedLogger; + @Nonnull + private final Ticker ticker; + + private final long startedAt; + + public Timed(@Nonnull Ticker ticker, @Nonnull String msg, @Nonnull List params, @Nonnull LogToLevelMethod delegatedLogger) { + this.ticker = Objects.requireNonNull(ticker); + this.msg = Objects.requireNonNull(msg); + this.params = List.copyOf(Objects.requireNonNull(params)); + this.delegatedLogger = Objects.requireNonNull(delegatedLogger); + this.startedAt = ticker.read(); + logStart(); + } + + private void logStart() { + delegatedLogger.log(MESSAGE_PREFIX_BEGIN + msg, params.toArray()); + } + + private void logEnd() { + delegatedLogger.log(MESSAGE_PREFIX_END + msg + MESSAGE_SUFFIX_TOOK, paramsForEnd()); + } + + @Override + public final void close() { + logEnd(); + } + + private Object[] paramsForEnd() { + if (params.isEmpty() || !(params.get(params.size() - 1) instanceof Throwable)) { + // if the last element is not a throwable, we can add the duration as the last element + return Stream.concat(params.stream(), Stream.of(lazy(this::durationString))).toArray(); + } + // if the last element is a throwable, we have to add the duration before the last element + return Stream.concat( + params.stream().limit(params.size() - 1), + Stream.of(lazy(this::durationString), + params.get(params.size() - 1))) + .toArray(); + } + + private String durationString() { + long duration = ticker.read() - startedAt; + if (duration < 1000) { + return duration + "ms"; + } else if (duration < 1000 * 60) { + long seconds = duration / 1000; + long millis = duration - seconds * 1000; + return seconds + "." + millis + "s"; + } else { + // output in the format 3m 4.321s + long minutes = duration / (1000 * 60); + long seconds = (duration - minutes * 1000 * 60) / 1000; + long millis = duration - minutes * 1000 * 60 - seconds * 1000; + return minutes + "m" + (seconds + millis > 0 ? " " + seconds + "." + millis + "s" : ""); + } + } + } + + public static final class NullStopWatchLogger extends Timed { + private static final NullStopWatchLogger INSTANCE = new NullStopWatchLogger(); + + private NullStopWatchLogger() { + super(Ticker.systemTicker(), "", List.of(), (m, a) -> {}); + } + } + + interface Ticker { + long read(); + + static Ticker systemTicker() { + return System::currentTimeMillis; + } + } + + static class TestTicker implements Ticker { + private long time = 0; + + @Override + public long read() { + return time; + } + + public void tickMillis(long millis) { + time += millis; + } + } + + public static class TimedExec { + @Nonnull + private final LogActiveMethod logActiveMethod; + @Nonnull + private final LogToLevelMethod logMethod; + @Nonnull + private final Ticker ticker; + @Nonnull + private final String message; + @Nonnull + private final Object[] args; + + public TimedExec(LogActiveMethod logActiveMethod, LogToLevelMethod logMethod, Ticker ticker, String message, Object... args) { + this.logActiveMethod = Objects.requireNonNull(logActiveMethod); + this.logMethod = Objects.requireNonNull(logMethod); + this.ticker = Objects.requireNonNull(ticker); + this.message = Objects.requireNonNull(message); + this.args = Objects.requireNonNull(args); + } + + public void run(ThrowingEx.Runnable r) { + try (Timed ignore = timed()) { + ThrowingEx.run(r); + } + } + + public T call(ThrowingEx.Supplier s) { + try (Timed ignore = timed()) { + return ThrowingEx.get(s); + } + } + + public void runChecked(ThrowingEx.Runnable r) throws Exception { + try (Timed ignore = timed()) { + r.run(); + } + } + + private Timed timed() { + if (logActiveMethod.isLogLevelActive()) { + return new Timed(ticker, message, List.of(args), logMethod); + } + return NullStopWatchLogger.INSTANCE; + } + } + + @FunctionalInterface + private interface LogActiveMethod { + boolean isLogLevelActive(); + } + + @FunctionalInterface + private interface LogToLevelMethod { + void log(String message, Object... args); + } +} diff --git a/lib/src/main/java/com/diffplug/spotless/npm/TsFmtFormatterStep.java b/lib/src/main/java/com/diffplug/spotless/npm/TsFmtFormatterStep.java index 4bd665c764..a83c3e202a 100644 --- a/lib/src/main/java/com/diffplug/spotless/npm/TsFmtFormatterStep.java +++ b/lib/src/main/java/com/diffplug/spotless/npm/TsFmtFormatterStep.java @@ -40,11 +40,11 @@ public class TsFmtFormatterStep { public static final String NAME = "tsfmt-format"; - public static FormatterStep create(Map versions, Provisioner provisioner, File projectDir, File buildDir, NpmPathResolver npmPathResolver, @Nullable TypedTsFmtConfigFile configFile, @Nullable Map inlineTsFmtSettings) { + public static FormatterStep create(Map versions, Provisioner provisioner, File projectDir, File buildDir, File cacheDir, NpmPathResolver npmPathResolver, @Nullable TypedTsFmtConfigFile configFile, @Nullable Map inlineTsFmtSettings) { requireNonNull(provisioner); requireNonNull(buildDir); return FormatterStep.createLazy(NAME, - () -> new State(NAME, versions, projectDir, buildDir, npmPathResolver, configFile, inlineTsFmtSettings), + () -> new State(NAME, versions, projectDir, buildDir, cacheDir, npmPathResolver, configFile, inlineTsFmtSettings), State::createFormatterFunc); } @@ -71,11 +71,10 @@ public static class State extends NpmFormatterStepStateBase implements Serializa @Nullable private final TypedTsFmtConfigFile configFile; - public State(String stepName, Map versions, File projectDir, File buildDir, NpmPathResolver npmPathResolver, @Nullable TypedTsFmtConfigFile configFile, @Nullable Map inlineTsFmtSettings) throws IOException { + public State(String stepName, Map versions, File projectDir, File buildDir, File cacheDir, NpmPathResolver npmPathResolver, @Nullable TypedTsFmtConfigFile configFile, @Nullable Map inlineTsFmtSettings) throws IOException { super(stepName, new NpmConfig( replaceDevDependencies(NpmResourceHelper.readUtf8StringFromClasspath(TsFmtFormatterStep.class, "/com/diffplug/spotless/npm/tsfmt-package.json"), new TreeMap<>(versions)), - "typescript-formatter", NpmResourceHelper.readUtf8StringFromClasspath(PrettierFormatterStep.class, "/com/diffplug/spotless/npm/common-serve.js", "/com/diffplug/spotless/npm/tsfmt-serve.js"), @@ -83,6 +82,7 @@ public State(String stepName, Map versions, File projectDir, Fil new NpmFormatterStepLocations( projectDir, buildDir, + cacheDir, npmPathResolver::resolveNpmExecutable, npmPathResolver::resolveNodeExecutable)); this.buildDir = requireNonNull(buildDir); diff --git a/lib/src/test/java/com/diffplug/spotless/npm/TimedLoggerTest.java b/lib/src/test/java/com/diffplug/spotless/npm/TimedLoggerTest.java new file mode 100644 index 0000000000..f08d4f1622 --- /dev/null +++ b/lib/src/test/java/com/diffplug/spotless/npm/TimedLoggerTest.java @@ -0,0 +1,269 @@ +/* + * Copyright 2023 DiffPlug + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.diffplug.spotless.npm; + +import static com.diffplug.spotless.npm.TimedLogger.MESSAGE_PREFIX_BEGIN; +import static com.diffplug.spotless.npm.TimedLogger.MESSAGE_PREFIX_END; +import static com.diffplug.spotless.npm.TimedLogger.MESSAGE_SUFFIX_TOOK; + +import java.util.Arrays; +import java.util.LinkedList; +import java.util.List; + +import org.assertj.core.api.Assertions; +import org.assertj.core.api.Condition; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.slf4j.Marker; +import org.slf4j.event.Level; +import org.slf4j.helpers.LegacyAbstractLogger; + +import com.diffplug.spotless.npm.TimedLogger.TestTicker; + +class TimedLoggerTest { + + private TestLogger testLogger; + + private TestTicker testTicker; + + private TimedLogger timedLogger; + + @BeforeEach + void setUp() { + testLogger = new TestLogger(); + testTicker = new TestTicker(); + timedLogger = TimedLogger.forLogger(testLogger, testTicker); + } + + @Test + void itDoesNotLogWhenLevelDisabled() { + + TestLogger logger = new TestLogger() { + @Override + public boolean isInfoEnabled() { + return false; + } + + @Override + public boolean isDebugEnabled() { + return false; + } + + @Override + public boolean isTraceEnabled() { + return false; + } + }; + TimedLogger timedLogger = TimedLogger.forLogger(logger); + + timedLogger.withInfo("This should not be logged").run(() -> Thread.sleep(1)); + logger.assertNoEvents(); + } + + @Test + void itLogsMillisWhenTakingMillis() { + timedLogger.withInfo("This should be logged").run(() -> testTicker.tickMillis(999)); + + testLogger.assertEvents(2); + testLogger.assertHasEventWithMessageAndArguments(MESSAGE_SUFFIX_TOOK, "999ms"); + } + + @Test + void itLogsSecondsOnlyWhenTakingSeconds() { + timedLogger.withInfo("This should be logged").run(() -> testTicker.tickMillis(2_000)); + + testLogger.assertEvents(2); + testLogger.assertHasEventWithMessageAndArguments(MESSAGE_SUFFIX_TOOK, "2.0s"); + } + + @Test + void itLogsMinutesOnlyWhenTakingMinutes() { + timedLogger.withInfo("This should be logged").run(() -> testTicker.tickMillis(2 * 60 * 1_000)); + + testLogger.assertEvents(2); + testLogger.assertHasEventWithMessageAndArguments(MESSAGE_SUFFIX_TOOK, "2m"); + } + + @Test + void itLogsMinutesAndSecondsWhenTakingMinutesAndSeconds() { + timedLogger.withInfo("This should be logged").run(() -> testTicker.tickMillis(2 * 60 * 1_000 + 3 * 1_000)); + + testLogger.assertEvents(2); + testLogger.assertHasEventWithMessageAndArguments(MESSAGE_SUFFIX_TOOK, "2m 3.0s"); + } + + @Test + void itLogsBeginAndEndPrefixes() { + timedLogger.withInfo("This should be logged").run(() -> testTicker.tickMillis(1)); + + testLogger.assertHasEventWithMessageAndArguments(MESSAGE_PREFIX_BEGIN); + testLogger.assertHasEventWithMessageAndArguments(MESSAGE_PREFIX_END, "1ms"); + } + + @Test + void itThrowsExceptionsInChecked() { + Assertions.assertThatThrownBy(() -> timedLogger.withInfo("This should be logged").runChecked(() -> { + throw new Exception("This is an exception"); + })).isInstanceOf(Exception.class).hasMessage("This is an exception"); + } + + @Test + void itLogsEvenWhenExceptionsAreThrown() { + Assertions.assertThatThrownBy(() -> timedLogger.withInfo("This should be logged").run(() -> { + testTicker.tickMillis(2); + throw new Exception("This is an exception"); + })).isInstanceOf(RuntimeException.class) + .hasMessageContaining("This is an exception") + .hasCauseInstanceOf(Exception.class); + + testLogger.assertEvents(2); + testLogger.assertHasEventWithMessageAndArguments(MESSAGE_PREFIX_BEGIN); + testLogger.assertHasEventWithMessageAndArguments(MESSAGE_PREFIX_END, "2ms"); + } + + @Test + void itReturnsValueOfCallableWhileStillLogging() { + String result = timedLogger.withInfo("This should be logged").call(() -> { + testTicker.tickMillis(2); + return "This is the result"; + }); + + Assertions.assertThat(result).isEqualTo("This is the result"); + + testLogger.assertEvents(2); + testLogger.assertHasEventWithMessageAndArguments(MESSAGE_PREFIX_BEGIN); + testLogger.assertHasEventWithMessageAndArguments(MESSAGE_PREFIX_END, "2ms"); + } + + private static class TestLogger extends LegacyAbstractLogger { + + private final List events = new LinkedList<>(); + + @Override + protected String getFullyQualifiedCallerName() { + return TestLogger.class.getName(); + } + + @Override + protected void handleNormalizedLoggingCall(Level level, Marker marker, String msg, Object[] arguments, Throwable throwable) { + events.add(new TestLoggingEvent(level, marker, msg, arguments, throwable)); + } + + @Override + public boolean isTraceEnabled() { + return true; + } + + @Override + public boolean isDebugEnabled() { + return true; + } + + @Override + public boolean isInfoEnabled() { + return true; + } + + @Override + public boolean isWarnEnabled() { + return true; + } + + @Override + public boolean isErrorEnabled() { + return true; + } + + public List getEvents() { + return events; + } + + public void assertNoEvents() { + Assertions.assertThat(getEvents()).isEmpty(); + } + + public void assertEvents(int eventCount) { + Assertions.assertThat(getEvents()).hasSize(eventCount); + } + + public void assertHasEventWithMessageAndArguments(String message, Object... arguments) { + + Assertions.assertThat(getEvents()).haveAtLeastOne(new Condition<>(event -> { + if (!event.msg().contains(message)) { + return false; + } + if (event.arguments().length != arguments.length) { + return false; + } + for (int i = 0; i < arguments.length; i++) { + if (!String.valueOf(event.arguments()[i]).equals(arguments[i])) { + return false; + } + } + return true; + }, "Event with message containing '%s' and arguments '%s'", message, Arrays.toString(arguments))); + } + } + + private static class TestLoggingEvent { + + private final Level level; + private final Marker marker; + private final String msg; + private final Object[] arguments; + private final Throwable throwable; + + public TestLoggingEvent(Level level, Marker marker, String msg, Object[] arguments, Throwable throwable) { + this.level = level; + this.marker = marker; + this.msg = msg; + this.arguments = arguments; + this.throwable = throwable; + } + + public Level level() { + return level; + } + + public Marker marker() { + return marker; + } + + public String msg() { + return msg; + } + + public Object[] arguments() { + return arguments; + } + + public Throwable throwable() { + return throwable; + } + + @Override + public String toString() { + return String.format( + "TestLoggingEvent[level=%s, marker=%s, msg=%s, arguments=%s, throwable=%s]", + this.level, + this.marker, + this.msg, + Arrays.toString(this.arguments), + this.throwable); + } + } + +} diff --git a/plugin-gradle/CHANGES.md b/plugin-gradle/CHANGES.md index 63aef7c8d9..41bb70e556 100644 --- a/plugin-gradle/CHANGES.md +++ b/plugin-gradle/CHANGES.md @@ -5,8 +5,11 @@ We adhere to the [keepachangelog](https://keepachangelog.com/en/1.0.0/) format ( ## [Unreleased] ### Added * `cleanthat` now has `includeDraft` option, to include draft mutators from composite mutators. ([#1574](https://github.com/diffplug/spotless/pull/1574)) +* `npm`-based formatters (`prettier`, `tsfmt` and `eslint`) now support caching of `node_modules` directory. + To enable it, provide `npmInstallCache()` option. ([#1590](https://github.com/diffplug/spotless/pull/1590)) ### Fixed * `json { jackson()` can now handle `Array` as a root element. ([#1585](https://github.com/diffplug/spotless/pull/1585)) +* Reduce logging-noise created by `npm`-based formatters ([#1590](https://github.com/diffplug/spotless/pull/1590) fixes [#1582](https://github.com/diffplug/spotless/issues/1582)) ### Changes * Bump default `cleanthat` version to latest `2.1` -> `2.6`. ([#1569](https://github.com/diffplug/spotless/pull/1569) and [#1574](https://github.com/diffplug/spotless/pull/1574)) diff --git a/plugin-gradle/README.md b/plugin-gradle/README.md index 106cf3dcfd..6c181c9330 100644 --- a/plugin-gradle/README.md +++ b/plugin-gradle/README.md @@ -67,7 +67,7 @@ Spotless supports all of Gradle's built-in performance features (incremental bui - [Javascript](#javascript) ([prettier](#prettier), [ESLint](#eslint-javascript)) - [JSON](#json) - Multiple languages - - [Prettier](#prettier) ([plugins](#prettier-plugins), [npm detection](#npm-detection), [`.npmrc` detection](#npmrc-detection)) + - [Prettier](#prettier) ([plugins](#prettier-plugins), [npm detection](#npm-detection), [`.npmrc` detection](#npmrc-detection), [caching `npm install` results](#caching-results-of-npm-install)) - javascript, jsx, angular, vue, flow, typescript, css, less, scss, html, json, graphql, markdown, ymaml - [clang-format](#clang-format) - c, c++, c#, objective-c, protobuf, javascript, java @@ -630,7 +630,8 @@ spotless { **Prerequisite: tsfmt requires a working NodeJS version** -For details, see the [npm detection](#npm-detection) and [`.npmrc` detection](#npmrc-detection) sections of prettier, which apply also to tsfmt. +For details, see the [npm detection](#npm-detection), [`.npmrc` detection](#npmrc-detection) and [caching results of `npm install`](#caching-results-of-npm-install) sections of prettier, which apply also to tsfmt. + ### ESLint (Typescript) @@ -678,7 +679,7 @@ spotless { **Prerequisite: ESLint requires a working NodeJS version** -For details, see the [npm detection](#npm-detection) and [`.npmrc` detection](#npmrc-detection) sections of prettier, which apply also to ESLint. +For details, see the [npm detection](#npm-detection), [`.npmrc` detection](#npmrc-detection) and [caching results of `npm install`](#caching-results-of-npm-install) sections of prettier, which apply also to ESLint. ## Javascript @@ -742,7 +743,7 @@ spotless { **Prerequisite: ESLint requires a working NodeJS version** -For details, see the [npm detection](#npm-detection) and [`.npmrc` detection](#npmrc-detection) sections of prettier, which apply also to ESLint. +For details, see the [npm detection](#npm-detection), [`.npmrc` detection](#npmrc-detection) and [caching results of `npm install`](#caching-results-of-npm-install) sections of prettier, which apply also to ESLint. ## JSON @@ -926,8 +927,6 @@ node- and npm-binaries dynamically installed by this plugin. See [this](https://github.com/diffplug/spotless/blob/main/plugin-gradle/src/test/resources/com/diffplug/gradle/spotless/NpmTestsWithoutNpmInstallationTest_gradle_node_plugin_example_1.gradle) or [this](https://github.com/diffplug/spotless/blob/main/plugin-gradle/src/test/resources/com/diffplug/gradle/spotless/NpmTestsWithoutNpmInstallationTest_gradle_node_plugin_example_2.gradle) example. -```gradle - ### `.npmrc` detection Spotless picks up npm configuration stored in a `.npmrc` file either in the project directory or in your user home. @@ -940,6 +939,22 @@ spotless { prettier().npmrc("$projectDir/config/.npmrc").config(...) ``` +### Caching results of `npm install` + +Spotless uses `npm` behind the scenes to install `prettier`. This can be a slow process, especially if you are using a slow internet connection or +if you need large plugins. You can instruct spotless to cache the results of the `npm install` calls, so that for the next installation, +it will not need to download the packages again, but instead reuse the cached version. + +```gradle +spotless { + typescript { + prettier().npmInstallCache() // will use the default cache directory (the build-directory of the respective module) + prettier().npmInstallCache("${rootProject.rootDir}/.gradle/spotless-npm-cache") // will use the specified directory (creating it if not existing) +``` + +Depending on your filesystem and the location of the cache directory, spotless will use hardlinks when caching the npm packages. If that is not +possible, it will fall back to copying the files. + ## clang-format [homepage](https://clang.llvm.org/docs/ClangFormat.html). [changelog](https://releases.llvm.org/download.html). `clang-format` is a formatter for c, c++, c#, objective-c, protobuf, javascript, and java. You can use clang-format in any language-specific format, but usually you will be creating a generic format. diff --git a/plugin-gradle/src/main/java/com/diffplug/gradle/spotless/FormatExtension.java b/plugin-gradle/src/main/java/com/diffplug/gradle/spotless/FormatExtension.java index 8012102200..fd613e82e1 100644 --- a/plugin-gradle/src/main/java/com/diffplug/gradle/spotless/FormatExtension.java +++ b/plugin-gradle/src/main/java/com/diffplug/gradle/spotless/FormatExtension.java @@ -522,12 +522,18 @@ public LicenseHeaderConfig licenseHeaderFile(Object licenseHeaderFile, String de } public abstract static class NpmStepConfig> { + + public static final String SPOTLESS_NPM_INSTALL_CACHE_DEFAULT_NAME = "spotless-npm-install-cache"; + @Nullable protected Object npmFile; @Nullable protected Object nodeFile; + @Nullable + protected Object npmInstallCache; + @Nullable protected Object npmrcFile; @@ -560,6 +566,18 @@ public T npmrc(final Object npmrcFile) { return (T) this; } + public T npmInstallCache(final Object npmInstallCache) { + this.npmInstallCache = npmInstallCache; + replaceStep(); + return (T) this; + } + + public T npmInstallCache() { + this.npmInstallCache = new File(project.getBuildDir(), SPOTLESS_NPM_INSTALL_CACHE_DEFAULT_NAME); + replaceStep(); + return (T) this; + } + File npmFileOrNull() { return fileOrNull(npmFile); } @@ -572,6 +590,10 @@ File npmrcFileOrNull() { return fileOrNull(npmrcFile); } + File npmModulesCacheOrNull() { + return fileOrNull(npmInstallCache); + } + private File fileOrNull(Object npmFile) { return npmFile != null ? project.file(npmFile) : null; } @@ -619,6 +641,7 @@ protected FormatterStep createStep() { provisioner(), project.getProjectDir(), project.getBuildDir(), + npmModulesCacheOrNull(), new NpmPathResolver(npmFileOrNull(), nodeFileOrNull(), npmrcFileOrNull(), Arrays.asList(project.getProjectDir(), project.getRootDir())), new com.diffplug.spotless.npm.PrettierConfig( this.prettierConfigFile != null ? project.file(this.prettierConfigFile) : null, diff --git a/plugin-gradle/src/main/java/com/diffplug/gradle/spotless/JavascriptExtension.java b/plugin-gradle/src/main/java/com/diffplug/gradle/spotless/JavascriptExtension.java index e829c2b53a..e8a76166bc 100644 --- a/plugin-gradle/src/main/java/com/diffplug/gradle/spotless/JavascriptExtension.java +++ b/plugin-gradle/src/main/java/com/diffplug/gradle/spotless/JavascriptExtension.java @@ -108,6 +108,7 @@ public FormatterStep createStep() { provisioner(), project.getProjectDir(), project.getBuildDir(), + npmModulesCacheOrNull(), new NpmPathResolver(npmFileOrNull(), nodeFileOrNull(), npmrcFileOrNull(), Arrays.asList(project.getProjectDir(), project.getRootDir())), eslintConfig()); } diff --git a/plugin-gradle/src/main/java/com/diffplug/gradle/spotless/TypescriptExtension.java b/plugin-gradle/src/main/java/com/diffplug/gradle/spotless/TypescriptExtension.java index a11f1b0c39..9f1d04abfd 100644 --- a/plugin-gradle/src/main/java/com/diffplug/gradle/spotless/TypescriptExtension.java +++ b/plugin-gradle/src/main/java/com/diffplug/gradle/spotless/TypescriptExtension.java @@ -82,31 +82,33 @@ public class TypescriptFormatExtension extends NpmStepConfig config) { + public TypescriptFormatExtension config(final Map config) { this.config = new TreeMap<>(requireNonNull(config)); replaceStep(); + return this; } - public void tsconfigFile(final Object path) { - configFile(TsConfigFileType.TSCONFIG, path); + public TypescriptFormatExtension tsconfigFile(final Object path) { + return configFile(TsConfigFileType.TSCONFIG, path); } - public void tslintFile(final Object path) { - configFile(TsConfigFileType.TSLINT, path); + public TypescriptFormatExtension tslintFile(final Object path) { + return configFile(TsConfigFileType.TSLINT, path); } - public void vscodeFile(final Object path) { - configFile(TsConfigFileType.VSCODE, path); + public TypescriptFormatExtension vscodeFile(final Object path) { + return configFile(TsConfigFileType.VSCODE, path); } - public void tsfmtFile(final Object path) { - configFile(TsConfigFileType.TSFMT, path); + public TypescriptFormatExtension tsfmtFile(final Object path) { + return configFile(TsConfigFileType.TSFMT, path); } - private void configFile(TsConfigFileType filetype, Object path) { + private TypescriptFormatExtension configFile(TsConfigFileType filetype, Object path) { this.configFileType = requireNonNull(filetype); this.configFilePath = requireNonNull(path); replaceStep(); + return this; } public FormatterStep createStep() { @@ -117,6 +119,7 @@ public FormatterStep createStep() { provisioner(), project.getProjectDir(), project.getBuildDir(), + npmModulesCacheOrNull(), new NpmPathResolver(npmFileOrNull(), nodeFileOrNull(), npmrcFileOrNull(), Arrays.asList(project.getProjectDir(), project.getRootDir())), typedConfigFile(), config); @@ -213,6 +216,7 @@ public FormatterStep createStep() { provisioner(), project.getProjectDir(), project.getBuildDir(), + npmModulesCacheOrNull(), new NpmPathResolver(npmFileOrNull(), nodeFileOrNull(), npmrcFileOrNull(), Arrays.asList(project.getProjectDir(), project.getRootDir())), eslintConfig()); } diff --git a/plugin-gradle/src/test/java/com/diffplug/gradle/spotless/JavascriptExtensionTest.java b/plugin-gradle/src/test/java/com/diffplug/gradle/spotless/JavascriptExtensionTest.java index 26354b93be..f1b00706d3 100644 --- a/plugin-gradle/src/test/java/com/diffplug/gradle/spotless/JavascriptExtensionTest.java +++ b/plugin-gradle/src/test/java/com/diffplug/gradle/spotless/JavascriptExtensionTest.java @@ -178,7 +178,7 @@ void formattingUsingStyleguide(String styleguide) throws Exception { " }", "}"); setFile("test.js").toResource(styleguidePath + "javascript-es6.dirty"); - gradleRunner().withArguments("--stacktrace", "spotlessApply").build(); + gradleRunner().forwardOutput().withArguments("--info", "--stacktrace", "spotlessApply").build(); assertFile("test.js").sameAsResource(styleguidePath + "javascript-es6.clean"); } } diff --git a/plugin-gradle/src/test/java/com/diffplug/gradle/spotless/NpmInstallCacheIntegrationTests.java b/plugin-gradle/src/test/java/com/diffplug/gradle/spotless/NpmInstallCacheIntegrationTests.java new file mode 100644 index 0000000000..3a690fbc55 --- /dev/null +++ b/plugin-gradle/src/test/java/com/diffplug/gradle/spotless/NpmInstallCacheIntegrationTests.java @@ -0,0 +1,249 @@ +/* + * Copyright 2023 DiffPlug + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.diffplug.gradle.spotless; + +import static com.diffplug.gradle.spotless.FormatExtension.NpmStepConfig.SPOTLESS_NPM_INSTALL_CACHE_DEFAULT_NAME; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; + +import org.assertj.core.api.Assertions; +import org.gradle.testkit.runner.BuildResult; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer.OrderAnnotation; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.io.TempDir; + +import com.diffplug.common.base.Errors; +import com.diffplug.spotless.tag.NpmTest; + +@TestMethodOrder(OrderAnnotation.class) +@NpmTest +class NpmInstallCacheIntegrationTests extends GradleIntegrationHarness { + + static File pertainingCacheDir; + + private static final File DEFAULT_DIR_FOR_NPM_INSTALL_CACHE_DO_NEVER_WRITE_TO_THIS = new File("."); + + @BeforeAll + static void beforeAll(@TempDir File pertainingCacheDir) { + NpmInstallCacheIntegrationTests.pertainingCacheDir = Errors.rethrow().get(pertainingCacheDir::getCanonicalFile); + } + + @Test + void prettierCachesNodeModulesToADefaultFolderWhenCachingEnabled() throws IOException { + File dir1 = newFolder("npm-prettier-1"); + File cacheDir = DEFAULT_DIR_FOR_NPM_INSTALL_CACHE_DO_NEVER_WRITE_TO_THIS; + BuildResult result = runPhpPrettierOnDir(dir1, cacheDir); + Assertions.assertThat(result.getOutput()) + .doesNotContain("Using cached node_modules for") + .contains("Caching node_modules for ") + .contains(Paths.get(dir1.getAbsolutePath(), "build", SPOTLESS_NPM_INSTALL_CACHE_DEFAULT_NAME).toString()); + + } + + @Test + void prettierCachesAndReusesNodeModulesInSpecificInstallCacheFolder() throws IOException { + File dir1 = newFolder("npm-prettier-1"); + File cacheDir = newFolder("npm-prettier-cache"); + BuildResult result = runPhpPrettierOnDir(dir1, cacheDir); + Assertions.assertThat(result.getOutput()).doesNotContainPattern("Using cached node_modules for .*\\Q" + cacheDir.getAbsolutePath() + "\\E"); + File dir2 = newFolder("npm-prettier-2"); + BuildResult result2 = runPhpPrettierOnDir(dir2, cacheDir); + Assertions.assertThat(result2.getOutput()).containsPattern("Using cached node_modules for .*\\Q" + cacheDir.getAbsolutePath() + "\\E"); + } + + @Test + void prettierDoesNotCacheNodeModulesIfNotExplicitlyEnabled() throws IOException { + File dir2 = newFolder("npm-prettier-1"); + BuildResult result = runPhpPrettierOnDir(dir2, null); + Assertions.assertThat(result.getOutput()) + .doesNotContainPattern("Using cached node_modules for .*") + .doesNotContainPattern("Caching node_modules for .*"); + } + + @Test + @Order(1) + void prettierCachesNodeModuleInGlobalInstallCacheDir() throws IOException { + File dir1 = newFolder("npm-prettier-global-1"); + File cacheDir = pertainingCacheDir; + BuildResult result = runPhpPrettierOnDir(dir1, cacheDir); + Assertions.assertThat(result.getOutput()) + .doesNotContainPattern("Using cached node_modules for .*\\Q" + cacheDir.getAbsolutePath() + "\\E") + .containsPattern("Caching node_modules for .*\\Q" + cacheDir.getAbsolutePath() + "\\E"); + } + + @Test + @Order(2) + void prettierUsesCachedNodeModulesFromGlobalInstallCacheDir() throws IOException { + File dir2 = newFolder("npm-prettier-global-2"); + File cacheDir = pertainingCacheDir; + BuildResult result = runPhpPrettierOnDir(dir2, cacheDir); + Assertions.assertThat(result.getOutput()) + .containsPattern("Using cached node_modules for .*\\Q" + cacheDir.getAbsolutePath() + "\\E") + .doesNotContainPattern("Caching node_modules for .*\\Q" + cacheDir.getAbsolutePath() + "\\E"); + } + + private BuildResult runPhpPrettierOnDir(File projDir, File cacheDir) throws IOException { + String baseDir = projDir.getName(); + String cacheDirEnabled = cacheDirEnabledStringForCacheDir(cacheDir); + setFile(baseDir + "/build.gradle").toLines( + "plugins {", + " id 'com.diffplug.spotless'", + "}", + "repositories { mavenCentral() }", + "def prettierConfig = [:]", + "prettierConfig['tabWidth'] = 3", + "prettierConfig['parser'] = 'php'", + "def prettierPackages = [:]", + "prettierPackages['prettier'] = '2.0.5'", + "prettierPackages['@prettier/plugin-php'] = '0.14.2'", + "spotless {", + " format 'php', {", + " target 'php-example.php'", + " prettier(prettierPackages).config(prettierConfig)" + cacheDirEnabled, + " }", + "}"); + setFile(baseDir + "/php-example.php").toResource("npm/prettier/plugins/php.dirty"); + final BuildResult spotlessApply = gradleRunner().withProjectDir(projDir).withArguments("--stacktrace", "--info", "spotlessApply").build(); + Assertions.assertThat(spotlessApply.getOutput()).contains("BUILD SUCCESSFUL"); + assertFile(baseDir + "/php-example.php").sameAsResource("npm/prettier/plugins/php.clean"); + return spotlessApply; + } + + @Test + @Order(3) + void tsfmtCachesNodeModuleInGlobalInstallCacheDir() throws IOException { + File dir1 = newFolder("npm-tsfmt-global-1"); + File cacheDir = pertainingCacheDir; + BuildResult result = runTsfmtOnDir(dir1, cacheDir); + Assertions.assertThat(result.getOutput()) + .doesNotContainPattern("Using cached node_modules for .*\\Q" + cacheDir.getAbsolutePath() + "\\E") + .containsPattern("Caching node_modules for .*\\Q" + cacheDir.getAbsolutePath() + "\\E"); + } + + @Test + @Order(4) + void tsfmtUsesCachedNodeModulesFromGlobalInstallCacheDir() throws IOException { + File dir2 = newFolder("npm-tsfmt-global-2"); + File cacheDir = pertainingCacheDir; + BuildResult result = runTsfmtOnDir(dir2, cacheDir); + Assertions.assertThat(result.getOutput()) + .containsPattern("Using cached node_modules for .*\\Q" + cacheDir.getAbsolutePath() + "\\E") + .doesNotContainPattern("Caching node_modules for .*\\Q" + cacheDir.getAbsolutePath() + "\\E"); + } + + @Test + void tsfmtDoesNotCacheNodeModulesIfNotExplicitlyEnabled() throws IOException { + File dir2 = newFolder("npm-tsfmt-1"); + BuildResult result = runTsfmtOnDir(dir2, null); + Assertions.assertThat(result.getOutput()) + .doesNotContainPattern("Using cached node_modules for .*") + .doesNotContainPattern("Caching node_modules for .*"); + } + + private BuildResult runTsfmtOnDir(File projDir, File cacheDir) throws IOException { + String baseDir = projDir.getName(); + String cacheDirEnabled = cacheDirEnabledStringForCacheDir(cacheDir); + setFile(baseDir + "/build.gradle").toLines( + "plugins {", + " id 'com.diffplug.spotless'", + "}", + "repositories { mavenCentral() }", + "def tsfmtconfig = [:]", + "tsfmtconfig['indentSize'] = 1", + "tsfmtconfig['convertTabsToSpaces'] = true", + "spotless {", + " typescript {", + " target 'test.ts'", + " tsfmt().config(tsfmtconfig)" + cacheDirEnabled, + " }", + "}"); + setFile(baseDir + "/test.ts").toResource("npm/tsfmt/tsfmt/tsfmt.dirty"); + final BuildResult spotlessApply = gradleRunner().withProjectDir(projDir).withArguments("--stacktrace", "--info", "spotlessApply").build(); + assertFile(baseDir + "/test.ts").sameAsResource("npm/tsfmt/tsfmt/tsfmt.clean"); + return spotlessApply; + } + + @Test + @Order(5) + void eslintCachesNodeModuleInGlobalInstallCacheDir() throws IOException { + File dir1 = newFolder("npm-eslint-global-1"); + File cacheDir = pertainingCacheDir; + BuildResult result = runEslintOnDir(dir1, cacheDir); + Assertions.assertThat(result.getOutput()) + .doesNotContainPattern("Using cached node_modules for .*\\Q" + cacheDir.getAbsolutePath() + "\\E") + .containsPattern("Caching node_modules for .*\\Q" + cacheDir.getAbsolutePath() + "\\E"); + } + + @Test + @Order(6) + void eslintUsesCachedNodeModulesFromGlobalInstallCacheDir() throws IOException { + File dir2 = newFolder("npm-eslint-global-2"); + File cacheDir = pertainingCacheDir; + BuildResult result = runEslintOnDir(dir2, cacheDir); + Assertions.assertThat(result.getOutput()) + .containsPattern("Using cached node_modules for .*\\Q" + cacheDir.getAbsolutePath() + "\\E") + .doesNotContainPattern("Caching node_modules for .*\\Q" + cacheDir.getAbsolutePath() + "\\E"); + } + + @Test + void eslintDoesNotCacheNodeModulesIfNotExplicitlyEnabled() throws IOException { + File dir2 = newFolder("npm-eslint-1"); + File cacheDir = null; + BuildResult result = runEslintOnDir(dir2, cacheDir); + Assertions.assertThat(result.getOutput()) + .doesNotContainPattern("Using cached node_modules for .*") + .doesNotContainPattern("Caching node_modules for .*"); + } + + private BuildResult runEslintOnDir(File projDir, File cacheDir) throws IOException { + String baseDir = projDir.getName(); + String cacheDirEnabled = cacheDirEnabledStringForCacheDir(cacheDir); + + setFile(baseDir + "/.eslintrc.js").toResource("npm/eslint/typescript/custom_rules/.eslintrc.js"); + setFile(baseDir + "/build.gradle").toLines( + "plugins {", + " id 'com.diffplug.spotless'", + "}", + "repositories { mavenCentral() }", + "spotless {", + " typescript {", + " target 'test.ts'", + " eslint().configFile('.eslintrc.js')" + cacheDirEnabled, + " }", + "}"); + setFile(baseDir + "/test.ts").toResource("npm/eslint/typescript/custom_rules/typescript.dirty"); + BuildResult spotlessApply = gradleRunner().withProjectDir(projDir).withArguments("--stacktrace", "--info", "spotlessApply").build(); + assertFile(baseDir + "/test.ts").sameAsResource("npm/eslint/typescript/custom_rules/typescript.clean"); + return spotlessApply; + } + + private static String cacheDirEnabledStringForCacheDir(File cacheDir) { + String cacheDirEnabled; + if (cacheDir == null) { + cacheDirEnabled = ""; + } else if (cacheDir == DEFAULT_DIR_FOR_NPM_INSTALL_CACHE_DO_NEVER_WRITE_TO_THIS) { + cacheDirEnabled = ".npmInstallCache()"; + } else { + cacheDirEnabled = ".npmInstallCache('" + cacheDir.getAbsolutePath() + "')"; + } + return cacheDirEnabled; + } +} diff --git a/plugin-maven/CHANGES.md b/plugin-maven/CHANGES.md index f8930c6873..f39449fb15 100644 --- a/plugin-maven/CHANGES.md +++ b/plugin-maven/CHANGES.md @@ -5,8 +5,11 @@ We adhere to the [keepachangelog](https://keepachangelog.com/en/1.0.0/) format ( ## [Unreleased] ### Added * `cleanthat` added `includeDraft` option, to include draft mutators from composite mutators. ([#1574](https://github.com/diffplug/spotless/pull/1574)) +* `npm`-based formatters (`prettier`, `tsfmt` and `eslint`) now support caching of `node_modules` directory. + To enable it, provide the `` option. ([#1590](https://github.com/diffplug/spotless/pull/1590)) ### Fixed * `` can now handle `Array` as a root element. ([#1585](https://github.com/diffplug/spotless/pull/1585)) +* Reduce logging-noise created by `npm`-based formatters ([#1590](https://github.com/diffplug/spotless/pull/1590) fixes [#1582](https://github.com/diffplug/spotless/issues/1582)) ### Changes * Bump default `cleanthat` version to latest `2.1` -> `2.6` ([#1569](https://github.com/diffplug/spotless/pull/1569) and [#1574](https://github.com/diffplug/spotless/pull/1574)) diff --git a/plugin-maven/README.md b/plugin-maven/README.md index 51ea9c23d7..c91c73f80a 100644 --- a/plugin-maven/README.md +++ b/plugin-maven/README.md @@ -54,7 +54,7 @@ user@machine repo % mvn spotless:check - [JSON](#json) - [YAML](#yaml) - Multiple languages - - [Prettier](#prettier) ([plugins](#prettier-plugins), [npm detection](#npm-detection), [`.npmrc` detection](#npmrc-detection)) + - [Prettier](#prettier) ([plugins](#prettier-plugins), [npm detection](#npm-detection), [`.npmrc` detection](#npmrc-detection), [caching `npm install` results](#caching-results-of-npm-install)) - [eclipse web tools platform](#eclipse-web-tools-platform) - **Language independent** - [Generic steps](#generic-steps) @@ -731,7 +731,7 @@ The auto-discovery of config files (up the file tree) will not work when using t **Prerequisite: tsfmt requires a working NodeJS version** -For details, see the [npm detection](#npm-detection) and [`.npmrc` detection](#npmrc-detection) sections of prettier, which apply also to tsfmt. +For details, see the [npm detection](#npm-detection), [`.npmrc` detection](#npmrc-detection) and [caching results of `npm install`](#caching-results-of-npm-install) sections of prettier, which apply also to tsfmt. ### ESLint (typescript) @@ -787,7 +787,7 @@ reference to a `tsconfig.json` is required. **Prerequisite: ESLint requires a working NodeJS version** -For details, see the [npm detection](#npm-detection) and [`.npmrc` detection](#npmrc-detection) sections of prettier, which apply also to ESLint. +For details, see the [npm detection](#npm-detection), [`.npmrc` detection](#npmrc-detection) and [caching results of `npm install`](#caching-results-of-npm-install) sections of prettier, which apply also to ESLint. ## Javascript @@ -863,7 +863,7 @@ The configuration is very similar to the [ESLint (Typescript)](#eslint-typescrip **Prerequisite: ESLint requires a working NodeJS version** -For details, see the [npm detection](#npm-detection) and [`.npmrc` detection](#npmrc-detection) sections of prettier, which apply also to ESLint. +For details, see the [npm detection](#npm-detection), [`.npmrc` detection](#npmrc-detection) and [caching results of `npm install`](#caching-results-of-npm-install) sections of prettier, which apply also to ESLint. ## JSON @@ -1113,6 +1113,21 @@ Alternatively you can supply spotless with a location of the `.npmrc` file to us /usr/local/shared/.npmrc ``` +### Caching results of `npm install` + +Spotless uses `npm` behind the scenes to install `prettier`. This can be a slow process, especially if you are using a slow internet connection or +if you need large plugins. You can instruct spotless to cache the results of the `npm install` calls, so that for the next installation, +it will not need to download the packages again, but instead reuse the cached version. + +```xml + + true + /usr/local/shared/.spotless-npm-install-cache +``` + +Depending on your filesystem and the location of the cache directory, spotless will use hardlinks when caching the npm packages. If that is not +possible, it will fall back to copying the files. + ## Eclipse web tools platform diff --git a/plugin-maven/src/main/java/com/diffplug/spotless/maven/generic/Prettier.java b/plugin-maven/src/main/java/com/diffplug/spotless/maven/generic/Prettier.java index 01ce2a5394..e92b2814bd 100644 --- a/plugin-maven/src/main/java/com/diffplug/spotless/maven/generic/Prettier.java +++ b/plugin-maven/src/main/java/com/diffplug/spotless/maven/generic/Prettier.java @@ -95,9 +95,10 @@ public FormatterStep newFormatterStep(FormatterStepConfig stepConfig) { // create the format step File baseDir = baseDir(stepConfig); File buildDir = buildDir(stepConfig); + File cacheDir = cacheDir(stepConfig); PrettierConfig prettierConfig = new PrettierConfig(configFileHandler, configInline); NpmPathResolver npmPathResolver = npmPathResolver(stepConfig); - return PrettierFormatterStep.create(devDependencies, stepConfig.getProvisioner(), baseDir, buildDir, npmPathResolver, prettierConfig); + return PrettierFormatterStep.create(devDependencies, stepConfig.getProvisioner(), baseDir, buildDir, cacheDir, npmPathResolver, prettierConfig); } private static IllegalArgumentException onlyOneConfig() { diff --git a/plugin-maven/src/main/java/com/diffplug/spotless/maven/javascript/AbstractEslint.java b/plugin-maven/src/main/java/com/diffplug/spotless/maven/javascript/AbstractEslint.java index b06d3079e7..ad113de833 100644 --- a/plugin-maven/src/main/java/com/diffplug/spotless/maven/javascript/AbstractEslint.java +++ b/plugin-maven/src/main/java/com/diffplug/spotless/maven/javascript/AbstractEslint.java @@ -67,8 +67,9 @@ public FormatterStep newFormatterStep(FormatterStepConfig stepConfig) { File buildDir = buildDir(stepConfig); File baseDir = baseDir(stepConfig); + File cacheDir = cacheDir(stepConfig); NpmPathResolver npmPathResolver = npmPathResolver(stepConfig); - return EslintFormatterStep.create(devDependencies, stepConfig.getProvisioner(), baseDir, buildDir, npmPathResolver, eslintConfig(stepConfig)); + return EslintFormatterStep.create(devDependencies, stepConfig.getProvisioner(), baseDir, buildDir, cacheDir, npmPathResolver, eslintConfig(stepConfig)); } private static IllegalArgumentException onlyOneConfig() { diff --git a/plugin-maven/src/main/java/com/diffplug/spotless/maven/npm/AbstractNpmFormatterStepFactory.java b/plugin-maven/src/main/java/com/diffplug/spotless/maven/npm/AbstractNpmFormatterStepFactory.java index e4a052106a..b0645c151e 100644 --- a/plugin-maven/src/main/java/com/diffplug/spotless/maven/npm/AbstractNpmFormatterStepFactory.java +++ b/plugin-maven/src/main/java/com/diffplug/spotless/maven/npm/AbstractNpmFormatterStepFactory.java @@ -16,9 +16,11 @@ package com.diffplug.spotless.maven.npm; import java.io.File; +import java.nio.file.Paths; import java.util.AbstractMap; import java.util.Arrays; import java.util.Collections; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Properties; @@ -32,6 +34,8 @@ public abstract class AbstractNpmFormatterStepFactory implements FormatterStepFactory { + public static final String SPOTLESS_NPM_INSTALL_CACHE_DEFAULT_NAME = "spotless-npm-install-cache"; + @Parameter private String npmExecutable; @@ -41,6 +45,9 @@ public abstract class AbstractNpmFormatterStepFactory implements FormatterStepFa @Parameter private String npmrc; + @Parameter + private String npmInstallCache; + protected File npm(FormatterStepConfig stepConfig) { File npm = npmExecutable != null ? stepConfig.getFileLocator().locateFile(npmExecutable) : null; return npm; @@ -60,6 +67,16 @@ protected File buildDir(FormatterStepConfig stepConfig) { return stepConfig.getFileLocator().getBuildDir(); } + protected File cacheDir(FormatterStepConfig stepConfig) { + if (this.npmInstallCache == null) { + return null; + } + if ("true".equals(this.npmInstallCache.toLowerCase(Locale.ROOT))) { + return new File(buildDir(stepConfig), SPOTLESS_NPM_INSTALL_CACHE_DEFAULT_NAME); + } + return Paths.get(this.npmInstallCache).toFile(); + } + protected File baseDir(FormatterStepConfig stepConfig) { return stepConfig.getFileLocator().getBaseDir(); } diff --git a/plugin-maven/src/main/java/com/diffplug/spotless/maven/typescript/Tsfmt.java b/plugin-maven/src/main/java/com/diffplug/spotless/maven/typescript/Tsfmt.java index 685d473072..396c688635 100644 --- a/plugin-maven/src/main/java/com/diffplug/spotless/maven/typescript/Tsfmt.java +++ b/plugin-maven/src/main/java/com/diffplug/spotless/maven/typescript/Tsfmt.java @@ -111,8 +111,9 @@ public FormatterStep newFormatterStep(FormatterStepConfig stepConfig) { File buildDir = buildDir(stepConfig); File baseDir = baseDir(stepConfig); + File cacheDir = cacheDir(stepConfig); NpmPathResolver npmPathResolver = npmPathResolver(stepConfig); - return TsFmtFormatterStep.create(devDependencies, stepConfig.getProvisioner(), baseDir, buildDir, npmPathResolver, configFile, configInline); + return TsFmtFormatterStep.create(devDependencies, stepConfig.getProvisioner(), baseDir, buildDir, cacheDir, npmPathResolver, configFile, configInline); } private static IllegalArgumentException onlyOneConfig() { diff --git a/plugin-maven/src/test/java/com/diffplug/spotless/maven/npm/NpmStepsWithNpmInstallCacheTest.java b/plugin-maven/src/test/java/com/diffplug/spotless/maven/npm/NpmStepsWithNpmInstallCacheTest.java new file mode 100644 index 0000000000..aae587aadb --- /dev/null +++ b/plugin-maven/src/test/java/com/diffplug/spotless/maven/npm/NpmStepsWithNpmInstallCacheTest.java @@ -0,0 +1,188 @@ +/* + * Copyright 2023 DiffPlug + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.diffplug.spotless.maven.npm; + +import static com.diffplug.spotless.maven.npm.AbstractNpmFormatterStepFactory.SPOTLESS_NPM_INSTALL_CACHE_DEFAULT_NAME; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.Test; + +import com.diffplug.spotless.ProcessRunner.Result; +import com.diffplug.spotless.maven.MavenIntegrationHarness; +import com.diffplug.spotless.tag.NpmTest; + +@NpmTest +public class NpmStepsWithNpmInstallCacheTest extends MavenIntegrationHarness { + + // TODO implement tests without cache and with various cache paths + // using only prettier is enough since the other cases are covered by gradle-side integration tests + + @Test + void prettierTypescriptWithoutCache() throws Exception { + String suffix = "ts"; + writePomWithPrettierSteps("**/*." + suffix, + "", + " 1.16.4", + " .prettierrc.yml", + ""); + Result result = run("typescript", suffix); + Assertions.assertThat(result.stdOutUtf8()).doesNotContain("Caching node_modules for").doesNotContain("Using cached node_modules for"); + } + + @Test + void prettierTypescriptWithDefaultCache() throws Exception { + String suffix = "ts"; + writePomWithPrettierSteps("**/*." + suffix, + "", + " 1.16.4", + " .prettierrc.yml", + " true", + ""); + Result result = run("typescript", suffix); + Assertions.assertThat(result.stdOutUtf8()) + .contains("Caching node_modules for") + .contains(SPOTLESS_NPM_INSTALL_CACHE_DEFAULT_NAME) + .doesNotContain("Using cached node_modules for"); + } + + @Test + void prettierTypescriptWithDefaultCacheIsReusedOnSecondRun() throws Exception { + String suffix = "ts"; + writePomWithPrettierSteps("**/*." + suffix, + "", + " 1.16.4", + " .prettierrc.yml", + " true", + ""); + Result result1 = run("typescript", suffix); + Assertions.assertThat(result1.stdOutUtf8()) + .contains("Caching node_modules for") + .contains(SPOTLESS_NPM_INSTALL_CACHE_DEFAULT_NAME) + .doesNotContain("Using cached node_modules for"); + + // recursively delete target folder to simulate a fresh run (except the default cache folder) + recursiveDelete(Paths.get(rootFolder().getAbsolutePath(), "target"), SPOTLESS_NPM_INSTALL_CACHE_DEFAULT_NAME); + + Result result2 = run("typescript", suffix); + Assertions.assertThat(result2.stdOutUtf8()) + .doesNotContain("Caching node_modules for") + .contains(SPOTLESS_NPM_INSTALL_CACHE_DEFAULT_NAME) + .contains("Using cached node_modules for"); + } + + @Test + void prettierTypescriptWithSpecificCache() throws Exception { + String suffix = "ts"; + File cacheDir = newFolder("cache-prettier-1"); + writePomWithPrettierSteps("**/*." + suffix, + "", + " 1.16.4", + " .prettierrc.yml", + " " + cacheDir.getAbsolutePath() + "", + ""); + Result result = run("typescript", suffix); + Assertions.assertThat(result.stdOutUtf8()) + .contains("Caching node_modules for") + .contains(Path.of(cacheDir.getAbsolutePath()).toAbsolutePath().toString()) + .doesNotContain("Using cached node_modules for"); + } + + @Test + void prettierTypescriptWithSpecificCacheIsUsedOnSecondRun() throws Exception { + String suffix = "ts"; + File cacheDir = newFolder("cache-prettier-1"); + writePomWithPrettierSteps("**/*." + suffix, + "", + " 1.16.4", + " .prettierrc.yml", + " " + cacheDir.getAbsolutePath() + "", + ""); + Result result1 = run("typescript", suffix); + Assertions.assertThat(result1.stdOutUtf8()) + .contains("Caching node_modules for") + .contains(Path.of(cacheDir.getAbsolutePath()).toAbsolutePath().toString()) + .doesNotContain("Using cached node_modules for"); + + // recursively delete target folder to simulate a fresh run + recursiveDelete(Paths.get(rootFolder().getAbsolutePath(), "target"), null); + + Result result2 = run("typescript", suffix); + Assertions.assertThat(result2.stdOutUtf8()) + .doesNotContain("Caching node_modules for") + .contains(Path.of(cacheDir.getAbsolutePath()).toAbsolutePath().toString()) + .contains("Using cached node_modules for"); + } + + private void recursiveDelete(Path path, String exclusion) throws IOException { + Files.walkFileTree(path, new RecursiveDelete(exclusion)); + } + + private Result run(String kind, String suffix) throws IOException, InterruptedException { + String path = prepareRun(kind, suffix); + Result result = mavenRunner().withArguments("spotless:apply").runNoError(); + assertFile(path).sameAsResource("npm/prettier/filetypes/" + kind + "/" + kind + ".clean"); + return result; + } + + private String prepareRun(String kind, String suffix) throws IOException { + String configPath = ".prettierrc.yml"; + setFile(configPath).toResource("npm/prettier/filetypes/" + kind + "/" + ".prettierrc.yml"); + String path = "src/main/" + kind + "/test." + suffix; + setFile(path).toResource("npm/prettier/filetypes/" + kind + "/" + kind + ".dirty"); + return path; + } + + private static class RecursiveDelete extends SimpleFileVisitor { + private final String exclusionDirectory; + + public RecursiveDelete(String exclusionDirectory) { + this.exclusionDirectory = exclusionDirectory; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (exclusionDirectory != null && dir.toFile().getName().equals(exclusionDirectory)) { + return FileVisitResult.SKIP_SUBTREE; + } + return super.preVisitDirectory(dir, attrs); + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + Files.delete(file); + return super.visitFile(file, attrs); + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + if (dir.toFile().listFiles().length != 0) { + // skip non-empty dir + return super.postVisitDirectory(dir, exc); + } + Files.delete(dir); + return super.postVisitDirectory(dir, exc); + } + } +} diff --git a/testlib/src/test/java/com/diffplug/spotless/npm/EslintFormatterStepTest.java b/testlib/src/test/java/com/diffplug/spotless/npm/EslintFormatterStepTest.java index 1bc0915ed3..631a4beaa7 100644 --- a/testlib/src/test/java/com/diffplug/spotless/npm/EslintFormatterStepTest.java +++ b/testlib/src/test/java/com/diffplug/spotless/npm/EslintFormatterStepTest.java @@ -64,6 +64,7 @@ void formattingUsingRulesetsFile(String ruleSetName) throws Exception { TestProvisioner.mavenCentral(), projectDir(), buildDir(), + null, npmPathResolver(), new EslintConfig(eslintRc, null)); @@ -107,6 +108,7 @@ void formattingUsingRulesetsFile(String ruleSetName) throws Exception { TestProvisioner.mavenCentral(), projectDir(), buildDir(), + null, npmPathResolver(), new EslintTypescriptConfig(eslintRc, null, tsconfigFile)); @@ -164,6 +166,7 @@ void formattingUsingInlineXoConfig() throws Exception { TestProvisioner.mavenCentral(), projectDir(), buildDir(), + null, npmPathResolver(), new EslintTypescriptConfig(null, esLintConfig, tsconfigFile)); diff --git a/testlib/src/test/java/com/diffplug/spotless/npm/NpmFormatterStepCommonTests.java b/testlib/src/test/java/com/diffplug/spotless/npm/NpmFormatterStepCommonTests.java index 60dd42801a..5eff34ef29 100644 --- a/testlib/src/test/java/com/diffplug/spotless/npm/NpmFormatterStepCommonTests.java +++ b/testlib/src/test/java/com/diffplug/spotless/npm/NpmFormatterStepCommonTests.java @@ -56,4 +56,5 @@ protected File projectDir() throws IOException { } return this.projectDir; } + } diff --git a/testlib/src/test/java/com/diffplug/spotless/npm/PrettierFormatterStepTest.java b/testlib/src/test/java/com/diffplug/spotless/npm/PrettierFormatterStepTest.java index acc756fa31..c0e587aa98 100644 --- a/testlib/src/test/java/com/diffplug/spotless/npm/PrettierFormatterStepTest.java +++ b/testlib/src/test/java/com/diffplug/spotless/npm/PrettierFormatterStepTest.java @@ -52,6 +52,7 @@ void formattingUsingConfigFile(String fileType) throws Exception { TestProvisioner.mavenCentral(), projectDir(), buildDir(), + null, npmPathResolver(), new PrettierConfig(prettierRc, null)); @@ -77,6 +78,7 @@ void parserInferenceBasedOnExplicitFilepathIsWorking() throws Exception { TestProvisioner.mavenCentral(), projectDir(), buildDir(), + null, npmPathResolver(), new PrettierConfig(null, ImmutableMap.of("filepath", "anyname.json"))); // should select parser based on this name @@ -97,6 +99,7 @@ void parserInferenceBasedOnFilenameIsWorking() throws Exception { TestProvisioner.mavenCentral(), projectDir(), buildDir(), + null, npmPathResolver(), new PrettierConfig(null, Collections.emptyMap())); @@ -112,6 +115,7 @@ void verifyPrettierErrorMessageIsRelayed() throws Exception { TestProvisioner.mavenCentral(), projectDir(), buildDir(), + null, npmPathResolver(), new PrettierConfig(null, ImmutableMap.of("parser", "postcss"))); try (StepHarnessWithFile stepHarness = StepHarnessWithFile.forStep(this, formatterStep)) { @@ -137,6 +141,7 @@ void runFormatTest(PrettierConfig config, String cleanFileNameSuffix) throws Exc TestProvisioner.mavenCentral(), projectDir(), buildDir(), + null, npmPathResolver(), config); // should select parser based on this name diff --git a/testlib/src/test/java/com/diffplug/spotless/npm/ShadowCopyTest.java b/testlib/src/test/java/com/diffplug/spotless/npm/ShadowCopyTest.java new file mode 100644 index 0000000000..1ef1b77caa --- /dev/null +++ b/testlib/src/test/java/com/diffplug/spotless/npm/ShadowCopyTest.java @@ -0,0 +1,206 @@ +/* + * Copyright 2023 DiffPlug + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.diffplug.spotless.npm; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import com.diffplug.spotless.ResourceHarness; + +class ShadowCopyTest extends ResourceHarness { + + public static final char[] CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789".toCharArray(); + private File shadowCopyRoot; + + private ShadowCopy shadowCopy; + + private final Random random = new Random(); + + @BeforeEach + void setUp() throws IOException { + shadowCopyRoot = newFolder("shadowCopyRoot"); + shadowCopy = new ShadowCopy(shadowCopyRoot); + } + + @Test + void anAddedEntryCanBeRetrieved() throws IOException { + File folderWithRandomFile = newFolderWithRandomFile(); + shadowCopy.addEntry("someEntry", folderWithRandomFile); + File shadowCopyFile = shadowCopy.getEntry("someEntry", folderWithRandomFile.getName()); + Assertions.assertThat(shadowCopyFile.listFiles()).hasSize(folderWithRandomFile.listFiles().length); + assertAllFilesAreEqualButNotSameAbsolutePath(folderWithRandomFile, shadowCopyFile); + } + + @Test + void twoAddedEntriesCanBeRetrieved() throws IOException { + File folderWithRandomFile = newFolderWithRandomFile(); + File folderWithRandomFile2 = newFolderWithRandomFile(); + shadowCopy.addEntry("someEntry", folderWithRandomFile); + shadowCopy.addEntry("someOtherEntry", folderWithRandomFile2); + File shadowCopyFile = shadowCopy.getEntry("someEntry", folderWithRandomFile.getName()); + File shadowCopyFile2 = shadowCopy.getEntry("someOtherEntry", folderWithRandomFile2.getName()); + Assertions.assertThat(shadowCopyFile.listFiles()).hasSize(folderWithRandomFile.listFiles().length); + Assertions.assertThat(shadowCopyFile2.listFiles()).hasSize(folderWithRandomFile2.listFiles().length); + assertAllFilesAreEqualButNotSameAbsolutePath(folderWithRandomFile, shadowCopyFile); + assertAllFilesAreEqualButNotSameAbsolutePath(folderWithRandomFile2, shadowCopyFile2); + } + + @Test + void addingTheSameEntryTwiceWorks() throws IOException { + File folderWithRandomFile = newFolderWithRandomFile(); + shadowCopy.addEntry("someEntry", folderWithRandomFile); + shadowCopy.addEntry("someEntry", folderWithRandomFile); + File shadowCopyFile = shadowCopy.getEntry("someEntry", folderWithRandomFile.getName()); + Assertions.assertThat(shadowCopyFile.listFiles()).hasSize(folderWithRandomFile.listFiles().length); + assertAllFilesAreEqualButNotSameAbsolutePath(folderWithRandomFile, shadowCopyFile); + } + + @Test + void changingAFolderAfterAddingItDoesNotChangeTheShadowCopy() throws IOException { + File folderWithRandomFile = newFolderWithRandomFile(); + shadowCopy.addEntry("someEntry", folderWithRandomFile); + + // now change the orig + Files.delete(folderWithRandomFile.listFiles()[0].toPath()); + File newRandomFile = new File(folderWithRandomFile, "replacedFile.txt"); + writeRandomStringOfLengthToFile(newRandomFile, 100); + + // now check that they are different + File shadowCopy = this.shadowCopy.getEntry("someEntry", folderWithRandomFile.getName()); + Assertions.assertThat(shadowCopy.listFiles()).hasSize(folderWithRandomFile.listFiles().length); + Assertions.assertThat(shadowCopy.listFiles()[0].getName()).isNotEqualTo(folderWithRandomFile.listFiles()[0].getName()); + } + + @Test + void addingTheSameEntryTwiceResultsInSecondEntryBeingRetained() throws IOException { + File folderWithRandomFile = newFolderWithRandomFile(); + shadowCopy.addEntry("someEntry", folderWithRandomFile); + + // now change the orig + Files.delete(folderWithRandomFile.listFiles()[0].toPath()); + File newRandomFile = new File(folderWithRandomFile, "replacedFile.txt"); + writeRandomStringOfLengthToFile(newRandomFile, 100); + + // and then add the same entry with new content again and check that they now are the same again + shadowCopy.addEntry("someEntry", folderWithRandomFile); + File shadowCopyFile = shadowCopy.getEntry("someEntry", folderWithRandomFile.getName()); + Assertions.assertThat(shadowCopyFile.listFiles()).hasSize(folderWithRandomFile.listFiles().length); + assertAllFilesAreEqualButNotSameAbsolutePath(folderWithRandomFile, shadowCopyFile); + } + + @Test + void aFolderCanBeCopiedUsingShadowCopy() throws IOException { + File folderWithRandomFile = newFolderWithRandomFile(); + shadowCopy.addEntry("someEntry", folderWithRandomFile); + File copiedFolder = newFolder("copyDest"); + File copiedEntry = shadowCopy.copyEntryInto("someEntry", folderWithRandomFile.getName(), copiedFolder); + + Assertions.assertThat(copiedEntry.listFiles()).hasSize(folderWithRandomFile.listFiles().length); + assertAllFilesAreEqualButNotSameAbsolutePath(folderWithRandomFile, copiedEntry); + } + + @Test + void aCopiedFolderIsDifferentFromShadowCopyEntry() throws IOException { + File folderWithRandomFile = newFolderWithRandomFile(); + shadowCopy.addEntry("someEntry", folderWithRandomFile); + File copiedFolder = newFolder("copyDest"); + File copiedEntry = shadowCopy.copyEntryInto("someEntry", folderWithRandomFile.getName(), copiedFolder); + + File shadowCopyFile = shadowCopy.getEntry("someEntry", folderWithRandomFile.getName()); + Assertions.assertThat(shadowCopyFile.listFiles()).hasSize(copiedEntry.listFiles().length); + assertAllFilesAreEqualButNotSameAbsolutePath(copiedEntry, shadowCopyFile); + } + + @Test + void anAddedEntryExistsAfterAdding() throws IOException { + File folderWithRandomFile = newFolderWithRandomFile(); + shadowCopy.addEntry("someEntry", folderWithRandomFile); + Assertions.assertThat(shadowCopy.entryExists("someEntry", folderWithRandomFile.getName())).isTrue(); + } + + @Test + void aEntryThatHasNotBeenAddedDoesNotExist() throws IOException { + File folderWithRandomFile = newFolderWithRandomFile(); + Assertions.assertThat(shadowCopy.entryExists("someEntry", folderWithRandomFile.getName())).isFalse(); + } + + private void assertAllFilesAreEqualButNotSameAbsolutePath(File expected, File actual) { + if (expected.isFile()) { + assertFileIsEqualButNotSameAbsolutePath(expected, actual); + } else { + assertDirectoryIsEqualButNotSameAbsolutePath(expected, actual); + } + } + + private void assertDirectoryIsEqualButNotSameAbsolutePath(File expected, File actual) { + Assertions.assertThat(actual.getAbsolutePath()).as("absolute path should be different").isNotEqualTo(expected.getAbsolutePath()); + Assertions.assertThat(actual.listFiles()).as("folder should have same amount of files").hasSize(expected.listFiles().length); + List actualContent = filesInAlphabeticalOrder(actual); + List expectedContent = filesInAlphabeticalOrder(expected); + + for (int i = 0; i < expectedContent.size(); i++) { + assertAllFilesAreEqualButNotSameAbsolutePath(expectedContent.get(i), actualContent.get(i)); + } + } + + private List filesInAlphabeticalOrder(File folder) { + if (!folder.isDirectory()) { + throw new IllegalArgumentException("folder must be a directory"); + } + return Arrays.stream(folder.listFiles()) + .sorted(Comparator.comparing(File::getName).thenComparing(File::getAbsolutePath)) + .collect(Collectors.toList()); + } + + private void assertFileIsEqualButNotSameAbsolutePath(File expected, File actual) { + Assertions.assertThat(actual).as("Files have same name").hasName(expected.getName()); + Assertions.assertThat(actual.getAbsolutePath()).as("absolute path is different").isNotEqualTo(expected.getAbsolutePath()); + Assertions.assertThat(actual).as("files have same content").hasSameTextualContentAs(expected, StandardCharsets.UTF_8); + } + + private File newFolderWithRandomFile() throws IOException { + File folder = newFolder(randomStringOfLength(10)); + File file = new File(folder, randomStringOfLength(10) + ".txt"); + writeRandomStringOfLengthToFile(file, 10); + return folder; + } + + private void writeRandomStringOfLengthToFile(File file, int length) throws IOException { + Files.write(file.toPath(), randomStringOfLength(length).getBytes(StandardCharsets.UTF_8)); + } + + private String randomStringOfLength(int length) { + // returns a string of length containing characters a-z, A-Z, 0-9 + + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + sb.append(CHARS[random.nextInt(CHARS.length)]); + } + return sb.toString(); + } + +} diff --git a/testlib/src/test/java/com/diffplug/spotless/npm/TsFmtFormatterStepTest.java b/testlib/src/test/java/com/diffplug/spotless/npm/TsFmtFormatterStepTest.java index a774c7c1ee..66fe6e05ac 100644 --- a/testlib/src/test/java/com/diffplug/spotless/npm/TsFmtFormatterStepTest.java +++ b/testlib/src/test/java/com/diffplug/spotless/npm/TsFmtFormatterStepTest.java @@ -59,6 +59,7 @@ void formattingUsingConfigFile(String formattingConfigFile) throws Exception { TestProvisioner.mavenCentral(), projectDir(), buildDir(), + null, npmPathResolver(), TypedTsFmtConfigFile.named(configFileNameWithoutExtension, configFile), Collections.emptyMap()); @@ -82,6 +83,7 @@ void formattingUsingInlineConfigWorks() throws Exception { TestProvisioner.mavenCentral(), projectDir(), buildDir(), + null, npmPathResolver(), null, inlineConfig);