Merge pull request #5660 from dependabot/jurre/yarn-berry

Initial yarn berry support

Author: jurre

Dockerfile

@@ -119,6 +119,9 @@ RUN curl -sL https://deb.nodesource.com/setup_16.x | bash - \
   && npm install -g [email protected] \
   && rm -rf ~/.npm
 
+# Install yarn berry and set it to a stable version
+RUN corepack enable \
+  && corepack prepare [email protected] --activate
 
 ### ELM
 
@@ -287,6 +290,9 @@ RUN bash /opt/pub/helpers/build
 
 COPY --chown=dependabot:dependabot npm_and_yarn/helpers /opt/npm_and_yarn/helpers
 RUN bash /opt/npm_and_yarn/helpers/build
+# Our native helpers pull in yarn 1, so we need to reset the version globally to
+# 3.2.3.
+RUN corepack prepare [email protected] --activate
 
 COPY --chown=dependabot:dependabot python/helpers /opt/python/helpers
 RUN bash /opt/python/helpers/build

npm_and_yarn/lib/dependabot/npm_and_yarn/file_fetcher.rb

@@ -42,6 +42,7 @@ def fetch_files
         fetched_files << lerna_json if lerna_json
         fetched_files << npmrc if npmrc
         fetched_files << yarnrc if yarnrc
+        fetched_files << yarnrc_yml if yarnrc_yml
         fetched_files += workspace_package_jsons
         fetched_files += lerna_packages
         fetched_files += path_dependencies(fetched_files)
@@ -53,8 +54,8 @@ def fetch_files
       def instrument_package_manager_version
         package_managers = {}
 
-        package_managers["npm"] =  Helpers.npm_version_numeric(package_lock.content) if package_lock
-        package_managers["yarn"] = 1 if yarn_lock
+        package_managers["npm"] = Helpers.npm_version_numeric(package_lock.content) if package_lock
+        package_managers["yarn"] = yarn_version if yarn_version
         package_managers["shrinkwrap"] = 1 if shrinkwrap
 
         Dependabot.instrument(
@@ -64,6 +65,22 @@ def instrument_package_manager_version
         )
       end
 
+      def yarn_version
+        return @yarn_version if defined?(@yarn_version)
+
+        package = JSON.parse(package_json.content)
+        if (pkgmanager = package.fetch("packageManager", nil))
+          get_yarn_version_from_path(pkgmanager)
+        elsif yarn_lock
+          1
+        end
+      end
+
+      def get_yarn_version_from_path(path)
+        version_match = path.match(/yarn@(?<version>\d+.\d+.\d+)/)
+        version_match&.named_captures&.fetch("version", nil)
+      end
+
       def package_json
         @package_json ||= fetch_file_from_host("package.json")
       end
@@ -118,6 +135,11 @@ def yarnrc
         @yarnrc
       end
 
+      def yarnrc_yml
+        @yarnrc_yml ||= fetch_file_if_present(".yarnrc.yml")&.
+                       tap { |f| f.support_file = true }
+      end
+
       def lerna_json
         @lerna_json ||= fetch_file_if_present("lerna.json")&.
                         tap { |f| f.support_file = true }

npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser.rb

@@ -3,6 +3,7 @@
 # See https://docs.npmjs.com/files/package.json for package.json format docs.
 
 require "dependabot/dependency"
+require "dependabot/experiments"
 require "dependabot/file_parsers"
 require "dependabot/file_parsers/base"
 require "dependabot/shared_helpers"
@@ -94,6 +95,7 @@ def build_dependency(file:, type:, name:, requirement:)
           manifest_name: file.name
         )
         version = version_for(name, requirement, file.name)
+
         return if lockfile_details && !version
         return if ignore_requirement?(requirement)
         return if workspace_package_names.include?(name)
@@ -326,6 +328,7 @@ def package_files
               dependency_files.
               select { |f| f.name.end_with?("package.json") }.
               reject { |f| f.name == "package.json" }.
+              reject { |f| f.name.include?("node_modules/") if Experiments.enabled?(:yarn_berry) }.
               reject(&:support_file?)
 
             [

npm_and_yarn/lib/dependabot/npm_and_yarn/file_parser/lockfile_parser.rb

@@ -96,6 +96,8 @@ def yarn_lock_dependencies
             parse_yarn_lock(yarn_lock).each do |req, details|
               next unless semver_version_for(details["version"])
               next if alias_package?(req)
+              next if Experiments.enabled?(:yarn_berry) && workspace_package?(req)
+              next if Experiments.enabled?(:yarn_berry) && req == "__metadata"
 
               # NOTE: The DependencySet will de-dupe our dependencies, so they
               # end up unique by name. That's not a perfect representation of
@@ -188,7 +190,15 @@ def semver_version_for(version_string)
         end
 
         def alias_package?(requirement)
-          requirement.include?("@npm:")
+          if Experiments.enabled?(:yarn_berry)
+            requirement.match?(/@npm:(.+@(?!npm))/)
+          else
+            requirement.include?("@npm:")
+          end
+        end
+
+        def workspace_package?(requirement)
+          requirement.include?("@workspace:")
         end
 
         def parse_package_lock(package_lock)

npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
+require "dependabot/experiments"
 require "dependabot/file_updaters"
 require "dependabot/file_updaters/base"
+require "dependabot/file_updaters/vendor_updater"
 require "dependabot/npm_and_yarn/dependency_files_filterer"
 require "dependabot/npm_and_yarn/sub_dependency_files_filterer"
 
@@ -53,11 +55,54 @@ def updated_dependency_files
           )
         end
 
+        if Experiments.enabled?(:yarn_berry)
+          base_dir = updated_files.first.directory
+          vendor_updater.updated_vendor_cache_files(base_directory: base_dir).each { |file| updated_files << file }
+          install_state_updater.updated_vendor_cache_files(base_directory: base_dir).each do |file|
+            updated_files << file
+          end
+        end
+
         updated_files
       end
 
       private
 
+      # Dynamically fetch the vendor cache folder from yarn
+      def vendor_cache_dir
+        return @vendor_cache_dir if defined?(@vendor_cache_dir)
+
+        @vendor_cache_dir = if File.exist?(".yarnrc.yml")
+                              YAML.load_file(".yarnrc.yml").fetch("cacheFolder", "./.yarn/cache")
+                            else
+                              "./.yarn/cache"
+                            end
+      end
+
+      def install_state_path
+        return @install_state_path if defined?(@install_state_path)
+
+        @install_state_path = if File.exist?(".yarnrc.yml")
+                                YAML.load_file(".yarnrc.yml").fetch("installStatePath", "./.yarn/install-state.gz")
+                              else
+                                "./.yarn/install-state.gz"
+                              end
+      end
+
+      def vendor_updater
+        Dependabot::FileUpdaters::VendorUpdater.new(
+          repo_contents_path: repo_contents_path,
+          vendor_dir: vendor_cache_dir
+        )
+      end
+
+      def install_state_updater
+        Dependabot::FileUpdaters::VendorUpdater.new(
+          repo_contents_path: repo_contents_path,
+          vendor_dir: install_state_path
+        )
+      end
+
       def filtered_dependency_files
         @filtered_dependency_files ||=
           if dependencies.select(&:top_level?).any?
@@ -175,6 +220,7 @@ def yarn_lockfile_updater
           YarnLockfileUpdater.new(
             dependencies: dependencies,
             dependency_files: dependency_files,
+            repo_contents_path: repo_contents_path,
             credentials: credentials
           )
       end

npm_and_yarn/lib/dependabot/npm_and_yarn/file_updater/yarn_lockfile_updater.rb

@@ -4,10 +4,12 @@
 
 require "dependabot/npm_and_yarn/file_updater"
 require "dependabot/npm_and_yarn/file_parser"
+require "dependabot/npm_and_yarn/helpers"
 require "dependabot/npm_and_yarn/update_checker/registry_finder"
 require "dependabot/npm_and_yarn/native_helpers"
 require "dependabot/shared_helpers"
 require "dependabot/errors"
+require "dependabot/experiments"
 
 # rubocop:disable Metrics/ClassLength
 module Dependabot
@@ -17,9 +19,10 @@ class YarnLockfileUpdater
         require_relative "npmrc_builder"
         require_relative "package_json_updater"
 
-        def initialize(dependencies:, dependency_files:, credentials:)
+        def initialize(dependencies:, dependency_files:, repo_contents_path:, credentials:)
           @dependencies = dependencies
           @dependency_files = dependency_files
+          @repo_contents_path = repo_contents_path
           @credentials = credentials
         end
 
@@ -35,7 +38,7 @@ def updated_yarn_lock_content(yarn_lock)
 
         private
 
-        attr_reader :dependencies, :dependency_files, :credentials
+        attr_reader :dependencies, :dependency_files, :repo_contents_path, :credentials
 
         UNREACHABLE_GIT = /ls-remote --tags --heads (?<url>.*)/.freeze
         TIMEOUT_FETCHING_PACKAGE =
@@ -51,21 +54,22 @@ def sub_dependencies
         end
 
         def updated_yarn_lock(yarn_lock)
-          SharedHelpers.in_a_temporary_directory do
+          base_dir = dependency_files.first.directory
+          SharedHelpers.in_a_temporary_repo_directory(base_dir, repo_contents_path) do
             write_temporary_dependency_files
             lockfile_name = Pathname.new(yarn_lock.name).basename.to_s
             path = Pathname.new(yarn_lock.name).dirname.to_s
             updated_files = run_current_yarn_update(
               path: path,
-              lockfile_name: lockfile_name
+              yarn_lock: yarn_lock
             )
             updated_files.fetch(lockfile_name)
           end
         rescue SharedHelpers::HelperSubprocessFailed => e
           handle_yarn_lock_updater_error(e, yarn_lock)
         end
 
-        def run_current_yarn_update(path:, lockfile_name:)
+        def run_current_yarn_update(path:, yarn_lock:)
           top_level_dependency_updates = top_level_dependencies.map do |d|
             {
               name: d.name,
@@ -76,12 +80,12 @@ def run_current_yarn_update(path:, lockfile_name:)
 
           run_yarn_updater(
             path: path,
-            lockfile_name: lockfile_name,
+            yarn_lock: yarn_lock,
             top_level_dependency_updates: top_level_dependency_updates
           )
         end
 
-        def run_previous_yarn_update(path:, lockfile_name:)
+        def run_previous_yarn_update(path:, yarn_lock:)
           previous_top_level_dependencies = top_level_dependencies.map do |d|
             {
               name: d.name,
@@ -94,22 +98,29 @@ def run_previous_yarn_update(path:, lockfile_name:)
 
           run_yarn_updater(
             path: path,
-            lockfile_name: lockfile_name,
+            yarn_lock: yarn_lock,
             top_level_dependency_updates: previous_top_level_dependencies
           )
         end
 
         # rubocop:disable Metrics/PerceivedComplexity
-        def run_yarn_updater(path:, lockfile_name:,
-                             top_level_dependency_updates:)
+        def run_yarn_updater(path:, yarn_lock:, top_level_dependency_updates:)
           SharedHelpers.with_git_configured(credentials: credentials) do
             Dir.chdir(path) do
               if top_level_dependency_updates.any?
-                run_yarn_top_level_updater(
-                  top_level_dependency_updates: top_level_dependency_updates
-                )
+                if yarn_berry?(yarn_lock)
+                  run_yarn_berry_top_level_updater(top_level_dependency_updates: top_level_dependency_updates,
+                                                   yarn_lock: yarn_lock)
+                else
+
+                  run_yarn_top_level_updater(
+                    top_level_dependency_updates: top_level_dependency_updates
+                  )
+                end
+              elsif yarn_berry?(yarn_lock)
+                run_yarn_berry_subdependency_updater(yarn_lock: yarn_lock)
               else
-                run_yarn_subdependency_updater(lockfile_name: lockfile_name)
+                run_yarn_subdependency_updater(yarn_lock: yarn_lock)
               end
             end
           end
@@ -133,6 +144,40 @@ def run_yarn_updater(path:, lockfile_name:,
 
         # rubocop:enable Metrics/PerceivedComplexity
 
+        def yarn_berry?(yarn_lock)
+          return false unless Experiments.enabled?(:yarn_berry)
+
+          yaml = YAML.safe_load(yarn_lock.content)
+          yaml.key?("__metadata")
+        rescue StandardError
+          false
+        end
+
+        def run_yarn_berry_top_level_updater(top_level_dependency_updates:, yarn_lock:)
+          updates = top_level_dependency_updates.collect do |dep|
+            # when there are multiple requirements, we're dealing with a
+            # workspace-like setup, where there are multiple package.json files
+            # that pull in the same dependency. It appears that these are always
+            # updated to a single new version, so we just pick the first one.
+            "#{dep[:name]}@#{dep[:requirements].first[:requirement]}"
+          end
+          command = "yarn add #{updates.join(' ')}"
+          Helpers.run_yarn_commands(command)
+          { yarn_lock.name => File.read(yarn_lock.name) }
+        end
+
+        def run_yarn_berry_subdependency_updater(yarn_lock:)
+          dep = sub_dependencies.first
+          update = "#{dep.name}@#{dep.version}"
+
+          Helpers.run_yarn_commands(
+            "yarn add #{update}",
+            "yarn dedupe #{dep.name}",
+            "yarn remove #{dep.name}"
+          )
+          { yarn_lock.name => File.read(yarn_lock.name) }
+        end
+
         def run_yarn_top_level_updater(top_level_dependency_updates:)
           SharedHelpers.run_helper_subprocess(
             command: NativeHelpers.helper_path,
@@ -144,7 +189,8 @@ def run_yarn_top_level_updater(top_level_dependency_updates:)
           )
         end
 
-        def run_yarn_subdependency_updater(lockfile_name:)
+        def run_yarn_subdependency_updater(yarn_lock:)
+          lockfile_name = Pathname.new(yarn_lock.name).basename.to_s
           SharedHelpers.run_helper_subprocess(
             command: NativeHelpers.helper_path,
             function: "yarn:updateSubdependency",
@@ -259,12 +305,11 @@ def resolvable_before_update?(yarn_lock)
 
           @resolvable_before_update[yarn_lock.name] =
             begin
-              SharedHelpers.in_a_temporary_directory do
+              base_dir = dependency_files.first.directory
+              SharedHelpers.in_a_temporary_repo_directory(base_dir, repo_contents_path) do
                 write_temporary_dependency_files(update_package_json: false)
-                lockfile_name = Pathname.new(yarn_lock.name).basename.to_s
                 path = Pathname.new(yarn_lock.name).dirname.to_s
-                run_previous_yarn_update(path: path,
-                                         lockfile_name: lockfile_name)
+                run_previous_yarn_update(path: path, yarn_lock: yarn_lock)
               end
 
               true

npm_and_yarn/lib/dependabot/npm_and_yarn/helpers.rb

@@ -15,6 +15,16 @@ def self.npm_version_numeric(lockfile_content)
       rescue JSON::ParserError
         6
       end
+
+      # Run any number of yarn commands while ensuring that `enableScripts` is
+      # set to false. Yarn commands should _not_ be ran outside of this helper
+      # to ensure that postinstall scripts are never executed, as they could
+      # contain malicious code.
+      def self.run_yarn_commands(*commands)
+        # We never want to execute postinstall scripts
+        SharedHelpers.run_shell_command("yarn config set enableScripts false")
+        commands.each { |cmd| SharedHelpers.run_shell_command(cmd) }
+      end
     end
   end
 end