Merge pull request #472 from scotje/921_bundler_gem_overrides

(PDK-921) Update PDK::Util::Bundler helpers to support gem switching

lib/pdk/cli/exec.rb

@@ -110,6 +110,10 @@ def add_spinner(message, opts = {})
           @spinner = TTY::Spinner.new("[:spinner] #{message}", opts.merge(PDK::CLI::Util.spinner_opts_for_platform))
         end
 
+        def update_environment(additional_env)
+          @environment.merge!(additional_env)
+        end
+
         def execute!
           # Start spinning if configured.
           @spinner.auto_spin if @spinner
@@ -193,12 +197,18 @@ def run_process_in_clean_env!
 
         def run_process!
           command_string = argv.join(' ')
+
           PDK.logger.debug(_("Executing '%{command}'") % { command: command_string })
+
           if context == :module
-            PDK.logger.debug(_("Command environment: GEM_HOME is '%{gem_home}' and GEM_PATH is '%{gem_path}'") % { gem_home: @process.environment['GEM_HOME'],
-                                                                                                                   gem_path: @process.environment['GEM_PATH'] })
+            PDK.logger.debug(_('Command environment:'))
+            @process.environment.each do |var, val|
+              PDK.logger.debug("  #{var}: #{val}")
+            end
           end
+
           start_time = Time.now
+
           begin
             @process.start
           rescue ChildProcess::LaunchError => e
@@ -215,7 +225,9 @@ def run_process!
             # Wait indfinitely if no timeout set.
             @process.wait
           end
+
           @duration = Time.now - start_time
+
           PDK.logger.debug(_("Execution of '%{command}' complete (duration: %{duration_in_seconds}s; exit code: %{exit_code})") %
             { command: command_string, duration_in_seconds: @duration, exit_code: @process.exit_code })
         end

lib/pdk/util/bundler.rb

@@ -1,4 +1,5 @@
 require 'bundler'
+require 'digest'
 require 'fileutils'
 require 'pdk/util'
 require 'pdk/cli/exec'
@@ -8,11 +9,11 @@ module Util
     module Bundler
       class BundleHelper; end
 
-      def self.ensure_bundle!
+      def self.ensure_bundle!(gem_overrides = {})
         bundle = BundleHelper.new
 
-        if already_bundled?(bundle.gemfile)
-          PDK.logger.debug(_('Bundle has already been installed. Skipping run.'))
+        if already_bundled?(bundle.gemfile, gem_overrides)
+          PDK.logger.debug(_('Bundler managed gems already up to date.'))
           return
         end
 
@@ -21,50 +22,50 @@ def self.ensure_bundle!
           return
         end
 
-        unless bundle.locked?
-          if PDK::Util.package_install?
-            # In packaged installs, try to use vendored Gemfile.lock as a starting point.
-            # The 'bundle install' below will pick up any new dependencies.
-            vendored_gemfile_lock = File.join(PDK::Util.package_cachedir, 'Gemfile.lock')
-
-            if File.exist?(vendored_gemfile_lock)
-              PDK.logger.debug(_("No Gemfile.lock found in module. Using vendored Gemfile.lock from '%{source}'.") % { source: vendored_gemfile_lock })
-              FileUtils.cp(vendored_gemfile_lock, File.join(PDK::Util.module_root, 'Gemfile.lock'))
-            end
-          else
-            # In non-packaged installs, just let bundler resolve deps as normal.
-            unless bundle.lock!
-              raise PDK::CLI::FatalError, _('Unable to resolve Gemfile dependencies.')
-            end
-          end
+        # Generate initial Gemfile.lock
+        if bundle.locked?
+          # Update puppet-related gem dependencies by re-resolving them specifically.
+          # If this is a packaged install, only consider already available gems at this point.
+          bundle.update_lock!(gem_overrides, local: PDK::Util.package_install?)
+        else
+          bundle.lock!(gem_overrides)
         end
 
-        unless bundle.installed?
-          unless bundle.install!
-            raise PDK::CLI::FatalError, _('Unable to install missing Gemfile dependencies.')
-          end
+        # Check for any still-unresolved dependencies. For packaged installs, this should
+        # only evaluate to false if the user has added custom gems that we don't vendor, in
+        # which case `bundle install` will resolve new dependencies as needed.
+        unless bundle.installed?(gem_overrides)
+          bundle.install!(gem_overrides)
         end
 
-        mark_as_bundled!(bundle.gemfile)
+        mark_as_bundled!(bundle.gemfile, gem_overrides)
       end
 
-      def self.already_bundled?(gemfile)
-        !(@bundled ||= {})[gemfile].nil?
+      def self.ensure_binstubs!(*gems)
+        bundle = BundleHelper.new
+
+        bundle.binstubs!(gems)
       end
 
-      def self.mark_as_bundled!(gemfile)
-        (@bundled ||= {})[gemfile] = true
+      def self.already_bundled?(gemfile, gem_overrides)
+        !(@bundled ||= {})[bundle_cache_key(gemfile, gem_overrides)].nil?
       end
 
-      def self.ensure_binstubs!(*gems)
-        bundle = BundleHelper.new
+      def self.mark_as_bundled!(gemfile, gem_overrides)
+        (@bundled ||= {})[bundle_cache_key(gemfile, gem_overrides)] = true
+      end
 
-        unless bundle.binstubs!(gems) # rubocop:disable Style/GuardClause
-          raise PDK::CLI::FatalError, _('Unable to install requested binstubs.')
-        end
+      def self.bundle_cache_key(gemfile, gem_overrides)
+        override_sig = (gem_overrides || {}).sort_by { |gem, _| gem.to_s }.to_s
+        Digest::MD5.hexdigest(gemfile.to_s + override_sig)
       end
+      private_class_method :bundle_cache_key
 
       class BundleHelper
+        def gemfile
+          @gemfile ||= PDK::Util.find_upwards('Gemfile')
+        end
+
         def gemfile?
           !gemfile.nil?
         end
@@ -73,13 +74,17 @@ def locked?
           !gemfile_lock.nil?
         end
 
-        def installed?
+        def installed?(gem_overrides = {})
           PDK.logger.debug(_('Checking for missing Gemfile dependencies.'))
 
           argv = ['check', "--gemfile=#{gemfile}"]
           argv << "--path=#{bundle_cachedir}" unless PDK::Util.package_install?
 
-          result = bundle_command(*argv).execute!
+          cmd = bundle_command(*argv).tap do |c|
+            c.update_environment(gemfile_env(gem_overrides)) unless gem_overrides.empty?
+          end
+
+          result = cmd.execute!
 
           unless result[:exit_code].zero?
             PDK.logger.debug(result.values_at(:stdout, :stderr).join("\n"))
@@ -88,69 +93,102 @@ def installed?
           result[:exit_code].zero?
         end
 
-        def lock!
-          spinner = TTY::Spinner.new("[:spinner] #{_('Resolving Gemfile dependencies.')}", PDK::CLI::Util.spinner_opts_for_platform)
-          spinner.auto_spin
+        def lock!(gem_overrides = {})
+          if PDK::Util.package_install?
+            # In packaged installs, use vendored Gemfile.lock as a starting point.
+            # Subsequent 'bundle install' will still pick up any new dependencies.
+            vendored_gemfile_lock = File.join(PDK::Util.package_cachedir, 'Gemfile.lock')
+
+            unless File.exist?(vendored_gemfile_lock)
+              raise PDK::CLI::FatalError, _('Vendored Gemfile.lock (%{source}) not found.') % {
+                source: vendored_gemfile_lock,
+              }
+            end
 
-          # After initial lockfile generation, re-resolve json gem to built-in
-          # version to avoid unncessary native compilation attempts.
-          lock_commands = [
-            bundle_command('lock'),
-            bundle_command('lock', '--update=json', '--local'),
-          ]
+            PDK.logger.debug(_('Using vendored Gemfile.lock from %{source}.') % { source: vendored_gemfile_lock })
+            FileUtils.cp(vendored_gemfile_lock, File.join(PDK::Util.module_root, 'Gemfile.lock'))
+
+            # Update the vendored lock with any overrides
+            update_lock!(gem_overrides, local: true) unless gem_overrides.empty?
+          else
+            argv = ['lock']
+
+            cmd = bundle_command(*argv).tap do |c|
+              c.add_spinner(_('Resolving Gemfile dependencies.'))
+              c.update_environment(gemfile_env(gem_overrides)) unless gem_overrides.empty?
+            end
 
-          results = lock_commands.collect do |cmd|
             result = cmd.execute!
 
             unless result[:exit_code].zero?
-              spinner.error
               PDK.logger.fatal(result.values_at(:stdout, :stderr).join("\n"))
-              break [result]
+              raise PDK::CLI::FatalError, _('Unable to resolve Gemfile dependencies.')
             end
+          end
+
+          # After initial lockfile generation, re-resolve json gem to built-in
+          # version to avoid unncessary native compilation attempts.
+          update_lock!({ json: nil }, local: true)
+
+          true
+        end
 
-            result
+        def update_lock!(gem_overrides, options = {})
+          return true if gem_overrides.empty?
+
+          PDK.logger.debug(_('Updating Gemfile dependencies.'))
+
+          update_gems = gem_overrides.keys.join(' ')
+
+          argv = ['lock', "--update=#{update_gems}"]
+          argv << '--local' if options && options[:local]
+
+          cmd = bundle_command(*argv).tap do |c|
+            c.update_environment(gemfile_env(gem_overrides))
           end
 
-          return false unless results.all? { |result| result[:exit_code].zero? }
+          result = cmd.execute!
+
+          unless result[:exit_code].zero?
+            PDK.logger.fatal(result.values_at(:stdout, :stderr).join("\n"))
+            raise PDK::CLI::FatalError, _('Unable to resolve Gemfile dependencies.')
+          end
 
-          spinner.success
           true
         end
 
-        def install!
+        def install!(gem_overrides = {})
           argv = ['install', "--gemfile=#{gemfile}", '-j4']
           argv << "--path=#{bundle_cachedir}" unless PDK::Util.package_install?
 
-          command = bundle_command(*argv).tap do |c|
+          cmd = bundle_command(*argv).tap do |c|
             c.add_spinner(_('Installing missing Gemfile dependencies.'))
+            c.update_environment(gemfile_env(gem_overrides)) unless gem_overrides.empty?
           end
 
-          result = command.execute!
+          result = cmd.execute!
 
           unless result[:exit_code].zero?
             PDK.logger.fatal(result.values_at(:stdout, :stderr).join("\n"))
+            raise PDK::CLI::FatalError, _('Unable to install missing Gemfile dependencies.')
           end
 
-          result[:exit_code].zero?
+          true
         end
 
         def binstubs!(gems)
           binstub_dir = File.join(File.dirname(gemfile), 'bin')
           return true if gems.all? { |gem| File.file?(File.join(binstub_dir, gem)) }
 
-          command = bundle_command('binstubs', *gems, '--force')
-
-          result = command.execute!
+          cmd = bundle_command('binstubs', *gems, '--force')
+          result = cmd.execute!
 
           unless result[:exit_code].zero?
             PDK.logger.fatal(_("Failed to generate binstubs for '%{gems}':\n%{output}") % { gems: gems.join(' '), output: result.values_at(:stdout, :stderr).join("\n") })
+            raise PDK::CLI::FatalError, _('Unable to install requested binstubs.')
           end
 
-          result[:exit_code].zero?
-        end
-
-        def gemfile
-          @gemfile ||= PDK::Util.find_upwards('Gemfile')
+          true
         end
 
         private
@@ -168,6 +206,20 @@ def gemfile_lock
         def bundle_cachedir
           @bundle_cachedir ||= PDK::Util.package_install? ? PDK::Util.package_cachedir : File.join(PDK::Util.cachedir)
         end
+
+        def gemfile_env(gem_overrides)
+          gemfile_env = {}
+
+          return gemfile_env unless gem_overrides.respond_to?(:each)
+
+          gem_overrides.each do |gem, version|
+            gemfile_env['PUPPET_GEM_VERSION'] = version if gem.respond_to?(:to_s) && gem.to_s == 'puppet'
+            gemfile_env['FACTER_GEM_VERSION'] = version if gem.respond_to?(:to_s) && gem.to_s == 'facter'
+            gemfile_env['HIERA_GEM_VERSION'] = version if gem.respond_to?(:to_s) && gem.to_s == 'hiera'
+          end
+
+          gemfile_env
+        end
       end
     end
   end

lib/pdk/util/ruby_version.rb

@@ -45,6 +45,7 @@ def scan_for_packaged_rubies
         end
 
         def default_ruby_version
+          # TODO: may not be a safe assumption that highest available version should be default
           versions.keys.sort { |a, b| Gem::Version.new(b) <=> Gem::Version.new(a) }.first
         end
 
@@ -66,11 +67,15 @@ def initialize(ruby_version = nil)
       def gem_path
         if PDK::Util.package_install?
           # Subprocesses use their own set of gems which are managed by pdk or
-          # installed with the package.
-          File.join(PDK::Util.package_cachedir, 'ruby', versions[ruby_version])
+          # installed with the package. We also include the separate gem path
+          # where our packaged multi-puppet installations live.
+          [
+            File.join(PDK::Util.package_cachedir, 'ruby', versions[ruby_version]),
+            File.join(PDK::Util.pdk_package_basedir, 'private', 'puppet', 'ruby', versions[ruby_version]),
+          ].join(File::PATH_SEPARATOR)
         else
           # This allows the subprocess to find the 'bundler' gem, which isn't
-          # in the cachedir above for gem installs.
+          # in GEM_HOME for gem installs.
           # TODO: There must be a better way to do this than shelling out to
           # gem...
           File.absolute_path(File.join(`gem which bundler`, '..', '..', '..', '..'))
@@ -91,10 +96,21 @@ def gem_home
 
       def available_puppet_versions
         return @available_puppet_versions unless @available_puppet_versions.nil?
-        puppet_spec_files = Dir[File.join(gem_path, 'specifications', '**', 'puppet*.gemspec')]
-        puppet_spec_files += Dir[File.join(gem_home, 'specifications', '**', 'puppet*.gemspec')]
-        puppet_specs = puppet_spec_files.map { |r| Gem::Specification.load(r) }
-        @available_puppet_versions = puppet_specs.select { |r| r.name == 'puppet' }.map { |r| r.version }.sort { |a, b| b <=> a }
+
+        puppet_spec_files = Dir[File.join(gem_home, 'specifications', '**', 'puppet*.gemspec')]
+
+        gem_path.split(File::PATH_SEPARATOR).each do |path|
+          puppet_spec_files += Dir[File.join(path, 'specifications', '**', 'puppet*.gemspec')]
+        end
+
+        puppet_specs = []
+
+        puppet_spec_files.each do |specfile|
+          spec = Gem::Specification.load(specfile)
+          puppet_specs << spec if spec.name == 'puppet'
+        end
+
+        @available_puppet_versions = puppet_specs.map(&:version).sort { |a, b| b <=> a }
       end
 
       private

lib/pdk/util/version.rb

@@ -1,6 +1,7 @@
 require 'pdk/version'
 require 'pdk/cli/exec'
 require 'pdk/util/git'
+require 'pdk/logger'
 
 module PDK
   module Util
@@ -33,6 +34,7 @@ def self.git_ref
       end
 
       def self.version_file
+        # FIXME: this gets called a LOT and doesn't currently get cached
         PDK::Util.find_upwards('PDK_VERSION', File.dirname(__FILE__))
       end
     end