Merge branch 'issue/flanders/pe-19049-add-meep-classification-feature…

…-flag' into pe-modules-next

Gemfile

@@ -2,8 +2,6 @@ source ENV['GEM_SOURCE'] || "https://rubygems.org"
 
 gemspec
 
-
-
 def location_for(place, fake_version = nil)
   if place =~ /^(git:[^#]*)#(.*)/
     [fake_version, { :git => $1, :branch => $2, :require => false }].compact
@@ -14,7 +12,6 @@ def location_for(place, fake_version = nil)
   end
 end
 
-
 # We don't put beaker in as a test dependency because we
 # don't want to create a transitive dependency
 group :acceptance_testing do
@@ -25,7 +22,6 @@ if ENV['GEM_SOURCE'] =~ /rubygems\.delivery\.puppetlabs\.net/
   gem "scooter", *location_for(ENV['SCOOTER_VERSION'] || '~> 3.0')
 end
 
-
 if File.exists? "#{__FILE__}.local"
   eval(File.read("#{__FILE__}.local"), binding)
 end

lib/beaker-pe.rb

@@ -18,7 +18,6 @@ module PE
   end
 end
 
-
 # Boilerplate DSL inclusion mechanism:
 # First we register our module with the Beaker DSL
 Beaker::DSL.register( Beaker::DSL::PE )

lib/beaker-pe/install/pe_utils.rb

@@ -26,6 +26,17 @@ module PEUtils
 
         # Version of PE when we switched from legacy installer to MEEP.
         MEEP_CUTOVER_VERSION = '2016.2.0'
+        # Version of PE when we switched to using meep for classification
+        # instead of PE node groups
+        MEEP_CLASSIFICATION_VERSION = '2017.2.0'
+        # PE-18799 temporary default used for meep classification check while
+        # we navigate the switchover.
+        # PE-18718 switch flag to true once beaker-pe, beaker-answers,
+        # beaker-pe-large-environments and pe_acceptance_tests are ready
+        DEFAULT_MEEP_CLASSIFICATION = false
+        MEEP_DATA_DIR = '/etc/puppetlabs/enterprise'
+        PE_CONF_FILE = "#{MEEP_DATA_DIR}/conf.d/pe.conf"
+        NODE_CONF_PATH = "#{MEEP_DATA_DIR}/conf.d/nodes"
 
         # @!macro [new] common_opts
         #   @param [Hash{Symbol=>String}] opts Options to alter execution.
@@ -177,7 +188,6 @@ def fetch_pe_on_mac(host, opts)
         def fetch_pe_on_windows(host, opts)
           path = host['pe_dir'] || opts[:pe_dir]
           local = File.directory?(path)
-          version = host['pe_ver'] || opts[:pe_ver_win]
           filename = "#{host['dist']}"
           extension = ".msi"
           if local
@@ -310,19 +320,12 @@ def deploy_frictionless_to_master(host)
             on dashboard, "cd /opt/puppet/share/puppet-dashboard && /opt/puppet/bin/bundle exec /opt/puppet/bin/rake node:addclass[#{master},#{klass}]"
             on master, puppet("agent -t"), :acceptable_exit_codes => [0,2]
           else
-            # the new hotness
-            begin
-              require 'scooter'
-            rescue LoadError => e
-              @logger.notify('WARNING: gem scooter is required for frictionless installation post 3.8')
-              raise e
-            end
-            dispatcher = Scooter::HttpDispatchers::ConsoleDispatcher.new(dashboard)
+            _console_dispatcher = get_console_dispatcher_for_beaker_pe!
 
             # Check if we've already created a frictionless agent node group
             # to avoid errors creating the same node group when the beaker hosts file contains
             # multiple hosts with the same platform
-            node_group = dispatcher.get_node_group_by_name('Beaker Frictionless Agent')
+            node_group = _console_dispatcher.get_node_group_by_name('Beaker Frictionless Agent')
             if node_group.nil? || node_group.empty?
               node_group = {}
               node_group['name'] = "Beaker Frictionless Agent"
@@ -334,7 +337,7 @@ def deploy_frictionless_to_master(host)
             # add the pe_repo platform class
             node_group['classes'][klass] = {}
 
-            dispatcher.create_new_node_group_model(node_group)
+            _console_dispatcher.create_new_node_group_model(node_group)
             on master, puppet("agent -t"), :acceptable_exit_codes => [0,2]
           end
         end
@@ -507,6 +510,11 @@ def do_install hosts, opts = {}
               on dashboard, "/opt/puppet/bin/rake -sf /opt/puppet/share/puppet-dashboard/Rakefile #{task} RAILS_ENV=production"
             end
 
+            # PE-18799 replace the version_is_less with a use_meep_for_classification? test
+            if use_meep_for_classification?(master[:pe_ver], options)
+              configure_puppet_agent_service(:ensure => 'stopped', :enabled => false)
+            end
+
             step "Final puppet agent run" do
               # Now that all hosts are in the dashbaord, run puppet one more
               # time to configure mcollective
@@ -634,6 +642,31 @@ def install_via_msi?(host)
           (host['platform'] =~ /windows/ && (version_is_less(host['pe_ver'], '2016.4.0') && !version_is_less(host['pe_ver'], '3.99'))) || (host['platform'] =~ /windows-2008r2/ && (version_is_less(host['pe_ver'], '2016.4.3') && !version_is_less(host['pe_ver'], '3.99')))
         end
 
+        # True if version is greater than or equal to MEEP_CLASSIFICATION_VERSION
+        # (PE-18718) AND the temporary feature flag is true.
+        #
+        # The temporary feature flag is pe_modules_next and can be set in
+        # the :answers hash given in beaker's host.cfg, inside a feature_flags
+        # hash. It will also be picked up from the environment as
+        # PE_MODULES_NEXT. (See register_feature_flags!())
+        #
+        # The :answers hash value will take precedence over the env variable.
+        #
+        # @param version String the current PE version
+        # @param opts Hash options hash to inspect for :answers
+        # @return Boolean true if version and flag allows for meep classification
+        #   feature.
+        def use_meep_for_classification?(version, opts)
+          # PE-19470 remove vv
+          register_feature_flags!(opts)
+
+          temporary_flag = feature_flag?('pe_modules_next', opts)
+          temporary_flag = DEFAULT_MEEP_CLASSIFICATION if temporary_flag.nil?
+          # ^^
+
+          !version_is_less(version, MEEP_CLASSIFICATION_VERSION) && temporary_flag
+        end
+
         # For PE 3.8.5 to PE 2016.1.2 they have an expired gpg key. This method is
         # for deb nodes to ignore the gpg-key expiration warning
         def ignore_gpg_key_warning_on_hosts(hosts, opts)
@@ -1063,6 +1096,204 @@ def fetch_and_push_pe(host, path, filename, extension, local_dir='tmp/pe')
           scp_to host, "#{local_dir}/#{filename}#{extension}", host['working_dir']
         end
 
+        # Being able to modify PE's classifier requires the Scooter gem and
+        # helpers which are in beaker-pe-large-environments.
+        def get_console_dispatcher_for_beaker_pe(raise_exception = false)
+          if !respond_to?(:get_dispatcher)
+            begin
+              # just in case scooter is present but beaker-pe-large-environments is not
+              # ...most likely this will raise a LoadError...
+              require 'scooter'
+              Scooter::HttpDispatchers::ConsoleDispatcher.new(dashboard)
+            rescue LoadError => e
+              logger.notify('WARNING: gem scooter is required for frictionless installation post 3.8')
+              raise e if raise_exception
+
+              return nil
+            end
+          else
+            get_dispatcher
+          end
+        end
+
+        # Will raise a LoadError if unable to require Scooter.
+        def get_console_dispatcher_for_beaker_pe!
+          get_console_dispatcher_for_beaker_pe(true)
+        end
+
+        # In PE versions >= 2017.1.0, allows you to configure the puppet agent
+        # service for all nodes.
+        #
+        # @param parameters [Hash] - agent profile parameters
+        # @option parameters [Boolean] :managed - whether or not to manage the
+        #   agent resource at all (Optional, defaults to true).
+        # @option parameters [String] :ensure - 'stopped', 'running'
+        # @option parameters [Boolean] :enabled - whether the service will be
+        #   enabled (for restarts)
+        # @raise [StandardError] if master version is less than 2017.1.0
+        def configure_puppet_agent_service(parameters)
+          raise(StandardError, "Can only manage puppet service in PE versions >= 2017.1.0; tried for #{master['pe_ver']}") if version_is_less(master['pe_ver'], '2017.1.0')
+          puppet_managed = parameters.include?(:managed) ? parameters[:managed] : true
+          puppet_ensure = parameters[:ensure]
+          puppet_enabled = parameters[:enabled]
+
+          msg = puppet_managed ?
+            "Configure agents '#{puppet_ensure}' and #{puppet_enabled ? 'enabled' : 'disabled'}" :
+            "Do not manage agents"
+
+          step msg do
+            # PE-18799 and remove this conditional
+            if use_meep_for_classification?(master[:pe_ver], options)
+              group_name = 'Puppet Enterprise Agent'
+              class_name = 'pe_infrastructure::agent'
+            else
+              group_name = 'PE Agent'
+              class_name = 'puppet_enterprise::profile::agent'
+            end
+
+            # update pe conf
+            # only the pe_infrastructure::agent parameters are relevant in pe.conf
+            update_pe_conf({
+              "pe_infrastructure::agent::puppet_service_managed" => puppet_managed,
+              "pe_infrastructure::agent::puppet_service_ensure" => puppet_ensure,
+              "pe_infrastructure::agent::puppet_service_enabled" => puppet_enabled,
+            })
+
+            if _console_dispatcher = get_console_dispatcher_for_beaker_pe
+              agent_group = _console_dispatcher.get_node_group_by_name(group_name)
+              agent_class = agent_group['classes'][class_name]
+              agent_class['puppet_service_managed'] = puppet_managed
+              agent_class['puppet_service_ensure'] = puppet_ensure
+              agent_class['puppet_service_enabled'] = puppet_enabled
+
+              _console_dispatcher.update_node_group(agent_group['id'], agent_group)
+            end
+          end
+        end
+
+        # Given a hash of parameters, updates the primary master's pe.conf, adding or
+        # replacing, or removing the given parameters.
+        #
+        # To remove a parameter, pass a nil as its value
+        #
+        # Handles stringifying and quoting namespaced keys, and also preparing non
+        # string values using Hocon::ConfigValueFactory.
+        #
+        # Logs the state of pe.conf before and after.
+        #
+        # @example
+        #   # Assuming pe.conf looks like:
+        #   # {
+        #   # "bar": "baz"
+        #   # "old": "item"
+        #   # }
+        #
+        #   update_pe_conf(
+        #     {
+        #       "foo" => "a",
+        #       "bar" => "b",
+        #       "old" => nil,
+        #     }
+        #   )
+        #
+        #   # Will produce a pe.conf like:
+        #   # {
+        #   # "bar": "b"
+        #   # "foo": "a"
+        #   # }
+        #
+        # @param parameters [Hash] Hash of parameters to be included in pe.conf.
+        # @param pe_conf_file [String] The file to update
+        #   (/etc/puppetlabs/enterprise/conf.d/pe.conf by default)
+        def update_pe_conf(parameters, pe_conf_file = PE_CONF_FILE)
+          step "Update #{pe_conf_file} with #{parameters}" do
+            hocon_file_edit_in_place_on(master, pe_conf_file) do |host,doc|
+              updated_doc = parameters.reduce(doc) do |pe_conf,param|
+                key, value = param
+
+                hocon_key = quoted_hocon_key(key)
+
+                hocon_value = case value
+                when String
+                  # ensure unquoted string values are quoted for uniformity
+                  then value.match(/^[^"]/) ? %Q{"#{value}"} : value
+                else Hocon::ConfigValueFactory.from_any_ref(value, nil)
+                end
+
+                updated = case value
+                when String
+                  pe_conf.set_value(hocon_key, hocon_value)
+                when nil
+                  pe_conf.remove_value(hocon_key)
+                else
+                  pe_conf.set_config_value(hocon_key, hocon_value)
+                end
+
+                updated
+              end
+
+              # return the modified document
+              updated_doc
+            end
+            on(master, "cat #{pe_conf_file}")
+          end
+        end
+
+        # If the key is unquoted and does not contain pathing ('.'),
+        # quote to ensure that puppet namespaces are protected
+        #
+        # @example
+        #   quoted_hocon_key("puppet_enterprise::database_host")
+        #   # => '"puppet_enterprise::database_host"'
+        #
+        def quoted_hocon_key(key)
+          case key
+          when /^[^"][^.]+/
+            then %Q{"#{key}"}
+          else key
+          end
+        end
+
+        # @return a Ruby object of any root key in pe.conf.
+        #
+        # @param key [String] to lookup
+        # @param pe_conf_path [String] defaults to /etc/puppetlabs/enterprise/conf.d/pe.conf
+        def get_unwrapped_pe_conf_value(key, pe_conf_path = PE_CONF_FILE)
+          file_contents = on(master, "cat #{pe_conf_path}").stdout
+          # Seem to need to use ConfigFactory instead of ConfigDocumentFactory
+          # to get something that we can read values from?
+          doc = Hocon::ConfigFactory.parse_string(file_contents)
+          hocon_key = quoted_hocon_key(key)
+          doc.has_path?(hocon_key) ?
+            doc.get_value(hocon_key).unwrapped :
+            nil
+        end
+
+        # Creates a new /etc/puppetlabs/enterprise/conf.d/nodes/*.conf file for the
+        # given host's certname, and adds the passed parameters, or updates with the
+        # passed parameters if the file already exists.
+        #
+        # Does not remove an empty file.
+        #
+        # @param host [Beaker::Host] to create a node file for
+        # @param parameters [Hash] of key value pairs to add to the nodes conf file
+        # @param node_conf_path [String] defaults to /etc/puppetlabs/enterprise/conf.d/nodes
+        def create_or_update_node_conf(host, parameters, node_conf_path = NODE_CONF_PATH)
+          node_conf_file = "#{node_conf_path}/#{host.node_name}.conf"
+          step "Create or Update #{node_conf_file} with #{parameters}" do
+            if !master.file_exist?(node_conf_file)
+              if !master.file_exist?(node_conf_path)
+                # potentially create the nodes directory
+                on(master, "mkdir #{node_conf_path}")
+              end
+              # The hocon gem will create a list of comma separated parameters
+              # on the same line unless we start with something in the file.
+              create_remote_file(master, node_conf_file, %Q|{\n}\n|)
+              on(master, "chown pe-puppet #{node_conf_file}")
+            end
+            update_pe_conf(parameters, node_conf_file)
+          end
+        end
       end
     end
   end