From 7dce71cdc462af218aec79df0b1b705dacfeb85e Mon Sep 17 00:00:00 2001 From: Josh Partlow Date: Sat, 31 Dec 2016 12:51:40 -0800 Subject: [PATCH 01/14] (maint) Remove unused variables The version variable is not used in the fetch_pe_on_windows method (and hopefully wasn't producing a useful side effect...) The removed snapshot and box keys in HOST_DEFAULTS were being overwritten by later keys in the same hash definition and were producing warnings... --- lib/beaker-pe/install/pe_utils.rb | 1 - spec/helpers.rb | 2 -- 2 files changed, 3 deletions(-) diff --git a/lib/beaker-pe/install/pe_utils.rb b/lib/beaker-pe/install/pe_utils.rb index 7cec06fe..31a35079 100644 --- a/lib/beaker-pe/install/pe_utils.rb +++ b/lib/beaker-pe/install/pe_utils.rb @@ -177,7 +177,6 @@ def fetch_pe_on_mac(host, opts) def fetch_pe_on_windows(host, opts) path = host['pe_dir'] || opts[:pe_dir] local = File.directory?(path) - version = host['pe_ver'] || opts[:pe_ver_win] filename = "#{host['dist']}" extension = ".msi" if local diff --git a/spec/helpers.rb b/spec/helpers.rb index 1ffb2e0f..049cd9a5 100644 --- a/spec/helpers.rb +++ b/spec/helpers.rb @@ -26,8 +26,6 @@ def fog_file_contents module HostHelpers HOST_DEFAULTS = { :platform => 'unix', - :snapshot => 'pe', - :box => 'box_name', :roles => ['agent'], :snapshot => 'snap', :ip => 'default.ip.address', From 6b9c6184a9e0b382809e0d492b964df5905b5963 Mon Sep 17 00:00:00 2001 From: Josh Partlow Date: Sat, 31 Dec 2016 12:53:12 -0800 Subject: [PATCH 02/14] (PE-19049,PE-18718,PE-18799) Provide a test method for meep classification so we can adjust tests and setup steps that need to work either with old pe node groups, or with meep. Ultimately the test is just based on version boundary. But while we are validating meep classification, we need to be able to toggle around a temporary feature flag: the pe_infrastructure::use_meep_for_classification parameter. The function checks to see if this has been passed into beaker via the hosts file answers hash. These are answers which beaker-answers would include in the pe.conf it generates. It can also be set from an ENV['PE_USE_MEEP_FOR_CLASSIFICATION'] variable. This will make it easier to setup temporary ci jobs. The answer file setting will take precedence over the environment variable. --- lib/beaker-pe/install/pe_utils.rb | 33 +++++++++++ spec/beaker-pe/install/pe_utils_spec.rb | 79 ++++++++++++++++++++++++- 2 files changed, 111 insertions(+), 1 deletion(-) diff --git a/lib/beaker-pe/install/pe_utils.rb b/lib/beaker-pe/install/pe_utils.rb index 31a35079..2fbf305c 100644 --- a/lib/beaker-pe/install/pe_utils.rb +++ b/lib/beaker-pe/install/pe_utils.rb @@ -26,6 +26,14 @@ module PEUtils # Version of PE when we switched from legacy installer to MEEP. MEEP_CUTOVER_VERSION = '2016.2.0' + # Version of PE when we switched to using meep for classification + # instead of PE node groups + MEEP_CLASSIFICATION_VERSION = '2017.2.0' + # PE-18799 temporary default used for meep classification check while + # we navigate the switchover. + # PE-18718 switch flag to true once beaker-pe, beaker-answers, + # beaker-pe-large-environments and pe_acceptance_tests are ready + DEFAULT_MEEP_CLASSIFICATION = false # @!macro [new] common_opts # @param [Hash{Symbol=>String}] opts Options to alter execution. @@ -633,6 +641,31 @@ def install_via_msi?(host) (host['platform'] =~ /windows/ && (version_is_less(host['pe_ver'], '2016.4.0') && !version_is_less(host['pe_ver'], '3.99'))) || (host['platform'] =~ /windows-2008r2/ && (version_is_less(host['pe_ver'], '2016.4.3') && !version_is_less(host['pe_ver'], '3.99'))) end + # True if version is greater than or equal to MEEP_CLASSIFICATION_VERSION + # (PE-18718) AND the temporary feature flag is true. + # + # The temporary feature flag is pe_modules_next and can be set in + # the :answers hash given in beaker's host.cfg, inside a feature_flags + # hash. It will also be picked up from the environment as + # PE_MODULES_NEXT. (See register_feature_flags!()) + # + # The :answers hash value will take precedence over the env variable. + # + # @param version String the current PE version + # @param opts Hash options hash to inspect for :answers + # @return Boolean true if version and flag allows for meep classification + # feature. + def use_meep_for_classification?(version, opts) + # PE-19470 remove vv + register_feature_flags!(opts) + + temporary_flag = feature_flag?('pe_modules_next', opts) + temporary_flag = DEFAULT_MEEP_CLASSIFICATION if temporary_flag.nil? + # ^^ + + !version_is_less(version, MEEP_CLASSIFICATION_VERSION) && temporary_flag + end + # For PE 3.8.5 to PE 2016.1.2 they have an expired gpg key. This method is # for deb nodes to ignore the gpg-key expiration warning def ignore_gpg_key_warning_on_hosts(hosts, opts) diff --git a/spec/beaker-pe/install/pe_utils_spec.rb b/spec/beaker-pe/install/pe_utils_spec.rb index 331dee24..3c5867da 100644 --- a/spec/beaker-pe/install/pe_utils_spec.rb +++ b/spec/beaker-pe/install/pe_utils_spec.rb @@ -10,10 +10,11 @@ class ClassMixedWithDSLInstallUtils include Beaker::DSL::Patterns include Beaker::DSL::PE - attr_accessor :hosts, :metadata + attr_accessor :hosts, :metadata, :options def initialize @metadata = {} + @options = {} end # Because some the methods now actually call out to the `step` method, we need to @@ -391,6 +392,82 @@ def slice_installer_options(host) end end + describe 'use_meep_for_classification?' do + let(:feature_flag) { nil } + let(:environment_feature_flag) { nil } + let(:answers) do + { + :answers => { + 'feature_flags' => { + 'pe_modules_next' => feature_flag, + }, + }, + } + end + let(:options) do + feature_flag.nil? ? + opts : + opts.merge(answers) + end + let(:host) { unixhost } + + before(:each) do + subject.options = options + if !environment_feature_flag.nil? + ENV['PE_MODULES_NEXT'] = environment_feature_flag + end + end + + after(:each) do + ENV.delete('PE_MODULES_NEXT') + end + + it { expect(subject.use_meep_for_classification?('2017.1.0', options)).to eq(false) } + it { expect(subject.use_meep_for_classification?('2017.2.0', options)).to eq(false) } + + context 'feature flag false' do + let(:feature_flag) { false } + + it { expect(subject.use_meep_for_classification?('2017.1.0', options)).to eq(false) } + it { expect(subject.use_meep_for_classification?('2017.2.0', options)).to eq(false) } + end + + context 'feature flag true' do + let(:feature_flag) { true } + + it { expect(subject.use_meep_for_classification?('2017.1.0', options)).to eq(false) } + it { expect(subject.use_meep_for_classification?('2017.2.0', options)).to eq(true) } + end + + context 'environment feature flag true' do + let(:environment_feature_flag) { 'true' } + + it { expect(subject.use_meep_for_classification?('2017.1.0', options)).to eq(false) } + it { expect(subject.use_meep_for_classification?('2017.2.0', options)).to eq(true) } + + context 'answers feature flag false' do + let(:feature_flag) { false } + + it { expect(subject.use_meep_for_classification?('2017.1.0', options)).to eq(false) } + it { expect(subject.use_meep_for_classification?('2017.2.0', options)).to eq(false) } + end + end + + context 'environment feature flag false' do + let(:environment_feature_flag) { 'false' } + + it { expect(subject.use_meep_for_classification?('2017.1.0', options)).to eq(false) } + it { expect(subject.use_meep_for_classification?('2017.2.0', options)).to eq(false) } + + context 'answers feature flag true' do + let(:feature_flag) { true } + + it { expect(subject.use_meep_for_classification?('2017.1.0', options)).to eq(false) } + it { expect(subject.use_meep_for_classification?('2017.2.0', options)).to eq(true) } + end + end + end + describe 'generate_installer_conf_file_for' do let(:master) { hosts.first } From 1dbf81980f1286395383bae19e78d6647a7d8a18 Mon Sep 17 00:00:00 2001 From: Josh Partlow Date: Mon, 2 Jan 2017 11:16:21 -0800 Subject: [PATCH 03/14] (maint) Require beaker directly in spec_helper The individual specs were already requiring beaker. This change just centralizes that into the spec_helper, and removes the beaker_test_helper now that we are using Beaker 3. --- Gemfile | 4 ---- spec/beaker-pe/install/pe_utils_spec.rb | 1 - .../pe-client-tools/config_file_helper_spec.rb | 1 - .../pe-client-tools/executable_helper_spec.rb | 1 - .../pe-client-tools/installer_helper_spec.rb | 3 ++- spec/beaker_test_helpers.rb | 17 ----------------- spec/spec_helper.rb | 4 ++-- 7 files changed, 4 insertions(+), 27 deletions(-) delete mode 100644 spec/beaker_test_helpers.rb diff --git a/Gemfile b/Gemfile index d1249eb7..a9b81eb9 100644 --- a/Gemfile +++ b/Gemfile @@ -2,8 +2,6 @@ source ENV['GEM_SOURCE'] || "https://rubygems.org" gemspec - - def location_for(place, fake_version = nil) if place =~ /^(git:[^#]*)#(.*)/ [fake_version, { :git => $1, :branch => $2, :require => false }].compact @@ -14,7 +12,6 @@ def location_for(place, fake_version = nil) end end - # We don't put beaker in as a test dependency because we # don't want to create a transitive dependency group :acceptance_testing do @@ -25,7 +22,6 @@ if ENV['GEM_SOURCE'] =~ /rubygems\.delivery\.puppetlabs\.net/ gem "scooter", *location_for(ENV['SCOOTER_VERSION'] || '~> 3.0') end - if File.exists? "#{__FILE__}.local" eval(File.read("#{__FILE__}.local"), binding) end diff --git a/spec/beaker-pe/install/pe_utils_spec.rb b/spec/beaker-pe/install/pe_utils_spec.rb index 3c5867da..a2a28a1c 100644 --- a/spec/beaker-pe/install/pe_utils_spec.rb +++ b/spec/beaker-pe/install/pe_utils_spec.rb @@ -1,5 +1,4 @@ require 'spec_helper' -require 'beaker' class ClassMixedWithDSLInstallUtils include Beaker::DSL::InstallUtils diff --git a/spec/beaker-pe/pe-client-tools/config_file_helper_spec.rb b/spec/beaker-pe/pe-client-tools/config_file_helper_spec.rb index 570530ea..252bff92 100644 --- a/spec/beaker-pe/pe-client-tools/config_file_helper_spec.rb +++ b/spec/beaker-pe/pe-client-tools/config_file_helper_spec.rb @@ -1,5 +1,4 @@ require 'spec_helper' -require 'beaker' class MixedWithConfigFileHelper include Beaker::DSL::PEClientTools::ConfigFileHelper diff --git a/spec/beaker-pe/pe-client-tools/executable_helper_spec.rb b/spec/beaker-pe/pe-client-tools/executable_helper_spec.rb index 36efe415..6053eebf 100644 --- a/spec/beaker-pe/pe-client-tools/executable_helper_spec.rb +++ b/spec/beaker-pe/pe-client-tools/executable_helper_spec.rb @@ -1,5 +1,4 @@ require 'spec_helper' -require 'beaker' require 'scooter' class MixedWithExecutableHelper diff --git a/spec/beaker-pe/pe-client-tools/installer_helper_spec.rb b/spec/beaker-pe/pe-client-tools/installer_helper_spec.rb index cb5a7a55..d3c5316c 100644 --- a/spec/beaker-pe/pe-client-tools/installer_helper_spec.rb +++ b/spec/beaker-pe/pe-client-tools/installer_helper_spec.rb @@ -1,9 +1,10 @@ require 'spec_helper' -require 'beaker/host' + class ClassPEClientToolsMixedWithPatterns include Beaker::DSL::InstallUtils::PEClientTools include Beaker::DSL::Patterns end + describe ClassPEClientToolsMixedWithPatterns do describe "#install_pe_client_tools_on" do let(:hosts) do diff --git a/spec/beaker_test_helpers.rb b/spec/beaker_test_helpers.rb deleted file mode 100644 index e136c6e7..00000000 --- a/spec/beaker_test_helpers.rb +++ /dev/null @@ -1,17 +0,0 @@ -# These are specifically to mock Beaker methods necessary for testing -# that will be available during runtime because this is never run separate -# from Beaker itself. -# -# Including Beaker as a dependency would not work as a solution to this issue, -# since that would make a cycle in the dependency graph, at least until -# Beaker 3.0 happens and this is no longer a dependency of Beaker's. -module BeakerTestHelpers - include Beaker::DSL -end - -module Beaker - module DSL - def self.register( helper ) - end - end -end \ No newline at end of file diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 3882a20e..88e890fb 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,5 +1,5 @@ require 'simplecov' -require 'beaker_test_helpers' +require 'beaker' require 'beaker-pe' require 'helpers' @@ -8,4 +8,4 @@ RSpec.configure do |config| config.include TestFileHelpers config.include HostHelpers -end \ No newline at end of file +end From 858e63a940229a324db11850753cda94513512bb Mon Sep 17 00:00:00 2001 From: Josh Partlow Date: Tue, 3 Jan 2017 14:30:09 -0800 Subject: [PATCH 04/14] (PE-19049) Modify how we obtain console dispatcher for frictionless platform configuration of the master node. Expect to use the beaker-pe-large-environments::classification#get_dispatcher() method, which will only be present during a pe_acceptance_tests run, when beaker-pe-large-environments is part of the gem bundle. --- lib/beaker-pe.rb | 1 - lib/beaker-pe/install/pe_utils.rb | 37 ++++++++++++++++++------- spec/beaker-pe/install/pe_utils_spec.rb | 14 ++++++++++ 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/lib/beaker-pe.rb b/lib/beaker-pe.rb index 19c7871d..e1c56a09 100644 --- a/lib/beaker-pe.rb +++ b/lib/beaker-pe.rb @@ -18,7 +18,6 @@ module PE end end - # Boilerplate DSL inclusion mechanism: # First we register our module with the Beaker DSL Beaker::DSL.register( Beaker::DSL::PE ) diff --git a/lib/beaker-pe/install/pe_utils.rb b/lib/beaker-pe/install/pe_utils.rb index 2fbf305c..118d1dd2 100644 --- a/lib/beaker-pe/install/pe_utils.rb +++ b/lib/beaker-pe/install/pe_utils.rb @@ -317,19 +317,12 @@ def deploy_frictionless_to_master(host) on dashboard, "cd /opt/puppet/share/puppet-dashboard && /opt/puppet/bin/bundle exec /opt/puppet/bin/rake node:addclass[#{master},#{klass}]" on master, puppet("agent -t"), :acceptable_exit_codes => [0,2] else - # the new hotness - begin - require 'scooter' - rescue LoadError => e - @logger.notify('WARNING: gem scooter is required for frictionless installation post 3.8') - raise e - end - dispatcher = Scooter::HttpDispatchers::ConsoleDispatcher.new(dashboard) + _console_dispatcher = get_console_dispatcher_for_beaker_pe! # Check if we've already created a frictionless agent node group # to avoid errors creating the same node group when the beaker hosts file contains # multiple hosts with the same platform - node_group = dispatcher.get_node_group_by_name('Beaker Frictionless Agent') + node_group = _console_dispatcher.get_node_group_by_name('Beaker Frictionless Agent') if node_group.nil? || node_group.empty? node_group = {} node_group['name'] = "Beaker Frictionless Agent" @@ -341,7 +334,7 @@ def deploy_frictionless_to_master(host) # add the pe_repo platform class node_group['classes'][klass] = {} - dispatcher.create_new_node_group_model(node_group) + _console_dispatcher.create_new_node_group_model(node_group) on master, puppet("agent -t"), :acceptable_exit_codes => [0,2] end end @@ -1095,6 +1088,30 @@ def fetch_and_push_pe(host, path, filename, extension, local_dir='tmp/pe') scp_to host, "#{local_dir}/#{filename}#{extension}", host['working_dir'] end + # Being able to modify PE's classifier requires the Scooter gem and + # helpers which are in beaker-pe-large-environments. + def get_console_dispatcher_for_beaker_pe(raise_exception = false) + if !respond_to?(:get_dispatcher) + begin + # just in case scooter is present but beaker-pe-large-environments is not + # ...most likely this will raise a LoadError... + require 'scooter' + Scooter::HttpDispatchers::ConsoleDispatcher.new(dashboard) + rescue LoadError => e + logger.notify('WARNING: gem scooter is required for frictionless installation post 3.8') + raise e if raise_exception + + return nil + end + else + get_dispatcher + end + end + + # Will raise a LoadError if unable to require Scooter. + def get_console_dispatcher_for_beaker_pe! + get_console_dispatcher_for_beaker_pe(true) + end end end end diff --git a/spec/beaker-pe/install/pe_utils_spec.rb b/spec/beaker-pe/install/pe_utils_spec.rb index a2a28a1c..f12874d2 100644 --- a/spec/beaker-pe/install/pe_utils_spec.rb +++ b/spec/beaker-pe/install/pe_utils_spec.rb @@ -1597,4 +1597,18 @@ def test_setup(mock_values) end end end + + describe 'get_console_dispatcher_for_beaker_pe' do + it { expect(subject.get_console_dispatcher_for_beaker_pe).to be_nil } + it do + expect { subject.get_console_dispatcher_for_beaker_pe(true) }.to( + raise_exception(LoadError, /cannot load.*scooter/) + ) + end + it do + expect { subject.get_console_dispatcher_for_beaker_pe! }.to( + raise_exception(LoadError, /cannot load.*scooter/) + ) + end + end end From 7154c0081060b7c687e719e716110a7129f28ce7 Mon Sep 17 00:00:00 2001 From: Josh Partlow Date: Fri, 6 Jan 2017 16:48:34 -0800 Subject: [PATCH 05/14] (maint) Use a Beaker::Host instance when mocking hosts The mock hosts being generated for tests where failing when :exec was called, despite the allow() in the helpers.rb make_host() function. Using a Beaker::Host resolved this. --- spec/beaker-pe/install/pe_utils_spec.rb | 6 +++--- spec/helpers.rb | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/spec/beaker-pe/install/pe_utils_spec.rb b/spec/beaker-pe/install/pe_utils_spec.rb index f12874d2..5bd458cd 100644 --- a/spec/beaker-pe/install/pe_utils_spec.rb +++ b/spec/beaker-pe/install/pe_utils_spec.rb @@ -364,7 +364,7 @@ def prep_host(host) end def slice_installer_options(host) - host.select { |k,v| [ :pe_installer_conf_file, :pe_installer_conf_setting].include?(k) } + host.host_hash.select { |k,v| [ :pe_installer_conf_file, :pe_installer_conf_setting].include?(k) } end context 'when version < 2016.2.0' do @@ -1602,12 +1602,12 @@ def test_setup(mock_values) it { expect(subject.get_console_dispatcher_for_beaker_pe).to be_nil } it do expect { subject.get_console_dispatcher_for_beaker_pe(true) }.to( - raise_exception(LoadError, /cannot load.*scooter/) + raise_exception(LoadError, /cannot load.*scooter/) ) end it do expect { subject.get_console_dispatcher_for_beaker_pe! }.to( - raise_exception(LoadError, /cannot load.*scooter/) + raise_exception(LoadError, /cannot load.*scooter/) ) end end diff --git a/spec/helpers.rb b/spec/helpers.rb index 049cd9a5..0acbd836 100644 --- a/spec/helpers.rb +++ b/spec/helpers.rb @@ -76,9 +76,9 @@ def make_host_opts name, opts end def make_host name, host_hash - host_hash = StringifyHash.new.merge(HOST_DEFAULTS.merge(host_hash)) + host_hash = Beaker::Options::OptionsHash.new.merge(HOST_DEFAULTS.merge(host_hash)) - host = make_opts.merge(host_hash) + host = Beaker::Host.create( name, host_hash, make_opts) allow(host).to receive( :name ).and_return( name ) allow(host).to receive( :hostname ).and_return( "#{name}.test" ) From ed5d149929c9c4f0d9d0cd5b9bce136bae8265c0 Mon Sep 17 00:00:00 2001 From: Josh Partlow Date: Tue, 3 Jan 2017 17:09:09 -0800 Subject: [PATCH 06/14] (PE-19049,PE-11353) Ensure puppet service is stopped in 2017.1+ builds We began managing the puppet service in 2017.1.0 and need to ensure it is stopped and disabled after installation, otherwise each subsequent puppet agent run will restart the agent service, causing potential havoc in smoke tests or other setup steps. --- lib/beaker-pe/install/pe_utils.rb | 104 ++++++++++++++++++++++++ spec/beaker-pe/install/pe_utils_spec.rb | 69 ++++++++++++++++ 2 files changed, 173 insertions(+) diff --git a/lib/beaker-pe/install/pe_utils.rb b/lib/beaker-pe/install/pe_utils.rb index 118d1dd2..f4f55910 100644 --- a/lib/beaker-pe/install/pe_utils.rb +++ b/lib/beaker-pe/install/pe_utils.rb @@ -34,6 +34,8 @@ module PEUtils # PE-18718 switch flag to true once beaker-pe, beaker-answers, # beaker-pe-large-environments and pe_acceptance_tests are ready DEFAULT_MEEP_CLASSIFICATION = false + MEEP_DATA_DIR = '/etc/puppetlabs/enterprise' + PE_CONF_FILE = "#{MEEP_DATA_DIR}/conf.d/pe.conf" # @!macro [new] common_opts # @param [Hash{Symbol=>String}] opts Options to alter execution. @@ -507,6 +509,11 @@ def do_install hosts, opts = {} on dashboard, "/opt/puppet/bin/rake -sf /opt/puppet/share/puppet-dashboard/Rakefile #{task} RAILS_ENV=production" end + # PE-18799 replace the version_is_less with a use_meep_for_classification? test + if !version_is_less(master[:pe_ver], '2017.1.0') + configure_puppet_agent_service(:ensure => 'stopped', :enabled => false) + end + step "Final puppet agent run" do # Now that all hosts are in the dashbaord, run puppet one more # time to configure mcollective @@ -1112,6 +1119,103 @@ def get_console_dispatcher_for_beaker_pe(raise_exception = false) def get_console_dispatcher_for_beaker_pe! get_console_dispatcher_for_beaker_pe(true) end + + # In PE versions >= 2017.1.0, allows you to configure the puppet agent + # service for all nodes. + # + # @param parameters [Hash] - agent profile parameters + # @option parameters [Boolean] :managed - whether or not to manage the + # agent resource at all (Optional, defaults to true). + # @option parameters [String] :ensure - 'stopped', 'running' + # @option parameters [Boolean] :enabled - whether the service will be + # enabled (for restarts) + # @raise [StandardError] if master version is less than 2017.1.0 + def configure_puppet_agent_service(parameters) + raise(StandardError, "Can only manage puppet service in PE versions >= 2017.1.0; tried for #{master['pe_ver']}") if version_is_less(master['pe_ver'], '2017.1.0') + puppet_managed = parameters.include?(:managed) ? parameters[:managed] : true + puppet_ensure = parameters[:ensure] + puppet_enabled = parameters[:enabled] + + msg = puppet_managed ? + "Configure agents '#{puppet_ensure}' and #{puppet_enabled ? 'enabled' : 'disabled'}" : + "Do not manage agents" + + step msg do + # PE-18799 and remove this conditional + if use_meep_for_classification?(master[:pe_ver], options) + group_name = 'Puppet Enterprise Agent' + class_name = 'pe_infrastructure::agent' + else + group_name = 'PE Agent' + class_name = 'puppet_enterprise::profile::agent' + end + + # update pe conf + # only the pe_infrastructure::agent parameters are relevant in pe.conf + update_pe_conf({ + "pe_infrastructure::agent::puppet_service_managed" => puppet_managed, + "pe_infrastructure::agent::puppet_service_ensure" => puppet_ensure, + "pe_infrastructure::agent::puppet_service_enabled" => puppet_enabled, + }) + + if _console_dispatcher = get_console_dispatcher_for_beaker_pe + agent_group = _console_dispatcher.get_node_group_by_name(group_name) + agent_class = agent_group['classes'][class_name] + agent_class['puppet_service_managed'] = puppet_managed + agent_class['puppet_service_ensure'] = puppet_ensure + agent_class['puppet_service_enabled'] = puppet_enabled + + _console_dispatcher.update_node_group(agent_group['id'], agent_group) + end + end + end + + # Given a hash of parameters, updates the primary master's pe.conf, adding or + # replacing the given parameters. + # + # Handles stringifying and quoting namespaced keys, and also preparing non + # string values using Hocon::ConfigValueFactory. + # + # Logs the state of pe.conf before and after. + # + # @param parameters [Hash] Hash of parameters to be included in pe.conf. + # @param pe_conf_file [String] The file to update + # (/etc/puppetlabs/enterprise/conf.d/pe.conf by default) + def update_pe_conf(parameters, pe_conf_file = PE_CONF_FILE) + step "Update #{pe_conf_file} with #{parameters}" do + hocon_file_edit_in_place_on(master, pe_conf_file) do |host,doc| + updated_doc = parameters.reduce(doc) do |pe_conf,param| + key, value = param + + hocon_key = case key + when /^[^"][^.]+/ + # if the key is unquoted and does not contain pathing ('.') + # quote to ensure that puppet namespaces are protected + # ("puppet_enterprise::database_host" for example...) + then %Q{"#{key}"} + else key + end + + hocon_value = case value + when String + # ensure unquoted string values are quoted for uniformity + then value.match(/^[^"]/) ? %Q{"#{value}"} : value + else Hocon::ConfigValueFactory.from_any_ref(value, nil) + end + + updated = value.kind_of?(String) ? + pe_conf.set_value(hocon_key, hocon_value) : + pe_conf.set_config_value(hocon_key, hocon_value) + + updated + end + + # return the modified document + updated_doc + end + on(master, 'cat /etc/puppetlabs/enterprise/conf.d/pe.conf') + end + end end end end diff --git a/spec/beaker-pe/install/pe_utils_spec.rb b/spec/beaker-pe/install/pe_utils_spec.rb index 5bd458cd..de191ed0 100644 --- a/spec/beaker-pe/install/pe_utils_spec.rb +++ b/spec/beaker-pe/install/pe_utils_spec.rb @@ -1611,4 +1611,73 @@ def test_setup(mock_values) ) end end + + describe 'configure_puppet_agent_service' do + let(:pe_version) { '2017.1.0' } + let(:master) { hosts[0] } + + before(:each) do + hosts.each { |h| h[:pe_ver] = pe_version } + allow( subject ).to receive( :hosts ).and_return( hosts ) + end + + it 'requires parameters' do + expect { subject.configure_puppet_agent_service }.to raise_error(ArgumentError, /wrong number/) + end + + context 'master prior to 2017.1.0' do + let(:pe_version) { '2016.5.1' } + + it 'raises an exception about version' do + expect { subject.configure_puppet_agent_service({}) }.to( + raise_error(StandardError, /Can only manage.*2017.1.0; tried.* 2016.5.1/) + ) + end + end + + context '2017.1.0 master' do + let(:pe_conf_path) { '/etc/puppetlabs/enterprise/conf.d/pe.conf' } + let(:pe_conf) do + <<-EOF +"node_roles": { + "pe_role::monolithic::primary_master": ["#{master.name}"], +} + EOF + end + let(:gold_pe_conf) do + <<-EOF +"node_roles": { + "pe_role::monolithic::primary_master": ["#{master.name}"], +} +"pe_infrastructure::agent::puppet_service_managed": true +"pe_infrastructure::agent::puppet_service_ensure": "stopped" +"pe_infrastructure::agent::puppet_service_enabled": false + EOF + end + + it do + # mock reading pe.conf + expect(master).to receive(:exec).with( + have_attributes(:command => match(%r{cat .*/pe\.conf})), + anything + ).and_return( + double('result', :stdout => pe_conf) + ) + + # mock hitting the console + dispatcher = double('dispatcher').as_null_object + expect(subject).to receive(:get_console_dispatcher_for_beaker_pe) + .and_return(dispatcher) + + # mock writing pe.conf and check for parameters + expect(subject).to receive(:create_remote_file).with( + master, + pe_conf_path, + gold_pe_conf + ) + + subject.configure_puppet_agent_service(:ensure => 'stopped', :enabled => false) + end + end + end end From d4bab316a6ce30c083b68232964d0e30e820cf00 Mon Sep 17 00:00:00 2001 From: Josh Partlow Date: Sat, 7 Jan 2017 21:07:38 -0800 Subject: [PATCH 07/14] (PE-19049) Can remove parameters from pe.conf ...using the update_pe_conf function. Since a null isn't meaningful for a hocon lookup parameter in pe.conf (or at least, I can't think of why it would be, at the moment), a nil can be used to remove a parameter that we want to clean up from the file. It's possible I am overlooking something tricky about nil, undef in hiera/lookup...it might be applicable to a nodes.conf file where we wanted to clear a parameter on a specific node that had been set in pe.conf, but this function only applies to pe.conf, so I think this is acceptable for now. --- lib/beaker-pe/install/pe_utils.rb | 34 +++++++++++-- spec/beaker-pe/install/pe_utils_spec.rb | 64 ++++++++++++++++++++++++- 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/lib/beaker-pe/install/pe_utils.rb b/lib/beaker-pe/install/pe_utils.rb index f4f55910..551e3f47 100644 --- a/lib/beaker-pe/install/pe_utils.rb +++ b/lib/beaker-pe/install/pe_utils.rb @@ -1171,13 +1171,36 @@ def configure_puppet_agent_service(parameters) end # Given a hash of parameters, updates the primary master's pe.conf, adding or - # replacing the given parameters. + # replacing, or removing the given parameters. + # + # To remove a parameter, pass a nil as its value # # Handles stringifying and quoting namespaced keys, and also preparing non # string values using Hocon::ConfigValueFactory. # # Logs the state of pe.conf before and after. # + # @example + # # Assuming pe.conf looks like: + # # { + # # "bar": "baz" + # # "old": "item" + # # } + # + # update_pe_conf( + # { + # "foo" => "a", + # "bar" => "b", + # "old" => nil, + # } + # ) + # + # # Will produce a pe.conf like: + # # { + # # "bar": "b" + # # "foo": "a" + # # } + # # @param parameters [Hash] Hash of parameters to be included in pe.conf. # @param pe_conf_file [String] The file to update # (/etc/puppetlabs/enterprise/conf.d/pe.conf by default) @@ -1203,9 +1226,14 @@ def update_pe_conf(parameters, pe_conf_file = PE_CONF_FILE) else Hocon::ConfigValueFactory.from_any_ref(value, nil) end - updated = value.kind_of?(String) ? - pe_conf.set_value(hocon_key, hocon_value) : + updated = case value + when String + pe_conf.set_value(hocon_key, hocon_value) + when nil + pe_conf.remove_value(hocon_key) + else pe_conf.set_config_value(hocon_key, hocon_value) + end updated end diff --git a/spec/beaker-pe/install/pe_utils_spec.rb b/spec/beaker-pe/install/pe_utils_spec.rb index de191ed0..d91bce1f 100644 --- a/spec/beaker-pe/install/pe_utils_spec.rb +++ b/spec/beaker-pe/install/pe_utils_spec.rb @@ -1655,7 +1655,7 @@ def test_setup(mock_values) EOF end - it do + it "modifies the agent puppet service settings in pe.conf" do # mock reading pe.conf expect(master).to receive(:exec).with( have_attributes(:command => match(%r{cat .*/pe\.conf})), @@ -1680,4 +1680,66 @@ def test_setup(mock_values) end end end + + describe 'update_pe_conf' do + let(:pe_version) { '2017.1.0' } + let(:master) { hosts[0] } + + before(:each) do + hosts.each { |h| h[:pe_ver] = pe_version } + allow( subject ).to receive( :hosts ).and_return( hosts ) + end + + it 'requires parameters' do + expect { subject.update_pe_conf}.to raise_error(ArgumentError, /wrong number/) + end + + context '2017.1.0 master' do + let(:pe_conf_path) { '/etc/puppetlabs/enterprise/conf.d/pe.conf' } + let(:pe_conf) do + <<-EOF +"node_roles": { + "pe_role::monolithic::primary_master": ["#{master.name}"], +} +"namespace::removed": "bye" +"namespace::changed": "old" + EOF + end + let(:gold_pe_conf) do + <<-EOF +"node_roles": { + "pe_role::monolithic::primary_master": ["#{master.name}"], +} + +"namespace::changed": "new" +"namespace::add": "hi" + EOF + end + + it "adds, changes and removes hocon parameters from pe.conf" do + # mock reading pe.conf + expect(master).to receive(:exec).with( + have_attributes(:command => match(%r{cat .*/pe\.conf})), + anything + ).and_return( + double('result', :stdout => pe_conf) + ) + + # mock writing pe.conf and check for parameters + expect(subject).to receive(:create_remote_file).with( + master, + pe_conf_path, + gold_pe_conf + ) + + subject.update_pe_conf( + { + "namespace::add" => "hi", + "namespace::changed" => "new", + "namespace::removed" => nil, + } + ) + end + end + end end From bb4094ae8c0c508fc3c0e584c2555ed7951f0671 Mon Sep 17 00:00:00 2001 From: Josh Partlow Date: Mon, 9 Jan 2017 14:24:24 -0800 Subject: [PATCH 08/14] (PE-19049) Add method to create or update a meep node.conf file This is necessary if we need to adjust parameters for a specific node rather than for all infrastructure via pe.conf. --- lib/beaker-pe/install/pe_utils.rb | 20 +++- spec/beaker-pe/install/pe_utils_spec.rb | 121 ++++++++++++++++++------ 2 files changed, 110 insertions(+), 31 deletions(-) diff --git a/lib/beaker-pe/install/pe_utils.rb b/lib/beaker-pe/install/pe_utils.rb index 551e3f47..fe3bb478 100644 --- a/lib/beaker-pe/install/pe_utils.rb +++ b/lib/beaker-pe/install/pe_utils.rb @@ -36,6 +36,7 @@ module PEUtils DEFAULT_MEEP_CLASSIFICATION = false MEEP_DATA_DIR = '/etc/puppetlabs/enterprise' PE_CONF_FILE = "#{MEEP_DATA_DIR}/conf.d/pe.conf" + NODE_CONF_PATH = "#{MEEP_DATA_DIR}/conf.d/nodes" # @!macro [new] common_opts # @param [Hash{Symbol=>String}] opts Options to alter execution. @@ -1227,7 +1228,7 @@ def update_pe_conf(parameters, pe_conf_file = PE_CONF_FILE) end updated = case value - when String + when String pe_conf.set_value(hocon_key, hocon_value) when nil pe_conf.remove_value(hocon_key) @@ -1244,6 +1245,23 @@ def update_pe_conf(parameters, pe_conf_file = PE_CONF_FILE) on(master, 'cat /etc/puppetlabs/enterprise/conf.d/pe.conf') end end + + def create_or_update_node_conf(host, parameters, node_conf_path = NODE_CONF_PATH) + node_conf_file = "#{node_conf_path}/#{host.puppet['certname']}.conf" + step "Create or Update #{node_conf_file} with #{parameters}" do + if !master.file_exist?(node_conf_file) + if !master.file_exist?(node_conf_path) + # potentially create the nodes directory + on(master, "mkdir #{node_conf_path}") + end + # The hocon gem will create a list of comma separated parameters + # on the same line unless we start with something in the file. + create_remote_file(master, node_conf_file, %Q|{\n}\n|) + on(master, "chown pe-puppet #{node_conf_file}") + end + update_pe_conf(parameters, node_conf_file) + end + end end end end diff --git a/spec/beaker-pe/install/pe_utils_spec.rb b/spec/beaker-pe/install/pe_utils_spec.rb index d91bce1f..7951e671 100644 --- a/spec/beaker-pe/install/pe_utils_spec.rb +++ b/spec/beaker-pe/install/pe_utils_spec.rb @@ -1612,6 +1612,25 @@ def test_setup(mock_values) end end + def assert_meep_conf_edit(input, output, path, &test) + # mock reading pe.conf + expect(master).to receive(:exec).with( + have_attributes(:command => match(%r{cat #{path}})), + anything + ).and_return( + double('result', :stdout => input) + ) + + # mock writing pe.conf and check for parameters + expect(subject).to receive(:create_remote_file).with( + master, + path, + output + ) + + yield + end + describe 'configure_puppet_agent_service' do let(:pe_version) { '2017.1.0' } let(:master) { hosts[0] } @@ -1656,27 +1675,14 @@ def test_setup(mock_values) end it "modifies the agent puppet service settings in pe.conf" do - # mock reading pe.conf - expect(master).to receive(:exec).with( - have_attributes(:command => match(%r{cat .*/pe\.conf})), - anything - ).and_return( - double('result', :stdout => pe_conf) - ) - # mock hitting the console dispatcher = double('dispatcher').as_null_object expect(subject).to receive(:get_console_dispatcher_for_beaker_pe) .and_return(dispatcher) - # mock writing pe.conf and check for parameters - expect(subject).to receive(:create_remote_file).with( - master, - pe_conf_path, - gold_pe_conf - ) - - subject.configure_puppet_agent_service(:ensure => 'stopped', :enabled => false) + assert_meep_conf_edit(pe_conf, gold_pe_conf, pe_conf_path) do + subject.configure_puppet_agent_service(:ensure => 'stopped', :enabled => false) + end end end end @@ -1713,31 +1719,86 @@ def test_setup(mock_values) "namespace::changed": "new" "namespace::add": "hi" +"namespace::add2": "other" EOF end it "adds, changes and removes hocon parameters from pe.conf" do - # mock reading pe.conf - expect(master).to receive(:exec).with( - have_attributes(:command => match(%r{cat .*/pe\.conf})), - anything - ).and_return( - double('result', :stdout => pe_conf) - ) + assert_meep_conf_edit(pe_conf, gold_pe_conf, pe_conf_path) do + subject.update_pe_conf( + { + "namespace::add" => "hi", + "namespace::changed" => "new", + "namespace::removed" => nil, + "namespace::add2" => "other", + } + ) + end + end + end + end + + describe 'create_or_update_node_conf' do + let(:pe_version) { '2017.1.0' } + let(:master) { hosts[0] } + let(:node) { hosts[1] } + let(:node_conf_path) { "/etc/puppetlabs/enterprise/conf.d/nodes/vm2.conf" } + let(:node_conf) do + <<-EOF +"namespace::removed": "bye" +"namespace::changed": "old" + EOF + end + let(:updated_node_conf) do + <<-EOF + +"namespace::changed": "new" +"namespace::add": "hi" + EOF + end + let(:created_node_conf) do + <<-EOF +{ + "namespace::one": "red" + "namespace::two": "blue" +} + EOF + end - # mock writing pe.conf and check for parameters - expect(subject).to receive(:create_remote_file).with( - master, - pe_conf_path, - gold_pe_conf + before(:each) do + hosts.each { |h| h[:pe_ver] = pe_version } + allow( subject ).to receive( :hosts ).and_return( hosts ) + end + + it 'requires parameters' do + expect { subject.update_pe_conf}.to raise_error(ArgumentError, /wrong number/) + end + + it 'creates a node file that did not exist' do + expect(master).to receive(:file_exist?).with(node_conf_path).and_return(false) + expect(master).to receive(:file_exist?).with("/etc/puppetlabs/enterprise/conf.d/nodes").and_return(false) + expect(subject).to receive(:create_remote_file).with(master, node_conf_path, %Q|{\n}\n|) + + assert_meep_conf_edit(%Q|{\n}\n|, created_node_conf, node_conf_path) do + subject.create_or_update_node_conf( + node, + { + "namespace::one" => "red", + "namespace::two" => "blue", + }, ) + end + end - subject.update_pe_conf( + it 'updates a node file that did exist' do + assert_meep_conf_edit(node_conf, updated_node_conf, node_conf_path) do + subject.create_or_update_node_conf( + node, { "namespace::add" => "hi", "namespace::changed" => "new", "namespace::removed" => nil, - } + }, ) end end From 02c0b3560db3b4cc82a8242bd3a3136e99e779f5 Mon Sep 17 00:00:00 2001 From: Josh Partlow Date: Mon, 23 Jan 2017 15:53:19 -0800 Subject: [PATCH 09/14] (PE-19049) Add helper method to read a hocon key from pe.conf --- lib/beaker-pe/install/pe_utils.rb | 52 ++++++++++++++++++++----- spec/beaker-pe/install/pe_utils_spec.rb | 38 ++++++++++++++++++ 2 files changed, 80 insertions(+), 10 deletions(-) diff --git a/lib/beaker-pe/install/pe_utils.rb b/lib/beaker-pe/install/pe_utils.rb index fe3bb478..2a004c31 100644 --- a/lib/beaker-pe/install/pe_utils.rb +++ b/lib/beaker-pe/install/pe_utils.rb @@ -1211,14 +1211,7 @@ def update_pe_conf(parameters, pe_conf_file = PE_CONF_FILE) updated_doc = parameters.reduce(doc) do |pe_conf,param| key, value = param - hocon_key = case key - when /^[^"][^.]+/ - # if the key is unquoted and does not contain pathing ('.') - # quote to ensure that puppet namespaces are protected - # ("puppet_enterprise::database_host" for example...) - then %Q{"#{key}"} - else key - end + hocon_key = quoted_hocon_key(key) hocon_value = case value when String @@ -1242,12 +1235,51 @@ def update_pe_conf(parameters, pe_conf_file = PE_CONF_FILE) # return the modified document updated_doc end - on(master, 'cat /etc/puppetlabs/enterprise/conf.d/pe.conf') + on(master, "cat #{pe_conf_file}") + end + end + + # If the key is unquoted and does not contain pathing ('.'), + # quote to ensure that puppet namespaces are protected + # + # @example + # quoted_hocon_key("puppet_enterprise::database_host") + # # => '"puppet_enterprise::database_host"' + # + def quoted_hocon_key(key) + case key + when /^[^"][^.]+/ + then %Q{"#{key}"} + else key end end + # @return a Ruby object of any root key in pe.conf. + # + # @param key [String] to lookup + # @param pe_conf_path [String] defaults to /etc/puppetlabs/enterprise/conf.d/pe.conf + def get_unwrapped_pe_conf_value(key, pe_conf_path = PE_CONF_FILE) + file_contents = on(master, "cat #{pe_conf_path}").stdout + # Seem to need to use ConfigFactory instead of ConfigDocumentFactory + # to get something that we can read values from? + doc = Hocon::ConfigFactory.parse_string(file_contents) + hocon_key = quoted_hocon_key(key) + doc.has_path?(hocon_key) ? + doc.get_value(hocon_key).unwrapped : + nil + end + + # Creates a new /etc/puppetlabs/enterprise/conf.d/nodes/*.conf file for the + # given host's certname, and adds the passed parameters, or updates with the + # passed parameters if the file already exists. + # + # Does not remove an empty file. + # + # @param host [Beaker::Host] to create a node file for + # @param parameters [Hash] of key value pairs to add to the nodes conf file + # @param node_conf_path [String] defaults to /etc/puppetlabs/enterprise/conf.d/nodes def create_or_update_node_conf(host, parameters, node_conf_path = NODE_CONF_PATH) - node_conf_file = "#{node_conf_path}/#{host.puppet['certname']}.conf" + node_conf_file = "#{node_conf_path}/#{host.node_name}.conf" step "Create or Update #{node_conf_file} with #{parameters}" do if !master.file_exist?(node_conf_file) if !master.file_exist?(node_conf_path) diff --git a/spec/beaker-pe/install/pe_utils_spec.rb b/spec/beaker-pe/install/pe_utils_spec.rb index 7951e671..e11cde1e 100644 --- a/spec/beaker-pe/install/pe_utils_spec.rb +++ b/spec/beaker-pe/install/pe_utils_spec.rb @@ -1803,4 +1803,42 @@ def assert_meep_conf_edit(input, output, path, &test) end end end + + describe "get_unwrapped_pe_conf_value" do + let(:pe_version) { '2017.1.0' } + let(:master) { hosts[0] } + let(:pe_conf_path) { "/etc/puppetlabs/enterprise/conf.d/pe.conf" } + let(:pe_conf) do + <<-EOF +"namespace::bool": true +"namespace::string": "stringy" +"namespace::array": ["of", "things"] +"namespace::hash": { + "foo": "a" + "bar": "b" +} + EOF + end + + before(:each) do + hosts.each { |h| h[:pe_ver] = pe_version } + allow( subject ).to receive( :hosts ).and_return( hosts ) + expect(master).to receive(:exec).with( + have_attributes(:command => match(%r{cat #{pe_conf_path}})), + anything + ).and_return( + double('result', :stdout => pe_conf) + ) + end + + it { expect(subject.get_unwrapped_pe_conf_value("namespace::bool")).to eq(true) } + it { expect(subject.get_unwrapped_pe_conf_value("namespace::string")).to eq("stringy") } + it { expect(subject.get_unwrapped_pe_conf_value("namespace::array")).to eq(["of","things"]) } + it do + expect(subject.get_unwrapped_pe_conf_value("namespace::hash")).to eq({ + 'foo' => 'a', + 'bar' => 'b', + }) + end + end end From 570694db731c95569dcfd69ebcaf7d5ae294e7fc Mon Sep 17 00:00:00 2001 From: Brandon High Date: Tue, 14 Feb 2017 00:17:56 +0000 Subject: [PATCH 10/14] (PE-19049) use_meep_for_classification for configure_puppet_agent_service This commit updates the condition on performing the `configure_puppet_agent_service` to be gated with `use_meep_for_classification` so that it will automatically be set to whatever arbitrary version we activate MEEP on. --- lib/beaker-pe/install/pe_utils.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/beaker-pe/install/pe_utils.rb b/lib/beaker-pe/install/pe_utils.rb index 2a004c31..7c49ef41 100644 --- a/lib/beaker-pe/install/pe_utils.rb +++ b/lib/beaker-pe/install/pe_utils.rb @@ -511,7 +511,7 @@ def do_install hosts, opts = {} end # PE-18799 replace the version_is_less with a use_meep_for_classification? test - if !version_is_less(master[:pe_ver], '2017.1.0') + if use_meep_for_classification?(master[:pe_ver], options) configure_puppet_agent_service(:ensure => 'stopped', :enabled => false) end From e3515a023e82f793678fd22026e6765ba82c0ec7 Mon Sep 17 00:00:00 2001 From: Josh Partlow Date: Thu, 9 Mar 2017 17:50:17 -0800 Subject: [PATCH 11/14] (PE-19049) Remove get_console_dispatcher_for_beaker_pe specs Because scooter can in fact be in the bundle based on the GEM_SOURCE setting, these specs can break, and it is not worth the effort reworking the tests to be conditional based on presence or absence of the scooter gem. RE-8616 should make the scooter gem public and then we don't need to dance around it like this. --- lib/beaker-pe/install/pe_utils.rb | 4 ++-- spec/beaker-pe/install/pe_utils_spec.rb | 14 -------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/lib/beaker-pe/install/pe_utils.rb b/lib/beaker-pe/install/pe_utils.rb index 7c49ef41..7619ea5d 100644 --- a/lib/beaker-pe/install/pe_utils.rb +++ b/lib/beaker-pe/install/pe_utils.rb @@ -1099,10 +1099,10 @@ def fetch_and_push_pe(host, path, filename, extension, local_dir='tmp/pe') # Being able to modify PE's classifier requires the Scooter gem and # helpers which are in beaker-pe-large-environments. def get_console_dispatcher_for_beaker_pe(raise_exception = false) + # XXX RE-8616, once scooter is public, we can remove this and just + # reference ConsoleDispatcher directly. if !respond_to?(:get_dispatcher) begin - # just in case scooter is present but beaker-pe-large-environments is not - # ...most likely this will raise a LoadError... require 'scooter' Scooter::HttpDispatchers::ConsoleDispatcher.new(dashboard) rescue LoadError => e diff --git a/spec/beaker-pe/install/pe_utils_spec.rb b/spec/beaker-pe/install/pe_utils_spec.rb index e11cde1e..61832089 100644 --- a/spec/beaker-pe/install/pe_utils_spec.rb +++ b/spec/beaker-pe/install/pe_utils_spec.rb @@ -1598,20 +1598,6 @@ def test_setup(mock_values) end end - describe 'get_console_dispatcher_for_beaker_pe' do - it { expect(subject.get_console_dispatcher_for_beaker_pe).to be_nil } - it do - expect { subject.get_console_dispatcher_for_beaker_pe(true) }.to( - raise_exception(LoadError, /cannot load.*scooter/) - ) - end - it do - expect { subject.get_console_dispatcher_for_beaker_pe! }.to( - raise_exception(LoadError, /cannot load.*scooter/) - ) - end - end - def assert_meep_conf_edit(input, output, path, &test) # mock reading pe.conf expect(master).to receive(:exec).with( From 2af29a4359033ced108ca9696adb49bd87f18f24 Mon Sep 17 00:00:00 2001 From: Brandon High Date: Fri, 24 Feb 2017 13:21:47 -0800 Subject: [PATCH 12/14] (PE-19438) Stop passing -c to upgrades from MEEP Prior to this commit we were essentially always passing in a config to the installer during upgrades because we typically provide some sort of custom answers for many tests. This meant that we were not really testing upgrades without a config file being passed in. This commit updates the beaker to not pass in a config/`-c` option on upgrades from a MEEP install. In order to pass in the custom answers, I have instead made use of the `update_pe_conf` method to inject the answers. --- lib/beaker-pe/install/pe_utils.rb | 36 ++++++++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/lib/beaker-pe/install/pe_utils.rb b/lib/beaker-pe/install/pe_utils.rb index 7619ea5d..84105545 100644 --- a/lib/beaker-pe/install/pe_utils.rb +++ b/lib/beaker-pe/install/pe_utils.rb @@ -131,9 +131,11 @@ def installer_cmd(host, opts) pe_cmd += " -y" end - # If there are no answer overrides, and we are doing an upgrade from 2016.2.0, + # If we are doing an upgrade from 2016.2.0, # we can assume there will be a valid pe.conf in /etc that we can re-use. - if opts[:answers].nil? && opts[:custom_answers].nil? && opts[:type] == :upgrade && !version_is_less(opts[:HOSTS][host.name][:pe_ver], '2016.2.0') + # We also expect that any custom_answers specified to beaker have been + # added to the pe.conf in /etc. + if opts[:type] == :upgrade && use_meep?(host[:previous_pe_ver]) "#{pe_cmd}" else "#{pe_cmd} #{host['pe_installer_conf_setting']}" @@ -450,9 +452,36 @@ def do_install hosts, opts = {} else prepare_host_installer_options(host) register_feature_flags!(opts) - generate_installer_conf_file_for(host, hosts, opts) + + enterprise_path = '/etc/puppetlabs/enterprise' + local_path = 'pe_conf' + if opts[:type] == :upgrade && use_meep?(host['previous_pe_ver']) + # In this scenario, Beaker runs the installer such that we make + # use of recovery code in the configure face of the installer. + if host['roles'].include?('master') + # merge answers into pe.conf + if opts[:answers] && !opts[:answers].empty? + update_pe_conf(opts[:answers]) + end + + if opts[:custom_answers] && !opts[:custom_answers].empty? + update_pe_conf(opts[:custom_answers]) + end + else + # scp conf.d to host + scp_to(host, "#{local_path}/conf.d", enterprise_path) + end + else + # Beaker creates a fresh pe.conf using beaker-answers, as if we were doing an install + generate_installer_conf_file_for(host, hosts, opts) + end + on host, installer_cmd(host, opts) configure_type_defaults_on(host) + if host['roles'].include?('master') + # scp conf.d over from master + scp_from(host, "#{enterprise_path}/conf.d", local_path) + end end end # On each agent, we ensure the certificate is signed @@ -960,6 +989,7 @@ def upgrade_pe_on upgrade_hosts, opts, path=nil hosts.each do |host| prep_host_for_upgrade(host, opts, path) end + do_install(sorted_hosts, opts.merge({:type => :upgrade, :set_console_password => set_console_password})) opts['upgrade'] = true end From 80b78e04ce8311af27bf78a62c9125ff0e167872 Mon Sep 17 00:00:00 2001 From: Brandon High Date: Thu, 9 Mar 2017 14:44:06 -0800 Subject: [PATCH 13/14] (PE-19438) Move pe.conf setup into descriptive function The `do_install` method was getting a bit cluttered with too many levels of logic, so I've moved the pe.conf setup steps out into their own method, `setup_pe_conf` --- lib/beaker-pe/install/pe_utils.rb | 66 ++++++++++++++++++------------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/lib/beaker-pe/install/pe_utils.rb b/lib/beaker-pe/install/pe_utils.rb index 84105545..28f2fcf0 100644 --- a/lib/beaker-pe/install/pe_utils.rb +++ b/lib/beaker-pe/install/pe_utils.rb @@ -37,6 +37,7 @@ module PEUtils MEEP_DATA_DIR = '/etc/puppetlabs/enterprise' PE_CONF_FILE = "#{MEEP_DATA_DIR}/conf.d/pe.conf" NODE_CONF_PATH = "#{MEEP_DATA_DIR}/conf.d/nodes" + BEAKER_MEEP_TMP = "pe_conf" # @!macro [new] common_opts # @param [Hash{Symbol=>String}] opts Options to alter execution. @@ -452,36 +453,11 @@ def do_install hosts, opts = {} else prepare_host_installer_options(host) register_feature_flags!(opts) - - enterprise_path = '/etc/puppetlabs/enterprise' - local_path = 'pe_conf' - if opts[:type] == :upgrade && use_meep?(host['previous_pe_ver']) - # In this scenario, Beaker runs the installer such that we make - # use of recovery code in the configure face of the installer. - if host['roles'].include?('master') - # merge answers into pe.conf - if opts[:answers] && !opts[:answers].empty? - update_pe_conf(opts[:answers]) - end - - if opts[:custom_answers] && !opts[:custom_answers].empty? - update_pe_conf(opts[:custom_answers]) - end - else - # scp conf.d to host - scp_to(host, "#{local_path}/conf.d", enterprise_path) - end - else - # Beaker creates a fresh pe.conf using beaker-answers, as if we were doing an install - generate_installer_conf_file_for(host, hosts, opts) - end + setup_pe_conf(host, hosts, opts) on host, installer_cmd(host, opts) configure_type_defaults_on(host) - if host['roles'].include?('master') - # scp conf.d over from master - scp_from(host, "#{enterprise_path}/conf.d", local_path) - end + download_pe_conf_if_master(host) end end # On each agent, we ensure the certificate is signed @@ -1324,6 +1300,42 @@ def create_or_update_node_conf(host, parameters, node_conf_path = NODE_CONF_PATH update_pe_conf(parameters, node_conf_file) end end + + def setup_pe_conf(host, hosts, opts={}) + if opts[:type] == :upgrade && use_meep?(host['previous_pe_ver']) + # In this scenario, Beaker runs the installer such that we make + # use of recovery code in the configure face of the installer. + if host['roles'].include?('master') + step "Updating #{MEEP_DATA_DIR}/conf.d with answers/custom_answers" do + # merge answers into pe.conf + if opts[:answers] && !opts[:answers].empty? + update_pe_conf(opts[:answers]) + end + + if opts[:custom_answers] && !opts[:custom_answers].empty? + update_pe_conf(opts[:custom_answers]) + end + end + else + step "Uploading #{BEAKER_MEEP_TMP}/conf.d that was generated on the master" do + # scp conf.d to host + scp_to(host, "#{BEAKER_MEEP_TMP}/conf.d", MEEP_DATA_DIR) + end + end + else + # Beaker creates a fresh pe.conf using beaker-answers, as if we were doing an install + generate_installer_conf_file_for(host, hosts, opts) + end + end + + def download_pe_conf_if_master(host) + if host['roles'].include?('master') + step "Downloading generated #{MEEP_DATA_DIR}/conf.d locally" do + # scp conf.d over from master + scp_from(host, "#{MEEP_DATA_DIR}/conf.d", BEAKER_MEEP_TMP) + end + end + end end end end From 1353f7bf3e46cb54c4139f96f4f846f6b15d68e2 Mon Sep 17 00:00:00 2001 From: Brandon High Date: Thu, 9 Mar 2017 15:20:37 -0800 Subject: [PATCH 14/14] (PE-19438) Mock `scp_from` for `do_install` The new functionality in `do_install` to copy the `conf.d` folder --- spec/beaker-pe/install/pe_utils_spec.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spec/beaker-pe/install/pe_utils_spec.rb b/spec/beaker-pe/install/pe_utils_spec.rb index 61832089..292c9b5f 100644 --- a/spec/beaker-pe/install/pe_utils_spec.rb +++ b/spec/beaker-pe/install/pe_utils_spec.rb @@ -897,6 +897,8 @@ def slice_installer_options(host) allow( subject ).to receive( :hosts ).and_return( hosts ) #create answers file per-host, except windows expect( subject ).to receive( :create_remote_file ).with( hosts[0], /answers/, /q/ ).once + # copy the pe.conf + expect( subject ).to receive( :scp_from ).and_return(true) #run installer on all hosts expect( subject ).to receive( :on ).with( hosts[0], /puppet-enterprise-installer/ ).once expect( subject ).to receive( :install_msi_on ).with ( any_args ) do | host, msi_path, msi_opts, opts | @@ -1041,6 +1043,7 @@ def slice_installer_options(host) allow( subject ).to receive( :configure_type_defaults_on ).with( host ) end + expect( subject ).to receive( :scp_from ).and_return(true) subject.do_install( hosts, opts ) end @@ -1106,6 +1109,7 @@ def slice_installer_options(host) allow( subject ).to receive( :configure_type_defaults_on ).with( host ) end + expect( subject ).to receive( :scp_from ).and_return(true) subject.do_install( hosts, opts ) end @@ -1176,6 +1180,7 @@ def slice_installer_options(host) allow( subject ).to receive( :configure_type_defaults_on ).with( host ) end + expect( subject ).to receive( :scp_from ).and_return(true) subject.do_install( hosts, opts ) end