Merge pull request #57 from nicklewis/improved-monolithic-install

(BKR-1058) Implement a simpler monolithic install method

lib/beaker-pe/install/pe_utils.rb

@@ -378,6 +378,122 @@ def deploy_frictionless_to_master(host)
         # @api private
         #
         def do_install hosts, opts = {}
+          # detect the kind of install we're doing
+          install_type = determine_install_type(hosts, opts)
+          case install_type
+          when :simple_monolithic
+            simple_monolithic_install(hosts.first, hosts.drop(1), opts)
+          when :simple_split
+            # This isn't implemented yet, so just do a generic install instead
+            #simple_split_install(hosts, opts)
+            generic_install(hosts, opts)
+          else
+            generic_install(hosts, opts)
+          end
+        end
+
+        def has_all_roles?(host, roles)
+          roles.all? {|role| host['roles'].include?(role)}
+        end
+
+        # Determine what kind of install is being performed
+        # @param [Array<Host>] hosts The sorted hosts to install or upgrade PE on
+        # @param [Hash{Symbol=>Symbol, String}] opts The options for how to install or upgrade PE
+        #
+        # @example
+        #   determine_install_type(hosts, {:type => :install, :pe_ver => '2017.2.0'})
+        #
+        # @return [Symbol]
+        #   One of :generic, :simple_monolithic, :simple_split
+        #   :simple_monolithic
+        #     returned when installing >=2016.4 with a monolithic master and
+        #     any number of frictionless agents
+        #   :simple_split
+        #     returned when installing >=2016.4 with a split install and any
+        #     number of frictionless agents
+        #   :generic
+        #     returned for any other install or upgrade
+        #
+        # @api private
+        def determine_install_type(hosts, opts)
+          # Do a generic install if this is masterless, not all the same PE version, an upgrade, or earlier than 2016.4
+          return :generic if opts[:masterless]
+          return :generic if hosts.map {|host| host['pe_ver']}.uniq.length > 1
+          return :generic if opts[:type] == :upgrade
+          return :generic if version_is_less(opts[:pe_ver] || hosts.first['pe_ver'], '2016.4')
+
+          mono_roles = ['master', 'database', 'dashboard']
+          if has_all_roles?(hosts.first, mono_roles) && hosts.drop(1).all? {|host| host['roles'].include?('frictionless')}
+            :simple_monolithic
+          elsif hosts[0]['roles'].include?('master') && hosts[1]['roles'].include?('database') && hosts[2]['roles'].include?('dashboard') && hosts.drop(3).all? {|host| host['roles'].include?('frictionless')}
+            :simple_split
+          else
+            :generic
+          end
+        end
+
+        # Install PE on a monolithic master and some number of frictionless agents.
+        # @param [Host] master The node to install the master on
+        # @param [Array<Host>] agents The nodes to install agents on
+        # @param [Hash{Symbol=>Symbol, String}] opts The options for how to install or upgrade PE
+        #
+        # @example
+        #   simple_monolithic_install(master, agents, {:type => :install, :pe_ver => '2017.2.0'})
+        #
+        # @return nil
+        #
+        # @api private
+        def simple_monolithic_install(master, agents, opts={})
+          step "Performing a standard monolithic install with frictionless agents"
+          all_hosts = [master, *agents]
+          configure_type_defaults_on(all_hosts)
+
+          # Set PE distribution on the master, create working dir
+          prepare_hosts([master], opts)
+          fetch_pe([master], opts)
+          prepare_host_installer_options(master)
+          generate_installer_conf_file_for(master, [master], opts)
+          on master, installer_cmd(master, opts)
+
+          step "Setup frictionless installer on the master" do
+            agents.each do |agent|
+              # If We're *not* running the classic installer, we want
+              # to make sure the master has packages for us.
+              if agent['platform'] != master['platform'] # only need to do this if platform differs
+                deploy_frictionless_to_master(agent)
+              end
+            end
+          end
+
+          step "Install agents" do
+            agents.group_by {|agent| installer_cmd(agent, opts)}.each do |cmd, agents|
+              on agents, cmd, :run_in_parallel => true
+            end
+          end
+
+          step "Stop puppet agents to avoid interfering with tests" do
+            stop_agent_on(all_hosts, :run_in_parallel => true)
+          end
+
+          step "Sign agent certificates" do
+            # This will sign all cert requests
+            sign_certificate_for(agents)
+          end
+
+          step "Run puppet to setup mcollective and pxp-agent" do
+            on all_hosts, puppet_agent('-t'), :acceptable_exit_codes => [0,2], :run_in_parallel => true
+
+            #Workaround for windows frictionless install, see BKR-943 for the reason
+            agents.select {|agent| agent['platform'] =~ /windows/}.each do |agent|
+              client_datadir = agent.puppet['client_datadir']
+              on(agent, puppet("resource file \"#{client_datadir}\" ensure=absent force=true"))
+            end
+          end
+        end
+
+        def generic_install hosts, opts = {}
+          step "Installing PE on a generic set of hosts"
+
           masterless = opts[:masterless]
           opts[:type] = opts[:type] || :install
           unless masterless

spec/beaker-pe/install/pe_utils_spec.rb

@@ -878,7 +878,64 @@ def slice_installer_options(host)
     end
   end
 
+  describe "#determine_install_type" do
+    let(:monolithic) { make_host('monolithic', :pe_ver => '2016.4', :roles => [ 'master', 'database', 'dashboard' ]) }
+    let(:master) { make_host('master', :pe_ver => '2016.4', :roles => [ 'master' ]) }
+    let(:puppetdb) { make_host('puppetdb', :pe_ver => '2016.4', :roles => [ 'database' ]) }
+    let(:console) { make_host('console', :pe_ver => '2016.4', :roles => [ 'dashboard' ]) }
+    let(:agent) { make_host('agent', :pe_ver => '2016.4', :roles => ['frictionless']) }
+
+    it 'identifies a monolithic install with frictionless agents' do
+      hosts = [monolithic, agent, agent, agent]
+      expect(subject.determine_install_type(hosts, {})).to eq(:simple_monolithic)
+    end
+
+    it 'identifies a monolithic install without frictionless agents' do
+      expect(subject.determine_install_type([monolithic], {})).to eq(:simple_monolithic)
+    end
+
+    it 'identifies a split install with frictionless agents' do
+      hosts = [master, puppetdb, console, agent, agent, agent]
+      expect(subject.determine_install_type(hosts, {})).to eq(:simple_split)
+    end
+
+    it 'identifies a split install without frictionless agents' do
+      hosts = [master, puppetdb, console]
+      expect(subject.determine_install_type(hosts, {})).to eq(:simple_split)
+    end
+
+    it 'identifies an install with multiple agent versions as generic' do
+      new_agent = make_host('agent', :pe_ver => '2017.2', :roles => ['frictionless'])
+      hosts = [monolithic, agent, new_agent]
+      expect(subject.determine_install_type(hosts, {})).to eq(:generic)
+    end
+
+    it 'identifies an upgrade as generic' do
+      hosts = [monolithic, agent, agent, agent]
+      expect(subject.determine_install_type(hosts, {:type => :upgrade})).to eq(:generic)
+    end
+
+    it 'identifies a legacy PE version as generic' do
+      old_monolithic = make_host('monolithic', :pe_ver => '3.8', :roles => [ 'master', 'database', 'dashboard' ])
+      old_agent = make_host('agent', :pe_ver => '3.8', :roles => ['frictionless'])
+      hosts = [old_monolithic, old_agent, old_agent, old_agent]
+      expect(subject.determine_install_type(hosts, {})).to eq(:generic)
+    end
+
+    it 'identifies a non-standard install as generic' do
+      hosts = [monolithic, master, agent, agent, agent]
+      expect(subject.determine_install_type(hosts, {})).to eq(:generic)
+    end
+  end
+
   describe 'do_install' do
+    it 'chooses to do a simple monolithic install when appropriate' do
+      expect(subject).to receive(:simple_monolithic_install)
+      allow(subject).to receive(:determine_install_type).and_return(:simple_monolithic)
+
+      subject.do_install([])
+    end
+
     it 'can perform a simple installation' do
       allow( subject ).to receive( :on ).and_return( Beaker::Result.new( {}, '' ) )
       allow( subject ).to receive( :fetch_pe ).and_return( true )
@@ -1186,6 +1243,78 @@ def slice_installer_options(host)
 
   end
 
+  describe 'simple_monolithic_install' do
+    let(:monolithic) { make_host('monolithic', :pe_ver => '2016.4', :platform => 'el-7-x86_64', :roles => [ 'master', 'database', 'dashboard' ]) }
+    let(:el_agent) { make_host('agent', :pe_ver => '2016.4', :platform => 'el-7-x86_64', :roles => ['frictionless']) }
+    let(:deb_agent) { make_host('agent', :pe_ver => '2016.4', :platform => 'debian-7-x86_64', :roles => ['frictionless']) }
+
+    before :each do
+      allow(subject).to receive(:on)
+      allow(subject).to receive(:configure_type_defaults_on)
+      allow(subject).to receive(:prepare_hosts)
+      allow(subject).to receive(:fetch_pe)
+      allow(subject).to receive(:prepare_host_installer_options)
+      allow(subject).to receive(:generate_installer_conf_file_for)
+      allow(subject).to receive(:deploy_frictionless_to_master)
+
+      allow(subject).to receive(:installer_cmd).with(monolithic, anything()).and_return("install master")
+      allow(subject).to receive(:installer_cmd).with(el_agent, anything()).and_return("install el agent")
+      allow(subject).to receive(:installer_cmd).with(deb_agent, anything()).and_return("install deb agent")
+
+      allow(subject).to receive(:stop_agent_on)
+      allow(subject).to receive(:sign_certificate_for)
+    end
+
+    describe 'configuring frictionless installer' do
+      it "skips the master's platform" do
+        expect(subject).not_to receive(:deploy_frictionless_to_master)
+
+        subject.simple_monolithic_install(monolithic, [el_agent, el_agent, el_agent])
+      end
+
+      it "adds frictionless install classes for other platforms" do
+        expect(subject).to receive(:deploy_frictionless_to_master).with(deb_agent)
+
+        subject.simple_monolithic_install(monolithic, [el_agent, deb_agent])
+      end
+    end
+
+    it 'installs on the master then on the agents' do
+      expect(subject).to receive(:on).with(monolithic, "install master").ordered
+      expect(subject).to receive(:on).with([el_agent, el_agent], "install el agent", anything()).ordered
+
+      subject.simple_monolithic_install(monolithic, [el_agent, el_agent])
+    end
+
+    it 'installs agents in parallel if their install command is the same' do
+      expect(subject).to receive(:on).with([el_agent, el_agent], "install el agent", :run_in_parallel => true)
+      expect(subject).to receive(:on).with([deb_agent, deb_agent], "install deb agent", :run_in_parallel => true)
+
+      subject.simple_monolithic_install(monolithic, [el_agent, el_agent, deb_agent, deb_agent])
+    end
+
+    it 'signs all certificates at once' do
+      agents = [el_agent, el_agent, deb_agent, deb_agent]
+      expect(subject).to receive(:sign_certificate_for).with(agents)
+
+      subject.simple_monolithic_install(monolithic, agents)
+    end
+
+    it 'stops the agents in parallel to avoid interference with tests' do
+      agents = [el_agent, el_agent, deb_agent, deb_agent]
+      expect(subject).to receive(:stop_agent_on).with([monolithic, *agents], :run_in_parallel => true)
+
+      subject.simple_monolithic_install(monolithic, agents)
+    end
+
+    it 'runs agents in parallel, only one time' do
+      agents = [el_agent, el_agent, deb_agent, deb_agent]
+      expect(subject).to receive(:on).with([monolithic, *agents], proc {|cmd| cmd.command == "puppet agent"}, hash_including(:run_in_parallel => true)).once
+
+      subject.simple_monolithic_install(monolithic, agents)
+    end
+  end
+
   describe 'do_higgs_install' do
 
     before :each do