(puppetlabsGH-121) Use the Puppet 4 loaders to load Puppet 3 functions

Previously functions were loaded with the autoloader classes however in the new
Puppet 4 API we use the environment loaders.  In particular we use the discover
method to discover all of the available "things", in this case functions, within
the environment.

`current_env.loaders.private_environment_loader.discover(:function)`

* Note this commit only loads the Puppet 3 API functions using the Puppet 4
loaders,  thus there are no test changes.  Later commits will load Puppet 4
functions

* Note we use the private loader, not the public one, because we want all of the
available things, not just the ones that are publicly exposed.  We still need
intellisense on private objects.

* Note that we monkey patch the ruby_legacy_function_instantiator (Puppet 3 API
functions) and ruby_function_instantiator (Puppet 4 API functions) as in
default Puppet any errors would terminate all function discovery.  Instead we
trap and log the error and move on.

* Due to PUP-9509 we need to monkey patch the cached loader to also load Puppet
3 API functions. Previously these were missing

* The Sidecar then uses the appropriate method to enumerate the available
functions based on the presence or absence of the feature flag.

lib/puppet-languageserver-sidecar/puppet_helper_pup4api.rb

@@ -147,6 +147,52 @@ def self.retrieve_types(cache, options = {})
       types
     end
 
+    # Loading via the Puppet 4 Language API
+    def self.available_object_types
+      [:function]
+    end
+
+    # Retrieve objects via the Puppet 4 API loaders
+    def self.retrieve_via_pup4_api(_cache, options = {})
+      PuppetLanguageServerSidecar.log_message(:debug, '[PuppetHelper::retrieve_via_pup4_api] Starting')
+
+      object_types = options[:object_types].nil? ? available_object_types : options[:object_types]
+      object_types.select! { |i| available_object_types.include?(i) }
+
+      result = {}
+      return compilation if object_types.empty?
+
+      result[:functions] = PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList.new if object_types.include?(:function)
+
+      current_env = current_environment
+      for_agent = options[:for_agent].nil? ? true : options[:for_agent]
+      loaders = Puppet::Pops::Loaders.new(current_env, for_agent)
+
+      context_overrides = {
+        :current_environment => current_env,
+        :loaders             => loaders,
+        :rich_data           => true
+      }
+
+      # TODO: Needed? Puppet[:tasks] = true
+      Puppet.override(context_overrides, 'LanguageServer Sidecar') do
+        current_env.loaders.private_environment_loader.discover(:function) if object_types.include?(:function)
+      end
+
+      if object_types.include?(:function)
+        # Enumerate V3 Functions from the monkey patching
+        Puppet::Parser::Functions.monkey_function_list
+                                 .select { |_k, i| path_has_child?(options[:root_path], i[:source_location][:source]) }
+                                 .each do |name, item|
+          obj = PuppetLanguageServerSidecar::Protocol::PuppetFunction.from_puppet(name, item)
+          result[:functions] << obj
+        end
+      end
+      PuppetLanguageServerSidecar.log_message(:debug, "[PuppetHelper::retrieve_via_pup4_api] Finished loading #{result[:functions].count} functions") if object_types.include?(:function)
+
+      result
+    end
+
     # Private functions
 
     def self.prune_resource_parameters(resources)

lib/puppet-languageserver-sidecar/puppet_monkey_patches_pup4api.rb

@@ -78,6 +78,92 @@ def newtype(name, options = {}, &block)
   end
 end
 
+# The Ruby Legacy Function Instantiator doesn't have any error catching upon loading and would normally cause the entire puppet
+# run to fail. However as we're a bit special, we can wrap the loader in rescue block and just continue on
+require 'puppet/pops/loader/ruby_legacy_function_instantiator'
+module Puppet
+  module Pops
+    module Loader
+      class RubyLegacyFunctionInstantiator
+        class << self
+          alias_method :original_create, :create
+        end
+
+        def self.create(loader, typed_name, source_ref, ruby_code_string)
+          original_create(loader, typed_name, source_ref, ruby_code_string)
+        rescue LoadError, StandardError => err
+          PuppetLanguageServerSidecar.log_message(:error, "[MonkeyPatch::Puppet::Pops::Loader::RubyLegacyFunctionInstantiator] Error loading legacy function #{typed_name.name}: #{err} #{err.backtrace}")
+        end
+      end
+    end
+  end
+end
+
+# The Ruby Function Instantiator doesn't have any error catching upon loading and would normally cause the entire puppet
+# run to fail. However as we're a bit special, we can wrap the loader in rescue block and just continue on
+require 'puppet/pops/loader/ruby_function_instantiator'
+module Puppet
+  module Pops
+    module Loader
+      class RubyFunctionInstantiator
+        class << self
+          alias_method :original_create, :create
+        end
+
+        def self.create(loader, typed_name, source_ref, ruby_code_string)
+          original_create(loader, typed_name, source_ref, ruby_code_string)
+        rescue LoadError, StandardError => err
+          PuppetLanguageServerSidecar.log_message(:error, "[MonkeyPatch::Puppet::Pops::Loader::RubyLegacyFunctionInstantiator] Error loading function #{typed_name.name}: #{err} #{err.backtrace}")
+        end
+      end
+    end
+  end
+end
+
+if Gem::Version.new(Puppet.version) >= Gem::Version.new('6.0.0')
+  # Due to PUP-9509, need to monkey patch the cache loader
+  # This need to be guarded on Puppet 6.0.0+
+  require 'puppet/pops/loader/module_loaders'
+  module Puppet
+    module Pops
+      module Loader
+        module ModuleLoaders
+          def self.cached_loader_from(parent_loader, loaders)
+            LibRootedFileBased.new(parent_loader,
+                                   loaders,
+                                   NAMESPACE_WILDCARD,
+                                   Puppet[:libdir],
+                                   'cached_puppet_lib',
+                                   %i[func_4x func_3x datatype])
+          end
+        end
+      end
+    end
+  end
+end
+
+# DEBUG It's hard to remember what locations are searched. This simple monkey patch
+# DEBUG will output what paths are being searched by what SmartPaths
+# module Puppet::Pops
+#   module Loader
+#     module ModuleLoaders
+#       class AbstractPathBasedModuleLoader
+#         alias_method :original_discover, :discover
+
+#         def discover(type, error_collector = nil, name_authority = Pcore::RUNTIME_NAME_AUTHORITY, &block)
+#           PuppetLanguageServerSidecar.log_message(:debug, "--- AbstractPathBasedModuleLoader::discover name=#{self.loader_name}")
+#           if name_authority == Pcore::RUNTIME_NAME_AUTHORITY
+#             smart_paths.effective_paths(type).each do |sp|
+#               PuppetLanguageServerSidecar.log_message(:debug, "Using SmartPath [#{sp.class.to_s}]Root=#{sp.root_path}")
+#             end
+#           end
+#           original_discover(type, error_collector, name_authority, &block)
+#         end
+#       end
+#     end
+#   end
+# end
+
 # MUST BE LAST!!!!!!
 # Suppress any warning messages to STDOUT.  It can pollute stdout when running in STDIO mode
 Puppet::Util::Log.newdesttype :null_logger do

lib/puppet_languageserver_sidecar.rb

@@ -212,7 +212,8 @@ def self.init_puppet_sidecar(options)
   def self.inject_workspace_as_module
     return false unless PuppetLanguageServerSidecar::Workspace.has_module_metadata?
 
-    Puppet.settings[:basemodulepath] = Puppet.settings[:basemodulepath] + ';' + PuppetLanguageServerSidecar::Workspace.root_path
+    # TODO: Is this really needed?
+    # Puppet.settings[:basemodulepath] = Puppet.settings[:basemodulepath] + ';' + PuppetLanguageServerSidecar::Workspace.root_path
 
     %w[puppet_modulepath_monkey_patches].each do |lib|
       begin
@@ -244,6 +245,8 @@ def self.inject_workspace_as_environment
   end
 
   def self.execute(options)
+    use_pup4api = featureflag?('pup4api')
+
     case options[:action].downcase
     when 'noop'
       []
@@ -254,7 +257,11 @@ def self.execute(options)
 
     when 'default_functions'
       cache = options[:disable_cache] ? PuppetLanguageServerSidecar::Cache::Null.new : PuppetLanguageServerSidecar::Cache::FileSystem.new
-      PuppetLanguageServerSidecar::PuppetHelper.retrieve_functions(cache)
+      if use_pup4api
+        PuppetLanguageServerSidecar::PuppetHelper.retrieve_via_pup4_api(cache, :object_types => [:function])[:functions]
+      else
+        PuppetLanguageServerSidecar::PuppetHelper.retrieve_functions(cache)
+      end
 
     when 'default_types'
       cache = options[:disable_cache] ? PuppetLanguageServerSidecar::Cache::Null.new : PuppetLanguageServerSidecar::Cache::FileSystem.new
@@ -294,8 +301,14 @@ def self.execute(options)
     when 'workspace_functions'
       null_cache = PuppetLanguageServerSidecar::Cache::Null.new
       return nil unless inject_workspace_as_module || inject_workspace_as_environment
-      PuppetLanguageServerSidecar::PuppetHelper.retrieve_functions(null_cache,
-                                                                   :root_path => PuppetLanguageServerSidecar::Workspace.root_path)
+      if use_pup4api
+        PuppetLanguageServerSidecar::PuppetHelper.retrieve_via_pup4_api(null_cache,
+                                                                        :root_path    => PuppetLanguageServerSidecar::Workspace.root_path,
+                                                                        :object_types => [:function])[:functions]
+      else
+        PuppetLanguageServerSidecar::PuppetHelper.retrieve_functions(null_cache,
+                                                                     :root_path => PuppetLanguageServerSidecar::Workspace.root_path)
+      end
 
     when 'workspace_types'
       null_cache = PuppetLanguageServerSidecar::Cache::Null.new