(puppetlabsGH-121) Load Puppet 3 and 4 API Functions via Puppet Strings

Currently the Sidecar can only detect and load Puppet 3 API functions.  The
newer Puppet 4 API functions use a different loader and schema, and importantly
have additional properties e.g. Puppet 4 API functions have one or more
signatures, whereas Puppet 3 API functions use arity.

In order to add the Puppet Strings loader, this commit will load Puppet 4 API
functions but present them to the Language Server as if they were Puppet 3.
A later commit will then change this behaviour so that all the metadata for
Puppet 4 API functions will be known by the Language Server, and Puppet 3 API
function metadata will be munged into the 4 API equivalent.

This commit:
* Extends the Sidecar protocol to add a function_version property to the Puppet
  Function schema.  This can be used later by the Language Server to determine
  how to handle the metadata.
* Adds a new method called retrieve_via_puppet_strings to the puppet_helper.
  This method queries the Puppet Loaders (vai Puppet-As-A-Library PAL) for all
  the files for particular puppet objects (functions in this case) and then gets
  the documentation about these files via the Puppet Strings helper
* The PAL files are only available on Puppet Gem 6 and above, so the feature
  flag is modified to only be active on Puppet version 6+
* Adds in a new method called 'discover_paths' on all PAL loaders.  The loaders
  themselves are normally used to load _something_ by name, however the Sidecar
  wants to load EVERYTHING. Generally this information is private to each
  loader. By adding this additional method, we can extract all of the
  discoverable paths, without needing to write our own loaders
* Removes the old function loading and monkey patches

lib/puppet-languageserver-sidecar/puppet_helper_puppetstrings.rb

@@ -34,6 +34,54 @@ def self.get_puppet_resource(typename, title = nil)
       result
     end
 
+    # Puppet Strings loading
+    def self.available_documentation_types
+      [:function]
+    end
+
+    # Retrieve objects via the Puppet 4 API loaders
+    def self.retrieve_via_puppet_strings(_cache, options = {})
+      PuppetLanguageServerSidecar.log_message(:debug, '[PuppetHelper::retrieve_via_puppet_strings] Starting')
+
+      object_types = options[:object_types].nil? ? available_documentation_types : options[:object_types]
+      object_types.select! { |i| available_documentation_types.include?(i) }
+
+      result = {}
+      return result if object_types.empty?
+
+      result[:functions] = PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList.new if object_types.include?(:function)
+
+      current_env = current_environment
+      for_agent = options[:for_agent].nil? ? true : options[:for_agent]
+      loaders = Puppet::Pops::Loaders.new(current_env, for_agent)
+
+      paths = []
+      paths.concat(discover_type_paths(:function, loaders)) if object_types.include?(:function)
+
+      paths.each do |path|
+        next unless path_has_child?(options[:root_path], path)
+        file_doc = PuppetLanguageServerSidecar::PuppetStringsHelper.file_documentation(path)
+        next if file_doc.nil?
+
+        if object_types.include?(:function) # rubocop:disable Style/IfUnlessModifier   This reads better
+          file_doc.functions.each { |item| result[:functions] << item }
+        end
+      end
+
+      # Remove Puppet3 functions which have a Puppet4 function already loaded
+      if object_types.include?(:function)
+        pup4_functions = result[:functions].select { |i| i.function_version == 4 }.map { |i| i.key }
+        result[:functions].reject! { |i| i.function_version == 3 && pup4_functions.include?(i.key) }
+      end
+
+      result.each { |key, item| PuppetLanguageServerSidecar.log_message(:debug, "[PuppetHelper::retrieve_via_puppet_strings] Finished loading #{item.count} #{key}") }
+      result
+    end
+
+    def self.discover_type_paths(type, loaders)
+      loaders.private_environment_loader.discover_paths(type)
+    end
+
     # Class and Defined Type loading
     def self.retrieve_classes(cache, options = {})
       PuppetLanguageServerSidecar.log_message(:debug, '[PuppetHelper::retrieve_classes] Starting')
@@ -65,57 +113,6 @@ def self.retrieve_classes(cache, options = {})
       classes
     end
 
-    # Function loading
-    def self.retrieve_functions(cache, options = {})
-      PuppetLanguageServerSidecar.log_message(:debug, '[PuppetHelper::load_functions] Starting')
-
-      autoloader = Puppet::Util::Autoload.new(self, 'puppet/parser/functions')
-      current_env = current_environment
-      funcs = PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList.new
-
-      # Functions that are already loaded (e.g. system default functions like alert)
-      # should already be populated so insert them into the function results
-      #
-      # Find the unique filename list
-      filenames = []
-      Puppet::Parser::Functions.monkey_function_list.each_value do |data|
-        filenames << data[:source_location][:source] unless data[:source_location].nil? || data[:source_location][:source].nil?
-      end
-      # Now add the functions in each file to the cache
-      filenames.uniq.compact.each do |filename|
-        Puppet::Parser::Functions.monkey_function_list
-                                 .select { |_k, i| filename.casecmp(i[:source_location][:source].to_s).zero? }
-                                 .select { |_k, i| path_has_child?(options[:root_path], i[:source_location][:source]) }
-                                 .each do |name, item|
-          obj = PuppetLanguageServerSidecar::Protocol::PuppetFunction.from_puppet(name, item)
-          funcs << obj
-        end
-      end
-
-      # Now we can load functions from the default locations
-      if autoloader.method(:files_to_load).arity.zero?
-        params = []
-      else
-        params = [current_env]
-      end
-      autoloader.files_to_load(*params).each do |file|
-        name = file.gsub(autoloader.path + '/', '')
-        begin
-          expanded_name = autoloader.expand(name)
-          absolute_name = Puppet::Util::Autoload.get_file(expanded_name, current_env)
-          raise("Could not find absolute path of function #{name}") if absolute_name.nil?
-          if path_has_child?(options[:root_path], absolute_name) # rubocop:disable Style/IfUnlessModifier  Nicer to read like this
-            funcs.concat(load_function_file(cache, name, absolute_name, autoloader, current_env))
-          end
-        rescue StandardError => e
-          PuppetLanguageServerSidecar.log_message(:error, "[PuppetHelper::load_functions] Error loading function #{file}: #{e} #{e.backtrace}")
-        end
-      end
-
-      PuppetLanguageServerSidecar.log_message(:debug, "[PuppetHelper::load_functions] Finished loading #{funcs.count} functions")
-      funcs
-    end
-
     def self.retrieve_types(cache, options = {})
       PuppetLanguageServerSidecar.log_message(:debug, '[PuppetHelper::retrieve_types] Starting')
 
@@ -252,44 +249,6 @@ def self.load_classes_from_manifest(cache, manifest_file)
     end
     private_class_method :load_classes_from_manifest
 
-    def self.load_function_file(cache, name, absolute_name, autoloader, env)
-      funcs = PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList.new
-
-      if cache.active?
-        cached_result = cache.load(absolute_name, PuppetLanguageServerSidecar::Cache::FUNCTIONS_SECTION)
-        unless cached_result.nil?
-          begin
-            funcs.from_json!(cached_result)
-            return funcs
-          rescue StandardError => e
-            PuppetLanguageServerSidecar.log_message(:warn, "[PuppetHelper::load_function_file] Error while deserializing #{absolute_name} from cache: #{e}")
-            funcs = PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList.new
-          end
-        end
-      end
-
-      unless autoloader.loaded?(name)
-        # This is an expensive call
-        unless autoloader.load(name, env) # rubocop:disable Style/IfUnlessModifier  Nicer to read like this
-          PuppetLanguageServerSidecar.log_message(:error, "[PuppetHelper::load_function_file] function #{absolute_name} did not load")
-        end
-      end
-
-      # Find the functions that were loaded based on source file name (case insensitive)
-      Puppet::Parser::Functions.monkey_function_list
-                               .select { |_k, i| absolute_name.casecmp(i[:source_location][:source].to_s).zero? }
-                               .each do |func_name, item|
-        obj = PuppetLanguageServerSidecar::Protocol::PuppetFunction.from_puppet(func_name, item)
-        obj.calling_source = absolute_name
-        funcs << obj
-      end
-      PuppetLanguageServerSidecar.log_message(:warn, "[PuppetHelper::load_function_file] file #{absolute_name} did load any functions") if funcs.count.zero?
-      cache.save(absolute_name, PuppetLanguageServerSidecar::Cache::FUNCTIONS_SECTION, funcs.to_json) if cache.active?
-
-      funcs
-    end
-    private_class_method :load_function_file
-
     def self.load_type_file(cache, name, absolute_name, autoloader, env)
       types = PuppetLanguageServer::Sidecar::Protocol::PuppetTypeList.new
       if cache.active?

lib/puppet-languageserver-sidecar/puppet_monkey_patches_puppetstrings.rb

@@ -1,54 +1,5 @@
 # frozen_string_literal: true
 
-# Monkey Patch 3.x functions so where know where they were loaded from
-require 'puppet/parser/functions'
-module Puppet
-  module Parser
-    module Functions
-      class << self
-        alias_method :original_newfunction, :newfunction
-        def newfunction(name, options = {}, &block)
-          # See if we've hooked elsewhere. This can happen while in debuggers (pry). If we're not in the previous caller
-          # stack then just use the last caller
-          monkey_index = Kernel.caller_locations.find_index { |loc| loc.path.match(/puppet_monkey_patches\.rb/) }
-          monkey_index = -1 if monkey_index.nil?
-          caller = Kernel.caller_locations[monkey_index + 1]
-          # Call the original new function method
-          result = original_newfunction(name, options, &block)
-          # Append the caller information
-          result[:source_location] = {
-            :source => caller.absolute_path,
-            :line   => caller.lineno - 1 # Convert to a zero based line number system
-          }
-          monkey_append_function_info(name, result)
-
-          result
-        end
-
-        def monkey_clear_function_info
-          @monkey_function_list = {}
-        end
-
-        def monkey_append_function_info(name, value)
-          @monkey_function_list = {} if @monkey_function_list.nil?
-          @monkey_function_list[name] = {
-            :arity           => value[:arity],
-            :name            => value[:name],
-            :type            => value[:type],
-            :doc             => value[:doc],
-            :source_location => value[:source_location]
-          }
-        end
-
-        def monkey_function_list
-          @monkey_function_list = {} if @monkey_function_list.nil?
-          @monkey_function_list.clone
-        end
-      end
-    end
-  end
-end
-
 # Add an additional method on Puppet Types to store their source location
 require 'puppet/type'
 module Puppet
@@ -80,6 +31,83 @@ def newtype(name, options = {}, &block)
   end
 end
 
+if Gem::Version.new(Puppet.version) >= Gem::Version.new('6.0.0')
+  # Due to PUP-9509, need to monkey patch the cache loader
+  # This need to be guarded on Puppet 6.0.0+
+  require 'puppet/pops/loader/module_loaders'
+  module Puppet
+    module Pops
+      module Loader
+        module ModuleLoaders
+          def self.cached_loader_from(parent_loader, loaders)
+            LibRootedFileBased.new(parent_loader,
+                                   loaders,
+                                   NAMESPACE_WILDCARD,
+                                   Puppet[:libdir],
+                                   'cached_puppet_lib',
+                                   %i[func_4x func_3x datatype])
+          end
+        end
+      end
+    end
+  end
+end
+
+module Puppet
+  module Pops
+    module Loader
+      class Loader
+        def discover_paths(type, name_authority = Pcore::RUNTIME_NAME_AUTHORITY)
+          if parent.nil?
+            []
+          else
+            parent.discover_paths(type, name_authority)
+          end
+        end
+      end
+    end
+  end
+end
+
+module Puppet
+  module Pops
+    module Loader
+      class DependencyLoader
+        def discover_paths(type, name_authority = Pcore::RUNTIME_NAME_AUTHORITY)
+          result = []
+
+          @dependency_loaders.each { |loader| result.concat(loader.discover_paths(type, name_authority)) }
+          result.concat(super)
+          result.uniq
+        end
+      end
+    end
+  end
+end
+
+module Puppet
+  module Pops
+    module Loader
+      module ModuleLoaders
+        class AbstractPathBasedModuleLoader
+          def discover_paths(type, name_authority = Pcore::RUNTIME_NAME_AUTHORITY)
+            result = []
+            if name_authority == Pcore::RUNTIME_NAME_AUTHORITY
+              smart_paths.effective_paths(type).each do |sp|
+                relative_paths(sp).each do |rp|
+                  result << File.join(sp.generic_path, rp)
+                end
+              end
+            end
+            result.concat(super)
+            result.uniq
+          end
+        end
+      end
+    end
+  end
+end
+
 # MUST BE LAST!!!!!!
 # Suppress any warning messages to STDOUT.  It can pollute stdout when running in STDIO mode
 Puppet::Util::Log.newdesttype :null_logger do

lib/puppet-languageserver-sidecar/puppet_strings_helper.rb

@@ -10,6 +10,7 @@ def self.file_documentation(path)
       return nil unless require_puppet_strings
       @helper_cache = FileDocumentationCache.new if @helper_cache.nil?
       return @helper_cache.document(path) if @helper_cache.path_exists?(path)
+
       PuppetLanguageServerSidecar.log_message(:debug, "[PuppetStringsHelper::file_documentation] Fetching documentation for #{path}")
 
       setup_yard!
@@ -80,16 +81,62 @@ def document(path)
     def populate_from_yard_registry!
       # Extract all of the information
       # Ref - https://github.com/puppetlabs/puppet-strings/blob/87a8e10f45bfeb7b6b8e766324bfb126de59f791/lib/puppet-strings/json.rb#L10-L16
+      populate_functions_from_yard_registry!
     end
 
     private
 
+    def populate_functions_from_yard_registry!
+      ::YARD::Registry.all(:puppet_function).map(&:to_hash).each do |item|
+        source_path = item[:file]
+        func_name = item[:name].to_s
+        @cache[source_path] = FileDocumentation.new(source_path) if @cache[source_path].nil?
+
+        obj                  = PuppetLanguageServer::Sidecar::Protocol::PuppetFunction.new
+        obj.key              = func_name
+        obj.source           = item[:file]
+        obj.calling_source   = obj.source
+        obj.line             = item[:line]
+        obj.doc              = item[:docstring][:text]
+        obj.arity            = -1 # We don't care about arity
+        obj.function_version = item[:type] == 'ruby4x' ? 4 : 3
+
+        # Try and determine the function call site from the source file
+        char = item[:source].index(":#{func_name}")
+        unless char.nil?
+          obj.char = char
+          obj.length = func_name.length + 1
+        end
+
+        case item[:type]
+        when 'ruby3x'
+          obj.function_version = 3
+          # This is a bit hacky but it works (probably).  Puppet-Strings doesn't rip this information out, but you do have the
+          # the source to query
+          obj.type = item[:source].match(/:rvalue/) ? :rvalue : :statement
+        when 'ruby4x'
+          obj.function_version = 4
+          # All ruby functions are statements
+          obj.type = :statement
+        else
+          PuppetLanguageServerSidecar.log_message(:error, "[#{self.class}] Unknown function type #{item[:type]}")
+        end
+
+        @cache[source_path].functions << obj
+      end
+    end
+  end
+
   class FileDocumentation
     # The path to file that has been documented
-    attr_reader :path
+    attr_accessor :path
+
+    # PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList object holding all functions
+    attr_accessor :functions
 
-    def initialize(path)
+    def initialize(path = nil)
       @path = path
+      @functions = PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList.new
     end
   end
 end

lib/puppet-languageserver/sidecar_protocol.rb

@@ -154,12 +154,15 @@ class PuppetFunction < BasePuppetObject
         attr_accessor :doc
         attr_accessor :arity
         attr_accessor :type
+        # The version of this function, typically 3 or 4.
+        attr_accessor :function_version
 
         def to_h
           super.to_h.merge(
-            'doc'   => doc,
-            'arity' => arity,
-            'type'  => type
+            'doc'              => doc,
+            'arity'            => arity,
+            'type'             => type,
+            'function_version' => function_version
           )
         end
 
@@ -169,6 +172,7 @@ def from_h!(value)
           self.doc = value['doc']
           self.arity = value['arity']
           self.type = value['type'].intern
+          self.function_version = value['function_version']
           self
         end
       end