diff --git a/lib/puppet-languageserver-sidecar/puppet_helper_puppetstrings.rb b/lib/puppet-languageserver-sidecar/puppet_helper_puppetstrings.rb index 4c16f401..d3c6e55b 100644 --- a/lib/puppet-languageserver-sidecar/puppet_helper_puppetstrings.rb +++ b/lib/puppet-languageserver-sidecar/puppet_helper_puppetstrings.rb @@ -34,6 +34,54 @@ def self.get_puppet_resource(typename, title = nil) result end + # Puppet Strings loading + def self.available_documentation_types + [:function] + end + + # Retrieve objects via the Puppet 4 API loaders + def self.retrieve_via_puppet_strings(_cache, options = {}) + PuppetLanguageServerSidecar.log_message(:debug, '[PuppetHelper::retrieve_via_puppet_strings] Starting') + + object_types = options[:object_types].nil? ? available_documentation_types : options[:object_types] + object_types.select! { |i| available_documentation_types.include?(i) } + + result = {} + return result if object_types.empty? + + result[:functions] = PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList.new if object_types.include?(:function) + + current_env = current_environment + for_agent = options[:for_agent].nil? ? true : options[:for_agent] + loaders = Puppet::Pops::Loaders.new(current_env, for_agent) + + paths = [] + paths.concat(discover_type_paths(:function, loaders)) if object_types.include?(:function) + + paths.each do |path| + next unless path_has_child?(options[:root_path], path) + file_doc = PuppetLanguageServerSidecar::PuppetStringsHelper.file_documentation(path) + next if file_doc.nil? + + if object_types.include?(:function) # rubocop:disable Style/IfUnlessModifier This reads better + file_doc.functions.each { |item| result[:functions] << item } + end + end + + # Remove Puppet3 functions which have a Puppet4 function already loaded + if object_types.include?(:function) + pup4_functions = result[:functions].select { |i| i.function_version == 4 }.map { |i| i.key } + result[:functions].reject! { |i| i.function_version == 3 && pup4_functions.include?(i.key) } + end + + result.each { |key, item| PuppetLanguageServerSidecar.log_message(:debug, "[PuppetHelper::retrieve_via_puppet_strings] Finished loading #{item.count} #{key}") } + result + end + + def self.discover_type_paths(type, loaders) + loaders.private_environment_loader.discover_paths(type) + end + # Class and Defined Type loading def self.retrieve_classes(cache, options = {}) PuppetLanguageServerSidecar.log_message(:debug, '[PuppetHelper::retrieve_classes] Starting') @@ -65,57 +113,6 @@ def self.retrieve_classes(cache, options = {}) classes end - # Function loading - def self.retrieve_functions(cache, options = {}) - PuppetLanguageServerSidecar.log_message(:debug, '[PuppetHelper::load_functions] Starting') - - autoloader = Puppet::Util::Autoload.new(self, 'puppet/parser/functions') - current_env = current_environment - funcs = PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList.new - - # Functions that are already loaded (e.g. system default functions like alert) - # should already be populated so insert them into the function results - # - # Find the unique filename list - filenames = [] - Puppet::Parser::Functions.monkey_function_list.each_value do |data| - filenames << data[:source_location][:source] unless data[:source_location].nil? || data[:source_location][:source].nil? - end - # Now add the functions in each file to the cache - filenames.uniq.compact.each do |filename| - Puppet::Parser::Functions.monkey_function_list - .select { |_k, i| filename.casecmp(i[:source_location][:source].to_s).zero? } - .select { |_k, i| path_has_child?(options[:root_path], i[:source_location][:source]) } - .each do |name, item| - obj = PuppetLanguageServerSidecar::Protocol::PuppetFunction.from_puppet(name, item) - funcs << obj - end - end - - # Now we can load functions from the default locations - if autoloader.method(:files_to_load).arity.zero? - params = [] - else - params = [current_env] - end - autoloader.files_to_load(*params).each do |file| - name = file.gsub(autoloader.path + '/', '') - begin - expanded_name = autoloader.expand(name) - absolute_name = Puppet::Util::Autoload.get_file(expanded_name, current_env) - raise("Could not find absolute path of function #{name}") if absolute_name.nil? - if path_has_child?(options[:root_path], absolute_name) # rubocop:disable Style/IfUnlessModifier Nicer to read like this - funcs.concat(load_function_file(cache, name, absolute_name, autoloader, current_env)) - end - rescue StandardError => e - PuppetLanguageServerSidecar.log_message(:error, "[PuppetHelper::load_functions] Error loading function #{file}: #{e} #{e.backtrace}") - end - end - - PuppetLanguageServerSidecar.log_message(:debug, "[PuppetHelper::load_functions] Finished loading #{funcs.count} functions") - funcs - end - def self.retrieve_types(cache, options = {}) PuppetLanguageServerSidecar.log_message(:debug, '[PuppetHelper::retrieve_types] Starting') @@ -252,44 +249,6 @@ def self.load_classes_from_manifest(cache, manifest_file) end private_class_method :load_classes_from_manifest - def self.load_function_file(cache, name, absolute_name, autoloader, env) - funcs = PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList.new - - if cache.active? - cached_result = cache.load(absolute_name, PuppetLanguageServerSidecar::Cache::FUNCTIONS_SECTION) - unless cached_result.nil? - begin - funcs.from_json!(cached_result) - return funcs - rescue StandardError => e - PuppetLanguageServerSidecar.log_message(:warn, "[PuppetHelper::load_function_file] Error while deserializing #{absolute_name} from cache: #{e}") - funcs = PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList.new - end - end - end - - unless autoloader.loaded?(name) - # This is an expensive call - unless autoloader.load(name, env) # rubocop:disable Style/IfUnlessModifier Nicer to read like this - PuppetLanguageServerSidecar.log_message(:error, "[PuppetHelper::load_function_file] function #{absolute_name} did not load") - end - end - - # Find the functions that were loaded based on source file name (case insensitive) - Puppet::Parser::Functions.monkey_function_list - .select { |_k, i| absolute_name.casecmp(i[:source_location][:source].to_s).zero? } - .each do |func_name, item| - obj = PuppetLanguageServerSidecar::Protocol::PuppetFunction.from_puppet(func_name, item) - obj.calling_source = absolute_name - funcs << obj - end - PuppetLanguageServerSidecar.log_message(:warn, "[PuppetHelper::load_function_file] file #{absolute_name} did load any functions") if funcs.count.zero? - cache.save(absolute_name, PuppetLanguageServerSidecar::Cache::FUNCTIONS_SECTION, funcs.to_json) if cache.active? - - funcs - end - private_class_method :load_function_file - def self.load_type_file(cache, name, absolute_name, autoloader, env) types = PuppetLanguageServer::Sidecar::Protocol::PuppetTypeList.new if cache.active? diff --git a/lib/puppet-languageserver-sidecar/puppet_monkey_patches_puppetstrings.rb b/lib/puppet-languageserver-sidecar/puppet_monkey_patches_puppetstrings.rb index f082ff78..4b092a17 100644 --- a/lib/puppet-languageserver-sidecar/puppet_monkey_patches_puppetstrings.rb +++ b/lib/puppet-languageserver-sidecar/puppet_monkey_patches_puppetstrings.rb @@ -1,54 +1,5 @@ # frozen_string_literal: true -# Monkey Patch 3.x functions so where know where they were loaded from -require 'puppet/parser/functions' -module Puppet - module Parser - module Functions - class << self - alias_method :original_newfunction, :newfunction - def newfunction(name, options = {}, &block) - # See if we've hooked elsewhere. This can happen while in debuggers (pry). If we're not in the previous caller - # stack then just use the last caller - monkey_index = Kernel.caller_locations.find_index { |loc| loc.path.match(/puppet_monkey_patches\.rb/) } - monkey_index = -1 if monkey_index.nil? - caller = Kernel.caller_locations[monkey_index + 1] - # Call the original new function method - result = original_newfunction(name, options, &block) - # Append the caller information - result[:source_location] = { - :source => caller.absolute_path, - :line => caller.lineno - 1 # Convert to a zero based line number system - } - monkey_append_function_info(name, result) - - result - end - - def monkey_clear_function_info - @monkey_function_list = {} - end - - def monkey_append_function_info(name, value) - @monkey_function_list = {} if @monkey_function_list.nil? - @monkey_function_list[name] = { - :arity => value[:arity], - :name => value[:name], - :type => value[:type], - :doc => value[:doc], - :source_location => value[:source_location] - } - end - - def monkey_function_list - @monkey_function_list = {} if @monkey_function_list.nil? - @monkey_function_list.clone - end - end - end - end -end - # Add an additional method on Puppet Types to store their source location require 'puppet/type' module Puppet @@ -80,6 +31,83 @@ def newtype(name, options = {}, &block) end end +if Gem::Version.new(Puppet.version) >= Gem::Version.new('6.0.0') + # Due to PUP-9509, need to monkey patch the cache loader + # This need to be guarded on Puppet 6.0.0+ + require 'puppet/pops/loader/module_loaders' + module Puppet + module Pops + module Loader + module ModuleLoaders + def self.cached_loader_from(parent_loader, loaders) + LibRootedFileBased.new(parent_loader, + loaders, + NAMESPACE_WILDCARD, + Puppet[:libdir], + 'cached_puppet_lib', + %i[func_4x func_3x datatype]) + end + end + end + end + end +end + +module Puppet + module Pops + module Loader + class Loader + def discover_paths(type, name_authority = Pcore::RUNTIME_NAME_AUTHORITY) + if parent.nil? + [] + else + parent.discover_paths(type, name_authority) + end + end + end + end + end +end + +module Puppet + module Pops + module Loader + class DependencyLoader + def discover_paths(type, name_authority = Pcore::RUNTIME_NAME_AUTHORITY) + result = [] + + @dependency_loaders.each { |loader| result.concat(loader.discover_paths(type, name_authority)) } + result.concat(super) + result.uniq + end + end + end + end +end + +module Puppet + module Pops + module Loader + module ModuleLoaders + class AbstractPathBasedModuleLoader + def discover_paths(type, name_authority = Pcore::RUNTIME_NAME_AUTHORITY) + result = [] + if name_authority == Pcore::RUNTIME_NAME_AUTHORITY + smart_paths.effective_paths(type).each do |sp| + relative_paths(sp).each do |rp| + result << File.join(sp.generic_path, rp) + end + end + end + result.concat(super) + result.uniq + end + end + end + end + end +end + # MUST BE LAST!!!!!! # Suppress any warning messages to STDOUT. It can pollute stdout when running in STDIO mode Puppet::Util::Log.newdesttype :null_logger do diff --git a/lib/puppet-languageserver-sidecar/puppet_strings_helper.rb b/lib/puppet-languageserver-sidecar/puppet_strings_helper.rb index 63226710..8d5067fa 100644 --- a/lib/puppet-languageserver-sidecar/puppet_strings_helper.rb +++ b/lib/puppet-languageserver-sidecar/puppet_strings_helper.rb @@ -10,6 +10,7 @@ def self.file_documentation(path) return nil unless require_puppet_strings @helper_cache = FileDocumentationCache.new if @helper_cache.nil? return @helper_cache.document(path) if @helper_cache.path_exists?(path) + PuppetLanguageServerSidecar.log_message(:debug, "[PuppetStringsHelper::file_documentation] Fetching documentation for #{path}") setup_yard! @@ -80,16 +81,62 @@ def document(path) def populate_from_yard_registry! # Extract all of the information # Ref - https://github.com/puppetlabs/puppet-strings/blob/87a8e10f45bfeb7b6b8e766324bfb126de59f791/lib/puppet-strings/json.rb#L10-L16 + populate_functions_from_yard_registry! end private + def populate_functions_from_yard_registry! + ::YARD::Registry.all(:puppet_function).map(&:to_hash).each do |item| + source_path = item[:file] + func_name = item[:name].to_s + @cache[source_path] = FileDocumentation.new(source_path) if @cache[source_path].nil? + + obj = PuppetLanguageServer::Sidecar::Protocol::PuppetFunction.new + obj.key = func_name + obj.source = item[:file] + obj.calling_source = obj.source + obj.line = item[:line] + obj.doc = item[:docstring][:text] + obj.arity = -1 # We don't care about arity + obj.function_version = item[:type] == 'ruby4x' ? 4 : 3 + + # Try and determine the function call site from the source file + char = item[:source].index(":#{func_name}") + unless char.nil? + obj.char = char + obj.length = func_name.length + 1 + end + + case item[:type] + when 'ruby3x' + obj.function_version = 3 + # This is a bit hacky but it works (probably). Puppet-Strings doesn't rip this information out, but you do have the + # the source to query + obj.type = item[:source].match(/:rvalue/) ? :rvalue : :statement + when 'ruby4x' + obj.function_version = 4 + # All ruby functions are statements + obj.type = :statement + else + PuppetLanguageServerSidecar.log_message(:error, "[#{self.class}] Unknown function type #{item[:type]}") + end + + @cache[source_path].functions << obj + end + end + end + class FileDocumentation # The path to file that has been documented - attr_reader :path + attr_accessor :path + + # PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList object holding all functions + attr_accessor :functions - def initialize(path) + def initialize(path = nil) @path = path + @functions = PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList.new end end end diff --git a/lib/puppet-languageserver/sidecar_protocol.rb b/lib/puppet-languageserver/sidecar_protocol.rb index 6e680370..7e448429 100644 --- a/lib/puppet-languageserver/sidecar_protocol.rb +++ b/lib/puppet-languageserver/sidecar_protocol.rb @@ -154,12 +154,15 @@ class PuppetFunction < BasePuppetObject attr_accessor :doc attr_accessor :arity attr_accessor :type + # The version of this function, typically 3 or 4. + attr_accessor :function_version def to_h super.to_h.merge( - 'doc' => doc, - 'arity' => arity, - 'type' => type + 'doc' => doc, + 'arity' => arity, + 'type' => type, + 'function_version' => function_version ) end @@ -169,6 +172,7 @@ def from_h!(value) self.doc = value['doc'] self.arity = value['arity'] self.type = value['type'].intern + self.function_version = value['function_version'] self end end diff --git a/lib/puppet_languageserver_sidecar.rb b/lib/puppet_languageserver_sidecar.rb index d9992dbc..b31339f1 100644 --- a/lib/puppet_languageserver_sidecar.rb +++ b/lib/puppet_languageserver_sidecar.rb @@ -65,6 +65,11 @@ def self.require_gems(options) PuppetEditorServices.log_message(:error, "The feature flag 'puppetstrings' has been specified but it is not capable due to Puppet Strings being unavailable. Turning off the flag.") flags -= ['puppetstrings'] end + if flags.include?('puppetstrings') && Gem::Version.new(Puppet.version) < Gem::Version.new('6.0.0') + # The puppetstrings flag is only valid on Puppet 6.0.0+ + PuppetEditorServices.log_message(:error, "The feature flag 'puppetstrings' has been specified but it is not capable due to low Puppet version (< 6). Turning off the flag.") + flags -= ['puppetstrings'] + end configure_featureflags(flags) end @@ -81,6 +86,7 @@ def self.require_gems(options) if featureflag?('puppetstrings') require_list << 'puppet_helper_puppetstrings' require_list << 'puppet_monkey_patches_puppetstrings' + require_list << 'puppet_strings_helper' else require_list << 'puppet_helper' require_list << 'puppet_monkey_patches' @@ -248,6 +254,8 @@ def self.inject_workspace_as_environment end def self.execute(options) + use_puppet_strings = featureflag?('puppetstrings') + case options[:action].downcase when 'noop' [] @@ -258,7 +266,11 @@ def self.execute(options) when 'default_functions' cache = options[:disable_cache] ? PuppetLanguageServerSidecar::Cache::Null.new : PuppetLanguageServerSidecar::Cache::FileSystem.new - PuppetLanguageServerSidecar::PuppetHelper.retrieve_functions(cache) + if use_puppet_strings + PuppetLanguageServerSidecar::PuppetHelper.retrieve_via_puppet_strings(cache, :object_types => [:function])[:functions] + else + PuppetLanguageServerSidecar::PuppetHelper.retrieve_functions(cache) + end when 'default_types' cache = options[:disable_cache] ? PuppetLanguageServerSidecar::Cache::Null.new : PuppetLanguageServerSidecar::Cache::FileSystem.new @@ -298,8 +310,16 @@ def self.execute(options) when 'workspace_functions' null_cache = PuppetLanguageServerSidecar::Cache::Null.new return nil unless inject_workspace_as_module || inject_workspace_as_environment - PuppetLanguageServerSidecar::PuppetHelper.retrieve_functions(null_cache, - :root_path => PuppetLanguageServerSidecar::Workspace.root_path) + + if use_puppet_strings + cache = options[:disable_cache] ? PuppetLanguageServerSidecar::Cache::Null.new : PuppetLanguageServerSidecar::Cache::FileSystem.new + PuppetLanguageServerSidecar::PuppetHelper.retrieve_via_puppet_strings(cache, + :object_types => [:function], + :root_path => PuppetLanguageServerSidecar::Workspace.root_path)[:functions] + else + PuppetLanguageServerSidecar::PuppetHelper.retrieve_functions(null_cache, + :root_path => PuppetLanguageServerSidecar::Workspace.root_path) + end when 'workspace_types' null_cache = PuppetLanguageServerSidecar::Cache::Null.new diff --git a/spec/languageserver/spec_helper.rb b/spec/languageserver/spec_helper.rb index e3e938a0..dfe4795b 100644 --- a/spec/languageserver/spec_helper.rb +++ b/spec/languageserver/spec_helper.rb @@ -70,6 +70,7 @@ def random_sidecar_puppet_function result.doc = 'doc' + rand(1000).to_s result.arity = rand(1000) result.type = ('type' + rand(1000).to_s).intern + result.function_version = rand(1) + 3 result end diff --git a/spec/languageserver/unit/puppet-languageserver/sidecar_protocol_spec.rb b/spec/languageserver/unit/puppet-languageserver/sidecar_protocol_spec.rb index 041dfdab..8aaea921 100644 --- a/spec/languageserver/unit/puppet-languageserver/sidecar_protocol_spec.rb +++ b/spec/languageserver/unit/puppet-languageserver/sidecar_protocol_spec.rb @@ -55,7 +55,7 @@ basepuppetobject_properties = [:key, :calling_source, :source, :line, :char, :length] nodegraph_properties = [:dot_content, :error_content] puppetclass_properties = [:doc, :parameters] - puppetfunction_properties = [:doc, :arity, :type] + puppetfunction_properties = [:doc, :arity, :type, :function_version] puppettype_properties = [:doc, :attributes] resource_properties = [:manifest]