(puppetlabsGH-121) Add documentation for Puppet 4 Functions

Previously the Puppet 4 API functions would be loaded however they would not
have any documentation notes as this was removed in the newer API.  Instead we
need to use Puppet Strings and YARD to generate the documentation.  This commit:

Note that Puppet Strings and YARD are only available in the PDK, not Puppet
Agent.  Once this feature is no longer hidden behind a feature flag, they
can be vendored into Editor Services

* Adds the puppet-strings gem for development purpsoes

* Sets up YARD via Puppet Strings which adds the Puppet Language parsing into
YARD

* Caches the result from YARD to speed up documentation resolution for multiple
items in the same file

* Post processes the function loader and adds the documetation for Puppet 4 API
functions

Gemfile

@@ -23,6 +23,10 @@ group :development do
   else
     gem 'puppet',                            :require => false
   end
+  # TODO Should this be vendored into Editor Services?
+  # The puppet-strings gem is not available in the Puppet Agent, but is in the PDK. We add it to the
+  # Gemfile here for testing and development.
+  gem "puppet-strings", "~> 2.0", :require => false
 
   case RUBY_PLATFORM
   when /darwin/

lib/puppet-languageserver-sidecar/puppet_helper_pup4api.rb

@@ -191,6 +191,9 @@ def self.retrieve_via_pup4_api(_cache, options = {})
         Puppet::Functions.monkey_function_list
                          .select { |_k, i| path_has_child?(options[:root_path], i[:source_location][:source]) }
                          .each do |name, item|
+          file_doc = PuppetLanguageServerSidecar::PuppetStringsHelper.file_documentation(item[:source_location][:source])
+
+          item[:doc] = file_doc[:functions][name][:doc] unless file_doc.nil? || file_doc[:functions].nil? || file_doc[:functions][name].nil?
           obj = PuppetLanguageServerSidecar::Protocol::PuppetFunction.from_puppet(name, item)
           result[:functions] << obj
         end

lib/puppet-languageserver-sidecar/puppet_strings_helper.rb

@@ -0,0 +1,82 @@
+module PuppetLanguageServerSidecar
+  module PuppetStringsHelper
+    def self.file_documentation(path)
+      return nil unless require_puppet_strings
+      return @helper_cache[path] unless @helper_cache.nil? || @helper_cache[path].nil?
+      PuppetLanguageServerSidecar.log_message(:debug, "[PuppetStringsHelper::file_documentation] Fetching documentation for #{path}")
+
+      setup_yard!
+
+      # For now, assume a single file path
+      search_patterns = [path]
+
+      # Format the arguments to YARD
+      args = ['doc']
+      args << '--no-output'
+      args << '--quiet'
+      args << '--no-stats'
+      args << '--no-progress'
+      args << '--no-save'
+      args << '--api public'
+      args << '--api private'
+      args << '--no-api'
+      args += search_patterns
+
+      # Run YARD
+      ::YARD::CLI::Yardoc.run(*args)
+
+      # Extract all of the information
+      # Ref - https://github.com/puppetlabs/puppet-strings/blob/87a8e10f45bfeb7b6b8e766324bfb126de59f791/lib/puppet-strings/json.rb#L10-L16
+      @helper_cache = {} if @helper_cache.nil?
+      extract_puppet_functions
+
+      @helper_cache[path]
+    end
+
+    def self.require_puppet_strings
+      return @puppet_strings_loaded unless @puppet_strings_loaded.nil?
+      begin
+        require 'puppet-strings'
+        require 'puppet-strings/yard'
+        require 'puppet-strings/json'
+        @puppet_strings_loaded = true
+      rescue LoadError => e
+        PuppetLanguageServerSidecar.log_message(:error, "[PuppetStringsHelper::require_puppet_strings] Unable to load puppet-strings gem: #{e}")
+        @puppet_strings_loaded = false
+      end
+      @puppet_strings_loaded
+    end
+    private_class_method :require_puppet_strings
+
+    def self.setup_yard!
+      unless @yard_setup # rubocop:disable Style/GuardClause
+        ::PuppetStrings::Yard.setup!
+        @yard_setup = true
+      end
+    end
+    private_class_method :setup_yard!
+
+    def self.assert_cache_has_file(source_path)
+      if @helper_cache[source_path].nil? # rubocop:disable Style/GuardClause
+        @helper_cache[source_path] = {
+          :functions => {}
+        }
+      end
+    end
+    private_class_method :assert_cache_has_file
+
+    def self.extract_puppet_functions
+      ::YARD::Registry.all(:puppet_function).map(&:to_hash).each do |item|
+        source_path = item[:file]
+        assert_cache_has_file(source_path)
+
+        func_name = item[:name].to_s
+        @helper_cache[source_path][:functions][func_name] = {} if @helper_cache[source_path][:functions][func_name].nil?
+
+        # Build the function documentation
+        @helper_cache[source_path][:functions][func_name][:doc] = item[:docstring][:text]
+      end
+    end
+    private_class_method :extract_puppet_functions
+  end
+end

lib/puppet_languageserver_sidecar.rb

@@ -77,6 +77,7 @@ def self.require_gems(options)
     if featureflag?('pup4api')
       require_list << 'puppet_helper_pup4api'
       require_list << 'puppet_monkey_patches_pup4api'
+      require_list << 'puppet_strings_helper'
     else
       require_list << 'puppet_helper'
       require_list << 'puppet_monkey_patches'

spec/languageserver-sidecar/integration/puppet-languageserver-sidecar/featureflag_pup4api/featureflag_pup4api_spec.rb

@@ -178,7 +178,7 @@ def with_temporary_file(content)
 
         # Make sure the function has the right properties
         func = child_with_key(deserial, :fixture_pup4_function)
-        expect(func.doc).to be_nil  # Currently we can't get the documentation for v4 functions
+        expect(func.doc).to match(/Example function using the Puppet 4 API in a module/)
         expect(func.source).to match(/valid_module_workspace/)
       end
     end
@@ -279,7 +279,7 @@ def with_temporary_file(content)
 
         # Make sure the function has the right properties
         func = child_with_key(deserial, :pup4_env_function)
-        expect(func.doc).to be_nil  # Currently we can't get the documentation for v4 functions
+        expect(func.doc).to match(/Example function using the Puppet 4 API in a module/)
         expect(func.source).to match(/valid_environment_workspace/)
       end
     end