(GH-121) Add on disk caching to Puppet Strings results

Previously the results of extracting puppet metadata needed to be calculated
whenever the Sidecar was run.  This commit modifies the Puppet Strings helper
to also use a caching mechanism to store the results and speed up metadata
gathering.

* Adds a clear! method to the cache, mainly for testing purposes.
* Adds integration tests to ensure that objects read from cache mirror the
  original results
* Adds serialisation and deserialisation methods to the FileDocumentation
  object which allows it to be cached and read back.

lib/puppet-languageserver-sidecar/cache/base.rb

@@ -5,6 +5,7 @@ module Cache
     CLASSES_SECTION = 'classes'
     FUNCTIONS_SECTION = 'functions'
     TYPES_SECTION = 'types'
+    PUPPETSTRINGS_SECTION = 'puppetstrings'
 
     class Base
       attr_reader :cache_options
@@ -24,6 +25,12 @@ def load(_absolute_path, _section)
       def save(_absolute_path, _section, _content_string)
         raise NotImplementedError
       end
+
+      # WARNING - This method is only intended for testing the cache
+      # and should not be used for normal operations
+      def clear!
+        raise NotImplementedError
+      end
     end
   end
 end

lib/puppet-languageserver-sidecar/cache/filesystem.rb

@@ -15,7 +15,7 @@ def initialize(_options = {})
         begin
           Dir.mkdir(@cache_dir) unless Dir.exist?(@cache_dir)
         rescue Errno::ENOENT => e
-          PuppetLanguageServerSidecar.log_message(:error, "[PuppetLanguageServerSidecar::Cache::File] An error occured while creating file cache.  Disabling cache: #{e}")
+          PuppetLanguageServerSidecar.log_message(:error, "[PuppetLanguageServerSidecar::Cache::FileSystem] An error occured while creating file cache.  Disabling cache: #{e}")
           @cache_dir = nil
         end
       end
@@ -37,20 +37,20 @@ def load(absolute_path, section)
 
         # Check that this is from the same language server version
         unless json_obj['sidecar_version'] == PuppetLanguageServerSidecar.version
-          PuppetLanguageServerSidecar.log_message(:debug, "[PuppetLanguageServerSidecar::load] Error loading #{absolute_path}: Expected sidecar_version version #{PuppetLanguageServerSidecar.version} but found #{json_obj['sidecar_version']}")
+          PuppetLanguageServerSidecar.log_message(:debug, "[PuppetLanguageServerSidecar::Cache::FileSystem.load] Error loading #{absolute_path}: Expected sidecar_version version #{PuppetLanguageServerSidecar.version} but found #{json_obj['sidecar_version']}")
           return nil
         end
         # Check that the source file hash matches
         content_hash = calculate_hash(absolute_path)
         if json_obj['file_hash'] != content_hash
-          PuppetLanguageServerSidecar.log_message(:debug, "[PuppetLanguageServerSidecar::load] Error loading #{absolute_path}: Expected file_hash of #{content_hash} but found #{json_obj['file_hash']}")
+          PuppetLanguageServerSidecar.log_message(:debug, "[PuppetLanguageServerSidecar::Cache::FileSystem.load] Error loading #{absolute_path}: Expected file_hash of #{content_hash} but found #{json_obj['file_hash']}")
           return nil
         end
-        PuppetLanguageServerSidecar.log_message(:debug, "[PuppetLanguageServerSidecar::load] Loading #{absolute_path} from cache")
+        PuppetLanguageServerSidecar.log_message(:debug, "[PuppetLanguageServerSidecar::Cache::FileSystem.load] Loading #{absolute_path} from cache")
 
         json_obj['data']
       rescue RuntimeError => e
-        PuppetLanguageServerSidecar.log_message(:debug, "[PuppetLanguageServerSidecar::load] Error loading #{absolute_path}: #{e}")
+        PuppetLanguageServerSidecar.log_message(:debug, "[PuppetLanguageServerSidecar::Cache::FileSystem.load] Error loading #{absolute_path}: #{e}")
         raise
       end
 
@@ -67,9 +67,16 @@ def save(absolute_path, section, content_string)
         content['path'] = absolute_path
         content['section'] = section
 
+        PuppetLanguageServerSidecar.log_message(:debug, "[PuppetLanguageServerSidecar::Cache::FileSystem.save] Saving #{absolute_path} to cache")
         save_file(cache_file, content.to_json)
       end
 
+      def clear!
+        return unless active?
+        PuppetLanguageServerSidecar.log_message(:warn, '[PuppetLanguageServerSidecar::Cache::FileSystem.clear] Filesystem based cache is being cleared')
+        FileUtils.rm(Dir.glob(File.join(cache_dir, '*')), :force => true)
+      end
+
       private
 
       def file_key(filepath, section)

lib/puppet-languageserver-sidecar/cache/null.rb

@@ -18,6 +18,10 @@ def load(*)
       def save(*)
         true
       end
+
+      def clear!
+        nil
+      end
     end
   end
 end

lib/puppet-languageserver-sidecar/puppet_helper_puppetstrings.rb

@@ -40,7 +40,7 @@ def self.available_documentation_types
     end
 
     # Retrieve objects via the Puppet 4 API loaders
-    def self.retrieve_via_puppet_strings(_cache, options = {})
+    def self.retrieve_via_puppet_strings(cache, options = {})
       PuppetLanguageServerSidecar.log_message(:debug, '[PuppetHelper::retrieve_via_puppet_strings] Starting')
 
       object_types = options[:object_types].nil? ? available_documentation_types : options[:object_types]
@@ -60,7 +60,7 @@ def self.retrieve_via_puppet_strings(_cache, options = {})
 
       paths.each do |path|
         next unless path_has_child?(options[:root_path], path)
-        file_doc = PuppetLanguageServerSidecar::PuppetStringsHelper.file_documentation(path)
+        file_doc = PuppetLanguageServerSidecar::PuppetStringsHelper.file_documentation(path, cache)
         next if file_doc.nil?
 
         if object_types.include?(:function) # rubocop:disable Style/IfUnlessModifier   This reads better

lib/puppet-languageserver-sidecar/puppet_strings_helper.rb

@@ -5,12 +5,17 @@ module PuppetStringsHelper
     # Returns a FileDocumentation object for a given path
     #
     # @param [String] path The absolute path to the file that will be documented
+    # @param [PuppetLanguageServerSidecar::Cache] cache A Sidecar cache which stores already parsed documents as serialised FileDocumentation objects
     # @return [FileDocumentation, nil] Returns the documentation for the path, or nil if it cannot be extracted
-    def self.file_documentation(path)
+    def self.file_documentation(path, cache = nil)
       return nil unless require_puppet_strings
       @helper_cache = FileDocumentationCache.new if @helper_cache.nil?
       return @helper_cache.document(path) if @helper_cache.path_exists?(path)
 
+      # Load from the permanent cache
+      @helper_cache.populate_from_sidecar_cache!(path, cache) unless cache.nil? || !cache.active?
+      return @helper_cache.document(path) if @helper_cache.path_exists?(path)
+
       PuppetLanguageServerSidecar.log_message(:debug, "[PuppetStringsHelper::file_documentation] Fetching documentation for #{path}")
 
       setup_yard!
@@ -36,6 +41,9 @@ def self.file_documentation(path)
       # Populate the documentation cache from the YARD information
       @helper_cache.populate_from_yard_registry!
 
+      # Save to the permanent cache
+      @helper_cache.save_to_sidecar_cache(path, cache) unless cache.nil? || !cache.active?
+
       # Return the documentation details
       @helper_cache.document(path)
     end
@@ -84,6 +92,22 @@ def populate_from_yard_registry!
       populate_functions_from_yard_registry!
     end
 
+    def populate_from_sidecar_cache!(path, cache)
+      cached_result = cache.load(path, PuppetLanguageServerSidecar::Cache::PUPPETSTRINGS_SECTION)
+      unless cached_result.nil? # rubocop:disable Style/GuardClause    Reads better this way
+        begin
+          obj = FileDocumentation.new.from_json!(cached_result)
+          @cache[path] = obj
+        rescue StandardError => e
+          PuppetLanguageServerSidecar.log_message(:warn, "[FileDocumentationCache::populate_from_sidecar_cache!] Error while deserializing #{path} from cache: #{e}")
+        end
+      end
+    end
+
+    def save_to_sidecar_cache(path, cache)
+      cache.save(path, PuppetLanguageServerSidecar::Cache::PUPPETSTRINGS_SECTION, document(path).to_json) if cache.active?
+    end
+
     private
 
     def populate_functions_from_yard_registry!
@@ -138,5 +162,40 @@ def initialize(path = nil)
       @path = path
       @functions = PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList.new
     end
+
+    # Serialisation
+    def to_h
+      {
+        'path'      => path,
+        'functions' => functions
+      }
+    end
+
+    def to_json(*options)
+      JSON.generate(to_h, options)
+    end
+
+    # Deserialisation
+    def from_json!(json_string)
+      obj = JSON.parse(json_string)
+
+      obj.keys.each do |key|
+        case key
+        when 'path'
+          # Simple deserialised object types
+          self.instance_variable_set("@#{key}", obj[key]) # rubocop:disable Style/RedundantSelf   Reads better this way
+        else
+          # Sidecar protocol list object types
+          prop = self.instance_variable_get("@#{key}") # rubocop:disable Style/RedundantSelf   Reads better this way
+
+          obj[key].each do |child_hash|
+            child = prop.child_type.new
+            # Let the sidecar deserialise for us
+            prop << child.from_h!(child_hash)
+          end
+        end
+      end
+      self
+    end
   end
 end

spec/languageserver-sidecar/integration/puppet-languageserver-sidecar/featureflag_puppetstrings/featureflag_puppetstrings_spec.rb

@@ -4,8 +4,6 @@
 
 describe 'PuppetLanguageServerSidecar with Feature Flag puppetstrings', :if => Gem::Version.new(Puppet.version) >= Gem::Version.new('6.0.0') do
   def run_sidecar(cmd_options)
-    cmd_options << '--no-cache'
-
     # Append the feature flag
     cmd_options << '--feature-flag=puppetstrings'
 
@@ -59,6 +57,37 @@ def should_not_contain_default_functions(deserial)
     expect(deserial).to_not contain_child_with_key(:'modname::default_mod_pup4_function')
   end
 
+  before(:each) do
+    # Purge the File Cache
+    cache = PuppetLanguageServerSidecar::Cache::FileSystem.new
+    cache.clear!
+  end
+
+  after(:all) do
+    # Purge the File Cache
+    cache = PuppetLanguageServerSidecar::Cache::FileSystem.new
+    cache.clear!
+  end
+
+  def expect_empty_cache
+    cache = PuppetLanguageServerSidecar::Cache::FileSystem.new
+    expect(Dir.exists?(cache.cache_dir)).to eq(true), "Expected the cache directory #{cache.cache_dir} to exist"
+    expect(Dir.glob(File.join(cache.cache_dir,'*')).count).to eq(0), "Expected the cache directory #{cache.cache_dir} to be empty"
+  end
+
+  def expect_populated_cache
+    cache = PuppetLanguageServerSidecar::Cache::FileSystem.new
+    expect(Dir.glob(File.join(cache.cache_dir,'*')).count).to be > 0, "Expected the cache directory #{cache.cache_dir} to be populated"
+  end
+
+  def expect_same_array_content(a, b)
+    expect(a.count).to eq(b.count), "Expected array with #{b.count} items to have #{a.count} items"
+
+    a.each_with_index do |item, index|
+      expect(item.to_json).to eq(b[index].to_json), "Expected item at index #{index} to have content #{item.to_json} but got #{b[index].to_json}"
+    end
+  end
+
   describe 'when running default_classes action' do
     let (:cmd_options) { ['--action', 'default_classes'] }
 
@@ -79,7 +108,9 @@ def should_not_contain_default_functions(deserial)
   describe 'when running default_functions action' do
     let (:cmd_options) { ['--action', 'default_functions'] }
 
-    it 'should return a deserializable function list with default functions' do
+    it 'should return a cachable deserializable function list with default functions' do
+      expect_empty_cache
+
       result = run_sidecar(cmd_options)
       deserial = PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList.new()
       expect { deserial.from_json!(result) }.to_not raise_error
@@ -98,6 +129,16 @@ def should_not_contain_default_functions(deserial)
       expect(deserial).to contain_child_with_key(:'environment::default_env_pup4_function')
       # These are defined in the fixtures/real_agent/cache/lib/puppet/functions/modname (module namespaced function)
       expect(deserial).to contain_child_with_key(:'modname::default_mod_pup4_function')
+
+      # Now run using cached information
+      expect_populated_cache
+
+      result2 = run_sidecar(cmd_options)
+      deserial2 = PuppetLanguageServer::Sidecar::Protocol::PuppetFunctionList.new()
+      expect { deserial2.from_json!(result2) }.to_not raise_error
+
+      # The second result should be the same as the first
+      expect_same_array_content(deserial, deserial2)
     end
   end