(FACT-2848) Improve cache performance

lib/facter/custom_facts/util/config.rb

@@ -78,7 +78,7 @@ def self.setup_default_cache_dir
         @facts_cache_dir = if windows_dir
                              File.join(windows_dir, 'PuppetLabs', 'facter', 'cache', 'cached_facts')
                            else
-                             '/opt/puppetlabs/facter/cache/cached_facts'
+                             '/Users/bogdan.irimie/cached_facts'
                            end
       end
 

lib/facter/custom_facts/util/directory_loader.rb

@@ -53,23 +53,26 @@ def load(collection)
       private
 
       def load_directory_entries(_collection)
-        cm = Facter::CacheManager.new
+        cache_reader = Facter::Cache::CacheReader.new
+        cache_augmenter = Facter::Cache::CacheAugmenter.new
         facts = []
         entries.each do |file|
           basename = File.basename(file)
           next if file_blocked?(basename)
 
-          if facts.find { |f| f.name == basename } && cm.fact_cache_enabled?(basename)
+          searched_fact = Facter::SearchedFact.new(basename, nil, [], nil, :file)
+          searched_fact.file = file
+          cache_augmenter.augment_with_cache_group([searched_fact])
+
+          if facts.find { |f| f.name == basename } && searched_fact.cache_group != nil
             Facter.log_exception(Exception.new("Caching is enabled for group \"#{basename}\" while "\
               'there are at least two external facts files with the same filename'))
           else
-            searched_fact = Facter::SearchedFact.new(basename, nil, [], nil, :file)
-            searched_fact.file = file
             facts << searched_fact
           end
         end
 
-        cm.resolve_facts(facts)
+        cache_reader.read_from_cache(facts)
       end
 
       def load_cached_facts(collection, cached_facts, weight)

lib/facter/facts/aix/identity/group.rb

@@ -8,7 +8,7 @@ class Group
         ALIASES = 'gid'
 
         def call_the_resolver
-          fact_value = Facter::Resolvers::PosxIdentity.resolve(:group)
+          fact_value = Facter::Resolvers::PosxIdentity.resolve(:cache_group)
 
           [Facter::ResolvedFact.new(FACT_NAME, fact_value), Facter::ResolvedFact.new(ALIASES, fact_value, :legacy)]
         end

lib/facter/facts/freebsd/identity/group.rb

@@ -8,7 +8,7 @@ class Group
         ALIASES = 'gid'
 
         def call_the_resolver
-          fact_value = Facter::Resolvers::PosxIdentity.resolve(:group)
+          fact_value = Facter::Resolvers::PosxIdentity.resolve(:cache_group)
 
           [Facter::ResolvedFact.new(FACT_NAME, fact_value), Facter::ResolvedFact.new(ALIASES, fact_value, :legacy)]
         end

lib/facter/facts/linux/identity/group.rb

@@ -8,7 +8,7 @@ class Group
         ALIASES = 'gid'
 
         def call_the_resolver
-          fact_value = Facter::Resolvers::PosxIdentity.resolve(:group)
+          fact_value = Facter::Resolvers::PosxIdentity.resolve(:cache_group)
 
           [Facter::ResolvedFact.new(FACT_NAME, fact_value), Facter::ResolvedFact.new(ALIASES, fact_value, :legacy)]
         end

lib/facter/facts/macosx/identity/group.rb

@@ -8,7 +8,7 @@ class Group
         ALIASES = 'gid'
 
         def call_the_resolver
-          fact_value = Facter::Resolvers::PosxIdentity.resolve(:group)
+          fact_value = Facter::Resolvers::PosxIdentity.resolve(:cache_group)
 
           [Facter::ResolvedFact.new(FACT_NAME, fact_value), Facter::ResolvedFact.new(ALIASES, fact_value, :legacy)]
         end

lib/facter/facts/solaris/identity/group.rb

@@ -8,7 +8,7 @@ class Group
         ALIASES = 'gid'
 
         def call_the_resolver
-          fact_value = Facter::Resolvers::PosxIdentity.resolve(:group)
+          fact_value = Facter::Resolvers::PosxIdentity.resolve(:cache_group)
 
           [Facter::ResolvedFact.new(FACT_NAME, fact_value), Facter::ResolvedFact.new(ALIASES, fact_value, :legacy)]
         end

lib/facter/framework/config/fact_groups.rb

@@ -33,30 +33,13 @@ def blocked_facts
       fact_list
     end
 
-    # Get the group name a fact is part of
-    def get_fact_group(fact_name)
-      fact = get_fact(fact_name)
-      return fact[:group] if fact
-
-      @groups.detect { |k, v| break k if Array(v).find { |f| fact_name =~ /^#{f}.*/ } }
-    end
-
     # Get config ttls for a given group
     def get_group_ttls(group_name)
       return unless (ttls = @groups_ttls.find { |g| g[group_name] })
 
       ttls_to_seconds(ttls[group_name])
     end
 
-    def get_fact(fact_name)
-      return @facts_ttls[fact_name] if @facts_ttls[fact_name]
-
-      result = @facts_ttls.select { |name, fact| break fact if fact_name =~ /^#{name}\..*/ }
-      return nil if result == {}
-
-      result
-    end
-
     private
 
     def load_groups_from_options
@@ -78,13 +61,15 @@ def load_facts_ttls
       @groups_ttls.reduce(:merge).each do |group, ttls|
         ttls = ttls_to_seconds(ttls)
         if @groups[group]
+          # the ttls is for a group
           @groups[group].each do |fact|
             if (@facts_ttls[fact] && @facts_ttls[fact][:ttls] < ttls) || @facts_ttls[fact].nil?
-              @facts_ttls[fact] = { ttls: ttls, group: group }
+              @facts_ttls[fact] = {ttls: ttls, cache_group: group }
             end
           end
         else
-          @facts_ttls[group] = { ttls: ttls, group: group }
+          # the ttls is for a fact not a group
+          @facts_ttls[group] = {ttls: ttls, cache_group: group }
         end
       end
     end

lib/facter/framework/core/cache/cache_augmenter.rb

@@ -0,0 +1,27 @@
+
+module Facter
+  module Cache
+    class CacheAugmenter
+
+      def initialize
+        @fact_groups = Facter::FactGroups.new
+      end
+
+      def augment_with_cache_group(searched_facts)
+        ttls = @fact_groups.facts_ttls
+
+        searched_facts.each do |fact|
+          fact_name = if fact.file
+                        File.basename(fact.file)
+                      else
+                        fact.name
+                      end
+
+          ttls.each do |fact_key, details|
+            fact.cache_group = details[:cache_group] if fact_name =~ /^#{fact_key}\..*/ || fact_name =~ /^#{fact_key}$/
+          end
+        end
+      end
+    end
+  end
+end

lib/facter/framework/core/cache/cache_reader.rb

@@ -0,0 +1,79 @@
+
+module Facter
+  module Cache
+    class CacheReader
+
+      def initialize
+        @groups = {}
+        @log = Log.new(self)
+        @cache_dir = LegacyFacter::Util::Config.facts_cache_dir
+      end
+
+      def read_from_cache(searched_facts)
+        return searched_facts, [] if (!File.directory?(@cache_dir) || !Options[:cache]) && Options[:ttls].any?
+
+        facts = []
+        searched_facts.delete_if do |searched_fact|
+          res = read_fact(searched_fact, searched_fact.cache_group) if searched_fact.cache_group
+          if res
+            facts << res
+            true
+          else
+            false
+          end
+        end
+
+        [searched_facts, facts.flatten]
+      end
+
+      private
+
+      def read_fact(searched_fact, fact_group)
+        data = nil
+        Facter::Framework::Benchmarking::Timer.measure(searched_fact.name, 'cached') do
+          data = read_group_json(fact_group)
+        end
+        return unless data
+
+        @log.debug("loading cached values for #{searched_fact.name} facts")
+
+        create_facts(searched_fact, data)
+      end
+
+      def read_group_json(group_name)
+        return @groups[group_name] if @groups.key?(group_name)
+
+        cache_file_name = File.join(@cache_dir, group_name)
+        data = nil
+        file = Util::FileHelper.safe_read(cache_file_name)
+        begin
+          data = JSON.parse(file)
+        rescue JSON::ParserError
+          delete_cache(group_name)
+        end
+        @groups[group_name] = data
+      end
+
+      def delete_cache(group_name)
+        cache_file_name = File.join(@cache_dir, group_name)
+        File.delete(cache_file_name) if File.readable?(cache_file_name)
+      end
+
+      def create_facts(searched_fact, data)
+        if searched_fact.type == :file
+          facts = []
+          data.each do |fact_name, fact_value|
+            fact = Facter::ResolvedFact.new(fact_name, fact_value, searched_fact.type,
+                                            searched_fact.user_query, searched_fact.filter_tokens, searched_fact.cache_group)
+            fact.file = searched_fact.file
+            facts << fact
+          end
+          facts
+        else
+          [Facter::ResolvedFact.new(searched_fact.name, data[searched_fact.name], searched_fact.type,
+                                    searched_fact.user_query, searched_fact.filter_tokens, searched_fact.cache_group)]
+        end
+      end
+    end
+  end
+end

lib/facter/framework/core/cache/cache_writer.rb

@@ -0,0 +1,68 @@
+
+module Facter
+  module Cache
+    class CacheWriter
+
+      def initialize
+        @log = Log.new(self)
+        @groups = {}
+        @fact_groups = Facter::FactGroups.new
+        @cache_dir = LegacyFacter::Util::Config.facts_cache_dir
+      end
+
+      def cache_facts(resolved_facts)
+        return unless Options[:cache] && Options[:ttls].any?
+
+        resolved_facts
+          .select { |resolved_fact| resolved_fact.cache_group != nil }
+          .group_by { |resolved_fact| resolved_fact.cache_group }
+          .each do |group_name, array_of_facts|
+          @groups[group_name] ||= {}
+          array_of_facts.each { |resolved_fact| @groups[group_name][resolved_fact.name] = resolved_fact.value}
+        end
+
+        begin
+          write_cache unless @groups.empty?
+        rescue Errno::EACCES => e
+          @log.warn("Could not write cache: #{e.message}")
+        end
+      end
+
+      private
+
+      def write_cache
+        unless File.directory?(@cache_dir)
+          require 'fileutils'
+          FileUtils.mkdir_p(@cache_dir)
+        end
+
+        @groups.each do |group_name, data|
+          next unless check_ttls?(group_name, @fact_groups.get_group_ttls(group_name))
+
+          cache_file_name = File.join(@cache_dir, group_name)
+          next if File.readable?(cache_file_name)
+
+          @log.debug("caching values for #{group_name} facts")
+          File.write(cache_file_name, JSON.pretty_generate(data))
+        end
+      end
+
+      def check_ttls?(group_name, ttls)
+        return false unless ttls
+
+        cache_file_name = File.join(@cache_dir, group_name)
+        if File.readable?(cache_file_name)
+          file_time = File.mtime(cache_file_name)
+          expire_date = file_time + ttls
+          return true if expire_date > Time.now
+
+          File.delete(cache_file_name)
+        end
+
+        @log.debug("#{group_name} facts cache file expired/missing")
+        true
+      end
+    end
+  end
+end
+