Support Job extension (#80)

support job extension: jobs can now inherit from another job, this will inherit a copy of the parent job's task list, and can be modified by methods task_list_prepend, task_list_prepend_at, task_list_append and task_list_append_at. when searching for task implementations, we will first look for implementations in the job's namespace, then in the parent's namespace and last of all without any namespace.
mercadolibre · Feb 27, 2023 · e711d40 · e711d40
1 parent 27a943c
commit e711d40
Show file tree

Hide file tree

Showing 34 changed files with 442 additions and 261 deletions.
diff --git a/.rubocop.yml b/.rubocop.yml
@@ -1,4 +1,4 @@
-Documentation:
+Style/Documentation:
   Enabled: false
 
 Style/FrozenStringLiteralComment:
@@ -10,10 +10,10 @@ Style/FrozenStringLiteralComment:
 Style/ClassAndModuleChildren:
   Enabled: false
 
-Metrics/LineLength:
+Layout/LineLength:
   Max: 120
 
-MethodLength:
+Metrics/MethodLength:
   Max: 20
 
 Metrics/AbcSize:
@@ -24,11 +24,12 @@ Metrics/ClassLength:
   Max: 200
 
 AllCops:
+  NewCops: enable
   Exclude:
     - 'vendor/**/*'
     - 'spec/**/*'
     - 'tmp/**/*'
     - 'config/**/*'
     - 'bin/**'
     - 'db/**/*'
-  TargetRubyVersion: 2.3
+  TargetRubyVersion: 2.7
diff --git a/Gemfile b/Gemfile
@@ -1,6 +1,6 @@
 source 'https://rubygems.org'
 
 gem 'codeclimate-test-reporter', group: :test, require: nil
-gem 'newrelic_rpm', '~> 4.0.0'
+gem 'newrelic_rpm'
 
 gemspec
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    little_monster (0.1.28)
+    little_monster (0.1.29)
       activesupport
       multi_json
       thor
@@ -17,40 +17,40 @@ GEM
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
-    addressable (2.8.0)
-      public_suffix (>= 2.0.2, < 5.0)
+    addressable (2.8.1)
+      public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
     aws-eventstream (1.2.0)
-    aws-partitions (1.600.0)
-    aws-sdk-core (3.131.2)
+    aws-partitions (1.716.0)
+    aws-sdk-core (3.170.0)
       aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.525.0)
-      aws-sigv4 (~> 1.1)
+      aws-partitions (~> 1, >= 1.651.0)
+      aws-sigv4 (~> 1.5)
       jmespath (~> 1, >= 1.6.1)
-    aws-sdk-sqs (1.51.1)
-      aws-sdk-core (~> 3, >= 3.127.0)
+    aws-sdk-sqs (1.53.0)
+      aws-sdk-core (~> 3, >= 3.165.0)
       aws-sigv4 (~> 1.1)
-    aws-sigv4 (1.5.0)
+    aws-sigv4 (1.5.2)
       aws-eventstream (~> 1, >= 1.0.2)
     byebug (11.1.3)
     codeclimate-test-reporter (1.0.7)
       simplecov
     coderay (1.1.3)
-    concurrent-ruby (1.1.10)
-    concurrent-ruby-edge (0.6.0)
-      concurrent-ruby (~> 1.1.6)
+    concurrent-ruby (1.2.2)
+    concurrent-ruby-edge (0.7.0)
+      concurrent-ruby (~> 1.2.0)
     diff-lcs (1.5.0)
     docile (1.4.0)
-    ethon (0.15.0)
+    ethon (0.16.0)
       ffi (>= 1.15.0)
-    faraday (2.3.0)
-      faraday-net_http (~> 2.0)
+    faraday (2.7.4)
+      faraday-net_http (>= 2.0, < 3.1)
       ruby2_keywords (>= 0.0.4)
-    faraday-net_http (2.0.3)
+    faraday-net_http (3.0.2)
     faraday-retry (2.0.0)
       faraday (~> 2.0)
     ffi (1.15.5)
-    gapic-common (0.10.0)
+    gapic-common (0.17.1)
       faraday (>= 1.9, < 3.a)
       faraday-retry (>= 1.0, < 3.a)
       google-protobuf (~> 3.14)
@@ -63,98 +63,106 @@ GEM
       google-cloud-errors (~> 1.0)
     google-cloud-env (1.6.0)
       faraday (>= 0.17.3, < 3.0)
-    google-cloud-errors (1.2.0)
-    google-cloud-pubsub (2.10.0)
+    google-cloud-errors (1.3.0)
+    google-cloud-pubsub (2.15.1)
       concurrent-ruby (~> 1.1)
       google-cloud-core (~> 1.5)
-      google-cloud-pubsub-v1 (~> 0.0)
-    google-cloud-pubsub-v1 (0.10.0)
-      gapic-common (>= 0.7, < 2.a)
+      google-cloud-pubsub-v1 (~> 0.8)
+      retriable (~> 3.1)
+    google-cloud-pubsub-v1 (0.15.1)
+      gapic-common (>= 0.17.1, < 2.a)
+      google-cloud-errors (~> 1.0)
+      google-iam-v1 (>= 0.4, < 2.a)
+    google-iam-v1 (0.4.0)
+      gapic-common (>= 0.17.1, < 2.a)
       google-cloud-errors (~> 1.0)
       grpc-google-iam-v1 (~> 1.1)
     google-protobuf (3.22.0)
-    googleapis-common-protos (1.3.12)
+    googleapis-common-protos (1.4.0)
       google-protobuf (~> 3.14)
       googleapis-common-protos-types (~> 1.2)
       grpc (~> 1.27)
-    googleapis-common-protos-types (1.3.2)
+    googleapis-common-protos-types (1.5.0)
       google-protobuf (~> 3.14)
-    googleauth (1.2.0)
+    googleauth (1.3.0)
       faraday (>= 0.17.3, < 3.a)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (>= 0.16, < 2.a)
-    grpc (1.46.3)
-      google-protobuf (~> 3.19)
+    grpc (1.52.0)
+      google-protobuf (~> 3.21)
       googleapis-common-protos-types (~> 1.0)
-    grpc-google-iam-v1 (1.1.1)
+    grpc-google-iam-v1 (1.2.0)
       google-protobuf (~> 3.14)
       googleapis-common-protos (>= 1.3.12, < 2.0)
       grpc (~> 1.27)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
-    jmespath (1.6.1)
-    jwt (2.4.1)
+    jmespath (1.6.2)
+    json (2.6.3)
+    jwt (2.7.0)
     memoist (0.16.2)
     method_source (1.0.0)
     minitest (5.17.0)
     multi_json (1.15.0)
-    newrelic_rpm (4.0.0.332)
-    oj (3.13.14)
+    newrelic_rpm (9.0.0)
+    oj (3.14.2)
     os (1.1.4)
     parallel (1.22.1)
-    parser (3.1.2.0)
+    parser (3.2.1.0)
       ast (~> 2.4.1)
-    pry (0.14.1)
+    pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (4.0.7)
+    public_suffix (5.0.1)
     rainbow (3.1.1)
     rake (13.0.6)
-    regexp_parser (2.5.0)
+    regexp_parser (2.7.0)
     require_all (3.0.0)
+    retriable (3.1.2)
     rexml (3.2.5)
-    rspec (3.11.0)
-      rspec-core (~> 3.11.0)
-      rspec-expectations (~> 3.11.0)
-      rspec-mocks (~> 3.11.0)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.0)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.1)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.2)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.1)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.3)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-support (3.11.0)
-    rubocop (1.30.1)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
+    rubocop (1.46.0)
+      json (~> 2.3)
       parallel (~> 1.10)
-      parser (>= 3.1.0.0)
+      parser (>= 3.2.0.0)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.18.0, < 2.0)
+      rubocop-ast (>= 1.26.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.18.0)
-      parser (>= 3.1.1.0)
+      unicode-display_width (>= 2.4.0, < 3.0)
+    rubocop-ast (1.26.0)
+      parser (>= 3.2.1.0)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.5)
     signet (0.17.0)
       addressable (~> 2.8)
       faraday (>= 0.17.5, < 3.a)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
-    simplecov (0.21.2)
+    simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
     thor (1.2.1)
-    tilt (2.0.10)
+    tilt (2.1.0)
     toiler (0.7.1)
       aws-sdk-sqs (>= 1.0.0, < 2.0.0)
       concurrent-ruby (~> 1.0, >= 1.0.0)
@@ -164,7 +172,7 @@ GEM
       ethon (>= 0.9.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.1.0)
+    unicode-display_width (2.4.2)
 
 PLATFORMS
   ruby
@@ -174,7 +182,7 @@ DEPENDENCIES
   byebug
   codeclimate-test-reporter
   little_monster!
-  newrelic_rpm (~> 4.0.0)
+  newrelic_rpm
   oj
   pry
   rake

diff --git a/lib/little_monster.rb b/lib/little_monster.rb
@@ -19,7 +19,7 @@ def init
     @@logger = @@env.test? ? Logger.new('/dev/null') : Toiler.logger
 
     @@logger.formatter = proc do |severity, datetime, _progname, msg|
-      MultiJson.dump(timestamp: datetime, severity: severity, message: msg) + "\n"
+      "#{MultiJson.dump(timestamp: datetime, severity: severity, message: msg)}\n"
     end
   end
 
@@ -61,6 +61,7 @@ def logger
 
   def method_missing(method, *args, &block)
     return @@config.public_send(method) if @@config.respond_to? method
+
     super method, *args, &block
   end
 end

diff --git a/lib/little_monster/config.rb b/lib/little_monster/config.rb
@@ -1,27 +1,8 @@
 module LittleMonster
   class Config
-    attr_accessor :api_url
-
-    attr_accessor :worker_concurrency
-    attr_accessor :worker_queue
-    attr_accessor :worker_provider
-
-    attr_accessor :formatter
-
-    attr_accessor :request_timeout
-
-    attr_accessor :default_request_retries
-    attr_accessor :default_request_retry_wait
-
-    attr_accessor :task_requests_retries
-    attr_accessor :task_requests_retry_wait
-
-    attr_accessor :job_requests_retries
-    attr_accessor :job_requests_retry_wait
-
-    attr_accessor :heartbeat_execution_interval
-
-    attr_accessor :default_job_retries
+    attr_accessor :api_url, :worker_concurrency, :worker_queue, :worker_provider, :formatter, :request_timeout,
+                  :default_request_retries, :default_request_retry_wait, :task_requests_retries, :task_requests_retry_wait,
+                  :job_requests_retries, :job_requests_retry_wait, :heartbeat_execution_interval, :default_job_retries
 
     def initialize(params = {})
       params.to_hash.each do |key, value|

diff --git a/lib/little_monster/core.rb b/lib/little_monster/core.rb
@@ -10,6 +10,7 @@
 require 'little_monster/core/errors/job_already_locked_error'
 require 'little_monster/core/errors/callback_failed_error'
 require 'little_monster/core/errors/ownership_lost_error'
+require 'little_monster/core/errors/task_not_found_error'
 
 require 'little_monster/core/tagged_logger'
 require 'little_monster/core/loggable' # must be required first to satisfy job and task dependencies

diff --git a/lib/little_monster/core/api.rb b/lib/little_monster/core/api.rb
@@ -34,7 +34,7 @@ def request(method, path, params = {}, retries: LittleMonster.default_request_re
         request_id = SecureRandom.uuid
         ret = 0
         res = nil
-        url = [LittleMonster.api_url.chomp('/'), path.sub(/\//, '')].join '/'
+        url = [LittleMonster.api_url.chomp('/'), path.sub(%r{/}, '')].join '/'
 
         params[:body] = MultiJson.dump params.fetch(:body, {}) unless params[:body].is_a? String
 
@@ -47,7 +47,7 @@ def request(method, path, params = {}, retries: LittleMonster.default_request_re
         begin
           res = Typhoeus.public_send method, url, params
           if !res.success? && (res.code < 400 || res.code >= 500)
-            raise FuryHttpApiError, "[type:request_failed][request_id:#{request_id}] request to #{res.effective_url} "\
+            raise FuryHttpApiError, "[type:request_failed][request_id:#{request_id}] request to #{res.effective_url} " \
                                     "failed with status #{res.code} code #{res.return_code} retry #{ret}"
           end
 
@@ -64,16 +64,15 @@ def request(method, path, params = {}, retries: LittleMonster.default_request_re
 
           if critical
             logger.error "[type:critical_request_failed][request_id:#{request_id}][url:#{url}][retries:#{ret}] request has reached max retries"
-            raise APIUnreachableError, "[request_id:#{request_id}] critical request to #{url} has fail, check little monster api"
+            raise APIUnreachableError,
+                  "[request_id:#{request_id}] critical request to #{url} has fail, check little monster api"
           end
         end
 
         res.define_singleton_method(:body) do
-          begin
-            MultiJson.load(res.options[:response_body], symbolize_keys: true)
-          rescue StandardError
-            res.options[:response_body]
-          end
+          MultiJson.load(res.options[:response_body], symbolize_keys: true)
+        rescue StandardError
+          res.options[:response_body]
         end
         res
       end

diff --git a/lib/little_monster/core/counters.rb b/lib/little_monster/core/counters.rb
@@ -20,6 +20,7 @@ def increase_counter(counter_name, unique_id, type, output = '')
   def counter(counter_name)
     resp = LittleMonster::Core::API.get("/jobs/#{job_id}/counters/#{counter_name}", {}, critical: true)
     raise MissedCounterError if resp.code == 404
+
     resp.body
   end
 

diff --git a/lib/little_monster/core/errors/task_not_found_error.rb b/lib/little_monster/core/errors/task_not_found_error.rb
@@ -0,0 +1,4 @@
+module LittleMonster::Core
+  class TaskNotFoundError < StandardError
+  end
+end