feat: add sidekiq_job_retry option

Author: ninoseki

docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
   redis:
     image: "redis/redis-stack:6.2.6-v10"

exe/miteru

@@ -3,7 +3,7 @@
 
 $LOAD_PATH.unshift("#{__dir__}/../lib")
 
-require 'miteru'
+require "miteru"
 
 ARGV.unshift(Miteru::CLI::App.default_task) unless Miteru::CLI::App.all_tasks.key?(ARGV[0])
 Miteru::CLI::App.start(ARGV)

lefthook.yml

@@ -5,5 +5,6 @@ pre-commit:
       run: bundle exec standardrb --fix {staged_files}
       stage_fixed: true
     actionlint:
-      glob: ".github/workflows/*.yaml"
+      root: ".github/workflows"
+      glob: "*.{yaml,yml}"
       run: actionlint

lib/miteru/commands/sidekiq.rb

@@ -10,7 +10,7 @@ class << self
         def included(thor)
           thor.class_eval do
             desc "sidekiq", "Start Sidekiq"
-            method_option :env, type: :string, default: "production", desc: "Environment"
+            method_option :env, type: :string, default: "production", desc: "Environment", aliases: "-e"
             method_option :concurrency, type: :numeric, default: 5, desc: "Sidekiq concurrency", aliases: "-c"
             def sidekiq
               require "sidekiq/cli"

lib/miteru/config.rb

@@ -14,14 +14,20 @@ class Config < Anyway::Config
       download_to: "/tmp",
       file_max_size: 1024 * 1024 * 100,
       file_extensions: [".zip", ".rar", ".7z", ".tar", ".gz"],
-      file_mime_types: ["application/zip", "application/vnd.rar", "application/x-7z-compressed", "application/x-tar",
-        "application/gzip"],
+      file_mime_types: [
+        "application/zip",
+        "application/vnd.rar",
+        "application/x-7z-compressed",
+        "application/x-tar",
+        "application/gzip"
+      ],
       api_timeout: 60,
       http_timeout: 60,
       download_timeout: 60,
       sentry_dsn: nil,
       sentry_trace_sample_rate: 0.25,
       sidekiq_redis_url: nil,
+      sidekiq_job_retry: 0,
       cache_redis_url: nil,
       cache_ex: nil,
       cache_prefix: "miteru:cache",
@@ -44,6 +50,39 @@ class Config < Anyway::Config
     # @!attribute [r] sidekiq_redis_url
     #   @return [String, nil]
 
+    # @!attribute [r] sidekiq_job_retry
+    #   @return [Integer]
+
+    # @!attribute [r] cache_redis_url
+    #   @return [String, nil]
+
+    # @!attribute [r] cache_ex
+    #   @return [Integer, nil]
+
+    # @!attribute [r] cache_prefix
+    #   @return [String]
+
+    # @!attribute [r] http_timeout
+    #   @return [Integer]
+
+    # @!attribute [r] api_timeout
+    #   @return [Integer]
+
+    # @!attribute [r] download_timeout
+    #   @return [Integer]
+
+    # @!attribute [rw] auto_download
+    #   @return [Boolean]
+
+    # @!attribute [rw] directory_traveling
+    #   @return [Boolean]
+
+    # @!attribute [rw] download_to
+    #   @return [String]
+
+    # @!attribute [rw] threads
+    #   @return [Integer]
+
     # @!attribute [r] cache_redis_url
     #   @return [String, nil]
 

lib/miteru/feeds/ayashige.rb

@@ -4,7 +4,7 @@ module Miteru
   class Feeds
     class Ayashige < Base
       def initialize(base_url = "https://ayashige.herokuapp.com")
-        super(base_url)
+        super
       end
 
       def urls

lib/miteru/feeds/phishing_database.rb

@@ -4,7 +4,7 @@ module Miteru
   class Feeds
     class PhishingDatabase < Base
       def initialize(base_url = "https://raw.githubusercontent.com")
-        super(base_url)
+        super
       end
 
       def urls

lib/miteru/feeds/tweetfeed.rb

@@ -4,7 +4,7 @@ module Miteru
   class Feeds
     class TweetFeed < Base
       def initialize(base_url = "https://api.tweetfeed.live")
-        super(base_url)
+        super
       end
 
       def urls

lib/miteru/feeds/urlscan.rb

@@ -7,7 +7,7 @@ class UrlScan < Base
       # @param [String] base_url
       #
       def initialize(base_url = "https://urlscan.io")
-        super(base_url)
+        super
 
         @headers = {"api-key": api_key}
       end

lib/miteru/feeds/urlscan_pro.rb

@@ -7,7 +7,7 @@ class UrlScanPro < Base
       # @param [String] base_url
       #
       def initialize(base_url = "https://urlscan.io")
-        super(base_url)
+        super
 
         @headers = {"api-key": api_key}
       end

lib/miteru/sidekiq/application.rb

@@ -6,6 +6,7 @@
 
 Sidekiq.configure_server do |config|
   config.redis = {url: Miteru.config.sidekiq_redis_url.to_s}
+  config.default_job_options = {"retry" => Miteru.config.sidekiq_job_retry}
 end
 
 Sidekiq.configure_client do |config|

lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "2.3.1"
+  VERSION = "2.3.2"
 end

miteru.gemspec

@@ -30,7 +30,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "coveralls_reborn", "~> 0.28"
   spec.add_development_dependency "fuubar", "~> 2.5.1"
   spec.add_development_dependency "mysql2", "~> 0.5.6"
-  spec.add_development_dependency "pg", "~> 1.5.7"
+  spec.add_development_dependency "pg", "~> 1.5.8"
   spec.add_development_dependency "rake", "~> 13.2.1"
   spec.add_development_dependency "rspec", "~> 3.13"
   spec.add_development_dependency "simplecov-lcov", "~> 0.8"
@@ -56,7 +56,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "rack-session", "2.0.0"
   spec.add_dependency "rackup", "2.1.0"
   spec.add_dependency "redis", "5.3.0"
-  spec.add_dependency "semantic_logger", "4.16.0"
+  spec.add_dependency "semantic_logger", "4.16.1"
   spec.add_dependency "sentry-ruby", "5.19.0"
   spec.add_dependency "sentry-sidekiq", "5.19.0"
   spec.add_dependency "sidekiq", "7.3.2"

spec/orchestrator_spec.rb

@@ -2,7 +2,7 @@
 
 class DummyFeed < Miteru::Feeds::Base
   def initialize(base_url = "http://example.com")
-    super(base_url)
+    super
   end
 
   def urls