v3.0.0

.github/workflows/ruby.yml

@@ -35,7 +35,7 @@ jobs:
           --health-retries=3
     strategy:
       matrix:
-        ruby: [3.1, 3.2, 3.3]
+        ruby: [3.2, 3.3]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Ruby
@@ -49,10 +49,8 @@ jobs:
       - name: Test with PostgreSQL
         env:
           MITERU_DATABASE: postgresql://postgres:postgres@localhost:5432/test
-        run: |
-          bundle exec rake
+        run: bundle exec rake
       - name: Test with MySQL
         env:
           MITERU_DATABASE: mysql2://mysql:[email protected]:3306/test
-        run: |
-          bundle exec rake
+        run: bundle exec rake

lib/miteru.rb

@@ -9,7 +9,6 @@
 # Core 3rd party libraries
 require "colorize"
 require "memo_wise"
-require "parallel"
 require "semantic_logger"
 require "sentry-ruby"
 
@@ -108,13 +107,6 @@ def cache
       @cache ||= Cache.new(Miteru.config.cache_redis_url)
     end
 
-    #
-    # @return [Boolean]
-    #
-    def sidekiq?
-      !Miteru.config.sidekiq_redis_url.nil?
-    end
-
     def sentry?
       !Miteru.config.sentry_dsn.nil?
     end

lib/miteru/commands/main.rb

@@ -13,7 +13,6 @@ def included(thor)
             method_option :directory_traveling, type: :boolean, default: false,
               desc: "Enable or disable directory traveling"
             method_option :download_to, type: :string, default: "/tmp", desc: "Directory to download phishing kits"
-            method_option :threads, type: :numeric, desc: "Number of threads to use", default: Parallel.processor_count
             method_option :verbose, type: :boolean, default: true
             desc "execute", "Execute the crawler"
             around :with_db_connection
@@ -22,10 +21,8 @@ def execute
                 config.auto_download = options["auto_download"]
                 config.directory_traveling = options["directory_traveling"]
                 config.download_to = options["download_to"]
-                config.threads = options["threads"]
                 config.verbose = options["verbose"]
               end
-
               Orchestrator.call
             end
             default_command :execute

lib/miteru/config.rb

@@ -26,7 +26,7 @@ class Config < Anyway::Config
       download_timeout: 60,
       sentry_dsn: nil,
       sentry_trace_sample_rate: 0.25,
-      sidekiq_redis_url: nil,
+      sidekiq_redis_url: "redis://localhost:6379",
       sidekiq_job_retry: 0,
       sidekiq_batch_size: 50,
       sidekiq_job_timeout: 600,
@@ -35,7 +35,6 @@ class Config < Anyway::Config
       cache_prefix: "miteru:cache",
       slack_channel: "#general",
       slack_webhook_url: nil,
-      threads: Parallel.processor_count,
       urlscan_api_key: nil,
       urlscan_submit_visibility: "public",
       urlscan_date_condition: "date:>now-1h",
@@ -50,7 +49,7 @@ class Config < Anyway::Config
     #   @return [Float]
 
     # @!attribute [r] sidekiq_redis_url
-    #   @return [String, nil]
+    #   @return [String]
 
     # @!attribute [r] sidekiq_job_retry
     #   @return [Integer]
@@ -88,9 +87,6 @@ class Config < Anyway::Config
     # @!attribute [rw] download_to
     #   @return [String]
 
-    # @!attribute [rw] threads
-    #   @return [Integer]
-
     # @!attribute [r] cache_redis_url
     #   @return [String, nil]
 

lib/miteru/database.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class V2Schema < ActiveRecord::Migration[7.2]
+class V2Schema < ActiveRecord::Migration[8.0]
   def change
     create_table :records, if_not_exists: true do |t|
       t.string :sha256, null: false, index: {unique: true}

lib/miteru/orchestrator.rb

@@ -4,32 +4,10 @@ module Miteru
   class Orchestrator < Service
     def call
       logger.info("#{non_cached_websites.length} websites loaded in total.") if verbose?
-
-      if sidekiq?
-        sidekiq_call
-      else
-        parallel_call
-      end
-    end
-
-    def sidekiq_call
       array_of_args = non_cached_websites.map { |website| [website.url, website.source] }
       Jobs::CrawleJob.perform_bulk(array_of_args, batch_size: Miteru.config.sidekiq_batch_size)
     end
 
-    def parallel_call
-      logger.info("Use #{threads} thread(s).") if verbose?
-      Parallel.each(non_cached_websites, in_threads: threads) do |website|
-        logger.info("Website:#{website.truncated_url} crawling started.") if verbose?
-        result = Crawler.result(website)
-        if result.success?
-          logger.info("Crawler:#{website.truncated_url} succeeded.")
-        else
-          logger.info("Crawler:#{website.truncated_url} failed - #{result.failure}.")
-        end
-      end
-    end
-
     #
     # @return [Array<Miteru::Website>]
     #

lib/miteru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Miteru
-  VERSION = "2.4.0"
+  VERSION = "3.0.0"
 end

miteru.gemspec

@@ -16,6 +16,8 @@ Gem::Specification.new do |spec|
   spec.homepage = "https://github.com/ninoseki/miteru"
   spec.license = "MIT"
 
+  spec.required_ruby_version = ">= 3.2"
+
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
@@ -34,12 +36,12 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 13.2.1"
   spec.add_development_dependency "rspec", "~> 3.13"
   spec.add_development_dependency "simplecov-lcov", "~> 0.8"
-  spec.add_development_dependency "standard", "~> 1.41.1"
+  spec.add_development_dependency "standard", "~> 1.42.0"
   spec.add_development_dependency "test-prof", "~> 1.4.2"
   spec.add_development_dependency "vcr", "~> 6.3.1"
   spec.add_development_dependency "webmock", "~> 3.24.0"
 
-  spec.add_dependency "activerecord", "7.2.2"
+  spec.add_dependency "activerecord", "8.0.0"
   spec.add_dependency "addressable", "2.8.7"
   spec.add_dependency "anyway_config", "2.6.4"
   spec.add_dependency "colorize", "1.1.0"
@@ -48,20 +50,19 @@ Gem::Specification.new do |spec|
   spec.add_dependency "dry-files", "1.1.0"
   spec.add_dependency "dry-monads", "1.6.0"
   spec.add_dependency "http", "5.2.0"
-  spec.add_dependency "memo_wise", "1.9.0"
+  spec.add_dependency "memo_wise", "1.10.0"
   spec.add_dependency "oga", "3.4"
-  spec.add_dependency "parallel", "1.26.3"
   spec.add_dependency "puma", "6.4.3"
   spec.add_dependency "rack", "3.1.8"
   spec.add_dependency "rack-session", "2.0.0"
-  spec.add_dependency "rackup", "2.2.0"
+  spec.add_dependency "rackup", "2.2.1"
   spec.add_dependency "redis", "5.3.0"
   spec.add_dependency "semantic_logger", "4.16.1"
   spec.add_dependency "sentry-ruby", "5.21.0"
   spec.add_dependency "sentry-sidekiq", "5.21.0"
-  spec.add_dependency "sidekiq", "7.3.4"
+  spec.add_dependency "sidekiq", "7.3.5"
   spec.add_dependency "slack-notifier", "2.4.0"
-  spec.add_dependency "sqlite3", "1.7.2"
+  spec.add_dependency "sqlite3", "2.2.0"
   spec.add_dependency "thor", "1.3.2"
   spec.add_dependency "thor-hollaback", "0.2.1"
   spec.add_dependency "uuidtools", "2.2.0"