diff --git a/.travis.yml b/.travis.yml
index b019a2df..7886e438 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,7 +10,14 @@ rvm:
   - jruby-9.1.16.0
 
 before_install:
-  - gem update --system
+  # For jruby we need to stick with rubygems 2.7.4 until
+  # https://github.com/rubygems/rubygems/issues/2188
+  # is fixed and released.
+  #
+  # Without this workaround, for jruby builds, rubygems
+  # activates jruby stdlib minitest (v5.4.1) instead of the
+  # bundled version (v5.11.3).
+  - if [ "${TRAVIS_RUBY_VERSION:0:5}" = "jruby" ]; then gem update --system 2.7.4; else gem update --system; fi
   - gem install bundler
 
 gemfile:
diff --git a/spec/acceptance/cache_store_config_for_allow2ban_spec.rb b/spec/acceptance/cache_store_config_for_allow2ban_spec.rb
new file mode 100644
index 00000000..385697bd
--- /dev/null
+++ b/spec/acceptance/cache_store_config_for_allow2ban_spec.rb
@@ -0,0 +1,65 @@
+require_relative "../spec_helper"
+
+describe "Cache store config when using allow2ban" do
+  before do
+    Rack::Attack.blocklist("allow2ban pentesters") do |request|
+      Rack::Attack::Allow2Ban.filter(request.ip, maxretry: 2, findtime: 30, bantime: 60) do
+        request.path.include?("scarce-resource")
+      end
+    end
+  end
+
+  it "gives error if no store was configured" do
+    assert_raises do
+      get "/"
+    end
+  end
+
+  it "gives error if incompatible store was configured" do
+    Rack::Attack.cache.store = Object.new
+
+    assert_raises do
+      get "/"
+    end
+  end
+
+  it "works with any object that responds to #read, #write and #increment" do
+    basic_store_class = Class.new do
+      attr_accessor :backend
+
+      def initialize
+        @backend = {}
+      end
+
+      def read(key)
+        @backend[key]
+      end
+
+      def write(key, value, options = {})
+        @backend[key] = value
+      end
+
+      def increment(key, count, options = {})
+        @backend[key] ||= 0
+        @backend[key] += 1
+      end
+    end
+
+    Rack::Attack.cache.store = basic_store_class.new
+
+    get "/"
+    assert_equal 200, last_response.status
+
+    get "/scarce-resource"
+    assert_equal 200, last_response.status
+
+    get "/scarce-resource"
+    assert_equal 200, last_response.status
+
+    get "/scarce-resource"
+    assert_equal 403, last_response.status
+
+    get "/"
+    assert_equal 403, last_response.status
+  end
+end
diff --git a/spec/acceptance/cache_store_config_for_fail2ban_spec.rb b/spec/acceptance/cache_store_config_for_fail2ban_spec.rb
new file mode 100644
index 00000000..320fb22d
--- /dev/null
+++ b/spec/acceptance/cache_store_config_for_fail2ban_spec.rb
@@ -0,0 +1,62 @@
+require_relative "../spec_helper"
+
+describe "Cache store config when using fail2ban" do
+  before do
+    Rack::Attack.blocklist("fail2ban pentesters") do |request|
+      Rack::Attack::Fail2Ban.filter(request.ip, maxretry: 2, findtime: 30, bantime: 60) do
+        request.path.include?("private-place")
+      end
+    end
+  end
+
+  it "gives error if no store was configured" do
+    assert_raises do
+      get "/"
+    end
+  end
+
+  it "gives error if incompatible store was configured" do
+    Rack::Attack.cache.store = Object.new
+
+    assert_raises do
+      get "/"
+    end
+  end
+
+  it "works with any object that responds to #read, #write and #increment" do
+    basic_store_class = Class.new do
+      attr_accessor :backend
+
+      def initialize
+        @backend = {}
+      end
+
+      def read(key)
+        @backend[key]
+      end
+
+      def write(key, value, options = {})
+        @backend[key] = value
+      end
+
+      def increment(key, count, options = {})
+        @backend[key] ||= 0
+        @backend[key] += 1
+      end
+    end
+
+    Rack::Attack.cache.store = basic_store_class.new
+
+    get "/"
+    assert_equal 200, last_response.status
+
+    get "/private-place"
+    assert_equal 403, last_response.status
+
+    get "/private-place"
+    assert_equal 403, last_response.status
+
+    get "/"
+    assert_equal 403, last_response.status
+  end
+end