diff --git a/test/capture_warnings.rb b/test/capture_warnings.rb
index 410040b08..5acdb3a04 100644
--- a/test/capture_warnings.rb
+++ b/test/capture_warnings.rb
@@ -10,49 +10,56 @@ def initialize(fail_on_warnings = true)
     @output_dir = File.join(app_root, 'tmp')
     FileUtils.mkdir_p(output_dir)
     @bundle_dir = File.join(app_root, 'bundle')
+    @output = STDOUT
   end
 
-  def before_tests
-    $stderr.reopen(stderr_file.path)
+  def execute!
     $VERBOSE = true
-    at_exit { $stderr.reopen(STDERR) }
-  end
-
-  def after_tests
-    stderr_file.rewind
-    lines = stderr_file.read.split("\n")
-    stderr_file.close!
+    $stderr.reopen(stderr_file.path)
 
-    $stderr.reopen(STDERR)
+    Minitest.after_run do
+      stderr_file.rewind
+      lines = stderr_file.read.split("\n")
+      stderr_file.close!
+      $stderr.reopen(STDERR)
+      after_tests(lines)
+    end
+  end
 
+  # rubocop:disable Metrics/AbcSize
+  def after_tests(lines)
     app_warnings, other_warnings = lines.partition { |line|
       line.include?(app_root) && !line.include?(bundle_dir)
     }
 
-    if app_warnings.any?
-      puts <<-WARNINGS
-#{'-' * 30} app warnings: #{'-' * 30}
-
-#{app_warnings.join("\n")}
+    header = "#{'-' * 22} app warnings: #{'-' * 22}"
+    output.puts
+    output.puts header
 
-#{'-' * 75}
-      WARNINGS
+    if app_warnings.any?
+      output.puts app_warnings.join("\n")
+    else
+      output.puts 'None. Yay!'
     end
 
     if other_warnings.any?
       File.write(File.join(output_dir, 'warnings.txt'), other_warnings.join("\n") << "\n")
-      puts
-      puts 'Non-app warnings written to tmp/warnings.txt'
-      puts
+      output.puts
+      output.puts 'Non-app warnings written to tmp/warnings.txt'
+      output.puts
     end
 
+    output.puts
+    output.puts '-' * header.size
+
     # fail the build...
     if fail_on_warnings && app_warnings.any?
       abort "Failing build due to app warnings: #{app_warnings.inspect}"
     end
   end
+  # rubocop:enable Metrics/AbcSize
 
   private
 
-  attr_reader :stderr_file, :app_root, :output_dir, :bundle_dir, :fail_on_warnings
+  attr_reader :stderr_file, :app_root, :output_dir, :bundle_dir, :fail_on_warnings, :output
 end
diff --git a/test/support/stream_capture.rb b/test/support/stream_capture.rb
index 2d7dfea9e..da6acba3b 100644
--- a/test/support/stream_capture.rb
+++ b/test/support/stream_capture.rb
@@ -3,6 +3,7 @@
 begin
   require 'active_support/testing/stream'
 rescue LoadError
+  require 'tempfile'
   module ActiveSupport
     module Testing
       module Stream #:nodoc:
diff --git a/test/test_helper.rb b/test/test_helper.rb
index 980b04f8c..a72640d83 100644
--- a/test/test_helper.rb
+++ b/test/test_helper.rb
@@ -9,24 +9,27 @@
 require 'fileutils'
 FileUtils.mkdir_p(File.expand_path('../../tmp/cache', __FILE__))
 
+gem 'minitest'
 require 'minitest/autorun'
-# Ensure backward compatibility with Minitest 4
-Minitest::Test = MiniTest::Unit::TestCase unless defined?(Minitest::Test)
-
-require 'capture_warnings'
-@capture_warnings = CaptureWarnings.new(fail_build = true)
-@capture_warnings.before_tests
-if Minitest.respond_to?(:after_run)
-  Minitest.after_run do
-    @capture_warnings.after_tests
-  end
+if defined?(Minitest::Test)
+  # Minitest 5
+  # https://github.com/seattlerb/minitest/blob/e21fdda9d/lib/minitest/autorun.rb
+  # https://github.com/seattlerb/minitest/blob/e21fdda9d/lib/minitest.rb#L45-L59
 else
-  at_exit do
-    STDOUT.puts 'Minitest.after_run not available.'
-    @capture_warnings.after_tests
+  # Minitest 4
+  # https://github.com/seattlerb/minitest/blob/644a52fd0/lib/minitest/autorun.rb
+  # https://github.com/seattlerb/minitest/blob/644a52fd0/lib/minitest/unit.rb#L768-L787
+  # Ensure backward compatibility with Minitest 4
+  Minitest = MiniTest unless defined?(Minitest)
+  Minitest::Test = MiniTest::Unit::TestCase
+  def Minitest.after_run(&block)
+    MiniTest::Unit.after_tests(&block)
   end
 end
 
+require 'capture_warnings'
+CaptureWarnings.new(_fail_build = true).execute!
+
 require 'active_model_serializers'
 
 require 'support/stream_capture'