Deal nicely with our somewhat magical no_input value

lib/benchee/benchmark/runner.ex

@@ -16,6 +16,8 @@ defmodule Benchee.Benchmark.Runner do
     ScenarioContext
   }
 
+  @no_input Benchmark.no_input()
+
   @doc """
   Executes the benchmarks defined before by first running the defined functions
   for `warmup` time without gathering results and them running them for `time`
@@ -75,7 +77,7 @@ defmodule Benchee.Benchmark.Runner do
         %{^input_name => {previous_job, previous_value}} when return_value !== previous_value ->
           raise Benchee.PreCheckError,
             message: """
-            all_same pre check failed for input #{inspect(input_name)}:
+            all_same pre check failed#{pre_check_failed_input_message(input_name)}:
             - #{previous_job} returned #{inspect(previous_value)}
             - #{scenario.job_name} returned #{inspect(return_value)}
             """
@@ -86,6 +88,9 @@ defmodule Benchee.Benchmark.Runner do
     end)
   end
 
+  defp pre_check_failed_input_message(@no_input), do: ""
+  defp pre_check_failed_input_message(input_name), do: " for input #{inspect(input_name)}"
+
   def measure_and_report_function_call_overhead(prtiner) do
     overhead = FunctionCallOverhead.measure()
     prtiner.function_call_overhead(overhead)
@@ -345,7 +350,6 @@ defmodule Benchee.Benchmark.Runner do
     RepeatedMeasurement.collect(scenario, scenario_context, collector)
   end
 
-  @no_input Benchmark.no_input()
   def main_function(function, @no_input), do: function
   def main_function(function, input), do: fn -> function.(input) end
 

samples/pre_check.exs

@@ -1,4 +1,19 @@
 a = 7
-Benchee.run(%{"a*2" => fn -> a * 2 end, "a+a" => fn -> a + a end}, time: 0, warmup: 0, pre_check: true)
-Benchee.run(%{"a*2" => fn -> a * 2 end, "a+a" => fn -> a + a end}, time: 0, warmup: 0, pre_check: :all_same)
-Benchee.run(%{"a*2" => fn -> a * 2 end, "a+a" => fn -> a + a + 1 end}, time: 0, warmup: 0, pre_check: :all_same)
+
+Benchee.run(%{"a*2" => fn -> a * 2 end, "a+a" => fn -> a + a end},
+  time: 0,
+  warmup: 0,
+  pre_check: true
+)
+
+Benchee.run(%{"a*2" => fn -> a * 2 end, "a+a" => fn -> a + a end},
+  time: 0,
+  warmup: 0,
+  pre_check: :all_same
+)
+
+Benchee.run(%{"a*2" => fn -> a * 2 end, "a+a" => fn -> a + a + 1 end},
+  time: 0,
+  warmup: 0,
+  pre_check: :all_same
+)

test/benchee/benchmark/runner_test.exs

@@ -1007,5 +1007,23 @@ defmodule Benchee.Benchmark.RunnerTest do
         |> Benchmark.collect(FakeBenchmarkPrinter)
       end
     end
+
+    test "the failure message handles the ominous no input value nicely" do
+      config = %{time: 0, warmup: 0, pre_check: :all_same}
+
+      message = """
+      all_same pre check failed:
+      - double returned 2
+      - triple returned 3
+      """
+
+      assert_raise Benchee.PreCheckError, message, fn ->
+        %Suite{configuration: config}
+        |> test_suite
+        |> Benchmark.benchmark("double", fn -> 2 end)
+        |> Benchmark.benchmark("triple", fn -> 3 end)
+        |> Benchmark.collect(FakeBenchmarkPrinter)
+      end
+    end
   end
 end