diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
index cb69c5a746804..6d1142e2b3e22 100644
--- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
+++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -863,7 +863,6 @@ class BuildPlugin implements Plugin<Project> {
         project.tasks.withType(RandomizedTestingTask) {task ->
             jvm "${project.runtimeJavaHome}/bin/java"
             parallelism System.getProperty('tests.jvms', project.rootProject.ext.defaultParallel)
-            ifNoTests System.getProperty('tests.ifNoTests', 'fail')
             onNonEmptyWorkDirectory 'wipe'
             leaveTemporary true
 
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java
index df7e71302152d..eddbe40c1a1a5 100644
--- a/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java
+++ b/buildSrc/src/main/java/org/elasticsearch/gradle/precommit/TestingConventionsTasks.java
@@ -25,7 +25,6 @@
 import org.gradle.api.file.FileTree;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.OutputFile;
-import org.gradle.api.tasks.SkipWhenEmpty;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.api.tasks.testing.Test;
 import org.gradle.api.tasks.util.PatternFilterable;
@@ -103,6 +102,22 @@ public void doCheck() throws IOException {
             final Map<String, Set<File>> classFilesPerRandomizedTestingTask = classFilesPerRandomizedTestingTask(allTestClassFiles);
             final Map<String, Set<File>> classFilesPerGradleTestTask = classFilesPerGradleTestTask();
 
+            Map<String, Set<Class<?>>> testClassesPerTask =
+                Stream.concat(
+                    classFilesPerGradleTestTask.entrySet().stream(),
+                    classFilesPerRandomizedTestingTask.entrySet().stream()
+                )
+                    .collect(
+                        Collectors.toMap(
+                            Map.Entry::getKey,
+                            entry -> entry.getValue().stream()
+                                .map(classes::get)
+                                .filter(implementsNamingConvention)
+                                .collect(Collectors.toSet())
+                        )
+                    );
+
+
             problems = collectProblems(
                 checkNoneExists(
                     "Test classes implemented by inner classes will not run",
@@ -118,6 +133,16 @@ public void doCheck() throws IOException {
                         .filter(this::seemsLikeATest)
                         .filter(implementsNamingConvention.negate())
                 ),
+                collectProblems(
+                    testClassesPerTask.entrySet().stream()
+                    .map( entry ->
+                        checkAtLeastOneExists(
+                            "test class in " + entry.getKey(),
+                            entry.getValue().stream()
+                        )
+                    )
+                    .collect(Collectors.joining())
+                ),
                 checkNoneExists(
                     "Test classes are not included in any enabled task (" +
                         Stream.concat(
@@ -215,7 +240,6 @@ private Class<? extends Task> getRandomizedTestingTask() {
     }
 
     @Input
-    @SkipWhenEmpty
     public Map<String, File> getTestClassNames() {
         if (testClassNames == null) {
             testClassNames = Boilerplate.getJavaSourceSets(getProject()).getByName("test").getOutput().getClassesDirs()
@@ -243,6 +267,14 @@ private String checkNoneExists(String message, Stream<? extends Class<?>> stream
         }
     }
 
+    private String checkAtLeastOneExists(String message, Stream<? extends Class<?>> stream) {
+        if (stream.findAny().isPresent()) {
+            return "";
+        } else {
+            return "Expected at least one " + message + ", but found none.\n";
+        }
+    }
+
     private boolean seemsLikeATest(Class<?> clazz) {
         try {
             ClassLoader classLoader = clazz.getClassLoader();