✨ Add additional fuzzing probes

checks/evaluation/fuzzing.go

@@ -18,13 +18,19 @@ import (
 	"github.com/ossf/scorecard/v4/checker"
 	sce "github.com/ossf/scorecard/v4/errors"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer"
 	"github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite"
+	"github.com/ossf/scorecard/v4/probes/fuzzedWithCppLibFuzzer"
 	"github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative"
+	"github.com/ossf/scorecard/v4/probes/fuzzedWithJavaJazzerFuzzer"
 	"github.com/ossf/scorecard/v4/probes/fuzzedWithOSSFuzz"
 	"github.com/ossf/scorecard/v4/probes/fuzzedWithOneFuzz"
 	"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedHaskell"
 	"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedJavascript"
 	"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedTypescript"
+	"github.com/ossf/scorecard/v4/probes/fuzzedWithPythonAtheris"
+	"github.com/ossf/scorecard/v4/probes/fuzzedWithRustCargofuzz"
+	"github.com/ossf/scorecard/v4/probes/fuzzedWithSwiftLibFuzzer"
 )
 
 // Fuzzing applies the score policy for the Fuzzing check.
@@ -35,12 +41,20 @@ func Fuzzing(name string,
 	expectedProbes := []string{
 		fuzzedWithClusterFuzzLite.Probe,
 		fuzzedWithGoNative.Probe,
+		fuzzedWithPythonAtheris.Probe,
+		fuzzedWithCLibFuzzer.Probe,
+		fuzzedWithCppLibFuzzer.Probe,
+		fuzzedWithRustCargofuzz.Probe,
+		fuzzedWithSwiftLibFuzzer.Probe,
+		fuzzedWithJavaJazzerFuzzer.Probe,
 		fuzzedWithOneFuzz.Probe,
 		fuzzedWithOSSFuzz.Probe,
 		fuzzedWithPropertyBasedHaskell.Probe,
 		fuzzedWithPropertyBasedJavascript.Probe,
 		fuzzedWithPropertyBasedTypescript.Probe,
 	}
+	// TODO: other packages to consider:
+	// - github.com/google/fuzztest
 
 	if !finding.UniqueProbesEqual(findings, expectedProbes) {
 		e := sce.WithMessage(sce.ErrScorecardInternal, "invalid probe results")

checks/evaluation/fuzzing_test.go

@@ -40,6 +40,30 @@ func TestFuzzing(t *testing.T) {
 					Probe:   "fuzzedWithGoNative",
 					Outcome: finding.OutcomeNegative,
 				},
+				{
+					Probe:   "fuzzedWithPythonAtheris",
+					Outcome: finding.OutcomeNegative,
+				},
+				{
+					Probe:   "fuzzedWithCLibFuzzer",
+					Outcome: finding.OutcomeNegative,
+				},
+				{
+					Probe:   "fuzzedWithCppLibFuzzer",
+					Outcome: finding.OutcomeNegative,
+				},
+				{
+					Probe:   "fuzzedWithRustCargofuzz",
+					Outcome: finding.OutcomeNegative,
+				},
+				{
+					Probe:   "fuzzedWithSwiftLibFuzzer",
+					Outcome: finding.OutcomeNegative,
+				},
+				{
+					Probe:   "fuzzedWithJavaJazzerFuzzer",
+					Outcome: finding.OutcomeNegative,
+				},
 				{
 					Probe:   "fuzzedWithOneFuzz",
 					Outcome: finding.OutcomeNegative,
@@ -63,7 +87,7 @@ func TestFuzzing(t *testing.T) {
 			},
 			result: scut.TestReturn{
 				Score:        checker.MinResultScore,
-				NumberOfWarn: 7,
+				NumberOfWarn: 13,
 			},
 		},
 		{
@@ -77,6 +101,30 @@ func TestFuzzing(t *testing.T) {
 					Probe:   "fuzzedWithGoNative",
 					Outcome: finding.OutcomePositive,
 				},
+				{
+					Probe:   "fuzzedWithPythonAtheris",
+					Outcome: finding.OutcomeNegative,
+				},
+				{
+					Probe:   "fuzzedWithCLibFuzzer",
+					Outcome: finding.OutcomeNegative,
+				},
+				{
+					Probe:   "fuzzedWithCppLibFuzzer",
+					Outcome: finding.OutcomeNegative,
+				},
+				{
+					Probe:   "fuzzedWithRustCargofuzz",
+					Outcome: finding.OutcomeNegative,
+				},
+				{
+					Probe:   "fuzzedWithSwiftLibFuzzer",
+					Outcome: finding.OutcomeNegative,
+				},
+				{
+					Probe:   "fuzzedWithJavaJazzerFuzzer",
+					Outcome: finding.OutcomeNegative,
+				},
 				{
 					Probe:   "fuzzedWithOneFuzz",
 					Outcome: finding.OutcomeNegative,

checks/fuzzing_test.go

@@ -53,7 +53,7 @@ func TestFuzzing(t *testing.T) {
 			wantErr: false,
 			expected: scut.TestReturn{
 				Error:         nil,
-				NumberOfWarn:  7,
+				NumberOfWarn:  13,
 				NumberOfDebug: 0,
 				NumberOfInfo:  0,
 				Score:         0,
@@ -110,7 +110,7 @@ func TestFuzzing(t *testing.T) {
 			wantFuzzErr: false,
 			expected: scut.TestReturn{
 				Error:         nil,
-				NumberOfWarn:  7,
+				NumberOfWarn:  13,
 				NumberOfDebug: 0,
 				NumberOfInfo:  0,
 				Score:         0,
@@ -121,7 +121,7 @@ func TestFuzzing(t *testing.T) {
 			wantFuzzErr: true,
 			expected: scut.TestReturn{
 				Error:         nil,
-				NumberOfWarn:  7,
+				NumberOfWarn:  13,
 				NumberOfDebug: 0,
 				NumberOfInfo:  0,
 				Score:         0,

checks/raw/fuzzing.go

@@ -35,6 +35,12 @@ const (
 	fuzzerPropertyBasedHaskell    = "HaskellPropertyBasedTesting"
 	fuzzerPropertyBasedJavaScript = "JavaScriptPropertyBasedTesting"
 	fuzzerPropertyBasedTypeScript = "TypeScriptPropertyBasedTesting"
+	fuzzerPythonAtheris           = "PythonAtherisFuzzer"
+	fuzzerCLibFuzzer              = "CLibFuzzer"
+	fuzzerCppLibFuzzer            = "CppLibFuzzer"
+	fuzzerSwiftLibFuzzer          = "SwiftLibFuzzer"
+	fuzzerRustCargoFuzz           = "RustCargoFuzzer"
+	fuzzerJavaJazzerFuzzer        = "JavaJazzerFuzzer"
 	// TODO: add more fuzzing check supports.
 )
 
@@ -47,8 +53,8 @@ type filesWithPatternStr struct {
 type languageFuzzConfig struct {
 	URL, Desc *string
 
-	// Pattern is according to path.Match.
-	filePattern string
+	// Patterns are according to path.Match.
+	filePatterns []string
 
 	funcPattern, Name string
 	// TODO: add more language fuzzing-related fields.
@@ -59,10 +65,10 @@ type languageFuzzConfig struct {
 var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{
 	// Default fuzz patterns for Go.
 	clients.Go: {
-		filePattern: "*_test.go",
-		funcPattern: `func\s+Fuzz\w+\s*\(\w+\s+\*testing.F\)`,
-		Name:        fuzzerBuiltInGo,
-		URL:         asPointer("https://go.dev/doc/fuzz/"),
+		filePatterns: []string{"*_test.go"},
+		funcPattern:  `func\s+Fuzz\w+\s*\(\w+\s+\*testing.F\)`,
+		Name:         fuzzerBuiltInGo,
+		URL:          asPointer("https://go.dev/doc/fuzz/"),
 		Desc: asPointer(
 			"Go fuzzing intelligently walks through the source code to report failures and find vulnerabilities."),
 	},
@@ -80,7 +86,7 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{
 	//
 	// This is not an exhaustive list.
 	clients.Haskell: {
-		filePattern: "*.hs",
+		filePatterns: []string{"*.hs", "*.lhs"},
 		// Look for direct imports of QuickCheck, Hedgehog, validity, or SmallCheck,
 		// or their indirect imports through the higher-level Hspec or Tasty testing frameworks.
 		funcPattern: `import\s+(qualified\s+)?Test\.((Hspec|Tasty)\.)?(QuickCheck|Hedgehog|Validity|SmallCheck)`,
@@ -96,7 +102,7 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{
 	//
 	// This is not an exhaustive list.
 	clients.JavaScript: {
-		filePattern: "*.js",
+		filePatterns: []string{"*.js"},
 		// Look for direct imports of fast-check.
 		funcPattern: `(from\s+['"]fast-check['"]|require\(\s*['"]fast-check['"]\s*\))`,
 		Name:        fuzzerPropertyBasedJavaScript,
@@ -105,14 +111,56 @@ var languageFuzzSpecs = map[clients.LanguageName]languageFuzzConfig{
 				"and test that specific properties are satisfied."),
 	},
 	clients.TypeScript: {
-		filePattern: "*.ts",
+		filePatterns: []string{"*.ts"},
 		// Look for direct imports of fast-check.
 		funcPattern: `(from\s+['"]fast-check['"]|require\(\s*['"]fast-check['"]\s*\))`,
 		Name:        fuzzerPropertyBasedTypeScript,
 		Desc: asPointer(
 			"Property-based testing in TypeScript generates test instances randomly or exhaustively " +
 				"and test that specific properties are satisfied."),
 	},
+	clients.Python: {
+		filePatterns: []string{"*.py"},
+		funcPattern:  `import atheris`,
+		Name:         fuzzerPythonAtheris,
+		Desc: asPointer(
+			"Python fuzzing by way of Atheris"),
+	},
+	clients.C: {
+		filePatterns: []string{"*.c"},
+		funcPattern:  `LLVMFuzzerTestOneInput`,
+		Name:         fuzzerCLibFuzzer,
+		Desc: asPointer(
+			"Fuzzed with C LibFuzzer"),
+	},
+	clients.Cpp: {
+		filePatterns: []string{"*.cc", "*.cpp"},
+		funcPattern:  `LLVMFuzzerTestOneInput`,
+		Name:         fuzzerCppLibFuzzer,
+		Desc: asPointer(
+			"Fuzzed with cpp LibFuzzer"),
+	},
+	clients.Rust: {
+		filePatterns: []string{"*.rs"},
+		funcPattern:  `libfuzzer_sys`,
+		Name:         fuzzerRustCargoFuzz,
+		Desc: asPointer(
+			"Fuzzed with Cargo-fuzz"),
+	},
+	clients.Java: {
+		filePatterns: []string{"*.java"},
+		funcPattern:  `com.code_intelligence.jazzer.api.FuzzedDataProvider;`,
+		Name:         fuzzerJavaJazzerFuzzer,
+		Desc: asPointer(
+			"Fuzzed with Jazzer fuzzer"),
+	},
+	clients.Swift: {
+		filePatterns: []string{"*.swift"},
+		funcPattern:  `LLVMFuzzerTestOneInput`,
+		Name:         fuzzerSwiftLibFuzzer,
+		Desc: asPointer(
+			"Fuzzed with Swift LibFuzzer"),
+	},
 	// TODO: add more language-specific fuzz patterns & configs.
 }
 
@@ -254,22 +302,26 @@ func checkFuzzFunc(c *checker.CheckRequest, lang clients.LanguageName) (bool, []
 	// Get patterns for file and func.
 	// We use the file pattern in the matcher to match the test files,
 	// and put the func pattern in var data to match file contents (func names).
-	filePattern, funcPattern := pattern.filePattern, pattern.funcPattern
-	matcher := fileparser.PathMatcher{
-		Pattern:       filePattern,
-		CaseSensitive: false,
-	}
-	data.pattern = funcPattern
-	err := fileparser.OnMatchingFileContentDo(c.RepoClient, matcher, getFuzzFunc, &data)
-	if err != nil {
-		return false, nil, fmt.Errorf("error when OnMatchingFileContentDo: %w", err)
+	filePatterns, funcPattern := pattern.filePatterns, pattern.funcPattern
+	var dataFiles []checker.File
+	for _, filePattern := range filePatterns {
+		matcher := fileparser.PathMatcher{
+			Pattern:       filePattern,
+			CaseSensitive: false,
+		}
+		data.pattern = funcPattern
+		err := fileparser.OnMatchingFileContentDo(c.RepoClient, matcher, getFuzzFunc, &data)
+		if err != nil {
+			return false, nil, fmt.Errorf("error when OnMatchingFileContentDo: %w", err)
+		}
+		dataFiles = append(dataFiles, data.files...)
 	}
 
-	if len(data.files) == 0 {
+	if len(dataFiles) == 0 {
 		// This means no fuzz funcs matched for this language.
 		return false, nil, nil
 	}
-	return true, data.files, nil
+	return true, dataFiles, nil
 }
 
 // This is the callback func for interface OnMatchingFileContentDo
@@ -322,7 +374,7 @@ func getProminentLanguages(langs []clients.Language) []clients.LanguageName {
 	// Languages that have lines of code above average will be considered prominent.
 	ret := []clients.LanguageName{}
 	for lName, loC := range langMap {
-		if loC >= avgLoC {
+		if loC >= avgLoC/4.0 {
 			lang := clients.LanguageName(strings.ToLower(string(lName)))
 			ret = append(ret, lang)
 		}

checks/raw/fuzzing_test.go

@@ -261,8 +261,8 @@ func Test_fuzzFileAndFuncMatchPattern(t *testing.T) {
 			expectedFileMatch: false,
 			expectedFuncMatch: false,
 			lang:              clients.LanguageName("not_a_supported_one"),
-			fileName:          "a_fuzz_test.py",
-			fileContent:       `def NotSupported (foo)`,
+			fileName:          "a_fuzz_test.php",
+			fileContent:       `function function-not-supported (foo)`,
 			wantErr:           true,
 		},
 	}
@@ -274,7 +274,10 @@ func Test_fuzzFileAndFuncMatchPattern(t *testing.T) {
 			if !ok && !tt.wantErr {
 				t.Errorf("retrieve supported language error")
 			}
-			fileMatchPattern := langSpecs.filePattern
+			var fileMatchPattern string
+			if len(langSpecs.filePatterns) > 0 {
+				fileMatchPattern = langSpecs.filePatterns[0]
+			}
 			fileMatch, err := path.Match(fileMatchPattern, tt.fileName)
 			if (fileMatch != tt.expectedFileMatch || err != nil) && !tt.wantErr {
 				t.Errorf("fileMatch = %v, want %v for %v", fileMatch, tt.expectedFileMatch, tt.name)

probes/entries.go

@@ -17,13 +17,19 @@ package probes
 import (
 	"github.com/ossf/scorecard/v4/checker"
 	"github.com/ossf/scorecard/v4/finding"
+	"github.com/ossf/scorecard/v4/probes/fuzzedWithCLibFuzzer"
 	"github.com/ossf/scorecard/v4/probes/fuzzedWithClusterFuzzLite"
+	"github.com/ossf/scorecard/v4/probes/fuzzedWithCppLibFuzzer"
 	"github.com/ossf/scorecard/v4/probes/fuzzedWithGoNative"
+	"github.com/ossf/scorecard/v4/probes/fuzzedWithJavaJazzerFuzzer"
 	"github.com/ossf/scorecard/v4/probes/fuzzedWithOSSFuzz"
 	"github.com/ossf/scorecard/v4/probes/fuzzedWithOneFuzz"
 	"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedHaskell"
 	"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedJavascript"
 	"github.com/ossf/scorecard/v4/probes/fuzzedWithPropertyBasedTypescript"
+	"github.com/ossf/scorecard/v4/probes/fuzzedWithPythonAtheris"
+	"github.com/ossf/scorecard/v4/probes/fuzzedWithRustCargofuzz"
+	"github.com/ossf/scorecard/v4/probes/fuzzedWithSwiftLibFuzzer"
 	"github.com/ossf/scorecard/v4/probes/securityPolicyContainsLinks"
 	"github.com/ossf/scorecard/v4/probes/securityPolicyContainsText"
 	"github.com/ossf/scorecard/v4/probes/securityPolicyContainsVulnerabilityDisclosure"
@@ -60,6 +66,12 @@ var (
 		fuzzedWithOSSFuzz.Run,
 		fuzzedWithOneFuzz.Run,
 		fuzzedWithGoNative.Run,
+		fuzzedWithPythonAtheris.Run,
+		fuzzedWithCLibFuzzer.Run,
+		fuzzedWithCppLibFuzzer.Run,
+		fuzzedWithSwiftLibFuzzer.Run,
+		fuzzedWithRustCargofuzz.Run,
+		fuzzedWithJavaJazzerFuzzer.Run,
 		fuzzedWithClusterFuzzLite.Run,
 		fuzzedWithPropertyBasedHaskell.Run,
 		fuzzedWithPropertyBasedTypescript.Run,

probes/fuzzedWithCLibFuzzer/def.yml

@@ -0,0 +1,32 @@
+# Copyright 2023 OpenSSF Scorecard Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+id: fuzzedWithCLibFuzzer
+short: Check that the project is fuzzed using LibFuzzer
+motivation: >
+  Fuzzing, or fuzz testing, is the practice of feeding unexpected or random data into a program to expose bugs.
+  Regular fuzzing is important to detect vulnerabilities that may be exploited by others, especially since attackers can also use fuzzing to find the same flaws.
+implementation: >
+ The implementation checks whether fo the presence of functions with the signature 'LLVMFuzzerTestOneInput' in .c files.
+outcome:
+  - If fuzzing functions are found, each finding is returned with OutcomePositive (1).
+  - If no fuzzing is detected, one finding with OutcomeNegative (0) is returned.
+remediation:
+  effort: Medium
+  text:
+    - Follow the steps in https://llvm.org/docs/LibFuzzer.html to enable fuzzing on your project.
+    - Over time, try to add fuzzing for more functionalities of your project.
+  markdown:
+    - Follow the steps in [https://llvm.org/docs/LibFuzzer.html](https://llvm.org/docs/LibFuzzer.html) to enable fuzzing on your project.
+    - Over time, try to add fuzzing for more functionalities of your project.