Static analysis tool providers from Prospector Project

Fixes SAP#730

src/main/java/com/sap/oss/phosphor/fosstars/data/DataProviderSelector.java

@@ -43,6 +43,7 @@
 import com.sap.oss.phosphor.fosstars.data.github.OwaspSecurityLibraries;
 import com.sap.oss.phosphor.fosstars.data.github.PackageManagement;
 import com.sap.oss.phosphor.fosstars.data.github.ProgrammingLanguages;
+import com.sap.oss.phosphor.fosstars.data.github.PylintDataProvider;
 import com.sap.oss.phosphor.fosstars.data.github.ReadmeInfo;
 import com.sap.oss.phosphor.fosstars.data.github.ReleasesFromGitHub;
 import com.sap.oss.phosphor.fosstars.data.github.SecurityReviewsFromOpenSSF;
@@ -237,6 +238,7 @@ public DataProviderSelector(GitHubDataFetcher fetcher, NVD nvd) throws IOExcepti
         new VulnerabilitiesFromOwaspDependencyCheck(),
         new VulnerabilitiesFromNpmAudit(nvd),
         new HasExecutableBinaries(fetcher),
+        new PylintDataProvider(fetcher),
         PROJECT_USAGE_PROVIDER,
         FUNCTIONALITY_PROVIDER,
         HANDLING_UNTRUSTED_DATA_LIKELIHOOD_PROVIDER,

src/main/java/com/sap/oss/phosphor/fosstars/data/github/BanditDataProvider.java

@@ -10,31 +10,35 @@
 import com.sap.oss.phosphor.fosstars.model.feature.oss.OssFeatures;
 import com.sap.oss.phosphor.fosstars.model.subject.oss.GitHubProject;
 import com.sap.oss.phosphor.fosstars.model.value.ValueHashSet;
-import com.sap.oss.phosphor.fosstars.util.Yaml;
 import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
+import java.util.HashMap;
 import java.util.Map;
-import java.util.Optional;
+import java.util.function.Predicate;
 import java.util.regex.Pattern;
-import org.apache.commons.collections4.IteratorUtils;
 
 /**
  * The data provider gathers info about how a project uses Bandit for static analysis. In
  * particular, it tries to fill out the following features:
  * <ul>
- *   <li>{@link OssFeatures#RUNS_BANDIT_SCANS}</li>
- *   <li>{@link OssFeatures#USES_BANDIT_SCAN_CHECKS}</li>
+ *  <li>{@link OssFeatures#RUNS_BANDIT_SCANS}</li>
+ *  <li>{@link OssFeatures#USES_BANDIT_SCAN_CHECKS}</li>
  * </ul>
  */
 public class BanditDataProvider extends AbstractStaticScanToolsDataProvider {
 
   /**
-   * A step in a GitHub action that triggers analysis with Bandit.
+   * A Predicate to check the any step in a GitHub action that triggers analysis with Bandit.
    */
-  private static final Pattern RUN_STEP_BANDIT_REGEX_PATTERN
-      = Pattern.compile("^.*bandit .*$", Pattern.DOTALL);
+  private static final Map<String, Predicate<String>> MATCH_BANDIT_PREDICATE = new HashMap<>();
+
+  static {
+    {
+      MATCH_BANDIT_PREDICATE.put("uses",
+          step -> Pattern.compile(".*bandit.*$", Pattern.DOTALL).matcher(step).matches());
+      MATCH_BANDIT_PREDICATE.put("run",
+          step -> Pattern.compile("^.*bandit .*$", Pattern.DOTALL).matcher(step).matches());
+    }
+  }
 
   /**
    * Initializes a data provider.
@@ -51,69 +55,14 @@ protected ValueSet fetchValuesFor(GitHubProject project) throws IOException {
 
     LocalRepository repository = GitHubDataFetcher.localRepositoryFor(project);
 
-    Value<Boolean> runsBandit = RUNS_BANDIT_SCANS.value(false);
-    Value<Boolean> usesBanditScanChecks = USES_BANDIT_SCAN_CHECKS.value(false);
-
     // ideally, we're looking for a GitHub action that runs Bandit scan on pull requests
     // but if we just find an action that runs Bandit scans, that's also fine
-    for (Path configPath : findGitHubActionsIn(repository)) {
-      try (InputStream content = Files.newInputStream(configPath)) {
-        Map<String, Object> githubAction = Yaml.readMap(content);
-        if (triggersScan(githubAction)) {
-          runsBandit = RUNS_BANDIT_SCANS.value(true);
-          if (runsOnPullRequests(githubAction)) {
-            usesBanditScanChecks = USES_BANDIT_SCAN_CHECKS.value(true);
-            break;
-          }
-        }
-      }
-    }
+    Visitor visitor = withVisitor();
+    browse(repository, MATCH_BANDIT_PREDICATE, visitor);
 
-    return ValueHashSet.from(runsBandit, usesBanditScanChecks);
-  }
-
-  @Override
-  public boolean triggersScan(Map<?, ?> githubAction) {
-    return Optional.ofNullable(githubAction.get("jobs"))
-        .filter(Map.class::isInstance)
-        .map(Map.class::cast)
-        .map(jobs -> jobs.values())
-        .filter(Iterable.class::isInstance)
-        .map(Iterable.class::cast)
-        .map(BanditDataProvider::scanJobs)
-        .orElse(false);
-  }
+    Value<Boolean> runsBandit = RUNS_BANDIT_SCANS.value(visitor.runCheck);
+    Value<Boolean> usesBanditScanChecks = USES_BANDIT_SCAN_CHECKS.value(visitor.usesCheck);
 
-  /**
-   * Checks if any step in a collection of jobs triggers a Bandit scan.
-   *
-   * @param jobs The collection of jobs from GitHub action.
-   * @return True if a step triggers a Bandit scan, false otherwise.
-   */
-  private static boolean scanJobs(Iterable<?> jobs) {
-    return IteratorUtils.toList(jobs.iterator()).stream()
-        .filter(Map.class::isInstance)
-        .map(Map.class::cast)
-        .map(job -> job.get("steps"))
-        .filter(Iterable.class::isInstance)
-        .map(Iterable.class::cast)
-        .anyMatch(BanditDataProvider::hasBanditRunStep);
-  }
-
-  /**
-   * Checks if a collection of steps from a GitHub action contains a step that triggers a Bandit
-   * scan.
-   *
-   * @param steps The steps to be checked.
-   * @return True if the steps contain a step that triggers a Bandit scan, false otherwise.
-   */
-  private static boolean hasBanditRunStep(Iterable<?> steps) {
-    return IteratorUtils.toList(steps.iterator()).stream()
-        .filter(Map.class::isInstance)
-        .map(Map.class::cast)
-        .map(step -> step.get("run"))
-        .filter(String.class::isInstance)
-        .map(String.class::cast)
-        .anyMatch(run -> RUN_STEP_BANDIT_REGEX_PATTERN.matcher(run).matches());
+    return ValueHashSet.from(runsBandit, usesBanditScanChecks);
   }
-}
+}

src/main/java/com/sap/oss/phosphor/fosstars/data/github/CodeqlDataProvider.java

@@ -10,31 +10,33 @@
 import com.sap.oss.phosphor.fosstars.model.feature.oss.OssFeatures;
 import com.sap.oss.phosphor.fosstars.model.subject.oss.GitHubProject;
 import com.sap.oss.phosphor.fosstars.model.value.ValueHashSet;
-import com.sap.oss.phosphor.fosstars.util.Yaml;
 import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
+import java.util.HashMap;
 import java.util.Map;
-import java.util.Optional;
-import org.apache.commons.collections4.IteratorUtils;
+import java.util.function.Predicate;
 
 /**
- * The data provider gathers info about how a project uses CodeQL for static analysis.
- * In particular, it tires to fill out the following features:
+ * The data provider gathers info about how a project uses CodeQL for static analysis. In
+ * particular, it tires to fill out the following features:
  * <ul>
- *   <li>{@link OssFeatures#RUNS_CODEQL_SCANS}</li>
- *   <li>{@link OssFeatures#USES_CODEQL_CHECKS}</li>
+ *  <li>{@link OssFeatures#RUNS_CODEQL_SCANS}</li>
+ *  <li>{@link OssFeatures#USES_CODEQL_CHECKS}</li>
  * </ul>
  *
  * @see LgtmDataProvider
  */
 public class CodeqlDataProvider extends AbstractStaticScanToolsDataProvider {
 
   /**
-   * A step in a GitHub action that triggers analysis with CodeQL.
+   * A predicate to check if any step in GitHub action that triggers analysis with CodeQL.
    */
-  private static final String CODEQL_ANALYZE_STEP_TASK = "github/codeql-action/analyze";
+  private static final Map<String, Predicate<String>> MATCH_CODEQL_ANALYZE_PREDICATE =
+      new HashMap<>();
+
+  static {
+    MATCH_CODEQL_ANALYZE_PREDICATE.put("uses",
+        uses -> uses.startsWith("github/codeql-action/analyze"));
+  }
 
   /**
    * Initializes a data provider.
@@ -51,56 +53,12 @@ protected ValueSet fetchValuesFor(GitHubProject project) throws IOException {
 
     LocalRepository repository = GitHubDataFetcher.localRepositoryFor(project);
 
-    Value<Boolean> runsCodeqlScans = RUNS_CODEQL_SCANS.value(false);
-    Value<Boolean> usesCodeqlChecks = USES_CODEQL_CHECKS.value(false);
-
-    // ideally, we're looking for a GitHub action that runs CodeQL scan on pull requests
-    // but if we just find an action that runs CodeQL scans, that's also fine
-    for (Path configPath : findGitHubActionsIn(repository)) {
-      try (InputStream content = Files.newInputStream(configPath)) {
-        Map<String, Object> githubAction = Yaml.readMap(content);
-        if (triggersScan(githubAction)) {
-          runsCodeqlScans = RUNS_CODEQL_SCANS.value(true);
-          if (runsOnPullRequests(githubAction)) {
-            usesCodeqlChecks = USES_CODEQL_CHECKS.value(true);
-            break;
-          }
-        }
-      }
-    }
-
-    return ValueHashSet.from(usesCodeqlChecks, runsCodeqlScans);
-  }
+    Visitor visitor = withVisitor();
+    browse(repository, MATCH_CODEQL_ANALYZE_PREDICATE, visitor);
 
-  @Override
-  public boolean triggersScan(Map<?, ?> githubAction) {
-    return Optional.ofNullable(githubAction.get("jobs"))
-        .filter(Map.class::isInstance)
-        .map(Map.class::cast)
-        .map(jobs -> jobs.get("analyze"))
-        .filter(Map.class::isInstance)
-        .map(Map.class::cast)
-        .map(jobs -> jobs.get("steps"))
-        .filter(Iterable.class::isInstance)
-        .map(Iterable.class::cast)
-        .map(CodeqlDataProvider::hasCodeqlAnalyzeStep)
-        .orElse(false);
-  }
+    Value<Boolean> runsCodeqlScans = RUNS_CODEQL_SCANS.value(visitor.runCheck);
+    Value<Boolean> usesCodeqlChecks = USES_CODEQL_CHECKS.value(visitor.usesCheck);
 
-  /**
-   * Checks if a collection of steps from a GitHub action contains a step that triggers
-   * a CodeQL scan.
-   *
-   * @param steps The steps to be checked.
-   * @return True if the steps contain a step that triggers a CodeQL scan, false otherwise.
-   */
-  private static boolean hasCodeqlAnalyzeStep(Iterable<?> steps) {
-    return IteratorUtils.toList(steps.iterator()).stream()
-        .filter(Map.class::isInstance)
-        .map(Map.class::cast)
-        .map(step -> step.get("uses"))
-        .filter(String.class::isInstance)
-        .map(String.class::cast)
-        .anyMatch(uses -> uses.startsWith(CODEQL_ANALYZE_STEP_TASK));
+    return ValueHashSet.from(runsCodeqlScans, usesCodeqlChecks);
   }
-}
+}

src/main/java/com/sap/oss/phosphor/fosstars/data/github/LocalRepository.java

@@ -234,6 +234,16 @@ public boolean hasFile(String file) {
     return Files.isRegularFile(info.path().resolve(Paths.get(file)));
   }
 
+  /**
+   * Resolve the repository path for specified file.
+   *
+   * @param file A path to the file.
+   * @return Optional resolved path within the {@link LocalRepository}.
+   */
+  public Optional<Path> path(String file) {
+    return Optional.ofNullable(info.path().resolve(Paths.get(file)));
+  }
+
   /**
    * Returns a stream of a file if it exists.
    *

src/main/java/com/sap/oss/phosphor/fosstars/data/github/PylintDataProvider.java

@@ -0,0 +1,65 @@
+package com.sap.oss.phosphor.fosstars.data.github;
+
+import static com.sap.oss.phosphor.fosstars.model.feature.oss.OssFeatures.RUNS_PYLINT_SCANS;
+import static com.sap.oss.phosphor.fosstars.model.feature.oss.OssFeatures.USES_PYLINT_SCAN_CHECKS;
+import static com.sap.oss.phosphor.fosstars.model.other.Utils.setOf;
+
+import com.sap.oss.phosphor.fosstars.data.AbstractStaticScanToolsDataProvider;
+import com.sap.oss.phosphor.fosstars.model.Value;
+import com.sap.oss.phosphor.fosstars.model.ValueSet;
+import com.sap.oss.phosphor.fosstars.model.feature.oss.OssFeatures;
+import com.sap.oss.phosphor.fosstars.model.subject.oss.GitHubProject;
+import com.sap.oss.phosphor.fosstars.model.value.ValueHashSet;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Predicate;
+import java.util.regex.Pattern;
+
+/**
+ * The data provider gathers info about how a project uses Bandit for static analysis. In
+ * particular, it tries to fill out the following features:
+ * <ul>
+ *   <li>{@link OssFeatures#RUNS_PYLINT_SCANS}</li>
+ *   <li>{@link OssFeatures#USES_PYLINT_SCAN_CHECKS}</li>
+ * </ul>
+ */
+public class PylintDataProvider extends AbstractStaticScanToolsDataProvider {
+
+  /**
+   * A predicate to check if the any step of a GitHub action that triggers analysis with Pylint.
+   */
+  private static final Map<String, Predicate<String>> MATCH_PYLINT_PREDICATE_MAP = new HashMap<>();
+
+  static {
+    MATCH_PYLINT_PREDICATE_MAP.put("uses",
+        step -> Pattern.compile("^.*pylint.*$", Pattern.DOTALL).matcher(step).matches());
+    MATCH_PYLINT_PREDICATE_MAP.put("run",
+        step -> Pattern.compile("^.*pylint .*$", Pattern.DOTALL).matcher(step).matches());
+    MATCH_PYLINT_PREDICATE_MAP.putAll(preCommitHookContextMap(hook -> hook.contains("pylint")));
+  }
+
+  /**
+   * Initializes a data provider.
+   *
+   * @param fetcher An interface to GitHub.
+   */
+  public PylintDataProvider(GitHubDataFetcher fetcher) {
+    super(fetcher, setOf(RUNS_PYLINT_SCANS, USES_PYLINT_SCAN_CHECKS));
+  }
+
+  @Override
+  protected ValueSet fetchValuesFor(GitHubProject project) throws IOException {
+    logger.info("Figuring out how the project uses pylint ...");
+
+    LocalRepository repository = GitHubDataFetcher.localRepositoryFor(project);
+
+    Visitor visitor = withVisitor();
+    browse(repository, MATCH_PYLINT_PREDICATE_MAP, visitor);
+
+    Value<Boolean> runsPylint = RUNS_PYLINT_SCANS.value(visitor.runCheck);
+    Value<Boolean> usesPylintScanChecks = USES_PYLINT_SCAN_CHECKS.value(visitor.usesCheck);
+
+    return ValueHashSet.from(runsPylint, usesPylintScanChecks);
+  }
+}

src/main/java/com/sap/oss/phosphor/fosstars/github/AbstractGitHubVisitor.java

@@ -0,0 +1,37 @@
+package com.sap.oss.phosphor.fosstars.github;
+
+import com.sap.oss.phosphor.fosstars.data.github.LocalRepository;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Predicate;
+
+/**
+ * An implementation of {@link GitHubVisitor} that does nothing.
+ */
+public abstract class AbstractGitHubVisitor implements GitHubVisitor {
+
+  @Override
+  public void visitPreCommitHook(LocalRepository repository,
+      Map<String, Predicate<String>> matchers, Set<Location> locations) throws IOException {
+    // do nothing
+  }
+
+  @Override
+  public void visitIniConfig(LocalRepository repository, Map<String, Predicate<String>> matchers,
+      Set<Location> locations) throws IOException {
+    // do nothing
+  }
+
+  @Override
+  public void visitSourceCode(LocalRepository repository, Map<String, Predicate<String>> matchers,
+      Set<Location> locations) throws IOException {
+    // don nothing
+  }
+
+  @Override
+  public void visitGitHubAction(LocalRepository repository, Map<String, Predicate<String>> matchers,
+      Set<Location> locations) throws IOException {
+    // do nothing
+  }
+}