Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Static analysis tool providers from Prospector Project #866

Merged
merged 12 commits into from
Sep 7, 2022
Merged

Static analysis tool providers from Prospector Project #866

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
7136ce8
Add more tests for Has Executbale Binaries
sourabhsparkala Aug 11, 2022
e37f9b0
Static analysis tool providers from Prospector Project
sourabhsparkala Aug 31, 2022
337f3d8
Dataprovider update to fix Pre commit hook config
sourabhsparkala Sep 1, 2022
1b03dec
Add test content search for Has Binaries provider
sourabhsparkala Sep 1, 2022
1ddab7d
Implement MyPy data provider
sourabhsparkala Sep 1, 2022
5a2a748
Fixing the compilation issues
sourabhsparkala Sep 1, 2022
d66a663
Implement the score for pylint, mypy
sourabhsparkala Sep 6, 2022
5e17df0
Address the unknown condition
sourabhsparkala Sep 6, 2022
2b1f3e8
Fix script tests
sourabhsparkala Sep 7, 2022
e305dc3
Update lib.sh to fix tests
sourabhsparkala Sep 7, 2022
6956e37
Merge branch 'master' into issue_730
sourabhsparkala Sep 7, 2022
c0c8ec4
Update src/main/java/com/sap/oss/phosphor/fosstars/data/AbstractStati…
sourabhsparkala Sep 7, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
388 changes: 375 additions & 13 deletions src/main/java/com/sap/oss/phosphor/fosstars/data/AbstractStaticScanToolsDataProvider.java
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions src/main/java/com/sap/oss/phosphor/fosstars/data/DataProviderSelector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import com.sap.oss.phosphor.fosstars.data.github.IsEclipse;
import com.sap.oss.phosphor.fosstars.data.github.LgtmDataProvider;
import com.sap.oss.phosphor.fosstars.data.github.LicenseInfo;
import com.sap.oss.phosphor.fosstars.data.github.MyPyDataProvider;
import com.sap.oss.phosphor.fosstars.data.github.NumberOfCommits;
import com.sap.oss.phosphor.fosstars.data.github.NumberOfContributors;
import com.sap.oss.phosphor.fosstars.data.github.NumberOfDependentProjectOnGitHub;
Expand All @@ -44,6 +45,7 @@
import com.sap.oss.phosphor.fosstars.data.github.OwaspSecurityLibraries;
import com.sap.oss.phosphor.fosstars.data.github.PackageManagement;
import com.sap.oss.phosphor.fosstars.data.github.ProgrammingLanguages;
import com.sap.oss.phosphor.fosstars.data.github.PylintDataProvider;
import com.sap.oss.phosphor.fosstars.data.github.ReadmeInfo;
import com.sap.oss.phosphor.fosstars.data.github.ReleasesFromGitHub;
import com.sap.oss.phosphor.fosstars.data.github.SecurityReviewsFromOpenSSF;
Expand Down Expand Up @@ -239,6 +241,8 @@ public DataProviderSelector(GitHubDataFetcher fetcher, NVD nvd) throws IOExcepti
new VulnerabilitiesFromOwaspDependencyCheck(),
new VulnerabilitiesFromNpmAudit(nvd),
new HasExecutableBinaries(fetcher),
new PylintDataProvider(fetcher),
new MyPyDataProvider(fetcher),
PROJECT_USAGE_PROVIDER,
FUNCTIONALITY_PROVIDER,
HANDLING_UNTRUSTED_DATA_LIKELIHOOD_PROVIDER,
Expand Down
25 changes: 0 additions & 25 deletions src/main/java/com/sap/oss/phosphor/fosstars/data/StaticAnalysisScanTool.java
View file
Open in desktop

This file was deleted.

94 changes: 22 additions & 72 deletions src/main/java/com/sap/oss/phosphor/fosstars/data/github/BanditDataProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,31 +10,36 @@
import com.sap.oss.phosphor.fosstars.model.feature.oss.OssFeatures;
import com.sap.oss.phosphor.fosstars.model.subject.oss.GitHubProject;
import com.sap.oss.phosphor.fosstars.model.value.ValueHashSet;
import com.sap.oss.phosphor.fosstars.util.Yaml;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import org.apache.commons.collections4.IteratorUtils;

/**
* The data provider gathers info about how a project uses Bandit for static analysis. In
* particular, it tries to fill out the following features:
* <ul>
* <li>{@link OssFeatures#RUNS_BANDIT_SCANS}</li>
* <li>{@link OssFeatures#USES_BANDIT_SCAN_CHECKS}</li>
* <li>{@link OssFeatures#RUNS_BANDIT_SCANS}</li>
* <li>{@link OssFeatures#USES_BANDIT_SCAN_CHECKS}</li>
* </ul>
*/
public class BanditDataProvider extends AbstractStaticScanToolsDataProvider {

/**
* A step in a GitHub action that triggers analysis with Bandit.
* A Predicate to check the any step in a GitHub action that triggers analysis with Bandit.
*/
private static final Pattern RUN_STEP_BANDIT_REGEX_PATTERN
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't it nice to keep this and use it in line 38 and 40 rather than creating Pattern twice.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is a slight difference in these regular patterns, the key to search for in the step and then matching the value of the key. But again you are right about the redundancy, i did not find a better way to integrate one predicate and check for everything.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@ManjunathMS35 lets fix this here #873 ?

= Pattern.compile("^.*bandit .*$", Pattern.DOTALL);
private static final Map<String, Predicate<String>> MATCH_BANDIT_PREDICATE = new HashMap<>();

static {
{
MATCH_BANDIT_PREDICATE.put("uses",
step -> Pattern.compile(".*bandit.*$", Pattern.DOTALL).matcher(step).matches());
MATCH_BANDIT_PREDICATE.put("run",
step -> Pattern.compile("^.*bandit .*$", Pattern.DOTALL).matcher(step).matches());
}
}

/**
* Initializes a data provider.
Expand All @@ -51,69 +56,14 @@ protected ValueSet fetchValuesFor(GitHubProject project) throws IOException {

LocalRepository repository = GitHubDataFetcher.localRepositoryFor(project);

Value<Boolean> runsBandit = RUNS_BANDIT_SCANS.value(false);
Value<Boolean> usesBanditScanChecks = USES_BANDIT_SCAN_CHECKS.value(false);

// ideally, we're looking for a GitHub action that runs Bandit scan on pull requests
// but if we just find an action that runs Bandit scans, that's also fine
for (Path configPath : findGitHubActionsIn(repository)) {
try (InputStream content = Files.newInputStream(configPath)) {
Map<String, Object> githubAction = Yaml.readMap(content);
if (triggersScan(githubAction)) {
runsBandit = RUNS_BANDIT_SCANS.value(true);
if (runsOnPullRequests(githubAction)) {
usesBanditScanChecks = USES_BANDIT_SCAN_CHECKS.value(true);
break;
}
}
}
}
Visitor visitor = withVisitor();
browse(repository, MATCH_BANDIT_PREDICATE, Collections.emptyMap(), visitor);

return ValueHashSet.from(runsBandit, usesBanditScanChecks);
}

@Override
public boolean triggersScan(Map<?, ?> githubAction) {
return Optional.ofNullable(githubAction.get("jobs"))
.filter(Map.class::isInstance)
.map(Map.class::cast)
.map(jobs -> jobs.values())
.filter(Iterable.class::isInstance)
.map(Iterable.class::cast)
.map(BanditDataProvider::scanJobs)
.orElse(false);
}
Value<Boolean> runsBandit = RUNS_BANDIT_SCANS.value(visitor.runCheck);
Value<Boolean> usesBanditScanChecks = USES_BANDIT_SCAN_CHECKS.value(visitor.usesCheck);

/**
* Checks if any step in a collection of jobs triggers a Bandit scan.
*
* @param jobs The collection of jobs from GitHub action.
* @return True if a step triggers a Bandit scan, false otherwise.
*/
private static boolean scanJobs(Iterable<?> jobs) {
return IteratorUtils.toList(jobs.iterator()).stream()
.filter(Map.class::isInstance)
.map(Map.class::cast)
.map(job -> job.get("steps"))
.filter(Iterable.class::isInstance)
.map(Iterable.class::cast)
.anyMatch(BanditDataProvider::hasBanditRunStep);
}

/**
* Checks if a collection of steps from a GitHub action contains a step that triggers a Bandit
* scan.
*
* @param steps The steps to be checked.
* @return True if the steps contain a step that triggers a Bandit scan, false otherwise.
*/
private static boolean hasBanditRunStep(Iterable<?> steps) {
return IteratorUtils.toList(steps.iterator()).stream()
.filter(Map.class::isInstance)
.map(Map.class::cast)
.map(step -> step.get("run"))
.filter(String.class::isInstance)
.map(String.class::cast)
.anyMatch(run -> RUN_STEP_BANDIT_REGEX_PATTERN.matcher(run).matches());
return ValueHashSet.from(runsBandit, usesBanditScanChecks);
}
}
}
83 changes: 21 additions & 62 deletions src/main/java/com/sap/oss/phosphor/fosstars/data/github/CodeqlDataProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,31 +10,34 @@
import com.sap.oss.phosphor.fosstars.model.feature.oss.OssFeatures;
import com.sap.oss.phosphor.fosstars.model.subject.oss.GitHubProject;
import com.sap.oss.phosphor.fosstars.model.value.ValueHashSet;
import com.sap.oss.phosphor.fosstars.util.Yaml;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
import org.apache.commons.collections4.IteratorUtils;
import java.util.function.Predicate;

/**
* The data provider gathers info about how a project uses CodeQL for static analysis.
* In particular, it tires to fill out the following features:
* The data provider gathers info about how a project uses CodeQL for static analysis. In
* particular, it tires to fill out the following features:
* <ul>
* <li>{@link OssFeatures#RUNS_CODEQL_SCANS}</li>
* <li>{@link OssFeatures#USES_CODEQL_CHECKS}</li>
* <li>{@link OssFeatures#RUNS_CODEQL_SCANS}</li>
* <li>{@link OssFeatures#USES_CODEQL_CHECKS}</li>
* </ul>
*
* @see LgtmDataProvider
*/
public class CodeqlDataProvider extends AbstractStaticScanToolsDataProvider {

/**
* A step in a GitHub action that triggers analysis with CodeQL.
* A predicate to check if any step in GitHub action that triggers analysis with CodeQL.
*/
private static final String CODEQL_ANALYZE_STEP_TASK = "github/codeql-action/analyze";
private static final Map<String, Predicate<String>> MATCH_CODEQL_ANALYZE_PREDICATE =
new HashMap<>();

static {
MATCH_CODEQL_ANALYZE_PREDICATE.put("uses",
uses -> uses.startsWith("github/codeql-action/analyze"));
}

/**
* Initializes a data provider.
Expand All @@ -51,56 +54,12 @@ protected ValueSet fetchValuesFor(GitHubProject project) throws IOException {

LocalRepository repository = GitHubDataFetcher.localRepositoryFor(project);

Value<Boolean> runsCodeqlScans = RUNS_CODEQL_SCANS.value(false);
Value<Boolean> usesCodeqlChecks = USES_CODEQL_CHECKS.value(false);

// ideally, we're looking for a GitHub action that runs CodeQL scan on pull requests
// but if we just find an action that runs CodeQL scans, that's also fine
for (Path configPath : findGitHubActionsIn(repository)) {
try (InputStream content = Files.newInputStream(configPath)) {
Map<String, Object> githubAction = Yaml.readMap(content);
if (triggersScan(githubAction)) {
runsCodeqlScans = RUNS_CODEQL_SCANS.value(true);
if (runsOnPullRequests(githubAction)) {
usesCodeqlChecks = USES_CODEQL_CHECKS.value(true);
break;
}
}
}
}

return ValueHashSet.from(usesCodeqlChecks, runsCodeqlScans);
}
Visitor visitor = withVisitor();
browse(repository, MATCH_CODEQL_ANALYZE_PREDICATE, Collections.emptyMap(), visitor);

@Override
public boolean triggersScan(Map<?, ?> githubAction) {
return Optional.ofNullable(githubAction.get("jobs"))
.filter(Map.class::isInstance)
.map(Map.class::cast)
.map(jobs -> jobs.get("analyze"))
.filter(Map.class::isInstance)
.map(Map.class::cast)
.map(jobs -> jobs.get("steps"))
.filter(Iterable.class::isInstance)
.map(Iterable.class::cast)
.map(CodeqlDataProvider::hasCodeqlAnalyzeStep)
.orElse(false);
}
Value<Boolean> runsCodeqlScans = RUNS_CODEQL_SCANS.value(visitor.runCheck);
Value<Boolean> usesCodeqlChecks = USES_CODEQL_CHECKS.value(visitor.usesCheck);

/**
* Checks if a collection of steps from a GitHub action contains a step that triggers
* a CodeQL scan.
*
* @param steps The steps to be checked.
* @return True if the steps contain a step that triggers a CodeQL scan, false otherwise.
*/
private static boolean hasCodeqlAnalyzeStep(Iterable<?> steps) {
return IteratorUtils.toList(steps.iterator()).stream()
.filter(Map.class::isInstance)
.map(Map.class::cast)
.map(step -> step.get("uses"))
.filter(String.class::isInstance)
.map(String.class::cast)
.anyMatch(uses -> uses.startsWith(CODEQL_ANALYZE_STEP_TASK));
return ValueHashSet.from(runsCodeqlScans, usesCodeqlChecks);
}
}
}
Loading