From dd6c5e090ec8e09c6146871c78440910efc2b848 Mon Sep 17 00:00:00 2001 From: Shenoy Date: Sat, 27 Jan 2024 12:23:03 +0530 Subject: [PATCH 01/12] Add function to generate purl from package advisory url Signed-off-by: Shenoy --- vulntotal/datasources/snyk.py | 49 +++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/vulntotal/datasources/snyk.py b/vulntotal/datasources/snyk.py index 4af4fa619..879bb44bb 100644 --- a/vulntotal/datasources/snyk.py +++ b/vulntotal/datasources/snyk.py @@ -14,6 +14,7 @@ import requests from bs4 import BeautifulSoup from packageurl import PackageURL +from urllib.parse import unquote_plus from vulntotal.validator import DataSource from vulntotal.validator import VendorData @@ -131,6 +132,54 @@ def generate_package_advisory_url(purl): package=purl_name, ) +def generate_purl(package_advisory_url): + """ + Generates purl from Package advisory url. + + Parameters: + package_advisory_url: URL of the package on Snyk. + + Returns: + A PackageURL instance representing the package + """ + package_advisory_url = unquote_plus(package_advisory_url.replace("https://security.snyk.io/package/", "")) + + package_url_split = package_advisory_url.split("/") + pkg_type = package_url_split[0] + + pkg_name = None + namespace = None + version = None + qualifiers = {} + + if pkg_type == "maven": + pkg_name = package_url_split[1].split(":")[1] + namespace = package_url_split[1].split(":")[0] + + elif pkg_type in ("golang", "composer"): + if package_url_split[1] == 'github.com': + pkg_name = package_url_split[-2] + namespace = f"{package_url_split[1]}/{package_url_split[2]}" + version = package_url_split[-1] + else: + pkg_name = package_url_split[-1] + namespace = package_url_split[-2] + + elif pkg_type == "linux": + pkg_name = package_url_split[-1] + qualifiers["distro"] = package_url_split[1] + + else: + pkg_name = package_url_split[-1] + + return PackageURL( + type=pkg_type, + name=pkg_name, + namespace=namespace, + version=version, + qualifiers=qualifiers + ) + def extract_html_json_advisories(package_advisories): """ From c85ae9b90d582405b771aaf649ddf3d503822e71 Mon Sep 17 00:00:00 2001 From: Shenoy Date: Sat, 27 Jan 2024 12:34:13 +0530 Subject: [PATCH 02/12] Add functions to generate cve url and parse the html advisory generated from the cve url Signed-off-by: Shenoy --- vulntotal/datasources/snyk.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/vulntotal/datasources/snyk.py b/vulntotal/datasources/snyk.py index 879bb44bb..c7a9d3755 100644 --- a/vulntotal/datasources/snyk.py +++ b/vulntotal/datasources/snyk.py @@ -252,6 +252,32 @@ def parse_html_advisory(advisory_html, snyk_id, affected, purl) -> VendorData: fixed_versions=fixed_versions, ) +def parse_cve_advisory_html(cve_advisory_html): + """ + Parse CVE HTML advisory from Snyk and extract list of vulnerabilities and corresponding packages for that CVE. + + Parameters: + advisory_html: A string of HTML containing the vulnerabilities for given CVE. + + Returns: + A dictionary with each item representing a vulnerability. Key of each item is the SNYK_ID and value is the package advisory url on snyk website + """ + cve_advisory_soup = BeautifulSoup(cve_advisory_html, 'html.parser') + vulns_table = cve_advisory_soup.find("tbody", class_="vue--table__tbody") + if not vulns_table: + return None + vulns_rows = vulns_table.find_all("tr", class_="vue--table__row") + vulns_list = {} + + for row in vulns_rows: + anchors = row.find_all("a", {"class" :"vue--anchor"}) + if len(anchors) != 2: + continue + snyk_id = anchors[0]['href'].split("/")[1] + package_advisory_url = f"https://security.snyk.io{anchors[1]['href']}" + vulns_list[snyk_id] = package_advisory_url + + return vulns_list def is_purl_in_affected(version, affected): return any(snyk_constraints_satisfied(affected_range, version) for affected_range in affected) @@ -259,3 +285,6 @@ def is_purl_in_affected(version, affected): def generate_advisory_payload(snyk_id): return f"https://security.snyk.io/vuln/{snyk_id}" + +def generate_payload_from_cve(cve_id): + return f"https://security.snyk.io/vuln?search={cve_id}" From a4555c8a6896d5b6c0f1bdb2e8773c60d7cc5bec Mon Sep 17 00:00:00 2001 From: Shenoy Date: Sat, 27 Jan 2024 13:18:04 +0530 Subject: [PATCH 03/12] Add function to SnykDataSource class to get advisories from cve Signed-off-by: Shenoy --- vulntotal/datasources/snyk.py | 36 +++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/vulntotal/datasources/snyk.py b/vulntotal/datasources/snyk.py index c7a9d3755..0cf71a44d 100644 --- a/vulntotal/datasources/snyk.py +++ b/vulntotal/datasources/snyk.py @@ -9,14 +9,14 @@ import logging from typing import Iterable -from urllib.parse import quote +from urllib.parse import quote, unquote_plus import requests from bs4 import BeautifulSoup from packageurl import PackageURL -from urllib.parse import unquote_plus from vulntotal.validator import DataSource +from vulntotal.validator import InvalidCVEError from vulntotal.validator import VendorData from vulntotal.vulntotal_utils import snyk_constraints_satisfied @@ -71,6 +71,38 @@ def datasource_advisory(self, purl) -> Iterable[VendorData]: if advisory_html: yield parse_html_advisory(advisory_html, snyk_id, affected, purl) + def datasource_advisory_from_cve(self, cve: str) -> Iterable[VendorData]: + """ + Fetch advisories from Snyk for a given CVE. + + Parameters: + cve : CVE ID + + Yields: + VendorData instance containing advisory information. + """ + if not cve.upper().startswith("CVE-"): + raise InvalidCVEError + + package_list = generate_payload_from_cve(cve) + response = self.fetch(package_list) + self._raw_dump = [response] + + # get list of vulnerabilities for cve id + vulns_list = parse_cve_advisory_html(response) + + # for each vulnerability get fixed version from snyk_id_url, get affected version from package_advisory_url + for snyk_id, package_advisory_url in vulns_list.items(): + package_advisories_list = self.fetch(package_advisory_url) + package_advisories = extract_html_json_advisories(package_advisories_list) + affected_versions = package_advisories[snyk_id] + advisory_payload = generate_advisory_payload(snyk_id) + advisory_html = self.fetch(advisory_payload) + self._raw_dump.append(advisory_html) + purl = generate_purl(package_advisory_url) + if advisory_html: + yield parse_html_advisory(advisory_html, snyk_id, affected_versions, purl) + @classmethod def supported_ecosystem(cls): return { From 55ff69a14200eb2771440eae050b4562534c57cf Mon Sep 17 00:00:00 2001 From: Shenoy Date: Sat, 27 Jan 2024 13:25:08 +0530 Subject: [PATCH 04/12] Add tests and test data for parse_cve_advisory_html function Signed-off-by: Shenoy --- vulntotal/tests/test_data/snyk/html/4.html | 190 ++++++++ .../test_data/snyk/html/4.html-expected.json | 8 + vulntotal/tests/test_data/snyk/html/5.html | 430 ++++++++++++++++++ .../test_data/snyk/html/5.html-expected.json | 31 ++ vulntotal/tests/test_snyk.py | 20 + 5 files changed, 679 insertions(+) create mode 100644 vulntotal/tests/test_data/snyk/html/4.html create mode 100644 vulntotal/tests/test_data/snyk/html/4.html-expected.json create mode 100644 vulntotal/tests/test_data/snyk/html/5.html create mode 100644 vulntotal/tests/test_data/snyk/html/5.html-expected.json diff --git a/vulntotal/tests/test_data/snyk/html/4.html b/vulntotal/tests/test_data/snyk/html/4.html new file mode 100644 index 000000000..6ab4ccc0a --- /dev/null +++ b/vulntotal/tests/test_data/snyk/html/4.html @@ -0,0 +1,190 @@ + + + + Vulnerability DB | Snyk + + +

Find out if you have vulnerabilities that put you at risk

+ Test your applications +
Toggle filtering controls
Expand this section

+ APPLICATION +

Expand this section

+ OPERATING SYSTEM +

+ Report a new vulnerability +
+ VULNERABILITY + + AFFECTS + + TYPE + + PUBLISHED +
  • C
+ Cross-site Scripting (XSS) +
+ org.webjars.bowergithub.sentsin:layui + + [0,] + + Maven + + 12 Jan 2024 +
  • C
+ Cross-site Scripting (XSS) +
+ org.webjars.bowergithub.diguoyihao:layui + + [0,] + + Maven + + 12 Jan 2024 +
  • C
+ Cross-site Scripting (XSS) +
+ org.webjars.bower:layui + + [0,] + + Maven + + 12 Jan 2024 +
  • C
+ Cross-site Scripting (XSS) +
+ org.webjars.bowergithub.layui:layui + + [0,] + + Maven + + 12 Jan 2024 +
  • C
+ Cross-site Scripting (XSS) +
+ org.webjars:layui + + [,2.7.6) + + Maven + + 12 Jan 2024 +
  • C
+ Cross-site Scripting (XSS) +
+ org.webjars.npm:layui + + [,2.7.6) + + Maven + + 12 Jan 2024 +
+ + diff --git a/vulntotal/tests/test_data/snyk/html/4.html-expected.json b/vulntotal/tests/test_data/snyk/html/4.html-expected.json new file mode 100644 index 000000000..5f5f93616 --- /dev/null +++ b/vulntotal/tests/test_data/snyk/html/4.html-expected.json @@ -0,0 +1,8 @@ +{ + "SNYK-JAVA-ORGWEBJARSBOWERGITHUBSENTSIN-6146043": "https://security.snyk.io/package/maven/org.webjars.bowergithub.sentsin%3Alayui", + "SNYK-JAVA-ORGWEBJARSBOWERGITHUBDIGUOYIHAO-6146042": "https://security.snyk.io/package/maven/org.webjars.bowergithub.diguoyihao%3Alayui", + "SNYK-JAVA-ORGWEBJARSBOWER-6146041": "https://security.snyk.io/package/maven/org.webjars.bower%3Alayui", + "SNYK-JAVA-ORGWEBJARSBOWERGITHUBLAYUI-6146040": "https://security.snyk.io/package/maven/org.webjars.bowergithub.layui%3Alayui", + "SNYK-JAVA-ORGWEBJARS-6146039": "https://security.snyk.io/package/maven/org.webjars%3Alayui", + "SNYK-JAVA-ORGWEBJARSNPM-6146038": "https://security.snyk.io/package/maven/org.webjars.npm%3Alayui" +} diff --git a/vulntotal/tests/test_data/snyk/html/5.html b/vulntotal/tests/test_data/snyk/html/5.html new file mode 100644 index 000000000..e52bffcea --- /dev/null +++ b/vulntotal/tests/test_data/snyk/html/5.html @@ -0,0 +1,430 @@ + + + + Vulnerability DB | Snyk + + +

Find out if you have vulnerabilities that put you at risk

+ Test your applications +
Toggle filtering controls
Expand this section

+ APPLICATION +

Expand this section

+ OPERATING SYSTEM +

+ Report a new vulnerability +
+ VULNERABILITY + + AFFECTS + + TYPE + + PUBLISHED +
  • M
+ CVE-2023-6237 +
+ libopenssl3 + + <3.0.8-150500.5.24.1 + + sles:15.5 + + 23 Jan 2024 +
  • M
+ CVE-2023-6237 +
+ libopenssl-3-devel + + <3.0.8-150500.5.24.1 + + sles:15.5 + + 23 Jan 2024 +
  • M
+ CVE-2023-6237 +
+ openssl-3 + + <3.0.8-150500.5.24.1 + + sles:15.5 + + 23 Jan 2024 +
  • L
+ CVE-2023-6237 +
+ openssl + + <3.0.12-r3 + + alpine:3.17 + + 17 Jan 2024 +
  • L
+ CVE-2023-6237 +
+ openssl + + <3.1.4-r4 + + alpine:3.18 + + 17 Jan 2024 +
  • L
+ CVE-2023-6237 +
+ openssl + + <3.1.4-r4 + + alpine:3.19 + + 17 Jan 2024 +
  • L
+ Resource Exhaustion +
+ openssl-perl + + * + + centos:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ openssl-perl + + * + + rhel:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ openssl-devel + + * + + centos:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ openssl-devel + + * + + rhel:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ openssl + + * + + centos:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ openssl + + * + + rhel:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ openssl-libs + + * + + centos:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ openssl-libs + + * + + rhel:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ edk2-tools-doc + + * + + centos:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ edk2-tools-doc + + * + + rhel:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ edk2-tools + + * + + centos:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ edk2-tools + + * + + rhel:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ edk2-aarch64 + + * + + centos:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ edk2-aarch64 + + * + + rhel:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ edk2 + + * + + centos:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ edk2 + + * + + rhel:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ edk2-ovmf + + * + + centos:9 + + 16 Jan 2024 +
  • L
+ Resource Exhaustion +
+ edk2-ovmf + + * + + rhel:9 + + 16 Jan 2024 +
  • M
+ Uncontrolled Resource Consumption ('Resource Exhaustion') +
+ pyopenssl + + [22.0.0,] + + pip + + 16 Jan 2024 +
  • M
+ Uncontrolled Resource Consumption ('Resource Exhaustion') +
+ openssl + + >=3.0.0 + + RubyGems + + 16 Jan 2024 +
  • M
+ Uncontrolled Resource Consumption ('Resource Exhaustion') +
+ openssl-src + + >=300.0.0+3.0.0 + + Cargo + + 16 Jan 2024 +
  • M
+ Uncontrolled Resource Consumption ('Resource Exhaustion') +
+ cryptography + + [35.0.0,] + + pip + + 16 Jan 2024 +
  • M
+ Uncontrolled Resource Consumption ('Resource Exhaustion') +
+ openssl + + [3.0.0,] + + Unmanaged (C/C++) + + 16 Jan 2024 +
  • L
+ CVE-2023-6237 +
+ openssl + + * + + debian:unstable + + 16 Jan 2024 +
+ + diff --git a/vulntotal/tests/test_data/snyk/html/5.html-expected.json b/vulntotal/tests/test_data/snyk/html/5.html-expected.json new file mode 100644 index 000000000..9b2a495a2 --- /dev/null +++ b/vulntotal/tests/test_data/snyk/html/5.html-expected.json @@ -0,0 +1,31 @@ +{ + "SNYK-SLES155-LIBOPENSSL3-6184655": "https://security.snyk.io/package/linux/sles:15.5/libopenssl3", + "SNYK-SLES155-LIBOPENSSL3DEVEL-6184653": "https://security.snyk.io/package/linux/sles:15.5/libopenssl-3-devel", + "SNYK-SLES155-OPENSSL3-6184652": "https://security.snyk.io/package/linux/sles:15.5/openssl-3", + "SNYK-ALPINE317-OPENSSL-6160001": "https://security.snyk.io/package/linux/alpine:3.17/openssl", + "SNYK-ALPINE318-OPENSSL-6160000": "https://security.snyk.io/package/linux/alpine:3.18/openssl", + "SNYK-ALPINE319-OPENSSL-6159994": "https://security.snyk.io/package/linux/alpine:3.19/openssl", + "SNYK-CENTOS9-OPENSSLPERL-6157924": "https://security.snyk.io/package/linux/centos:9/openssl-perl", + "SNYK-RHEL9-OPENSSLPERL-6157922": "https://security.snyk.io/package/linux/rhel:9/openssl-perl", + "SNYK-CENTOS9-OPENSSLDEVEL-6157920": "https://security.snyk.io/package/linux/centos:9/openssl-devel", + "SNYK-RHEL9-OPENSSLDEVEL-6157919": "https://security.snyk.io/package/linux/rhel:9/openssl-devel", + "SNYK-CENTOS9-OPENSSL-6157917": "https://security.snyk.io/package/linux/centos:9/openssl", + "SNYK-RHEL9-OPENSSL-6157915": "https://security.snyk.io/package/linux/rhel:9/openssl", + "SNYK-CENTOS9-OPENSSLLIBS-6157913": "https://security.snyk.io/package/linux/centos:9/openssl-libs", + "SNYK-RHEL9-OPENSSLLIBS-6157911": "https://security.snyk.io/package/linux/rhel:9/openssl-libs", + "SNYK-CENTOS9-EDK2TOOLSDOC-6157909": "https://security.snyk.io/package/linux/centos:9/edk2-tools-doc", + "SNYK-RHEL9-EDK2TOOLSDOC-6157908": "https://security.snyk.io/package/linux/rhel:9/edk2-tools-doc", + "SNYK-CENTOS9-EDK2TOOLS-6157906": "https://security.snyk.io/package/linux/centos:9/edk2-tools", + "SNYK-RHEL9-EDK2TOOLS-6157904": "https://security.snyk.io/package/linux/rhel:9/edk2-tools", + "SNYK-CENTOS9-EDK2AARCH64-6157902": "https://security.snyk.io/package/linux/centos:9/edk2-aarch64", + "SNYK-RHEL9-EDK2AARCH64-6157900": "https://security.snyk.io/package/linux/rhel:9/edk2-aarch64", + "SNYK-CENTOS9-EDK2-6157898": "https://security.snyk.io/package/linux/centos:9/edk2", + "SNYK-RHEL9-EDK2-6157896": "https://security.snyk.io/package/linux/rhel:9/edk2", + "SNYK-CENTOS9-EDK2OVMF-6157895": "https://security.snyk.io/package/linux/centos:9/edk2-ovmf", + "SNYK-RHEL9-EDK2OVMF-6157893": "https://security.snyk.io/package/linux/rhel:9/edk2-ovmf", + "SNYK-PYTHON-PYOPENSSL-6157250": "https://security.snyk.io/package/pip/pyopenssl", + "SNYK-RUBY-OPENSSL-6157246": "https://security.snyk.io/package/rubygems/openssl", + "SNYK-RUST-OPENSSLSRC-6157249": "https://security.snyk.io/package/cargo/openssl-src", + "SNYK-PYTHON-CRYPTOGRAPHY-6157248": "https://security.snyk.io/package/pip/cryptography", + "SNYK-DEBIANUNSTABLE-OPENSSL-6157245": "https://security.snyk.io/package/linux/debian:unstable/openssl" +} diff --git a/vulntotal/tests/test_snyk.py b/vulntotal/tests/test_snyk.py index f4f221b39..9f796c50a 100644 --- a/vulntotal/tests/test_snyk.py +++ b/vulntotal/tests/test_snyk.py @@ -91,3 +91,23 @@ def test_parse_html_advisory_3(self): ).to_dict() expected_file = f"{file}-expected.json" util_tests.check_results_against_json(result, expected_file) + + def test_parse_cve_advisory_html_0(self): + file = self.get_test_loc("html/4.html") + with open(file) as f: + page = f.read() + result = snyk.parse_cve_advisory_html( + page + ).to_dict() + expected_file = f"{file}-expected.json" + util_tests.check_results_against_json(result, expected_file) + + def test_parse_cve_advisory_html_1(self): + file = self.get_test_loc("html/5.html") + with open(file) as f: + page = f.read() + result = snyk.parse_cve_advisory_html( + page + ).to_dict() + expected_file = f"{file}-expected.json" + util_tests.check_results_against_json(result, expected_file) From 15de6392d9565a9dd24fc0b80befd75de5230a65 Mon Sep 17 00:00:00 2001 From: Shenoy Date: Sat, 27 Jan 2024 13:27:49 +0530 Subject: [PATCH 05/12] Add test for generate_purl function Signed-off-by: Shenoy --- vulntotal/tests/test_snyk.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/vulntotal/tests/test_snyk.py b/vulntotal/tests/test_snyk.py index 9f796c50a..abefe5f69 100644 --- a/vulntotal/tests/test_snyk.py +++ b/vulntotal/tests/test_snyk.py @@ -51,6 +51,36 @@ def test_generate_package_advisory_url(self): "https://security.snyk.io/api/listing?search=firefox&type=unmanaged", ] util_tests.check_results_against_expected(results, expected) + + def test_generate_purl(self): + package_advisory_urls = [ + "https://security.snyk.io/package/pip/jinja2", + "https://security.snyk.io/package/maven/org.apache.tomcat%3Atomcat", + "https://security.snyk.io/package/npm/semver-regex", + "https://security.snyk.io/package/composer/bolt%2Fcore", + "https://security.snyk.io/package/linux/debain:11/trafficserver", + "https://security.snyk.io/package/nuget/moment.js", + "https://security.snyk.io/package/cocoapods/ffmpeg", + "https://security.snyk.io/package/hex/coherence", + "https://security.snyk.io/package/rubygems/log4j-jars" + ] + + results = [ + PackageURL.to_string(snyk.generate_purl(package_advisory_url)) for package_advisory_url in package_advisory_urls + ] + + expected = [ + "pkg:pip/jinja2", + "pkg:maven/org.apache.tomcat/tomcat", + "pkg:npm/semver-regex", + "pkg:composer/bolt/core", + "pkg:linux/trafficserver?distro=debain:11", + "pkg:nuget/moment.js", + "pkg:cocoapods/ffmpeg", + "pkg:hex/coherence", + "pkg:rubygems/log4j-jars", + ] + util_tests.check_results_against_expected(results, expected) def test_parse_html_advisory_0(self): file = self.get_test_loc("html/0.html") From 31d09c2ea78390c949e925b7431c88dde002dde0 Mon Sep 17 00:00:00 2001 From: Shenoy Date: Sat, 27 Jan 2024 15:17:10 +0530 Subject: [PATCH 06/12] Remove to_dict in snyk test file where not required Signed-off-by: Shenoy --- vulntotal/tests/test_snyk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vulntotal/tests/test_snyk.py b/vulntotal/tests/test_snyk.py index abefe5f69..a21dc7213 100644 --- a/vulntotal/tests/test_snyk.py +++ b/vulntotal/tests/test_snyk.py @@ -128,7 +128,7 @@ def test_parse_cve_advisory_html_0(self): page = f.read() result = snyk.parse_cve_advisory_html( page - ).to_dict() + ) expected_file = f"{file}-expected.json" util_tests.check_results_against_json(result, expected_file) @@ -138,6 +138,6 @@ def test_parse_cve_advisory_html_1(self): page = f.read() result = snyk.parse_cve_advisory_html( page - ).to_dict() + ) expected_file = f"{file}-expected.json" util_tests.check_results_against_json(result, expected_file) From db2bb0436b970d25a86ce1d72006fc3ad7006d8b Mon Sep 17 00:00:00 2001 From: Shenoy Date: Tue, 30 Jan 2024 16:54:02 +0530 Subject: [PATCH 07/12] Format code using Black Signed-off-by: Shenoy --- vulntotal/datasources/snyk.py | 46 ++++++++++++++++++----------------- vulntotal/tests/test_snyk.py | 17 ++++++------- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/vulntotal/datasources/snyk.py b/vulntotal/datasources/snyk.py index 0cf71a44d..9995bc8d6 100644 --- a/vulntotal/datasources/snyk.py +++ b/vulntotal/datasources/snyk.py @@ -83,18 +83,18 @@ def datasource_advisory_from_cve(self, cve: str) -> Iterable[VendorData]: """ if not cve.upper().startswith("CVE-"): raise InvalidCVEError - + package_list = generate_payload_from_cve(cve) response = self.fetch(package_list) self._raw_dump = [response] # get list of vulnerabilities for cve id vulns_list = parse_cve_advisory_html(response) - + # for each vulnerability get fixed version from snyk_id_url, get affected version from package_advisory_url for snyk_id, package_advisory_url in vulns_list.items(): package_advisories_list = self.fetch(package_advisory_url) - package_advisories = extract_html_json_advisories(package_advisories_list) + package_advisories = extract_html_json_advisories(package_advisories_list) affected_versions = package_advisories[snyk_id] advisory_payload = generate_advisory_payload(snyk_id) advisory_html = self.fetch(advisory_payload) @@ -164,6 +164,7 @@ def generate_package_advisory_url(purl): package=purl_name, ) + def generate_purl(package_advisory_url): """ Generates purl from Package advisory url. @@ -174,8 +175,10 @@ def generate_purl(package_advisory_url): Returns: A PackageURL instance representing the package """ - package_advisory_url = unquote_plus(package_advisory_url.replace("https://security.snyk.io/package/", "")) - + package_advisory_url = unquote_plus( + package_advisory_url.replace("https://security.snyk.io/package/", "") + ) + package_url_split = package_advisory_url.split("/") pkg_type = package_url_split[0] @@ -183,15 +186,15 @@ def generate_purl(package_advisory_url): namespace = None version = None qualifiers = {} - + if pkg_type == "maven": pkg_name = package_url_split[1].split(":")[1] namespace = package_url_split[1].split(":")[0] elif pkg_type in ("golang", "composer"): - if package_url_split[1] == 'github.com': + if package_url_split[1] == "github.com": pkg_name = package_url_split[-2] - namespace = f"{package_url_split[1]}/{package_url_split[2]}" + namespace = f"{package_url_split[1]}/{package_url_split[2]}" version = package_url_split[-1] else: pkg_name = package_url_split[-1] @@ -205,12 +208,8 @@ def generate_purl(package_advisory_url): pkg_name = package_url_split[-1] return PackageURL( - type=pkg_type, - name=pkg_name, - namespace=namespace, - version=version, - qualifiers=qualifiers - ) + type=pkg_type, name=pkg_name, namespace=namespace, version=version, qualifiers=qualifiers + ) def extract_html_json_advisories(package_advisories): @@ -284,33 +283,35 @@ def parse_html_advisory(advisory_html, snyk_id, affected, purl) -> VendorData: fixed_versions=fixed_versions, ) + def parse_cve_advisory_html(cve_advisory_html): """ Parse CVE HTML advisory from Snyk and extract list of vulnerabilities and corresponding packages for that CVE. - + Parameters: advisory_html: A string of HTML containing the vulnerabilities for given CVE. - + Returns: A dictionary with each item representing a vulnerability. Key of each item is the SNYK_ID and value is the package advisory url on snyk website """ - cve_advisory_soup = BeautifulSoup(cve_advisory_html, 'html.parser') + cve_advisory_soup = BeautifulSoup(cve_advisory_html, "html.parser") vulns_table = cve_advisory_soup.find("tbody", class_="vue--table__tbody") if not vulns_table: - return None + return None vulns_rows = vulns_table.find_all("tr", class_="vue--table__row") vulns_list = {} - + for row in vulns_rows: - anchors = row.find_all("a", {"class" :"vue--anchor"}) + anchors = row.find_all("a", {"class": "vue--anchor"}) if len(anchors) != 2: continue - snyk_id = anchors[0]['href'].split("/")[1] + snyk_id = anchors[0]["href"].split("/")[1] package_advisory_url = f"https://security.snyk.io{anchors[1]['href']}" vulns_list[snyk_id] = package_advisory_url - + return vulns_list + def is_purl_in_affected(version, affected): return any(snyk_constraints_satisfied(affected_range, version) for affected_range in affected) @@ -318,5 +319,6 @@ def is_purl_in_affected(version, affected): def generate_advisory_payload(snyk_id): return f"https://security.snyk.io/vuln/{snyk_id}" + def generate_payload_from_cve(cve_id): return f"https://security.snyk.io/vuln?search={cve_id}" diff --git a/vulntotal/tests/test_snyk.py b/vulntotal/tests/test_snyk.py index a21dc7213..ffdf92e7a 100644 --- a/vulntotal/tests/test_snyk.py +++ b/vulntotal/tests/test_snyk.py @@ -51,7 +51,7 @@ def test_generate_package_advisory_url(self): "https://security.snyk.io/api/listing?search=firefox&type=unmanaged", ] util_tests.check_results_against_expected(results, expected) - + def test_generate_purl(self): package_advisory_urls = [ "https://security.snyk.io/package/pip/jinja2", @@ -62,11 +62,12 @@ def test_generate_purl(self): "https://security.snyk.io/package/nuget/moment.js", "https://security.snyk.io/package/cocoapods/ffmpeg", "https://security.snyk.io/package/hex/coherence", - "https://security.snyk.io/package/rubygems/log4j-jars" + "https://security.snyk.io/package/rubygems/log4j-jars", ] results = [ - PackageURL.to_string(snyk.generate_purl(package_advisory_url)) for package_advisory_url in package_advisory_urls + PackageURL.to_string(snyk.generate_purl(package_advisory_url)) + for package_advisory_url in package_advisory_urls ] expected = [ @@ -126,9 +127,7 @@ def test_parse_cve_advisory_html_0(self): file = self.get_test_loc("html/4.html") with open(file) as f: page = f.read() - result = snyk.parse_cve_advisory_html( - page - ) + result = snyk.parse_cve_advisory_html(page) expected_file = f"{file}-expected.json" util_tests.check_results_against_json(result, expected_file) @@ -136,8 +135,6 @@ def test_parse_cve_advisory_html_1(self): file = self.get_test_loc("html/5.html") with open(file) as f: page = f.read() - result = snyk.parse_cve_advisory_html( - page - ) + result = snyk.parse_cve_advisory_html(page) expected_file = f"{file}-expected.json" - util_tests.check_results_against_json(result, expected_file) + util_tests.check_results_against_json(result, expected_file) From c3ba2876a781b848927ac07cd4856fa6d90b9f81 Mon Sep 17 00:00:00 2001 From: Shenoy Date: Tue, 30 Jan 2024 17:04:14 +0530 Subject: [PATCH 08/12] Sort imports correctly using isort Signed-off-by: Shenoy --- vulntotal/datasources/snyk.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vulntotal/datasources/snyk.py b/vulntotal/datasources/snyk.py index 9995bc8d6..9b48d28ed 100644 --- a/vulntotal/datasources/snyk.py +++ b/vulntotal/datasources/snyk.py @@ -9,7 +9,8 @@ import logging from typing import Iterable -from urllib.parse import quote, unquote_plus +from urllib.parse import quote +from urllib.parse import unquote_plus import requests from bs4 import BeautifulSoup From a5ce04446304bd58466f982c2ab238de9fc7aa28 Mon Sep 17 00:00:00 2001 From: Shenoy Date: Sat, 10 Feb 2024 15:25:57 +0530 Subject: [PATCH 09/12] Modify generate_purl function in snyk Signed-off-by: Shenoy --- vulntotal/datasources/snyk.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/vulntotal/datasources/snyk.py b/vulntotal/datasources/snyk.py index 9b48d28ed..66883d51d 100644 --- a/vulntotal/datasources/snyk.py +++ b/vulntotal/datasources/snyk.py @@ -185,32 +185,38 @@ def generate_purl(package_advisory_url): pkg_name = None namespace = None - version = None qualifiers = {} if pkg_type == "maven": pkg_name = package_url_split[1].split(":")[1] namespace = package_url_split[1].split(":")[0] - elif pkg_type in ("golang", "composer"): + elif pkg_type == "composer": + pkg_name = package_url_split[-1] + namespace = package_url_split[-2] + + elif pkg_type == "golang": if package_url_split[1] == "github.com": - pkg_name = package_url_split[-2] - namespace = f"{package_url_split[1]}/{package_url_split[2]}" - version = package_url_split[-1] - else: + ns_start = package_advisory_url.find("github.com") + ns_end = package_advisory_url.rfind("/") + namespace = package_advisory_url[ns_start:ns_end] pkg_name = package_url_split[-1] + + elif pkg_type == "npm": + # handle scoped npm packages + if "@" in package_advisory_url: namespace = package_url_split[-2] + pkg_name = package_url_split[-1] + elif pkg_type == "linux": pkg_name = package_url_split[-1] qualifiers["distro"] = package_url_split[1] - else: + elif pkg_type in ("cocoapods", "hex", "nuget", "pip", "rubygems", "unmanaged"): pkg_name = package_url_split[-1] - return PackageURL( - type=pkg_type, name=pkg_name, namespace=namespace, version=version, qualifiers=qualifiers - ) + return PackageURL(type=pkg_type, name=pkg_name, namespace=namespace) def extract_html_json_advisories(package_advisories): From 9d9a1af07939d7bd14c5fc2e8a608abbab852b48 Mon Sep 17 00:00:00 2001 From: Shenoy Date: Sat, 10 Feb 2024 15:29:19 +0530 Subject: [PATCH 10/12] Add additional tests for golang and npm Signed-off-by: Shenoy --- vulntotal/tests/test_snyk.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/vulntotal/tests/test_snyk.py b/vulntotal/tests/test_snyk.py index ffdf92e7a..34a87aea3 100644 --- a/vulntotal/tests/test_snyk.py +++ b/vulntotal/tests/test_snyk.py @@ -57,12 +57,19 @@ def test_generate_purl(self): "https://security.snyk.io/package/pip/jinja2", "https://security.snyk.io/package/maven/org.apache.tomcat%3Atomcat", "https://security.snyk.io/package/npm/semver-regex", + "https://security.snyk.io/package/npm/@urql%2Fnext", + "https://security.snyk.io/package/npm/@lobehub%2Fchat", + "https://security.snyk.io/package/npm/meshcentral", "https://security.snyk.io/package/composer/bolt%2Fcore", "https://security.snyk.io/package/linux/debain:11/trafficserver", + "https://security.snyk.io/package/linux/almalinux:8/rpm-plugin-fapolicyd", "https://security.snyk.io/package/nuget/moment.js", "https://security.snyk.io/package/cocoapods/ffmpeg", "https://security.snyk.io/package/hex/coherence", "https://security.snyk.io/package/rubygems/log4j-jars", + "https://security.snyk.io/package/golang/github.com%2Fgrafana%2Fgrafana%2Fpkg%2Fservices%2Fsqlstore%2Fmigrator", + "https://security.snyk.io/package/golang/github.com%2Fanswerdev%2Fanswer%2Finternal%2Frepo%2Factivity", + "https://security.snyk.io/package/golang/github.com%2Fargoproj%2Fargo-cd%2Fv2%2Futil%2Fhelm", ] results = [ @@ -74,13 +81,21 @@ def test_generate_purl(self): "pkg:pip/jinja2", "pkg:maven/org.apache.tomcat/tomcat", "pkg:npm/semver-regex", + "pkg:npm/%40urql/next", + "pkg:npm/%40lobehub/chat", + "pkg:npm/meshcentral", "pkg:composer/bolt/core", - "pkg:linux/trafficserver?distro=debain:11", + "pkg:linux/trafficserver", + "pkg:linux/rpm-plugin-fapolicyd", "pkg:nuget/moment.js", "pkg:cocoapods/ffmpeg", "pkg:hex/coherence", "pkg:rubygems/log4j-jars", + "pkg:golang/github.com/grafana/grafana/pkg/services/sqlstore/migrator", + "pkg:golang/github.com/answerdev/answer/internal/repo/activity", + "pkg:golang/github.com/argoproj/argo-cd/v2/util/helm", ] + util_tests.check_results_against_expected(results, expected) def test_parse_html_advisory_0(self): From f23acebc07464b5beff9b71ac9c04f9297d1fc28 Mon Sep 17 00:00:00 2001 From: Shenoy Date: Sun, 18 Feb 2024 11:05:52 +0530 Subject: [PATCH 11/12] Add non-Github Go packages to test, modify generate_purl function, improve error handling Signed-off-by: Shenoy --- vulntotal/datasources/snyk.py | 13 +++++++------ vulntotal/tests/test_snyk.py | 5 ++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/vulntotal/datasources/snyk.py b/vulntotal/datasources/snyk.py index 66883d51d..418cd8830 100644 --- a/vulntotal/datasources/snyk.py +++ b/vulntotal/datasources/snyk.py @@ -101,7 +101,7 @@ def datasource_advisory_from_cve(self, cve: str) -> Iterable[VendorData]: advisory_html = self.fetch(advisory_payload) self._raw_dump.append(advisory_html) purl = generate_purl(package_advisory_url) - if advisory_html: + if advisory_html and purl: yield parse_html_advisory(advisory_html, snyk_id, affected_versions, purl) @classmethod @@ -196,11 +196,8 @@ def generate_purl(package_advisory_url): namespace = package_url_split[-2] elif pkg_type == "golang": - if package_url_split[1] == "github.com": - ns_start = package_advisory_url.find("github.com") - ns_end = package_advisory_url.rfind("/") - namespace = package_advisory_url[ns_start:ns_end] - pkg_name = package_url_split[-1] + pkg_name = package_url_split[-1] + namespace = "/".join(package_url_split[1:-1]) elif pkg_type == "npm": # handle scoped npm packages @@ -216,6 +213,10 @@ def generate_purl(package_advisory_url): elif pkg_type in ("cocoapods", "hex", "nuget", "pip", "rubygems", "unmanaged"): pkg_name = package_url_split[-1] + if pkg_type is None or pkg_name is None: + logger.error("Invalid package advisory url, package type or name is missing") + return + return PackageURL(type=pkg_type, name=pkg_name, namespace=namespace) diff --git a/vulntotal/tests/test_snyk.py b/vulntotal/tests/test_snyk.py index 34a87aea3..3da312e35 100644 --- a/vulntotal/tests/test_snyk.py +++ b/vulntotal/tests/test_snyk.py @@ -93,7 +93,10 @@ def test_generate_purl(self): "pkg:rubygems/log4j-jars", "pkg:golang/github.com/grafana/grafana/pkg/services/sqlstore/migrator", "pkg:golang/github.com/answerdev/answer/internal/repo/activity", - "pkg:golang/github.com/argoproj/argo-cd/v2/util/helm", + "pkg:golang/go.etcd.io/etcd/v3/auth", + "pkg:golang/gopkg.in/kubernetes/kubernetes.v0/pkg/registry/pod", + "pkg:golang/gogs.io/gogs/internal/db", + "pkg:golang/golang.org/x/crypto/ssh", ] util_tests.check_results_against_expected(results, expected) From 4874cbedeb636a8468e33d29c107a85c6630e0f4 Mon Sep 17 00:00:00 2001 From: Shenoy Date: Sun, 18 Feb 2024 11:35:11 +0530 Subject: [PATCH 12/12] Modify generate_purl to comply with purl spec Signed-off-by: Shenoy --- vulntotal/datasources/snyk.py | 3 ++- vulntotal/tests/test_snyk.py | 9 ++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/vulntotal/datasources/snyk.py b/vulntotal/datasources/snyk.py index 418cd8830..5b2418071 100644 --- a/vulntotal/datasources/snyk.py +++ b/vulntotal/datasources/snyk.py @@ -179,6 +179,7 @@ def generate_purl(package_advisory_url): package_advisory_url = unquote_plus( package_advisory_url.replace("https://security.snyk.io/package/", "") ) + supported_ecosystems = {v: k for (k, v) in SnykDataSource.supported_ecosystem().items()} package_url_split = package_advisory_url.split("/") pkg_type = package_url_split[0] @@ -217,7 +218,7 @@ def generate_purl(package_advisory_url): logger.error("Invalid package advisory url, package type or name is missing") return - return PackageURL(type=pkg_type, name=pkg_name, namespace=namespace) + return PackageURL(type=supported_ecosystems[pkg_type], name=pkg_name, namespace=namespace) def extract_html_json_advisories(package_advisories): diff --git a/vulntotal/tests/test_snyk.py b/vulntotal/tests/test_snyk.py index 3da312e35..871d8a27b 100644 --- a/vulntotal/tests/test_snyk.py +++ b/vulntotal/tests/test_snyk.py @@ -69,7 +69,10 @@ def test_generate_purl(self): "https://security.snyk.io/package/rubygems/log4j-jars", "https://security.snyk.io/package/golang/github.com%2Fgrafana%2Fgrafana%2Fpkg%2Fservices%2Fsqlstore%2Fmigrator", "https://security.snyk.io/package/golang/github.com%2Fanswerdev%2Fanswer%2Finternal%2Frepo%2Factivity", - "https://security.snyk.io/package/golang/github.com%2Fargoproj%2Fargo-cd%2Fv2%2Futil%2Fhelm", + "https://security.snyk.io/package/golang/go.etcd.io%2Fetcd%2Fv3%2Fauth", + "https://security.snyk.io/package/golang/gopkg.in%2Fkubernetes%2Fkubernetes.v0%2Fpkg%2Fregistry%2Fpod", + "https://security.snyk.io/package/golang/gogs.io%2Fgogs%2Finternal%2Fdb", + "https://security.snyk.io/package/golang/golang.org%2Fx%2Fcrypto%2Fssh", ] results = [ @@ -78,7 +81,7 @@ def test_generate_purl(self): ] expected = [ - "pkg:pip/jinja2", + "pkg:pypi/jinja2", "pkg:maven/org.apache.tomcat/tomcat", "pkg:npm/semver-regex", "pkg:npm/%40urql/next", @@ -90,7 +93,7 @@ def test_generate_purl(self): "pkg:nuget/moment.js", "pkg:cocoapods/ffmpeg", "pkg:hex/coherence", - "pkg:rubygems/log4j-jars", + "pkg:gem/log4j-jars", "pkg:golang/github.com/grafana/grafana/pkg/services/sqlstore/migrator", "pkg:golang/github.com/answerdev/answer/internal/repo/activity", "pkg:golang/go.etcd.io/etcd/v3/auth",