Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: update rootfs to scan resource path #840

Merged
merged 4 commits into from
Aug 7, 2023
Merged

fix: update rootfs to scan resource path #840

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7afbd31
fix: update rootfs to scan resource path
philcali Aug 1, 2023
2dd4249
fix: regression test adapted to fix
philcali Aug 3, 2023
8203bae
Merge pull request #1 from nexB/main
philcali Aug 7, 2023
5eb9259
chore: adding changelog entry
philcali Aug 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ v32.6.0 (unreleased)
a VulnerableCode database.
https://github.com/nexB/scancode.io/issues/835

- Fix root filesystem scanning for installed packages and archived Linux distributions.
Allows the scan to discover system packages from `rpmdb.sqlite` and other sources.
https://github.com/nexB/scancode.io/pull/840

v32.5.0 (2023-08-02)
--------------------

Expand Down
3 changes: 2 additions & 1 deletion scanpipe/pipes/rootfs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,8 @@ def _create_system_package(project, purl, package):
codebase_resources = project.codebaseresources.all()

for install_file in installed_files:
rootfs_path = pipes.normalize_path(install_file.path)
install_file_path = install_file.get_path(strip_root=True)
rootfs_path = pipes.normalize_path(install_file_path)
logger.info(f" installed file rootfs_path: {rootfs_path}")

try:
Expand Down
215 changes: 4 additions & 211 deletions scanpipe/tests/data/basic-rootfs_root_filesystems.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,6 @@
"datasource_id": "",
"file_references": [],
"missing_resources": [
"/basic-rootfs.tar.gz-extract/usr/share/doc/libncurses5/copyright"
],
"modified_resources": [],
"affected_by_vulnerabilities": []
Expand Down Expand Up @@ -288,7 +287,6 @@
"datasource_id": "",
"file_references": [],
"missing_resources": [
"/basic-rootfs.tar.gz-extract/usr/share/doc/libndp0/copyright"
],
"modified_resources": [],
"affected_by_vulnerabilities": []
Expand Down Expand Up @@ -486,7 +484,7 @@
"path": "basic-rootfs.tar.gz-extract/usr/share/doc/libncurses5/copyright",
"type": "file",
"name": "copyright",
"status": "application-package",
"status": "system-package",
"tag": "",
"extension": "",
"md5": "bd73d1dbbd2e6374358baa205d6d9e66",
Expand All @@ -508,126 +506,8 @@
"holders": [],
"authors": [],
"package_data": [
{
"md5": null,
"name": "libncurses5",
"purl": "pkg:deb/libncurses5",
"sha1": null,
"type": "deb",
"holder": "Free Software Foundation, Inc.\nPradeep Padala\nX Consortium\nThe Regents of the University of California\nThomas E. Dickey",
"sha256": null,
"sha512": null,
"parties": [],
"subpath": null,
"vcs_url": null,
"version": null,
"keywords": [],
"copyright": "Copyright (c) 1998-2016 Free Software Foundation, Inc.\nCopyright (c) 2001 by Pradeep Padala\nCopyright (c) 1994 X Consortium\nCopyright (c) 1980, 1991, 1992, 1993 The Regents of the University of California\nCopyright 1996-2007 by Thomas E. Dickey",
"namespace": null,
"extra_data": {},
"qualifiers": {},
"description": null,
"notice_text": null,
"api_data_url": null,
"dependencies": [],
"download_url": null,
"homepage_url": null,
"release_date": null,
"code_view_url": null,
"datasource_id": "debian_copyright_in_package",
"file_references": [],
"source_packages": [],
"bug_tracking_url": null,
"primary_language": null,
"license_detections": [
{
"matches": [
{
"score": 100.0,
"matcher": "2-aho",
"end_line": 45,
"rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/x11-fsf.LICENSE",
"start_line": 23,
"matched_text": "Permission is hereby granted, free of charge, to any person obtaining a\ncopy of this software and associated documentation files (the\n\"Software\"), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, distribute with modifications, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included\nin all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS\nOR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\nIN NO EVENT SHALL THE ABOVE COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,\nDAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR\nOTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR\nTHE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\nExcept as contained in this notice, the name(s) of the above copyright\nholders shall not be used in advertising or otherwise to promote the\nsale, use or other dealings in this Software without prior written\nauthorization.",
"match_coverage": 100.0,
"matched_length": 200,
"rule_relevance": 100,
"rule_identifier": "x11-fsf.LICENSE",
"license_expression": "x11-fsf"
}
],
"identifier": "x11_fsf-5f3d72c2-fa6a-2f7b-b859-17e7567c1724",
"license_expression": "x11-fsf"
},
{
"matches": [
{
"score": 100.0,
"matcher": "2-aho",
"end_line": 70,
"rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/x11-xconsortium_2.RULE",
"start_line": 50,
"matched_text": "Permission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to\ndeal in the Software without restriction, including without limitation the\nrights to use, copy, modify, merge, publish, distribute, sublicense, and/or\nsell copies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in\nall copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nX CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN\nAN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC-\nTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\nExcept as contained in this notice, the name of the X Consortium shall not\nbe used in advertising or otherwise to promote the sale, use or other deal-\nings in this Software without prior written authorization from the X Consor-\ntium.",
"match_coverage": 100.0,
"matched_length": 201,
"rule_relevance": 100,
"rule_identifier": "x11-xconsortium_2.RULE",
"license_expression": "x11-xconsortium"
}
],
"identifier": "x11_xconsortium-8bc3e205-5f29-ecad-90bc-2f492c65be46",
"license_expression": "x11-xconsortium"
},
{
"matches": [
{
"score": 100.0,
"matcher": "2-aho",
"end_line": 98,
"rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/bsd-new_19.RULE",
"start_line": 76,
"matched_text": "Redistribution and use in source and binary forms, with or without\nmodification, are permitted provided that the following conditions\nare met:\n1. Redistributions of source code must retain the above copyright\n notice, this list of conditions and the following disclaimer.\n2. Redistributions in binary form must reproduce the above copyright\n notice, this list of conditions and the following disclaimer in the\n documentation and/or other materials provided with the distribution.\n3. Neither the name of the University nor the names of its contributors\n may be used to endorse or promote products derived from this software\n without specific prior written permission.\n\nTHIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND\nANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE\nIMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE\nARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE\nFOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL\nDAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS\nOR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)\nHOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT\nLIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY\nOUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF\nSUCH DAMAGE.",
"match_coverage": 100.0,
"matched_length": 213,
"rule_relevance": 100,
"rule_identifier": "bsd-new_19.RULE",
"license_expression": "bsd-new"
}
],
"identifier": "bsd_new-ccc98c3a-92d4-e7a3-e0ba-798328cb6b98",
"license_expression": "bsd-new"
},
{
"matches": [
{
"score": 100.0,
"matcher": "2-aho",
"end_line": 127,
"rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/x11-fsf_7.RULE",
"start_line": 105,
"matched_text": "Permission is hereby granted, free of charge, to any person obtaining a\ncopy of this software and associated documentation files (the\n\"Software\"), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be included\nin all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS\nOR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.\nIN NO EVENT SHALL THE ABOVE LISTED COPYRIGHT HOLDER(S) BE LIABLE FOR ANY\nCLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,\nTORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE\nSOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n\nExcept as contained in this notice, the name(s) of the above copyright\nholders shall not be used in advertising or otherwise to promote the\nsale, use or other dealings in this Software without prior written\nauthorization.",
"match_coverage": 100.0,
"matched_length": 199,
"rule_relevance": 100,
"rule_identifier": "x11-fsf_7.RULE",
"license_expression": "x11-fsf"
}
],
"identifier": "x11_fsf-c2b8535e-1b81-42d8-945b-b026f349228d",
"license_expression": "x11-fsf"
}
],
"repository_download_url": null,
"repository_homepage_url": null,
"other_license_detections": [],
"other_license_expression": null,
"declared_license_expression": "x11-fsf AND x11-xconsortium AND bsd-new",
"extracted_license_statement": null,
"other_license_expression_spdx": null,
"declared_license_expression_spdx": "X11-distribute-modifications-variant AND X11 AND BSD-3-Clause"
}
],
"for_packages": [],
"for_packages": ["pkg:deb/[email protected]?architecture=amd64"],
"emails": [],
"urls": [],
"extra_data": {}
Expand Down Expand Up @@ -667,7 +547,7 @@
"path": "basic-rootfs.tar.gz-extract/usr/share/doc/libndp0/copyright",
"type": "file",
"name": "copyright",
"status": "application-package",
"status": "system-package",
"tag": "",
"extension": "",
"md5": "3f4ecdd67d5b9427cdc66847bdd11cf4",
Expand All @@ -689,95 +569,8 @@
"holders": [],
"authors": [],
"package_data": [
{
"md5": null,
"name": "libndp0",
"purl": "pkg:deb/libndp0",
"sha1": null,
"type": "deb",
"holder": "Jiri Pirko\nAndrew Ayer",
"sha256": null,
"sha512": null,
"parties": [],
"subpath": null,
"vcs_url": null,
"version": null,
"keywords": [],
"copyright": "Copyright 2013 Jiri Pirko <[email protected]>\nCopyright 2014 Andrew Ayer <[email protected]>",
"namespace": null,
"extra_data": {},
"qualifiers": {},
"description": null,
"notice_text": null,
"api_data_url": null,
"dependencies": [],
"download_url": null,
"homepage_url": null,
"release_date": null,
"code_view_url": null,
"datasource_id": "debian_copyright_in_package",
"file_references": [],
"source_packages": [],
"bug_tracking_url": null,
"primary_language": null,
"license_detections": [],
"repository_download_url": null,
"repository_homepage_url": null,
"other_license_detections": [
{
"matches": [
{
"score": 100.0,
"matcher": "1-hash",
"end_line": 13,
"rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/lgpl-2.1-plus_108.RULE",
"start_line": 13,
"matched_text": "License: lgpl-2.1+",
"match_coverage": 100.0,
"matched_length": 4,
"rule_relevance": 100,
"rule_identifier": "lgpl-2.1-plus_108.RULE",
"license_expression": "lgpl-2.1-plus"
},
{
"score": 100.0,
"matcher": "2-aho",
"end_line": 26,
"rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/lgpl-2.1-plus_93.RULE",
"start_line": 14,
"matched_text": "This program is free software; you can redistribute it and/or modify it\nunder the terms of the GNU Lesser General Public License as published\nby the Free Software Foundation; either version 2.1 of the License, or\n(at your option) any later version.\n\nThis program is distributed in the hope that it will be useful,\nbut WITHOUT ANY WARRANTY; without even the implied warranty of\nMERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser\nGeneral Public License for more details.\n\nYou should have received a copy of the GNU Lesser General Public License\nalong with this program; if not, write to the Free Software Foundation,\nInc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA",
"match_coverage": 100.0,
"matched_length": 117,
"rule_relevance": 100,
"rule_identifier": "lgpl-2.1-plus_93.RULE",
"license_expression": "lgpl-2.1-plus"
},
{
"score": 100.0,
"matcher": "2-aho",
"end_line": 30,
"rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/lgpl-2.1_314.RULE",
"start_line": 24,
"matched_text": "You should have received a copy of the GNU Lesser General Public License\nalong with this program; if not, write to the Free Software Foundation,\nInc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA\n\nOn Debian systems, the full text of the GNU Lesser General Public\nLicense version 2.1 can be found in the file\n`/usr/share/common-licenses/LGPL-2.1'.",
"match_coverage": 100.0,
"matched_length": 64,
"rule_relevance": 100,
"rule_identifier": "lgpl-2.1_314.RULE",
"license_expression": "lgpl-2.1"
}
],
"identifier": "lgpl_2_1_plus_and_lgpl_2_1-1018860f-b475-01e3-bdd4-39bf3444650e",
"license_expression": "lgpl-2.1-plus AND lgpl-2.1"
}
],
"other_license_expression": "lgpl-2.1-plus AND lgpl-2.1",
"declared_license_expression": "lgpl-2.1-plus AND lgpl-2.1",
"extracted_license_statement": "- LGPL-2.1+\n- LGPL-2.1+\n- LGPL-2.1+\n",
"other_license_expression_spdx": "LGPL-2.1-or-later AND LGPL-2.1-only",
"declared_license_expression_spdx": "LGPL-2.1-or-later AND LGPL-2.1-only"
}
],
"for_packages": [],
"for_packages": ["pkg:deb/[email protected]?architecture=amd64"],
"emails": [],
"urls": [],
"extra_data": {}
Expand Down