411 toolkit upgrade 2

.gitignore

@@ -40,6 +40,7 @@ local
 policies.yml
 *.rdb
 *.aof
+.vscode
 
 # This is only created when packaging for external redistribution
 /thirdparty/

scanpipe/models.py

@@ -1769,7 +1769,7 @@ def create_from_data(cls, project, package_data):
         If one of the values of the required fields is not available, a "ProjectError"
         is created instead of a new DiscoveredPackage instance.
         """
-        required_fields = ["type", "name", "version"]
+        required_fields = ["type", "name"]
         missing_values = [
             field_name
             for field_name in required_fields

scanpipe/pipelines/scan_package.py

@@ -58,11 +58,8 @@ def steps(cls):
         "--url",
     ] + [
         "--classify",
-        "--consolidate",
         "--is-license-text",
-        "--license-clarity-score",
         "--summary",
-        "--summary-key-files",
     ]
 
     def get_package_archive_input(self):
@@ -121,8 +118,8 @@ def build_inventory_from_scan(self):
         """
         project = self.project
         scanned_codebase = scancode.get_virtual_codebase(project, str(self.scan_output))
-        scancode.create_codebase_resources(project, scanned_codebase)
         scancode.create_discovered_packages(project, scanned_codebase)
+        scancode.create_codebase_resources(project, scanned_codebase)
 
     def make_summary_from_scan_results(self):
         """

scanpipe/pipes/__init__.py

@@ -73,15 +73,27 @@ def update_or_create_package(project, package_data, codebase_resource=None):
     """
     Gets, updates or creates a DiscoveredPackage then returns it.
     Uses the `project` and `package_data` mapping to lookup and creates the
-    DiscoveredPackage using its Package URL as a unique key.
+    DiscoveredPackage using its Package URL and package_uid as a unique key.
     """
     purl_data = DiscoveredPackage.extract_purl_data(package_data)
+    package_uid = package_data.get("package_uid")
+    purl_data_and_package_uid = {
+        **purl_data,
+        "extra_data": {"package_uid": package_uid},
+    }
 
     try:
-        package = DiscoveredPackage.objects.get(project=project, **purl_data)
+        package = DiscoveredPackage.objects.get(
+            project=project,
+            **purl_data_and_package_uid
+        )
     except DiscoveredPackage.DoesNotExist:
         package = None
 
+    package_data = {
+        **package_data,
+        "extra_data": {"package_uid": package_uid},
+    }
     if package:
         package.update_from_data(package_data)
 

scanpipe/pipes/codebase.py

@@ -53,6 +53,8 @@ def get_tree(resource, fields, codebase=None):
     return resource_dict
 
 
+# TODO: Walking the ProjectCodebase is broken as we do not have a consistent way
+# to get the root of a codebase.
 class ProjectCodebase:
     """
     Represents the codebase of a project stored in the database.

scanpipe/pipes/docker.py

@@ -22,7 +22,6 @@
 
 import logging
 import posixpath
-from functools import partial
 from pathlib import Path
 
 from container_inspector.image import Image
@@ -152,23 +151,21 @@ def scan_image_for_system_packages(project, image, detect_licenses=True):
         raise rootfs.DistroNotFound(f"Distro not found.")
 
     distro_id = image.distro.identifier
-    if distro_id not in rootfs.PACKAGE_GETTER_BY_DISTRO:
+    if distro_id not in rootfs.SUPPORTED_DISTROS:
         raise rootfs.DistroNotSupported(f'Distro "{distro_id}" is not supported.')
 
-    package_getter = partial(
-        rootfs.PACKAGE_GETTER_BY_DISTRO[distro_id],
-        distro=distro_id,
-        detect_licenses=detect_licenses,
-    )
-
-    installed_packages = image.get_installed_packages(package_getter)
+    installed_packages = image.get_installed_packages(rootfs.package_getter)
 
     for i, (purl, package, layer) in enumerate(installed_packages):
         logger.info(f"Creating package #{i}: {purl}")
         created_package = pipes.update_or_create_package(project, package.to_dict())
 
+        installed_files = []
+        if hasattr(package, "resources"):
+            installed_files = package.resources
+
         # We have no files for this installed package, we cannot go further.
-        if not package.installed_files:
+        if not installed_files:
             logger.info(f"  No installed_files for: {purl}")
             continue
 
@@ -177,8 +174,9 @@ def scan_image_for_system_packages(project, image, detect_licenses=True):
 
         codebase_resources = project.codebaseresources.all()
 
-        for install_file in package.installed_files:
-            install_file_path = pipes.normalize_path(install_file.path)
+        for install_file in installed_files:
+            install_file_path = install_file.get_path(strip_root=True)
+            install_file_path = pipes.normalize_path(install_file_path)
             layer_rootfs_path = posixpath.join(
                 layer.layer_id,
                 install_file_path.strip("/"),

scanpipe/pipes/output.py

@@ -168,7 +168,7 @@ def get_headers(self, project):
     def get_packages(self, project):
         from scanpipe.api.serializers import DiscoveredPackageSerializer
 
-        packages = project.discoveredpackages.all()
+        packages = project.discoveredpackages.all().order_by("type", "name")
 
         for obj in packages.iterator():
             yield self.encode(DiscoveredPackageSerializer(obj).data)
@@ -280,9 +280,9 @@ def _add_xlsx_worksheet(workbook, worksheet_name, rows, fields):
 # https://github.com/nexB/scancode-toolkit/pull/2381
 # https://github.com/nexB/scancode-toolkit/issues/2350
 mappings_key_by_fieldname = {
-    "copyrights": "value",
-    "holders": "value",
-    "authors": "value",
+    "copyrights": "copyright",
+    "holders": "holder",
+    "authors": "author",
     "emails": "email",
     "urls": "url",
 }

scanpipe/pipes/rootfs.py

@@ -23,36 +23,32 @@
 import fnmatch
 import logging
 import os
-from functools import partial
 
 from django.core.exceptions import ObjectDoesNotExist
 from django.db.models import Q
 
 import attr
 from commoncode.ignore import default_ignores
 from container_inspector.distro import Distro
+from packagedcode import plugin_package
 
 from scanpipe import pipes
-from scanpipe.pipes import alpine
-from scanpipe.pipes import debian
-from scanpipe.pipes import rpm
-from scanpipe.pipes import windows
 
 logger = logging.getLogger(__name__)
 
-PACKAGE_GETTER_BY_DISTRO = {
-    "alpine": alpine.package_getter,
-    "debian": partial(debian.package_getter, distro="debian"),
-    "ubuntu": partial(debian.package_getter, distro="ubuntu"),
-    "rhel": rpm.package_getter,
-    "centos": rpm.package_getter,
-    "fedora": rpm.package_getter,
-    "sles": rpm.package_getter,
-    "opensuse": rpm.package_getter,
-    "opensuse-tumbleweed": rpm.package_getter,
-    "photon": rpm.package_getter,
-    "windows": windows.package_getter,
-}
+SUPPORTED_DISTROS = [
+    "alpine",
+    "debian",
+    "ubuntu",
+    "rhel",
+    "centos",
+    "fedora",
+    "sles",
+    "opensuse",
+    "opensuse-tumbleweed",
+    "photon",
+    "windows",
+]
 
 
 class DistroNotFound(Exception):
@@ -175,6 +171,14 @@ def has_hash_diff(install_file, codebase_resource):
     hash_types = ["sha512", "sha256", "sha1", "md5"]
 
     for hash_type in hash_types:
+        # Find a suitable hash type that is present on both install_file and
+        # codebase_resource, skip otherwise.
+        if not (
+            hasattr(install_file, hash_type)
+            and hasattr(codebase_resource, hash_type)
+        ):
+            continue
+
         install_file_sum = getattr(install_file, hash_type)
         codebase_resource_sum = getattr(codebase_resource, hash_type)
         hashes_differ = all(
@@ -190,6 +194,15 @@ def has_hash_diff(install_file, codebase_resource):
     return False
 
 
+def package_getter(root_dir, **kwargs):
+    """
+    Returns installed package objects.
+    """
+    packages = plugin_package.get_installed_packages(root_dir)
+    for package in packages:
+        yield package.purl, package
+
+
 def scan_rootfs_for_system_packages(project, rootfs, detect_licenses=True):
     """
     Given a `project` Project and a `rootfs` RootFs, scan the `rootfs` for
@@ -203,23 +216,23 @@ def scan_rootfs_for_system_packages(project, rootfs, detect_licenses=True):
         raise DistroNotFound(f"Distro not found.")
 
     distro_id = rootfs.distro.identifier
-    if distro_id not in PACKAGE_GETTER_BY_DISTRO:
+    if distro_id not in SUPPORTED_DISTROS:
         raise DistroNotSupported(f'Distro "{distro_id}" is not supported.')
 
-    package_getter = partial(
-        PACKAGE_GETTER_BY_DISTRO[distro_id],
-        distro=distro_id,
-        detect_licenses=detect_licenses,
-    )
+    logger.info(f"rootfs location: {rootfs.location}")
 
     installed_packages = rootfs.get_installed_packages(package_getter)
 
     for i, (purl, package) in enumerate(installed_packages):
         logger.info(f"Creating package #{i}: {purl}")
         created_package = pipes.update_or_create_package(project, package.to_dict())
 
+        installed_files = []
+        if hasattr(package, "resources"):
+            installed_files = package.resources
+
         # We have no files for this installed package, we cannot go further.
-        if not package.installed_files:
+        if not installed_files:
             logger.info(f"  No installed_files for: {purl}")
             continue
 
@@ -228,7 +241,7 @@ def scan_rootfs_for_system_packages(project, rootfs, detect_licenses=True):
 
         codebase_resources = project.codebaseresources.all()
 
-        for install_file in package.installed_files:
+        for install_file in installed_files:
             rootfs_path = pipes.normalize_path(install_file.path)
             logger.info(f"   installed file rootfs_path: {rootfs_path}")