Tag about files and companions correctly #825 (#837)

Tag about files and LICENSE/NOTICE files accompanying the ABOUT
file as "about-mapped", and ignore these files from the scanning
step.

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

scanpipe/models.py

@@ -1943,6 +1943,13 @@ def location_path(self):
         path = Path(str(self.path).strip("/"))
         return self.project.codebase_path / path
 
+    @property
+    def name_without_extension(self):
+        """Return the name of the resource without it's extension."""
+        if self.extension:
+            return self.name.replace(self.extension, "")
+        return self.name
+
     @property
     def location(self):
         """Return the location of the resource as a string."""

scanpipe/pipelines/deploy_to_develop.py

@@ -175,6 +175,5 @@ def flag_mapped_resources_and_ignored_directories(self):
 
     def scan_mapped_from_for_files(self):
         """Scan mapped ``from/`` files for copyrights, licenses, emails, and urls."""
-        resource_qs = self.project.codebaseresources
-        mapped_from_files = resource_qs.from_codebase().files().has_relation()
-        scancode.scan_for_files(self.project, mapped_from_files)
+        scan_files = d2d.get_from_files_for_scanning(self.project.codebaseresources)
+        scancode.scan_for_files(self.project, scan_files)

scanpipe/pipes/d2d.py

@@ -124,6 +124,15 @@ def get_best_path_matches(to_resource, matches):
     return matches
 
 
+def get_from_files_for_scanning(resources):
+    """
+    Return resources in the "from/" side which has been mapped to the "to/"
+    side, but are not mapped using ABOUT files.
+    """
+    mapped_from_files = resources.from_codebase().files().has_relation()
+    return mapped_from_files.filter(~Q(status=flag.ABOUT_MAPPED))
+
+
 def _map_checksum_resource(to_resource, from_resources, checksum_field):
     checksum_value = getattr(to_resource, checksum_field)
     matches = from_resources.filter(**{checksum_field: checksum_value})
@@ -644,8 +653,8 @@ def _map_about_file_resource(project, about_file_resource, to_resources):
             map_type="about_file",
         )
 
-    codebase_resources.update(status=flag.MAPPED)
-    about_file_resource.update(status=flag.MAPPED)
+    codebase_resources.update(status=flag.ABOUT_MAPPED)
+    about_file_resource.update(status=flag.ABOUT_MAPPED)
 
 
 def map_about_files(project, logger=None):
@@ -664,14 +673,19 @@ def map_about_files(project, logger=None):
     for about_file_resource in from_about_files:
         _map_about_file_resource(project, about_file_resource, to_resources)
 
+        about_file_companions = about_file_resource.siblings().filter(
+            name__contains=about_file_resource.name_without_extension
+        )
+        about_file_companions.update(status=flag.ABOUT_MAPPED)
+
 
 def map_javascript_post_purldb_match(project, logger=None):
     """Map minified javascript file based on existing PurlDB match."""
     project_files = project.codebaseresources.files()
 
     to_resources = project_files.to_codebase()
 
-    to_resources_dot_map = to_resources.filter(status="matched-to-purldb").filter(
+    to_resources_dot_map = to_resources.filter(status=flag.MATCHED_TO_PURLDB).filter(
         extension=".map"
     )
 

scanpipe/pipes/flag.py

@@ -45,6 +45,7 @@
 COMPLIANCE_LICENSES = "compliance-licenses"
 COMPLIANCE_SOURCEMIRROR = "compliance-sourcemirror"
 
+ABOUT_MAPPED = "about-mapped"
 MAPPED = "mapped"
 MATCHED_TO_PURLDB = "matched-to-purldb"
 TOO_MANY_MAPS = "too-many-maps"