Refine the CycloneDX properties implementation #637

Signed-off-by: Thomas Druez <[email protected]>
aboutcode-org · Mar 16, 2023 · 2dced62 · 2dced62
1 parent 45b3d63
commit 2dced62
Show file tree

Hide file tree

Showing 9 changed files with 50 additions and 29 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -12,6 +12,7 @@ v33.0.0 (unreleased)
   https://github.com/nexB/scancode.io/issues/583
 
 - Add fields in CycloneDX BOM output using the component properties.
+  See registered properties at https://github.com/nexB/aboutcode-cyclonedx-taxonomy
   https://github.com/nexB/scancode.io/issues/637
 
 - Refine the "Command Line Interface" documentation about the ``scanpipe`` command

diff --git a/scanpipe/cyclonedx/__init__.py b/scanpipe/cyclonedx/__init__.py
@@ -164,7 +164,7 @@ def get_properties_data(component):
         property_name = component_property.name
         property_value = component_property.value
         if property_name.startswith(prefix) and property_value not in EMPTY_VALUES:
-            field_name = property_name.removeprefix(prefix)
+            field_name = property_name.replace(prefix, "", 1)
             properties_data[field_name] = property_value
 
     return properties_data

diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py
@@ -272,6 +272,9 @@ def run_command(cmd, log_output=False):
 def remove_prefix(text, prefix):
     """
     Remove the `prefix` from `text`.
+    Note that build-in `removeprefix` was added in Python3.9 but we need to keep
+    this one for Python3.8 support.
+    https://docs.python.org/3.9/library/stdtypes.html#str.removeprefix
     """
     if text.startswith(prefix):
         prefix_len = len(prefix)

diff --git a/scanpipe/tests/__init__.py b/scanpipe/tests/__init__.py
@@ -79,7 +79,7 @@
     "version": "3.118",
     "qualifiers": {"arch": "all"},
     "subpath": None,
-    "primary_language": None,
+    "primary_language": "bash",
     "description": "add and remove users and groups",
     "release_date": "1999-10-10",
     "parties": [
@@ -93,7 +93,8 @@
     ],
     "keywords": ["admin"],
     "homepage_url": "https://packages.debian.org",
-    "download_url": None,
+    "download_url": "https://download.url/package.zip",
+    "filename": "package.zip",
     "size": "849",
     "sha1": None,
     "md5": "76cf50f29e47676962645632737365a7",

diff --git a/scanpipe/tests/data/cyclonedx/nested.bom.json b/scanpipe/tests/data/cyclonedx/nested.bom.json
@@ -141,18 +141,22 @@
                 }
               ],
               "properties": [
-                  {
-                    "name": "aboutcode:download_url",
-                    "value": "https://download.url/package.zip"
-                  },
-                  {
-                    "name": "aboutcode:filename",
-                    "value": "package.zip"
-                  },
-                  {
-                    "name": "aboutcode:primary_language",
-                    "value": "Python"
-                  }
+                {
+                  "name": "aboutcode:download_url",
+                  "value": "https://download.url/package.zip"
+                },
+                {
+                  "name": "aboutcode:filename",
+                  "value": "package.zip"
+                },
+                {
+                  "name": "aboutcode:primary_language",
+                  "value": "Python"
+                },
+                {
+                  "name": "aboutcode:homepage_url",
+                  "value": "https://home.page"
+                }
               ],
               "externalReferences": [
                 {

diff --git a/scanpipe/tests/test_cyclonedx.py b/scanpipe/tests/test_cyclonedx.py
@@ -73,8 +73,18 @@ def test_scanpipe_cyclonedx_bom_attributes_to_dict(self):
                         "name": "aboutcode:download_url",
                         "value": "https://download.url/package.zip",
                     },
-                    {"name": "aboutcode:filename", "value": "package.zip"},
-                    {"name": "aboutcode:primary_language", "value": "Python"},
+                    {
+                        "name": "aboutcode:filename",
+                        "value": "package.zip",
+                    },
+                    {
+                        "name": "aboutcode:primary_language",
+                        "value": "Python",
+                    },
+                    {
+                        "name": "aboutcode:homepage_url",
+                        "value": "https://home.page",
+                    },
                 ],
                 "licenses": [
                     {

diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py
@@ -1450,19 +1450,21 @@ def test_scanpipe_discovered_package_model_as_cyclonedx(self):
         self.assertEqual(purl, str(cyclonedx_component.bom_ref))
         self.assertEqual(purl, cyclonedx_component.purl)
         self.assertEqual(1, len(cyclonedx_component.licenses))
-        self.assertEqual(
-            "GPL-2.0-only AND GPL-2.0-or-later AND LicenseRef-scancode-unknown",
-            cyclonedx_component.licenses[0].expression,
-        )
+        expected = "GPL-2.0-only AND GPL-2.0-or-later AND LicenseRef-scancode-unknown"
+        self.assertEqual(expected, cyclonedx_component.licenses[0].expression)
         self.assertEqual(package_data1["copyright"], cyclonedx_component.copyright)
         self.assertEqual(package_data1["description"], cyclonedx_component.description)
         self.assertEqual(1, len(cyclonedx_component.hashes))
         self.assertEqual(package_data1["md5"], cyclonedx_component.hashes[0].content)
 
-        properties = cyclonedx_component.properties
-        self.assertEqual(1, len(properties))
-        self.assertEqual("aboutcode:homepage_url", properties[0].name)
-        self.assertEqual("https://packages.debian.org", properties[0].value)
+        properties = {prop.name: prop.value for prop in cyclonedx_component.properties}
+        expected_properties = {
+            "aboutcode:download_url": "https://download.url/package.zip",
+            "aboutcode:filename": "package.zip",
+            "aboutcode:homepage_url": "https://packages.debian.org",
+            "aboutcode:primary_language": "bash",
+        }
+        self.assertEqual(expected_properties, properties)
 
         external_references = cyclonedx_component.external_references
         self.assertEqual(1, len(external_references))

diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py
@@ -773,7 +773,7 @@ def test_scanpipe_inspect_manifest_pipeline_cyclonedx_integration_test(self):
                 "license_expression": (
                     "lgpl-3.0-plus AND openssl-exception-lgpl-3.0-plus"
                 ),
-                "homepage_url": "",
+                "homepage_url": "https://home.page",
                 "bug_tracking_url": "",
                 "vcs_url": "",
                 "extra_data": "",

diff --git a/scanpipe/tests/test_pipes.py b/scanpipe/tests/test_pipes.py
@@ -1149,14 +1149,14 @@ def test_scanpipe_pipes_update_or_create_package(self):
         p1 = Project.objects.create(name="Analysis")
         package = pipes.update_or_create_package(p1, package_data1)
         self.assertEqual("pkg:deb/debian/[email protected]?arch=all", package.purl)
-        self.assertEqual("", package.primary_language)
+        self.assertEqual("bash", package.primary_language)
         self.assertEqual(datetime.date(1999, 10, 10), package.release_date)
 
         updated_data = dict(package_data1)
-        updated_data["primary_language"] = "Python"
+        updated_data["notice_text"] = "NOTICE"
         updated_package = pipes.update_or_create_package(p1, updated_data)
         self.assertEqual("pkg:deb/debian/[email protected]?arch=all", updated_package.purl)
-        self.assertEqual("Python", updated_package.primary_language)
+        self.assertEqual("NOTICE", updated_package.notice_text)
         self.assertEqual(package.pk, updated_package.pk)
 
         resource1 = CodebaseResource.objects.create(project=p1, path="filename.ext")