Refine code and add unit tests #609

Signed-off-by: Thomas Druez <[email protected]>

scanpipe/pipelines/load_inventory.py

@@ -58,7 +58,7 @@ def build_inventory_from_scans(self):
             tool_name = input.get_tool_name_from_scan_headers(scan_data)
 
             if tool_name == "scancode-toolkit":
-                scancode.load_inventory_from_scan(self.project, input_path)
+                scancode.load_inventory_from_toolkit_scan(self.project, input_path)
             elif tool_name == "scanpipe":
                 scancode.load_inventory_from_scanpipe(self.project, scan_data)
             else:

scanpipe/pipelines/scan_package.py

@@ -112,11 +112,13 @@ def run_scancode(self):
         if not scan_output_path.exists():
             raise FileNotFoundError("ScanCode output not available.")
 
-    def load_inventory_from_scan(self):
+    def load_inventory_from_toolkit_scan(self):
         """
         Process a JSON Scan results file to populate codebase resources and packages.
         """
-        scancode.load_inventory_from_scan(self.project, self.scan_output_location)
+        scancode.load_inventory_from_toolkit_scan(
+            self.project, self.scan_output_location
+        )
 
     def make_summary_from_scan_results(self):
         """

scanpipe/pipes/scancode.py

@@ -607,7 +607,7 @@ def make_results_summary(project, scan_results_location):
     return summary
 
 
-def load_inventory_from_scan(project, input_location):
+def load_inventory_from_toolkit_scan(project, input_location):
     """
     Create packages, dependencies, and resources loaded from the ScanCode-toolkit scan
     results located at `input_location`.

scanpipe/tests/data/asgiref-3.3.0_scancode_scan.json → scanpipe/tests/data/asgiref-3.3.0_toolkit_scan.json

@@ -144,9 +144,9 @@
       "notice_text": null,
       "source_packages": [],
       "extra_data": {
-        "Documentation": "Documentation, https://asgi.readthedocs.io/",
-        "Further Documentation": "Further Documentation, https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions",
-        "Changelog": "Changelog, https://github.com/django/asgiref/blob/master/CHANGELOG.txt"
+        "Documentation": "https://asgi.readthedocs.io/",
+        "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions",
+        "Changelog": "https://github.com/django/asgiref/blob/master/CHANGELOG.txt"
       },
       "repository_homepage_url": "https://pypi.org/project/asgiref",
       "repository_download_url": "https://pypi.org/packages/source/a/asgiref/asgiref-3.3.0.tar.gz",
@@ -214,9 +214,9 @@
       "notice_text": null,
       "source_packages": [],
       "extra_data": {
-        "Documentation": "Documentation, https://asgi.readthedocs.io/",
-        "Further Documentation": "Further Documentation, https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions",
-        "Changelog": "Changelog, https://github.com/django/asgiref/blob/master/CHANGELOG.txt"
+        "Documentation": "https://asgi.readthedocs.io/",
+        "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions",
+        "Changelog": "https://github.com/django/asgiref/blob/master/CHANGELOG.txt"
       },
       "repository_homepage_url": "https://pypi.org/project/asgiref",
       "repository_download_url": "https://pypi.org/packages/source/a/asgiref/asgiref-3.3.0.tar.gz",
@@ -474,9 +474,9 @@
             }
           ],
           "extra_data": {
-            "Documentation": "Documentation, https://asgi.readthedocs.io/",
-            "Further Documentation": "Further Documentation, https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions",
-            "Changelog": "Changelog, https://github.com/django/asgiref/blob/master/CHANGELOG.txt"
+            "Documentation": "https://asgi.readthedocs.io/",
+            "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions",
+            "Changelog": "https://github.com/django/asgiref/blob/master/CHANGELOG.txt"
           },
           "dependencies": [
             {
@@ -1331,9 +1331,9 @@
             }
           ],
           "extra_data": {
-            "Documentation": "Documentation, https://asgi.readthedocs.io/",
-            "Further Documentation": "Further Documentation, https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions",
-            "Changelog": "Changelog, https://github.com/django/asgiref/blob/master/CHANGELOG.txt"
+            "Documentation": "https://asgi.readthedocs.io/",
+            "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions",
+            "Changelog": "https://github.com/django/asgiref/blob/master/CHANGELOG.txt"
           },
           "dependencies": [
             {

scanpipe/tests/regen_test_data.py

@@ -68,7 +68,7 @@ def test_regen_asgiref_test_files(self):
         self.assertEqual(0, exitcode)
 
         # Scan results
-        test_file_location = self.data_location / "asgiref-3.3.0_scan.json"
+        test_file_location = self.data_location / "asgiref-3.3.0_scanpipe_output.json"
         result_file = output.to_json(project1)
         result_json = json.loads(Path(result_file).read_text())
         test_file_location.write_text(json.dumps(result_json, indent=2))

scanpipe/tests/test_models.py

@@ -1243,15 +1243,20 @@ def test_scanpipe_codebase_resource_children(self):
         ]
         self.assertEqual(expected, [resource.path for resource in children])
 
+    def test_scanpipe_codebase_resource_add_package(self):
+        resource = CodebaseResource.objects.create(project=self.project1, path="file")
+        package = DiscoveredPackage.create_from_data(self.project1, package_data1)
+        resource.add_package(package)
+        self.assertEqual(1, resource.discovered_packages.count())
+        self.assertEqual(package, resource.discovered_packages.get())
+
     def test_scanpipe_codebase_resource_create_and_add_package(self):
-        codebase_resource = CodebaseResource.objects.create(
-            project=self.project1, path="filename.ext"
-        )
-        package = codebase_resource.create_and_add_package(package_data1)
+        resource = CodebaseResource.objects.create(project=self.project1, path="file")
+        package = resource.create_and_add_package(package_data1)
         self.assertEqual(self.project1, package.project)
         self.assertEqual("pkg:deb/debian/[email protected]?arch=all", str(package))
-        self.assertEqual(1, codebase_resource.discovered_packages.count())
-        self.assertEqual(package, codebase_resource.discovered_packages.get())
+        self.assertEqual(1, resource.discovered_packages.count())
+        self.assertEqual(package, resource.discovered_packages.get())
 
     def test_scanpipe_discovered_package_model_queryset_methods(self):
         DiscoveredPackage.create_from_data(self.project1, package_data1)

scanpipe/tests/test_pipelines.py

@@ -562,9 +562,9 @@ def test_scanpipe_rootfs_pipeline_integration_test(self):
 
     def test_scanpipe_load_inventory_pipeline_integration_test(self):
         pipeline_name = "load_inventory"
-        project1 = Project.objects.create(name="Analysis")
+        project1 = Project.objects.create(name="Tool: scancode-toolkit")
 
-        input_location = self.data_location / "asgiref-3.3.0_scancode_scan.json"
+        input_location = self.data_location / "asgiref-3.3.0_toolkit_scan.json"
         project1.copy_input_from(input_location)
 
         run = project1.add_pipeline(pipeline_name)
@@ -583,6 +583,22 @@ def test_scanpipe_load_inventory_pipeline_integration_test(self):
         )
         self.assertPipelineResultEqual(expected_file, result_file)
 
+        # Using the ScanCode.io JSON output as the input
+        project2 = Project.objects.create(name="Tool: scanpipe")
+
+        input_location = self.data_location / "asgiref-3.3.0_scanpipe_output.json"
+        project2.copy_input_from(input_location)
+
+        run = project2.add_pipeline(pipeline_name)
+        pipeline = run.make_pipeline_instance()
+
+        exitcode, out = pipeline.execute()
+        self.assertEqual(0, exitcode, msg=out)
+
+        self.assertEqual(18, project2.codebaseresources.count())
+        self.assertEqual(2, project2.discoveredpackages.count())
+        self.assertEqual(4, project2.discovereddependencies.count())
+
     @mock.patch("scanpipe.pipes.vulnerablecode.is_available")
     @mock.patch("scanpipe.pipes.vulnerablecode.is_configured")
     @mock.patch("scanpipe.pipes.vulnerablecode.get_vulnerabilities_by_purl")

scanpipe/tests/test_pipes.py

@@ -106,13 +106,13 @@ def test_scanpipe_pipes_input_get_tool_name_from_scan_headers(self):
         tool_name = input.get_tool_name_from_scan_headers(scan_data={"headers": []})
         self.assertIsNone(tool_name)
 
-        input_location = self.data_location / "asgiref-3.3.0_scan.json"
+        input_location = self.data_location / "asgiref-3.3.0_scanpipe_output.json"
         tool_name = input.get_tool_name_from_scan_headers(
             scan_data=json.loads(input_location.read_text())
         )
         self.assertEqual("scanpipe", tool_name)
 
-        input_location = self.data_location / "asgiref-3.3.0_scancode_scan.json"
+        input_location = self.data_location / "asgiref-3.3.0_toolkit_scan.json"
         tool_name = input.get_tool_name_from_scan_headers(
             scan_data=json.loads(input_location.read_text())
         )
@@ -405,7 +405,7 @@ def noop(*args, **kwargs):
     @expectedFailure
     def test_scanpipe_pipes_scancode_virtual_codebase(self):
         project = Project.objects.create(name="asgiref")
-        input_location = self.data_location / "asgiref-3.3.0_scan.json"
+        input_location = self.data_location / "asgiref-3.3.0_scanpipe_output.json"
         virtual_codebase = scancode.get_virtual_codebase(project, input_location)
         self.assertEqual(19, len(virtual_codebase.resources.keys()))
 
@@ -439,10 +439,10 @@ def test_scanpipe_pipes_scancode_virtual_codebase(self):
 
     def test_scanpipe_pipes_scancode_create_codebase_resources_inject_policy(self):
         project = Project.objects.create(name="asgiref")
-        # We are using `asgiref-3.3.0_scancode_scan.json` instead of
-        # `asgiref-3.3.0_scan.json` because `asgiref-3.3.0_scan.json` is not
-        # exactly the same format as a scancode-toolkit scan
-        input_location = self.data_location / "asgiref-3.3.0_scancode_scan.json"
+        # We are using `asgiref-3.3.0_toolkit_scan.json` instead of
+        # `asgiref-3.3.0_scanpipe_output.json` because it is not exactly the same
+        # format as a scancode-toolkit scan
+        input_location = self.data_location / "asgiref-3.3.0_toolkit_scan.json"
         virtual_codebase = scancode.get_virtual_codebase(project, input_location)
 
         scanpipe_app.license_policies_index = license_policies_index
@@ -502,10 +502,27 @@ def test_scanpipe_pipes_scancode_make_results_summary(self):
         summary = scancode.make_results_summary(project, scan_results_location)
         self.assertEqual(10, len(summary.keys()))
 
+    def test_scanpipe_pipes_scancode_load_inventory_from_toolkit_scan(self):
+        project = Project.objects.create(name="Analysis")
+        input_location = self.data_location / "asgiref-3.3.0_toolkit_scan.json"
+        scancode.load_inventory_from_toolkit_scan(project, input_location)
+        self.assertEqual(18, project.codebaseresources.count())
+        self.assertEqual(2, project.discoveredpackages.count())
+        self.assertEqual(4, project.discovereddependencies.count())
+
+    def test_scanpipe_pipes_scancode_load_inventory_from_scanpipe(self):
+        project = Project.objects.create(name="Analysis")
+        input_location = self.data_location / "asgiref-3.3.0_scanpipe_output.json"
+        scan_data = json.loads(input_location.read_text())
+        scancode.load_inventory_from_scanpipe(project, scan_data)
+        self.assertEqual(18, project.codebaseresources.count())
+        self.assertEqual(2, project.discoveredpackages.count())
+        self.assertEqual(4, project.discovereddependencies.count())
+
     def test_scanpipe_pipes_scancode_assemble_packages(self):
         project = Project.objects.create(name="Analysis")
         project_scan_location = self.data_location / "package_assembly_codebase.json"
-        scancode.load_inventory_from_scan(project, project_scan_location)
+        scancode.load_inventory_from_toolkit_scan(project, project_scan_location)
 
         self.assertEqual(0, project.discoveredpackages.count())
         scancode.assemble_packages(project)
@@ -538,7 +555,7 @@ def _replace_path(virtual_tree_children):
         call_command("loaddata", fixtures, **{"verbosity": 0})
         project = Project.objects.get(name="asgiref")
 
-        scan_results = self.data_location / "asgiref-3.3.0_scan.json"
+        scan_results = self.data_location / "asgiref-3.3.0_scanpipe_output.json"
         virtual_codebase = scancode.get_virtual_codebase(project, scan_results)
         project_codebase = codebase.ProjectCodebase(project)