diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 69b29e500..65ecb8646 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -519,3 +519,18 @@ def make_results_summary(project, scan_results_location): summary["key_files_packages"] = key_files_packages return summary + + +def replace_root_path_and_name(scanned_codebase): + """ + Set the name and path of the root Resource of `scanned_codebase` to "." + and strip the root prefix from the paths of the remaining Resources. + """ + for scanned_resource in scanned_codebase.walk(): + if scanned_resource.is_root: + scanned_resource.path = "." + scanned_resource.name = "." + else: + scanned_resource.path = scanned_resource.get_path(strip_root=True) + scanned_resource.save(scanned_codebase) + return scanned_codebase diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index d09d01c3f..3cbb9973e 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -262,7 +262,7 @@ def test_scanpipe_scan_package_pipeline_integration_test(self): exitcode, _ = pipeline.execute() self.assertEqual(0, exitcode) - self.assertEqual(4, project1.codebaseresources.count()) + self.assertEqual(5, project1.codebaseresources.count()) self.assertEqual(1, project1.discoveredpackages.count()) scancode_file = project1.get_latest_output(filename="scancode") diff --git a/scanpipe/tests/test_pipes.py b/scanpipe/tests/test_pipes.py index 4425b2ab1..063ebed1d 100644 --- a/scanpipe/tests/test_pipes.py +++ b/scanpipe/tests/test_pipes.py @@ -35,6 +35,7 @@ from django.test import override_settings from commoncode.archive import extract_tar +from commoncode.resource import VirtualCodebase from scancode.interrupt import TimeoutError as InterruptTimeoutError from scanpipe.models import CodebaseResource @@ -523,9 +524,11 @@ def test_scanpipe_pipes_scancode_virtual_codebase(self): scancode.create_codebase_resources(project, virtual_codebase) scancode.create_discovered_packages(project, virtual_codebase) - self.assertEqual(18, CodebaseResource.objects.count()) + self.assertEqual(19, CodebaseResource.objects.count()) self.assertEqual(1, DiscoveredPackage.objects.count()) - # Make sure the root is not created as a CodebaseResource, walk(skip_root=True) + # Make sure the root CodebaseResource has been created as "." + self.assertTrue(CodebaseResource.objects.filter(path=".", name=".").exists()) + # Make sure that the root was not created with a path of "codebase" self.assertFalse(CodebaseResource.objects.filter(path="codebase").exists()) # Make sure the root is properly stripped, see `.get_path(strip_root=True)` @@ -541,7 +544,7 @@ def test_scanpipe_pipes_scancode_virtual_codebase(self): # The functions can be called again and existing objects are skipped scancode.create_codebase_resources(project, virtual_codebase) scancode.create_discovered_packages(project, virtual_codebase) - self.assertEqual(18, CodebaseResource.objects.count()) + self.assertEqual(19, CodebaseResource.objects.count()) self.assertEqual(1, DiscoveredPackage.objects.count()) def test_scanpipe_pipes_scancode_create_codebase_resources_inject_policy(self): @@ -597,6 +600,39 @@ def test_scanpipe_pipes_scancode_make_results_summary(self): summary = scancode.make_results_summary(project, scan_results_location) self.assertEqual(10, len(summary.keys())) + def test_scanpipe_pipes_scancode_replace_root_path_and_name(self): + input_location = self.data_location / "asgiref-3.3.0_scan.json" + virtual_codebase = VirtualCodebase(location=input_location) + virtual_codebase_root_replaced = scancode.replace_root_path_and_name( + virtual_codebase + ) + expected_topdown_paths = [ + ".", + "asgiref-3.3.0-py3-none-any.whl", + "asgiref-3.3.0-py3-none-any.whl-extract", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref/__init__.py", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref/compatibility.py", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref/current_thread_executor.py", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref/local.py", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref/server.py", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref/sync.py", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref/testing.py", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref/timeout.py", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref/wsgi.py", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/LICENSE", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/METADATA", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/RECORD", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/top_level.txt", + "asgiref-3.3.0-py3-none-any.whl-extract/asgiref-3.3.0.dist-info/WHEEL", + ] + results = [ + resource.path + for resource in virtual_codebase_root_replaced.walk(topdown=True) + ] + self.assertEqual(expected_topdown_paths, results) + def test_scanpipe_pipes_codebase_get_tree(self): fixtures = self.data_location / "asgiref-3.3.0_fixtures.json" call_command("loaddata", fixtures, **{"verbosity": 0}) @@ -604,11 +640,17 @@ def test_scanpipe_pipes_codebase_get_tree(self): scan_results = self.data_location / "asgiref-3.3.0_scan.json" virtual_codebase = scancode.get_virtual_codebase(project, scan_results) + # Rename the root Resource to "." and remove the root prefix from the + # paths of the remaining Resources to mirror how ScanCode.io creates + # Resources. + virtual_codebase_root_replaced = scancode.replace_root_path_and_name( + virtual_codebase + ) project_codebase = codebase.ProjectCodebase(project) fields = ["name", "path"] virtual_tree = codebase.get_tree( - virtual_codebase.root, fields, codebase=virtual_codebase + virtual_codebase.root, fields, codebase=virtual_codebase_root_replaced ) project_tree = codebase.get_tree(project_codebase.root, fields)