Add input_sources used to fetch inputs to JSON results output #351

Signed-off-by: Thomas Druez <[email protected]>

CHANGELOG.rst

@@ -4,6 +4,9 @@ Changelog
 Unreleased
 ----------
 
+- Add input_sources used to fetch inputs to JSON results output.
+  https://github.com/nexB/scancode.io/issues/351
+
 - Refactor the update_or_create_package pipe to support the ProjectError system
   and fix a database transaction error.
   https://github.com/nexB/scancode.io/issues/381

scanpipe/api/serializers.py

@@ -110,7 +110,7 @@ class ProjectSerializer(
     webhook_url = serializers.CharField(write_only=True, required=False)
     next_run = serializers.CharField(source="get_next_run", read_only=True)
     runs = RunSerializer(many=True, read_only=True)
-    input_sources = serializers.SerializerMethodField()
+    input_sources = serializers.JSONField(source="input_sources_list", read_only=True)
     codebase_resources_summary = serializers.SerializerMethodField()
     discovered_package_summary = serializers.SerializerMethodField()
 
@@ -151,12 +151,6 @@ class Meta:
             "discovered_package_summary",
         ]
 
-    def get_input_sources(self, project):
-        return [
-            {"filename": filename, "source": source}
-            for filename, source in project.input_sources.items()
-        ]
-
     def get_codebase_resources_summary(self, project):
         queryset = project.codebaseresources.all()
         return count_group_by(queryset, "status")

scanpipe/models.py

@@ -563,6 +563,13 @@ def clear_tmp_directory(self):
         shutil.rmtree(self.tmp_path, ignore_errors=True)
         self.tmp_path.mkdir(exist_ok=True)
 
+    @property
+    def input_sources_list(self):
+        return [
+            {"filename": filename, "source": source}
+            for filename, source in self.input_sources.items()
+        ]
+
     def inputs(self, pattern="**/*"):
         """
         Returns all files and directories path of the input/ directory matching

scanpipe/pipes/output.py

@@ -159,7 +159,7 @@ def get_headers(self, project):
             "notice": SCAN_NOTICE,
             "uuid": project.uuid,
             "created_date": project.created_date,
-            "input_files": project.input_files,
+            "input_sources": project.input_sources_list,
             "runs": runs.data,
             "extra_data": project.extra_data,
         }

scanpipe/tests/data/is-npm-1.0.0_scan_codebase.json

@@ -3,8 +3,11 @@
     {
       "tool_name": "scanpipe",
       "notice": "Generated with ScanCode.io and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied.\nNo content created from ScanCode.io should be considered or used as legal advice.\nConsult an Attorney for any legal advice.\nScanCode.io is a free software code scanning tool from nexB Inc. and others\nlicensed under the Apache License version 2.0.\nScanCode is a trademark of nexB Inc.\nVisit https://github.com/nexB/scancode.io for support and download.\n",
-      "input_files": [
-        "is-npm-1.0.0.tgz"
+      "input_sources": [
+        {
+          "filename": "is-npm-1.0.0.tgz",
+          "source": "https://download.url"
+        }
       ],
       "runs": [
         {
@@ -37,7 +40,6 @@
       "release_date": null,
       "homepage_url": "",
       "download_url": "https://registry.npmjs.org/is-npm/-/is-npm-1.0.0.tgz",
-      "size": null,
       "sha1": "",
       "md5": "",
       "bug_tracking_url": "",
@@ -77,7 +79,6 @@
     {
       "for_packages": [],
       "path": "is-npm-1.0.0.tgz",
-      "size": 756,
       "sha1": "f2fb63a65e4905b406c86072765a1a4dc793b9f4",
       "md5": "effb58881c1aca12da9d3a18335bdc3f",
       "extra_data": {},
@@ -104,7 +105,6 @@
     {
       "for_packages": [],
       "path": "is-npm-1.0.0.tgz-extract",
-      "size": 96,
       "sha1": "",
       "md5": "",
       "extra_data": {},
@@ -131,7 +131,6 @@
     {
       "for_packages": [],
       "path": "is-npm-1.0.0.tgz-extract/package",
-      "size": 160,
       "sha1": "",
       "md5": "",
       "extra_data": {},
@@ -158,7 +157,6 @@
     {
       "for_packages": [],
       "path": "is-npm-1.0.0.tgz-extract/package/index.js",
-      "size": 147,
       "sha1": "ecce1fc677fa7d20fc5593b63db45e45c5e8768a",
       "md5": "bc4b18b0c8c32b94883d6fc1d675e919",
       "extra_data": {},
@@ -187,7 +185,6 @@
         "pkg:npm/[email protected]"
       ],
       "path": "is-npm-1.0.0.tgz-extract/package/package.json",
-      "size": 563,
       "sha1": "943efb8abda9b9cdaa4ea73f5709563b948af50b",
       "md5": "c843e88ecb274d5d573c71be330bff8b",
       "extra_data": {},
@@ -214,7 +211,6 @@
     {
       "for_packages": [],
       "path": "is-npm-1.0.0.tgz-extract/package/readme.md",
-      "size": 477,
       "sha1": "b77832e53cb8239edd733c2ca02b1fc01ca3c2b6",
       "md5": "a743e0abf08c28a37ecc4bef4dc02f8c",
       "extra_data": {},

scanpipe/tests/test_api.py

@@ -132,9 +132,14 @@ def test_scanpipe_api_project_detail(self):
         }
         self.assertEqual(expected, response.data["discovered_package_summary"])
 
-        self.project1.add_input_source(filename="file", source="uploaded", save=True)
+        self.project1.add_input_source(filename="file1", source="uploaded")
+        self.project1.add_input_source(filename="file2", source="https://download.url")
+        self.project1.save()
         response = self.csrf_client.get(self.project1_detail_url)
-        expected = [{"filename": "file", "source": "uploaded"}]
+        expected = [
+            {"filename": "file1", "source": "uploaded"},
+            {"filename": "file2", "source": "https://download.url"},
+        ]
         self.assertEqual(expected, response.data["input_sources"])
 
     @mock.patch("requests.get")

scanpipe/tests/test_models.py

@@ -175,6 +175,16 @@ def test_scanpipe_project_model_reset(self):
         self.assertTrue(self.project1.codebase_path.exists())
         self.assertTrue(self.project1.tmp_path.exists())
 
+    def test_scanpipe_project_model_input_sources_list_property(self):
+        self.project1.add_input_source(filename="file1", source="uploaded")
+        self.project1.add_input_source(filename="file2", source="https://download.url")
+
+        expected = [
+            {"filename": "file1", "source": "uploaded"},
+            {"filename": "file2", "source": "https://download.url"},
+        ]
+        self.assertEqual(expected, self.project1.input_sources_list)
+
     def test_scanpipe_project_model_inputs_and_input_files_and_input_root(self):
         self.assertEqual([], list(self.project1.inputs()))
         self.assertEqual([], self.project1.input_files)

scanpipe/tests/test_pipelines.py

@@ -282,8 +282,10 @@ def test_scanpipe_scan_codebase_pipeline_integration_test(self):
         pipeline_name = "scan_codebase"
         project1 = Project.objects.create(name="Analysis")
 
+        filename = "is-npm-1.0.0.tgz"
         input_location = self.data_location / "is-npm-1.0.0.tgz"
         project1.copy_input_from(input_location)
+        project1.add_input_source(filename, "https://download.url", save=True)
 
         run = project1.add_pipeline(pipeline_name)
         pipeline = run.make_pipeline_instance()