From 270b9894e6e70d9482d6403ac6b41383810f3cff Mon Sep 17 00:00:00 2001 From: Thomas Druez Date: Tue, 14 Mar 2023 21:32:36 -0100 Subject: [PATCH] Refine code and add unit tests #609 Signed-off-by: Thomas Druez --- scanpipe/pipelines/load_inventory.py | 2 +- scanpipe/pipelines/scan_package.py | 6 ++-- scanpipe/pipes/scancode.py | 2 +- ...asgiref-3.3.0_load_inventory_expected.json | 24 ++++++------- ...son => asgiref-3.3.0_scanpipe_output.json} | 0 ...n.json => asgiref-3.3.0_toolkit_scan.json} | 24 ++++++------- scanpipe/tests/regen_test_data.py | 2 +- scanpipe/tests/test_models.py | 17 +++++---- scanpipe/tests/test_pipelines.py | 20 +++++++++-- scanpipe/tests/test_pipes.py | 35 ++++++++++++++----- 10 files changed, 86 insertions(+), 46 deletions(-) rename scanpipe/tests/data/{asgiref-3.3.0_scan.json => asgiref-3.3.0_scanpipe_output.json} (100%) rename scanpipe/tests/data/{asgiref-3.3.0_scancode_scan.json => asgiref-3.3.0_toolkit_scan.json} (98%) diff --git a/scanpipe/pipelines/load_inventory.py b/scanpipe/pipelines/load_inventory.py index af5e110dd..a65002fdd 100644 --- a/scanpipe/pipelines/load_inventory.py +++ b/scanpipe/pipelines/load_inventory.py @@ -58,7 +58,7 @@ def build_inventory_from_scans(self): tool_name = input.get_tool_name_from_scan_headers(scan_data) if tool_name == "scancode-toolkit": - scancode.load_inventory_from_scan(self.project, input_path) + scancode.load_inventory_from_toolkit_scan(self.project, input_path) elif tool_name == "scanpipe": scancode.load_inventory_from_scanpipe(self.project, scan_data) else: diff --git a/scanpipe/pipelines/scan_package.py b/scanpipe/pipelines/scan_package.py index 834c59aa7..4e1f7672f 100644 --- a/scanpipe/pipelines/scan_package.py +++ b/scanpipe/pipelines/scan_package.py @@ -112,11 +112,13 @@ def run_scancode(self): if not scan_output_path.exists(): raise FileNotFoundError("ScanCode output not available.") - def load_inventory_from_scan(self): + def load_inventory_from_toolkit_scan(self): """ Process a JSON Scan results file to populate codebase resources and packages. """ - scancode.load_inventory_from_scan(self.project, self.scan_output_location) + scancode.load_inventory_from_toolkit_scan( + self.project, self.scan_output_location + ) def make_summary_from_scan_results(self): """ diff --git a/scanpipe/pipes/scancode.py b/scanpipe/pipes/scancode.py index 3783bcec8..0ca3b8034 100644 --- a/scanpipe/pipes/scancode.py +++ b/scanpipe/pipes/scancode.py @@ -607,7 +607,7 @@ def make_results_summary(project, scan_results_location): return summary -def load_inventory_from_scan(project, input_location): +def load_inventory_from_toolkit_scan(project, input_location): """ Create packages, dependencies, and resources loaded from the ScanCode-toolkit scan results located at `input_location`. diff --git a/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json b/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json index 07a40fef0..420df1e77 100644 --- a/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json +++ b/scanpipe/tests/data/asgiref-3.3.0_load_inventory_expected.json @@ -74,9 +74,9 @@ "notice_text": "", "source_packages": [], "extra_data": { - "Changelog": "Changelog, https://github.com/django/asgiref/blob/master/CHANGELOG.txt", - "Documentation": "Documentation, https://asgi.readthedocs.io/", - "Further Documentation": "Further Documentation, https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" + "Changelog": "https://github.com/django/asgiref/blob/master/CHANGELOG.txt", + "Documentation": "https://asgi.readthedocs.io/", + "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758", "manifest_path": "", @@ -138,9 +138,9 @@ "notice_text": "", "source_packages": [], "extra_data": { - "Changelog": "Changelog, https://github.com/django/asgiref/blob/master/CHANGELOG.txt", - "Documentation": "Documentation, https://asgi.readthedocs.io/", - "Further Documentation": "Further Documentation, https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" + "Changelog": "https://github.com/django/asgiref/blob/master/CHANGELOG.txt", + "Documentation": "https://asgi.readthedocs.io/", + "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, "package_uid": "pkg:pypi/asgiref@3.3.0?uuid=fixed-uid-done-for-testing-5642512d1758", "manifest_path": "", @@ -266,9 +266,9 @@ "copyright": null, "namespace": null, "extra_data": { - "Changelog": "Changelog, https://github.com/django/asgiref/blob/master/CHANGELOG.txt", - "Documentation": "Documentation, https://asgi.readthedocs.io/", - "Further Documentation": "Further Documentation, https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" + "Changelog": "https://github.com/django/asgiref/blob/master/CHANGELOG.txt", + "Documentation": "https://asgi.readthedocs.io/", + "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, "qualifiers": {}, "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", @@ -745,9 +745,9 @@ "copyright": null, "namespace": null, "extra_data": { - "Changelog": "Changelog, https://github.com/django/asgiref/blob/master/CHANGELOG.txt", - "Documentation": "Documentation, https://asgi.readthedocs.io/", - "Further Documentation": "Further Documentation, https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" + "Changelog": "https://github.com/django/asgiref/blob/master/CHANGELOG.txt", + "Documentation": "https://asgi.readthedocs.io/", + "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions" }, "qualifiers": {}, "description": "ASGI specs, helper code, and adapters\nasgiref\n=======\n\n.. image:: https://api.travis-ci.org/django/asgiref.svg\n :target: https://travis-ci.org/django/asgiref\n\n.. image:: https://img.shields.io/pypi/v/asgiref.svg\n :target: https://pypi.python.org/pypi/asgiref\n\nASGI is a standard for Python asynchronous web apps and servers to communicate\nwith each other, and positioned as an asynchronous successor to WSGI. You can\nread more at https://asgi.readthedocs.io/en/latest/\n\nThis package includes ASGI base libraries, such as:\n\n* Sync-to-async and async-to-sync function wrappers, ``asgiref.sync``\n* Server base classes, ``asgiref.server``\n* A WSGI-to-ASGI adapter, in ``asgiref.wsgi``\n\n\nFunction wrappers\n-----------------\n\nThese allow you to wrap or decorate async or sync functions to call them from\nthe other style (so you can call async functions from a synchronous thread,\nor vice-versa).\n\nIn particular:\n\n* AsyncToSync lets a synchronous subthread stop and wait while the async\n function is called on the main thread's event loop, and then control is\n returned to the thread when the async function is finished.\n\n* SyncToAsync lets async code call a synchronous function, which is run in\n a threadpool and control returned to the async coroutine when the synchronous\n function completes.\n\nThe idea is to make it easier to call synchronous APIs from async code and\nasynchronous APIs from synchronous code so it's easier to transition code from\none style to the other. In the case of Channels, we wrap the (synchronous)\nDjango view system with SyncToAsync to allow it to run inside the (asynchronous)\nASGI server.\n\nNote that exactly what threads things run in is very specific, and aimed to\nkeep maximum compatibility with old synchronous code. See\n\"Synchronous code & Threads\" below for a full explanation. By default,\n``sync_to_async`` will run all synchronous code in the program in the same\nthread for safety reasons; you can disable this for more performance with\n``@sync_to_async(thread_sensitive=False)``, but make sure that your code does\nnot rely on anything bound to threads (like database connections) when you do.\n\n\nThreadlocal replacement\n-----------------------\n\nThis is a drop-in replacement for ``threading.local`` that works with both\nthreads and asyncio Tasks. Even better, it will proxy values through from a\ntask-local context to a thread-local context when you use ``sync_to_async``\nto run things in a threadpool, and vice-versa for ``async_to_sync``.\n\nIf you instead want true thread- and task-safety, you can set\n``thread_critical`` on the Local object to ensure this instead.\n\n\nServer base classes\n-------------------\n\nIncludes a ``StatelessServer`` class which provides all the hard work of\nwriting a stateless server (as in, does not handle direct incoming sockets\nbut instead consumes external streams or sockets to work out what is happening).\n\nAn example of such a server would be a chatbot server that connects out to\na central chat server and provides a \"connection scope\" per user chatting to\nit. There's only one actual connection, but the server has to separate things\ninto several scopes for easier writing of the code.\n\nYou can see an example of this being used in `frequensgi `_.\n\n\nWSGI-to-ASGI adapter\n--------------------\n\nAllows you to wrap a WSGI application so it appears as a valid ASGI application.\n\nSimply wrap it around your WSGI application like so::\n\n asgi_application = WsgiToAsgi(wsgi_application)\n\nThe WSGI application will be run in a synchronous threadpool, and the wrapped\nASGI application will be one that accepts ``http`` class messages.\n\nPlease note that not all extended features of WSGI may be supported (such as\nfile handles for incoming POST bodies).\n\n\nDependencies\n------------\n\n``asgiref`` requires Python 3.5 or higher.\n\n\nContributing\n------------\n\nPlease refer to the\n`main Channels contributing docs `_.\n\n\nTesting\n'''''''\n\nTo run tests, make sure you have installed the ``tests`` extra with the package::\n\n cd asgiref/\n pip install -e .[tests]\n pytest\n\n\nBuilding the documentation\n''''''''''''''''''''''''''\n\nThe documentation uses `Sphinx `_::\n\n cd asgiref/docs/\n pip install sphinx\n\nTo build the docs, you can use the default tools::\n\n sphinx-build -b html . _build/html # or `make html`, if you've got make set up\n cd _build/html\n python -m http.server\n\n...or you can use ``sphinx-autobuild`` to run a server and rebuild/reload\nyour documentation changes automatically::\n\n pip install sphinx-autobuild\n sphinx-autobuild . _build/html\n\n\nImplementation Details\n----------------------\n\nSynchronous code & threads\n''''''''''''''''''''''''''\n\nThe ``asgiref.sync`` module provides two wrappers that let you go between\nasynchronous and synchronous code at will, while taking care of the rough edges\nfor you.\n\nUnfortunately, the rough edges are numerous, and the code has to work especially\nhard to keep things in the same thread as much as possible. Notably, the\nrestrictions we are working with are:\n\n* All synchronous code called through ``SyncToAsync`` and marked with\n ``thread_sensitive`` should run in the same thread as each other (and if the\n outer layer of the program is synchronous, the main thread)\n\n* If a thread already has a running async loop, ``AsyncToSync`` can't run things\n on that loop if it's blocked on synchronous code that is above you in the\n call stack.\n\nThe first compromise you get to might be that ``thread_sensitive`` code should\njust run in the same thread and not spawn in a sub-thread, fulfilling the first\nrestriction, but that immediately runs you into the second restriction.\n\nThe only real solution is to essentially have a variant of ThreadPoolExecutor\nthat executes any ``thread_sensitive`` code on the outermost synchronous\nthread - either the main thread, or a single spawned subthread.\n\nThis means you now have two basic states:\n\n* If the outermost layer of your program is synchronous, then all async code\n run through ``AsyncToSync`` will run in a per-call event loop in arbitary\n sub-threads, while all ``thread_sensitive`` code will run in the main thread.\n\n* If the outermost layer of your program is asynchronous, then all async code\n runs on the main thread's event loop, and all ``thread_sensitive`` synchronous\n code will run in a single shared sub-thread.\n\nCruicially, this means that in both cases there is a thread which is a shared\nresource that all ``thread_sensitive`` code must run on, and there is a chance\nthat this thread is currently blocked on its own ``AsyncToSync`` call. Thus,\n``AsyncToSync`` needs to act as an executor for thread code while it's blocking.\n\nThe ``CurrentThreadExecutor`` class provides this functionality; rather than\nsimply waiting on a Future, you can call its ``run_until_future`` method and\nit will run submitted code until that Future is done. This means that code\ninside the call can then run code on your thread.\n\n\nMaintenance and Security\n------------------------\n\nTo report security issues, please contact security@djangoproject.com. For GPG\nsignatures and more security process information, see\nhttps://docs.djangoproject.com/en/dev/internals/security/.\n\nTo report bugs or request new features, please open a new GitHub issue.\n\nThis repository is part of the Channels project. For the shepherd and maintenance team, please see the\n`main Channels readme `_.", diff --git a/scanpipe/tests/data/asgiref-3.3.0_scan.json b/scanpipe/tests/data/asgiref-3.3.0_scanpipe_output.json similarity index 100% rename from scanpipe/tests/data/asgiref-3.3.0_scan.json rename to scanpipe/tests/data/asgiref-3.3.0_scanpipe_output.json diff --git a/scanpipe/tests/data/asgiref-3.3.0_scancode_scan.json b/scanpipe/tests/data/asgiref-3.3.0_toolkit_scan.json similarity index 98% rename from scanpipe/tests/data/asgiref-3.3.0_scancode_scan.json rename to scanpipe/tests/data/asgiref-3.3.0_toolkit_scan.json index d7a4ca2fb..6aa541c3e 100644 --- a/scanpipe/tests/data/asgiref-3.3.0_scancode_scan.json +++ b/scanpipe/tests/data/asgiref-3.3.0_toolkit_scan.json @@ -144,9 +144,9 @@ "notice_text": null, "source_packages": [], "extra_data": { - "Documentation": "Documentation, https://asgi.readthedocs.io/", - "Further Documentation": "Further Documentation, https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions", - "Changelog": "Changelog, https://github.com/django/asgiref/blob/master/CHANGELOG.txt" + "Documentation": "https://asgi.readthedocs.io/", + "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions", + "Changelog": "https://github.com/django/asgiref/blob/master/CHANGELOG.txt" }, "repository_homepage_url": "https://pypi.org/project/asgiref", "repository_download_url": "https://pypi.org/packages/source/a/asgiref/asgiref-3.3.0.tar.gz", @@ -214,9 +214,9 @@ "notice_text": null, "source_packages": [], "extra_data": { - "Documentation": "Documentation, https://asgi.readthedocs.io/", - "Further Documentation": "Further Documentation, https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions", - "Changelog": "Changelog, https://github.com/django/asgiref/blob/master/CHANGELOG.txt" + "Documentation": "https://asgi.readthedocs.io/", + "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions", + "Changelog": "https://github.com/django/asgiref/blob/master/CHANGELOG.txt" }, "repository_homepage_url": "https://pypi.org/project/asgiref", "repository_download_url": "https://pypi.org/packages/source/a/asgiref/asgiref-3.3.0.tar.gz", @@ -474,9 +474,9 @@ } ], "extra_data": { - "Documentation": "Documentation, https://asgi.readthedocs.io/", - "Further Documentation": "Further Documentation, https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions", - "Changelog": "Changelog, https://github.com/django/asgiref/blob/master/CHANGELOG.txt" + "Documentation": "https://asgi.readthedocs.io/", + "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions", + "Changelog": "https://github.com/django/asgiref/blob/master/CHANGELOG.txt" }, "dependencies": [ { @@ -1331,9 +1331,9 @@ } ], "extra_data": { - "Documentation": "Documentation, https://asgi.readthedocs.io/", - "Further Documentation": "Further Documentation, https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions", - "Changelog": "Changelog, https://github.com/django/asgiref/blob/master/CHANGELOG.txt" + "Documentation": "https://asgi.readthedocs.io/", + "Further Documentation": "https://docs.djangoproject.com/en/stable/topics/async/#async-adapter-functions", + "Changelog": "https://github.com/django/asgiref/blob/master/CHANGELOG.txt" }, "dependencies": [ { diff --git a/scanpipe/tests/regen_test_data.py b/scanpipe/tests/regen_test_data.py index 431c8bedf..e64703673 100644 --- a/scanpipe/tests/regen_test_data.py +++ b/scanpipe/tests/regen_test_data.py @@ -68,7 +68,7 @@ def test_regen_asgiref_test_files(self): self.assertEqual(0, exitcode) # Scan results - test_file_location = self.data_location / "asgiref-3.3.0_scan.json" + test_file_location = self.data_location / "asgiref-3.3.0_scanpipe_output.json" result_file = output.to_json(project1) result_json = json.loads(Path(result_file).read_text()) test_file_location.write_text(json.dumps(result_json, indent=2)) diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 99751d1ca..acbdf5404 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -1243,15 +1243,20 @@ def test_scanpipe_codebase_resource_children(self): ] self.assertEqual(expected, [resource.path for resource in children]) + def test_scanpipe_codebase_resource_add_package(self): + resource = CodebaseResource.objects.create(project=self.project1, path="file") + package = DiscoveredPackage.create_from_data(self.project1, package_data1) + resource.add_package(package) + self.assertEqual(1, resource.discovered_packages.count()) + self.assertEqual(package, resource.discovered_packages.get()) + def test_scanpipe_codebase_resource_create_and_add_package(self): - codebase_resource = CodebaseResource.objects.create( - project=self.project1, path="filename.ext" - ) - package = codebase_resource.create_and_add_package(package_data1) + resource = CodebaseResource.objects.create(project=self.project1, path="file") + package = resource.create_and_add_package(package_data1) self.assertEqual(self.project1, package.project) self.assertEqual("pkg:deb/debian/adduser@3.118?arch=all", str(package)) - self.assertEqual(1, codebase_resource.discovered_packages.count()) - self.assertEqual(package, codebase_resource.discovered_packages.get()) + self.assertEqual(1, resource.discovered_packages.count()) + self.assertEqual(package, resource.discovered_packages.get()) def test_scanpipe_discovered_package_model_queryset_methods(self): DiscoveredPackage.create_from_data(self.project1, package_data1) diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 3691fb7b9..1f5d4e7c0 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -562,9 +562,9 @@ def test_scanpipe_rootfs_pipeline_integration_test(self): def test_scanpipe_load_inventory_pipeline_integration_test(self): pipeline_name = "load_inventory" - project1 = Project.objects.create(name="Analysis") + project1 = Project.objects.create(name="Tool: scancode-toolkit") - input_location = self.data_location / "asgiref-3.3.0_scancode_scan.json" + input_location = self.data_location / "asgiref-3.3.0_toolkit_scan.json" project1.copy_input_from(input_location) run = project1.add_pipeline(pipeline_name) @@ -583,6 +583,22 @@ def test_scanpipe_load_inventory_pipeline_integration_test(self): ) self.assertPipelineResultEqual(expected_file, result_file) + # Using the ScanCode.io JSON output as the input + project2 = Project.objects.create(name="Tool: scanpipe") + + input_location = self.data_location / "asgiref-3.3.0_scanpipe_output.json" + project2.copy_input_from(input_location) + + run = project2.add_pipeline(pipeline_name) + pipeline = run.make_pipeline_instance() + + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) + + self.assertEqual(18, project2.codebaseresources.count()) + self.assertEqual(2, project2.discoveredpackages.count()) + self.assertEqual(4, project2.discovereddependencies.count()) + @mock.patch("scanpipe.pipes.vulnerablecode.is_available") @mock.patch("scanpipe.pipes.vulnerablecode.is_configured") @mock.patch("scanpipe.pipes.vulnerablecode.get_vulnerabilities_by_purl") diff --git a/scanpipe/tests/test_pipes.py b/scanpipe/tests/test_pipes.py index d874435c0..ccfd8b8c4 100644 --- a/scanpipe/tests/test_pipes.py +++ b/scanpipe/tests/test_pipes.py @@ -106,13 +106,13 @@ def test_scanpipe_pipes_input_get_tool_name_from_scan_headers(self): tool_name = input.get_tool_name_from_scan_headers(scan_data={"headers": []}) self.assertIsNone(tool_name) - input_location = self.data_location / "asgiref-3.3.0_scan.json" + input_location = self.data_location / "asgiref-3.3.0_scanpipe_output.json" tool_name = input.get_tool_name_from_scan_headers( scan_data=json.loads(input_location.read_text()) ) self.assertEqual("scanpipe", tool_name) - input_location = self.data_location / "asgiref-3.3.0_scancode_scan.json" + input_location = self.data_location / "asgiref-3.3.0_toolkit_scan.json" tool_name = input.get_tool_name_from_scan_headers( scan_data=json.loads(input_location.read_text()) ) @@ -405,7 +405,7 @@ def noop(*args, **kwargs): @expectedFailure def test_scanpipe_pipes_scancode_virtual_codebase(self): project = Project.objects.create(name="asgiref") - input_location = self.data_location / "asgiref-3.3.0_scan.json" + input_location = self.data_location / "asgiref-3.3.0_scanpipe_output.json" virtual_codebase = scancode.get_virtual_codebase(project, input_location) self.assertEqual(19, len(virtual_codebase.resources.keys())) @@ -439,10 +439,10 @@ def test_scanpipe_pipes_scancode_virtual_codebase(self): def test_scanpipe_pipes_scancode_create_codebase_resources_inject_policy(self): project = Project.objects.create(name="asgiref") - # We are using `asgiref-3.3.0_scancode_scan.json` instead of - # `asgiref-3.3.0_scan.json` because `asgiref-3.3.0_scan.json` is not - # exactly the same format as a scancode-toolkit scan - input_location = self.data_location / "asgiref-3.3.0_scancode_scan.json" + # We are using `asgiref-3.3.0_toolkit_scan.json` instead of + # `asgiref-3.3.0_scanpipe_output.json` because it is not exactly the same + # format as a scancode-toolkit scan + input_location = self.data_location / "asgiref-3.3.0_toolkit_scan.json" virtual_codebase = scancode.get_virtual_codebase(project, input_location) scanpipe_app.license_policies_index = license_policies_index @@ -502,10 +502,27 @@ def test_scanpipe_pipes_scancode_make_results_summary(self): summary = scancode.make_results_summary(project, scan_results_location) self.assertEqual(10, len(summary.keys())) + def test_scanpipe_pipes_scancode_load_inventory_from_toolkit_scan(self): + project = Project.objects.create(name="Analysis") + input_location = self.data_location / "asgiref-3.3.0_toolkit_scan.json" + scancode.load_inventory_from_toolkit_scan(project, input_location) + self.assertEqual(18, project.codebaseresources.count()) + self.assertEqual(2, project.discoveredpackages.count()) + self.assertEqual(4, project.discovereddependencies.count()) + + def test_scanpipe_pipes_scancode_load_inventory_from_scanpipe(self): + project = Project.objects.create(name="Analysis") + input_location = self.data_location / "asgiref-3.3.0_scanpipe_output.json" + scan_data = json.loads(input_location.read_text()) + scancode.load_inventory_from_scanpipe(project, scan_data) + self.assertEqual(18, project.codebaseresources.count()) + self.assertEqual(2, project.discoveredpackages.count()) + self.assertEqual(4, project.discovereddependencies.count()) + def test_scanpipe_pipes_scancode_assemble_packages(self): project = Project.objects.create(name="Analysis") project_scan_location = self.data_location / "package_assembly_codebase.json" - scancode.load_inventory_from_scan(project, project_scan_location) + scancode.load_inventory_from_toolkit_scan(project, project_scan_location) self.assertEqual(0, project.discoveredpackages.count()) scancode.assemble_packages(project) @@ -538,7 +555,7 @@ def _replace_path(virtual_tree_children): call_command("loaddata", fixtures, **{"verbosity": 0}) project = Project.objects.get(name="asgiref") - scan_results = self.data_location / "asgiref-3.3.0_scan.json" + scan_results = self.data_location / "asgiref-3.3.0_scanpipe_output.json" virtual_codebase = scancode.get_virtual_codebase(project, scan_results) project_codebase = codebase.ProjectCodebase(project)