diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index d7775a298b2..c0da1d326ac 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -37,6 +37,12 @@ v33.0.0 (next next, roadmap)
v32.1.0 (next, roadmap)
----------------------------
+New CLI options:
+
+- A new CLI option ``--package-only`` has been added which performs
+ a faster package scan by skipping the package assembly step and
+ also skipping license/copyright detection on package metadata.
+
Major API/other changes:
- Output Format Version updated to 3.1.0 (minor version bump)
diff --git a/docs/source/rst_snippets/basic_options.rst b/docs/source/rst_snippets/basic_options.rst
index b4c3eb408ec..d01fbf72a6c 100644
--- a/docs/source/rst_snippets/basic_options.rst
+++ b/docs/source/rst_snippets/basic_options.rst
@@ -33,6 +33,10 @@ documenting a program's options. For example:
--system-package Scan ```` for installed system package
databases.
+--package-only Scan ```` for system and application
+ only for package metadata, without license/
+ copyright detection and package assembly.
+
-e, --email Scan ```` for emails.
Sub-Options:
diff --git a/src/packagedcode/about.py b/src/packagedcode/about.py
index 3f33c6838db..5a1192fdd28 100644
--- a/src/packagedcode/about.py
+++ b/src/packagedcode/about.py
@@ -47,7 +47,7 @@ class AboutFileHandler(models.DatafileHandler):
documentation_url = 'https://aboutcode-toolkit.readthedocs.io/en/latest/specification.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
"""
Yield one or more Package manifest objects given a file ``location`` pointing to a
package archive, manifest or similar.
@@ -90,7 +90,7 @@ def parse(cls, location):
file_references.append(models.FileReference(path=about_resource))
# FIXME: we should put the unprocessed attributes in extra data
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=package_type,
namespace=package_ns,
@@ -103,6 +103,7 @@ def parse(cls, location):
download_url=download_url,
file_references=file_references,
)
+ yield models.PackageData.from_data(package_data, package_only)
@classmethod
def assemble(cls, package_data, resource, codebase, package_adder):
diff --git a/src/packagedcode/alpine.py b/src/packagedcode/alpine.py
index 59223cf72da..1b155a90c97 100644
--- a/src/packagedcode/alpine.py
+++ b/src/packagedcode/alpine.py
@@ -63,11 +63,12 @@ class AlpineInstalledDatabaseHandler(models.DatafileHandler):
description = 'Alpine Linux installed package database'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
yield from parse_alpine_installed_db(
location=location,
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
+ package_only=package_only,
)
@classmethod
@@ -134,9 +135,14 @@ class AlpineApkbuildHandler(models.DatafileHandler):
documentation_url = 'https://wiki.alpinelinux.org/wiki/APKBUILD_Reference'
@classmethod
- def parse(cls, location):
- package_data = parse_apkbuild(location, strict=True)
- cls.populate_license_fields(package_data)
+ def parse(cls, location, package_only=False):
+ package_data = parse_apkbuild(
+ location=location,
+ strict=True,
+ package_only=package_only
+ )
+ if not package_only:
+ cls.populate_license_fields(package_data)
if package_data:
yield package_data
@@ -165,7 +171,7 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder)
)
-def parse_alpine_installed_db(location, datasource_id, package_type):
+def parse_alpine_installed_db(location, datasource_id, package_type, package_only=False):
"""
Yield PackageData objects from an installed database file at `location`
or None. Typically found at '/lib/apk/db/installed' in an Alpine
@@ -179,6 +185,7 @@ def parse_alpine_installed_db(location, datasource_id, package_type):
package_fields=package_fields,
datasource_id=datasource_id,
package_type=package_type,
+ package_only=package_only,
)
@@ -241,7 +248,7 @@ def get_alpine_installed_db_fields(location):
])
-def parse_apkbuild(location, strict=False):
+def parse_apkbuild(location, strict=False, package_only=False):
"""
Return a PackageData object from an APKBUILD file at ``location`` or None.
@@ -256,6 +263,7 @@ def parse_apkbuild(location, strict=False):
datasource_id=AlpineApkbuildHandler.datasource_id,
package_type=AlpineApkbuildHandler.default_package_type,
strict=strict,
+ package_only=package_only,
)
@@ -732,7 +740,7 @@ def fix_apkbuild(text):
return text
-def parse_apkbuild_text(text, datasource_id, package_type, strict=False):
+def parse_apkbuild_text(text, datasource_id, package_type, strict=False, package_only=False):
"""
Return a PackageData object from an APKBUILD text context or None. Only
consider variables with a name listed in the ``names`` set.
@@ -761,7 +769,8 @@ def parse_apkbuild_text(text, datasource_id, package_type, strict=False):
package = build_package_data(
variables,
datasource_id=datasource_id,
- package_type=package_type
+ package_type=package_type,
+ package_only=package_only,
)
if package and unresolved:
@@ -800,7 +809,7 @@ def parse_pkginfo(location):
raise NotImplementedError
-def build_package_data(package_fields, datasource_id, package_type):
+def build_package_data(package_fields, datasource_id, package_type, package_only=False):
"""
Return a PackageData object from a ``package_fields`` iterable of (name,
value) tuples.
@@ -850,7 +859,16 @@ def build_package_data(package_fields, datasource_id, package_type):
converted_fields.update(converted)
- return models.PackageData.from_dict(converted_fields)
+ fields_not_required = ["current_file", "current_dir"]
+ for field in fields_not_required:
+ value = converted_fields.get(field)
+ if value:
+ converted_fields.pop(field)
+
+ return models.PackageData.from_data(
+ package_data=converted_fields,
+ package_only=package_only,
+ )
#####################################
# Note: all handlers MUST accept **kwargs as they also receive the current data
diff --git a/src/packagedcode/bower.py b/src/packagedcode/bower.py
index fe8c197cc87..7189b001741 100644
--- a/src/packagedcode/bower.py
+++ b/src/packagedcode/bower.py
@@ -25,7 +25,7 @@ class BowerJsonHandler(models.DatafileHandler):
documentation_url = 'https://bower.io'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with io.open(location, encoding='utf-8') as loc:
package_data = json.load(loc)
@@ -87,7 +87,7 @@ def parse(cls, location):
)
)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=name,
@@ -98,5 +98,6 @@ def parse(cls, location):
parties=parties,
homepage_url=homepage_url,
vcs_url=vcs_url,
- dependencies=dependencies
+ dependencies=dependencies,
)
+ yield models.PackageData.from_data(package_data, package_only)
diff --git a/src/packagedcode/build.py b/src/packagedcode/build.py
index b9e77dddd62..1c7fb152a7d 100644
--- a/src/packagedcode/build.py
+++ b/src/packagedcode/build.py
@@ -55,7 +55,7 @@ class AutotoolsConfigureHandler(models.NonAssemblableDatafileHandler):
documentation_url = 'https://www.gnu.org/software/automake/'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
# we use the parent directory as a package name
name = fileutils.file_name(fileutils.parent_directory(location))
# we could use checksums as version in the future
@@ -67,12 +67,13 @@ def parse(cls, location):
# there are dependencies we could use
# dependencies = []
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=name,
version=version,
)
+ yield models.PackageData.from_data(package_data, package_only)
@@ -104,6 +105,7 @@ def assemble(cls, package_data, resource, codebase, package_adder):
package = models.Package.from_package_data(
package_data=package_data,
datafile_path=resource.path,
+ package_only=True,
)
if TRACE:
@@ -135,8 +137,7 @@ def assemble(cls, package_data, resource, codebase, package_adder):
yield resource
@classmethod
- def parse(cls, location):
-
+ def parse(cls, location, package_only=False):
# Thanks to Starlark being a Python dialect, we can use `ast` to parse it
with open(location, 'rb') as f:
tree = ast.parse(f.read())
@@ -188,23 +189,28 @@ def parse(cls, location):
if TRACE:
logger_debug(f"build: parse: license_files: {license_files}")
- package_data = models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=name,
+ extracted_license_statement=license_files,
+ )
+ # `package_only` is True as we do the license detection
+ # on assembly
+ yield models.PackageData.from_data(
+ package_data=package_data,
+ package_only=True,
)
-
- package_data.extracted_license_statement = license_files
- yield package_data
else:
# If we don't find anything in the pkgdata file, we yield a Package
# with the parent directory as the name
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=fileutils.file_name(fileutils.parent_directory(location))
)
+ yield models.PackageData.from_data(package_data, package_only)
@classmethod
def assign_package_to_resources(cls, package, resource, codebase, package_adder, skip_name=None):
@@ -326,7 +332,7 @@ class BuckMetadataBzlHandler(BaseStarlarkManifestHandler):
documentation_url = 'https://buck.build/'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=True):
with open(location, 'rb') as f:
tree = ast.parse(f.read())
@@ -378,7 +384,7 @@ def parse(cls, location):
):
# TODO: Create function that determines package type from download URL,
# then create a package of that package type from the metadata info
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=metadata_fields.get('upstream_type', cls.default_package_type),
name=metadata_fields.get('name'),
@@ -388,6 +394,7 @@ def parse(cls, location):
homepage_url=metadata_fields.get('upstream_address', ''),
# TODO: Store 'upstream_hash` somewhere
)
+ yield models.PackageData.from_data(package_data, package_only=True)
if (
'package_type'
@@ -401,7 +408,7 @@ def parse(cls, location):
and 'vcs_commit_hash'
in metadata_fields
):
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=metadata_fields.get('package_type', cls.default_package_type),
name=metadata_fields.get('name'),
@@ -414,6 +421,7 @@ def parse(cls, location):
sha1=metadata_fields.get('download_archive_sha1', ''),
extra_data=dict(vcs_commit_hash=metadata_fields.get('vcs_commit_hash', ''))
)
+ yield models.PackageData.from_data(package_data, package_only=True)
@classmethod
def assign_package_to_resources(cls, package, resource, codebase, package_adder):
diff --git a/src/packagedcode/build_gradle.py b/src/packagedcode/build_gradle.py
index 308abe5bc62..9c4882ccf04 100644
--- a/src/packagedcode/build_gradle.py
+++ b/src/packagedcode/build_gradle.py
@@ -59,9 +59,9 @@ class BuildGradleHandler(models.DatafileHandler):
description = 'Gradle build script'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
dependencies = get_dependencies(location)
- return build_package(cls, dependencies)
+ return build_package(cls, dependencies, package_only)
# TODO: handle complex cases of nested builds with many packages
@classmethod
@@ -328,7 +328,7 @@ def get_dependencies(build_gradle_location):
return list(get_dependencies_from_parse_tree(parse_tree))
-def build_package(cls, dependencies):
+def build_package(cls, dependencies, package_only=False):
"""
Yield PackageData from a ``dependencies`` list of mappings.
"""
@@ -364,10 +364,11 @@ def build_package(cls, dependencies):
)
)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=BuildGradleHandler.default_primary_language,
dependencies=package_dependencies,
)
+ yield models.PackageData.from_data(package_data, package_only)
diff --git a/src/packagedcode/cargo.py b/src/packagedcode/cargo.py
index 18e19e55219..9113a9040b6 100644
--- a/src/packagedcode/cargo.py
+++ b/src/packagedcode/cargo.py
@@ -104,7 +104,7 @@ class CargoTomlHandler(CargoBaseHandler):
documentation_url = 'https://doc.rust-lang.org/cargo/reference/manifest.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
package_data = toml.load(location, _dict=dict)
core_package_data = package_data.get('package', {})
workspace = package_data.get('workspace', {})
@@ -149,7 +149,7 @@ def parse(cls, location):
if workspace:
extra_data["workspace"] = workspace
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=name,
@@ -166,6 +166,7 @@ def parse(cls, location):
dependencies=dependencies,
extra_data=extra_data,
)
+ yield models.PackageData.from_data(package_data, package_only)
CARGO_ATTRIBUTE_MAPPING = {
@@ -200,7 +201,7 @@ class CargoLockHandler(CargoBaseHandler):
# ]
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
cargo_lock = toml.load(location, _dict=dict)
dependencies = []
package = cargo_lock.get('package', [])
@@ -221,12 +222,13 @@ def parse(cls, location):
)
)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
dependencies=dependencies,
)
+ yield models.PackageData.from_data(package_data, package_only)
def dependency_mapper(dependencies, scope='dependencies'):
diff --git a/src/packagedcode/chef.py b/src/packagedcode/chef.py
index 5c378cdce4e..d5df6a2f6c4 100644
--- a/src/packagedcode/chef.py
+++ b/src/packagedcode/chef.py
@@ -183,14 +183,18 @@ def is_datafile(cls, location, filetypes=tuple()):
return not parent.endswith('dist-info')
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
"""
Yield one or more Package manifest objects given a file ``location``
pointing to a package archive, manifest or similar.
"""
with io.open(location, encoding='utf-8') as loc:
package_data = json.load(loc)
- yield build_package(package_data, datasource_id=cls.datasource_id)
+ yield build_package(
+ package_data=package_data,
+ datasource_id=cls.datasource_id,
+ package_only=package_only,
+ )
class ChefMetadataRbHandler(BaseChefMetadataHandler):
@@ -202,7 +206,7 @@ class ChefMetadataRbHandler(BaseChefMetadataHandler):
documentation_url = 'https://docs.chef.io/config_rb_metadata/'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with io.open(location, encoding='utf-8') as loc:
file_contents = loc.read()
@@ -213,10 +217,14 @@ def parse(cls, location):
ChefMetadataFormatter()
)
package_data = json.loads(formatted_file_contents)
- yield build_package(package_data, datasource_id=cls.datasource_id)
+ yield build_package(
+ package_data=package_data,
+ datasource_id=cls.datasource_id,
+ package_only=package_only,
+ )
-def build_package(package_data, datasource_id):
+def build_package(package_data, datasource_id, package_only=False):
"""
Return a PackageData object from a package_data mapping from a metadata.json
or similar or None.
@@ -261,7 +269,7 @@ def build_package(package_data, datasource_id):
)
)
- return models.PackageData(
+ package_data = dict(
datasource_id=datasource_id,
type=ChefMetadataJsonHandler.default_package_type,
name=name,
@@ -275,3 +283,4 @@ def build_package(package_data, datasource_id):
primary_language='Ruby',
**get_urls(name, version),
)
+ return models.PackageData.from_data(package_data, package_only)
diff --git a/src/packagedcode/cocoapods.py b/src/packagedcode/cocoapods.py
index 9773ae13638..8ca3617800b 100644
--- a/src/packagedcode/cocoapods.py
+++ b/src/packagedcode/cocoapods.py
@@ -216,7 +216,7 @@ class PodspecHandler(BasePodHandler):
documentation_url = 'https://guides.cocoapods.org/syntax/podspec.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
"""
Yield one or more Package manifest objects given a file ``location``
pointing to a package archive, manifest or similar.
@@ -258,7 +258,7 @@ def parse(cls, location):
homepage_url=homepage_url,
vcs_url=vcs_url)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=name,
@@ -273,6 +273,7 @@ def parse(cls, location):
parties=parties,
**urls,
)
+ yield models.PackageData.from_data(package_data, package_only)
class PodfileHandler(PodspecHandler):
@@ -293,7 +294,7 @@ class PodfileLockHandler(BasePodHandler):
documentation_url = 'https://guides.cocoapods.org/using/the-podfile.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
"""
Yield PackageData from a YAML Podfile.lock.
"""
@@ -337,12 +338,13 @@ def parse(cls, location):
)
)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
dependencies=dependencies,
)
+ yield models.PackageData.from_data(package_data, package_only)
class PodspecJsonHandler(models.DatafileHandler):
@@ -354,7 +356,7 @@ class PodspecJsonHandler(models.DatafileHandler):
documentation_url = 'https://guides.cocoapods.org/syntax/podspec.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with open(location) as psj:
data = json.load(psj)
@@ -423,7 +425,7 @@ def parse(cls, location):
name=name,
version=version, homepage_url=homepage_url, vcs_url=vcs_url)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
primary_language=cls.default_primary_language,
type=cls.default_package_type,
@@ -437,6 +439,7 @@ def parse(cls, location):
download_url=download_url,
**urls,
)
+ yield models.PackageData.from_data(package_data, package_only)
def get_urls(name=None, version=None, homepage_url=None, vcs_url=None, **kwargs):
diff --git a/src/packagedcode/conan.py b/src/packagedcode/conan.py
index 7d4b637f9bc..44321c581a7 100644
--- a/src/packagedcode/conan.py
+++ b/src/packagedcode/conan.py
@@ -123,7 +123,7 @@ class ConanFileHandler(models.DatafileHandler):
documentation_url = "https://docs.conan.io/2.0/reference/conanfile.html"
@classmethod
- def _parse(cls, conan_recipe):
+ def _parse(cls, conan_recipe, package_only=False):
try:
tree = ast.parse(conan_recipe)
recipe_class_def = next(
@@ -150,7 +150,7 @@ def _parse(cls, conan_recipe):
dependencies = get_dependencies(parser.requires)
- return models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
@@ -163,13 +163,14 @@ def _parse(cls, conan_recipe):
extracted_license_statement=parser.license,
dependencies=dependencies,
)
+ return models.PackageData.from_data(package_data, package_only)
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with io.open(location, encoding="utf-8") as loc:
conan_recipe = loc.read()
- yield cls._parse(conan_recipe)
+ yield cls._parse(conan_recipe, package_only)
class ConanDataHandler(models.DatafileHandler):
@@ -184,7 +185,7 @@ class ConanDataHandler(models.DatafileHandler):
)
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with io.open(location, encoding="utf-8") as loc:
conan_data = loc.read()
@@ -203,7 +204,7 @@ def parse(cls, location):
elif isinstance(source_urls, list):
url = source_urls[0]
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
@@ -212,6 +213,8 @@ def parse(cls, location):
download_url=url,
sha256=sha256,
)
+ yield models.PackageData.from_data(package_data, package_only)
+
@classmethod
def assemble(
@@ -245,7 +248,7 @@ def assemble(
] = conanfile_package_data.get("extracted_license_statement")
datafile_path = resource.path
- pkg_data = models.PackageData.from_dict(package_data_dict)
+ pkg_data = models.PackageData.from_data(package_data_dict)
if pkg_data.purl:
package = models.Package.from_package_data(
diff --git a/src/packagedcode/conda.py b/src/packagedcode/conda.py
index 545b3a1fb53..225940d69fc 100644
--- a/src/packagedcode/conda.py
+++ b/src/packagedcode/conda.py
@@ -79,7 +79,7 @@ def assign_package_to_resources(cls, package, resource, codebase, package_adder)
)
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
metayaml = get_meta_yaml_data(location)
package_element = metayaml.get('package') or {}
package_name = package_element.get('name')
@@ -118,7 +118,7 @@ def parse(cls, location):
)
)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=package_name,
@@ -131,6 +131,7 @@ def parse(cls, location):
extracted_license_statement=extracted_license_statement,
dependencies=dependencies,
)
+ yield models.PackageData.from_data(package_data, package_only)
def get_meta_yaml_data(location):
diff --git a/src/packagedcode/cran.py b/src/packagedcode/cran.py
index ef274105da6..827ff38f9d3 100644
--- a/src/packagedcode/cran.py
+++ b/src/packagedcode/cran.py
@@ -30,7 +30,7 @@ class CranDescriptionFileHandler(models.DatafileHandler):
documentation_url = 'https://r-pkgs.org/description.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
cran_desc = get_cran_description(location)
name = cran_desc.get('Package')
@@ -93,7 +93,7 @@ def parse(cls, location):
# TODO: Let's handle the release date as a Date type
# release_date = cran_desc.get('Date/Publication'),
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=name,
@@ -105,6 +105,7 @@ def parse(cls, location):
dependencies=package_dependencies,
repository_homepage_url=f'https://cran.r-project.org/package={name}',
)
+ yield models.PackageData.from_data(package_data, package_only)
# FIXME: THIS IS NOT YAML but RFC 822
diff --git a/src/packagedcode/debian.py b/src/packagedcode/debian.py
index 5f417243e49..e49ade1c232 100644
--- a/src/packagedcode/debian.py
+++ b/src/packagedcode/debian.py
@@ -60,7 +60,7 @@ class DebianDebPackageHandler(models.DatafileHandler):
documentation_url = 'https://manpages.debian.org/unstable/dpkg-dev/deb.5.en.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
yield build_package_data_from_package_filename(
filename=fileutils.file_name(location),
datasource_id=cls.datasource_id,
@@ -83,7 +83,7 @@ class DebianSourcePackageMetadataTarballHandler(models.DatafileHandler):
documentation_url = 'https://manpages.debian.org/unstable/dpkg-dev/deb.5.en.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
# strip extension
filename, _, _ = location.rpartition('.tar')
yield build_package_data_from_package_filename(
@@ -108,7 +108,7 @@ class DebianSourcePackageTarballHandler(models.DatafileHandler):
documentation_url = 'https://manpages.debian.org/unstable/dpkg-dev/deb.5.en.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
# strip extension
filename, _, _ = location.rpartition('.tar')
yield build_package_data_from_package_filename(
@@ -132,13 +132,14 @@ class DebianControlFileInExtractedDebHandler(models.DatafileHandler):
documentation_url = 'https://www.debian.org/doc/debian-policy/ch-controlfields.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
# TODO: we cannot know the distro from the name only
yield build_package_data(
debian_data=get_paragraph_data_from_file(location=location),
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
distro='debian',
+ package_only=package_only,
)
@classmethod
@@ -158,7 +159,7 @@ class DebianControlFileInSourceHandler(models.DatafileHandler):
documentation_url = 'https://www.debian.org/doc/debian-policy/ch-controlfields.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
# NOTE: a control file in a source repo or debina.tar tarball can contain more than one package
debian_packages = []
for debian_data in get_paragraphs_data_from_file(location=location):
@@ -167,6 +168,7 @@ def parse(cls, location):
debian_data=debian_data,
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
+ package_only=package_only,
)
)
@@ -191,7 +193,7 @@ class DebianDscFileHandler(models.DatafileHandler):
documentation_url = 'https://wiki.debian.org/dsc'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
# this is typically signed
debian_data = get_paragraph_data_from_file(
location=location,
@@ -207,6 +209,7 @@ def parse(cls, location):
debian_data=debian_data,
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
+ package_only=package_only,
)
package_data.update_purl_fields(package_data=package_data_from_file)
yield package_data
@@ -225,7 +228,7 @@ class DebianInstalledStatusDatabaseHandler(models.DatafileHandler):
documentation_url = 'https://www.debian.org/doc/debian-policy/ch-controlfields.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
# note that we do not know yet the distro at this stage
# we could get it... but we get that later during assemble()
debian_packages = []
@@ -235,6 +238,7 @@ def parse(cls, location):
debian_data=debian_data,
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
+ package_only=package_only,
)
)
@@ -392,7 +396,7 @@ class DebianDistrolessInstalledDatabaseHandler(models.DatafileHandler):
documentation_url = 'https://www.debian.org/doc/debian-policy/ch-controlfields.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
"""
Yield installed PackageData objects given a ``location``
var/lib/dpkg/status.d/ file as found in a distroless container
@@ -406,6 +410,7 @@ def parse(cls, location):
debian_data=debian_data,
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
+ package_only=package_only,
)
)
@@ -475,7 +480,7 @@ class DebianInstalledFilelistHandler(models.DatafileHandler):
description = 'Debian installed file paths list'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
return parse_debian_files_list(
location=location,
datasource_id=cls.datasource_id,
@@ -501,7 +506,7 @@ class DebianInstalledMd5sumFilelistHandler(models.DatafileHandler):
documentation_url = 'https://www.debian.org/doc/manuals/debian-handbook/sect.package-meta-information.en.html#sect.configuration-scripts'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
return parse_debian_files_list(
location=location,
datasource_id=cls.datasource_id,
@@ -526,7 +531,7 @@ class DebianMd5sumFilelistInPackageHandler(models.DatafileHandler):
documentation_url = 'https://www.debian.org/doc/manuals/debian-handbook/sect.package-meta-information.en.html#sect.configuration-scripts'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
return parse_debian_files_list(
location=location,
datasource_id=cls.datasource_id,
@@ -561,7 +566,7 @@ def build_package_data_from_package_filename(filename, datasource_id, package_ty
if isinstance(version, DebVersion):
version = str(version)
- return models.PackageData(
+ package_data = dict(
datasource_id=datasource_id,
type=package_type,
name=deb.name,
@@ -569,6 +574,7 @@ def build_package_data_from_package_filename(filename, datasource_id, package_ty
version=version,
qualifiers=qualifiers,
)
+ return models.PackageData.from_data(package_data)
def parse_debian_files_list(location, datasource_id, package_type):
@@ -609,16 +615,17 @@ def parse_debian_files_list(location, datasource_id, package_type):
if not file_references:
return
- yield models.PackageData(
+ package_data = dict(
datasource_id=datasource_id,
type=package_type,
name=name,
qualifiers=qualifiers,
file_references=file_references,
)
+ yield models.PackageData.from_data(package_data)
-def build_package_data(debian_data, datasource_id, package_type='deb', distro=None):
+def build_package_data(debian_data, datasource_id, package_type='deb', distro=None, package_only=False):
"""
Return a PackageData object from a package_data mapping (from a dpkg status
or similar file) or None.
@@ -696,7 +703,7 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
source_packages.append(source_pkg_purl)
- return models.PackageData(
+ package_data = dict(
datasource_id=datasource_id,
type=package_type,
namespace=distro,
@@ -711,6 +718,7 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No
parties=parties,
extra_data=extra_data,
)
+ return models.PackageData.from_data(package_data, package_only)
def parse_debian_maintainers(maintainer):
diff --git a/src/packagedcode/debian_copyright.py b/src/packagedcode/debian_copyright.py
index 510abf620d6..7f08592fe8d 100644
--- a/src/packagedcode/debian_copyright.py
+++ b/src/packagedcode/debian_copyright.py
@@ -95,7 +95,7 @@ def is_datafile(cls, location, filetypes=tuple(), strict=False):
return True
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
debian_copyright = parse_copyright_file(location)
license_fields = DebianLicenseFields.get_license_fields(
debian_copyright=debian_copyright
@@ -111,20 +111,27 @@ def parse(cls, location):
# no name otherwise for now
name = None
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=name,
- extracted_license_statement=license_fields.extracted_license_statement,
- declared_license_expression=license_fields.declared_license_expression,
- declared_license_expression_spdx=license_fields.declared_license_expression_spdx,
- license_detections=license_fields.license_detections,
- other_license_expression=license_fields.other_license_expression,
- other_license_expression_spdx=license_fields.other_license_expression_spdx,
- other_license_detections=license_fields.other_license_detections,
- copyright=debian_copyright.get_copyright(),
)
+ if not package_only:
+ license_data = dict(
+ extracted_license_statement=license_fields.extracted_license_statement,
+ declared_license_expression=license_fields.declared_license_expression,
+ declared_license_expression_spdx=license_fields.declared_license_expression_spdx,
+ license_detections=license_fields.license_detections,
+ other_license_expression=license_fields.other_license_expression,
+ other_license_expression_spdx=license_fields.other_license_expression_spdx,
+ other_license_detections=license_fields.other_license_detections,
+ copyright=debian_copyright.get_copyright(),
+ )
+ package_data.update(license_data)
+
+ yield models.PackageData.from_data(package_data, package_only)
+
@attr.s
class DebianLicenseFields:
@@ -279,14 +286,14 @@ def assemble(cls, package_data, resource, codebase, package_adder):
yield from super().assemble(package_data, resource, codebase, package_adder)
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
"""
Gets license/copyright information from file like
other copyright files, but also gets purl fields if
present in copyright filename, if obtained from
upstream metadata archive.
"""
- package_data = list(super().parse(location)).pop()
+ package_data = list(super().parse(location, package_only)).pop()
package_data_from_file = build_package_data_from_metadata_filename(
filename=os.path.basename(location),
datasource_id=cls.datasource_id,
diff --git a/src/packagedcode/distro.py b/src/packagedcode/distro.py
index 6336e6b3111..7e572948a03 100644
--- a/src/packagedcode/distro.py
+++ b/src/packagedcode/distro.py
@@ -25,7 +25,7 @@ class EtcOsReleaseHandler(models.NonAssemblableDatafileHandler):
documentation_url = 'https://www.freedesktop.org/software/systemd/man/os-release.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
distro = Distro.from_os_release_file(location)
distro_identifier = distro.identifier
pretty_name = distro.pretty_name and distro.pretty_name.lower() or ''
@@ -53,13 +53,14 @@ def parse(cls, location):
version = distro.version_id
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
namespace=namespace,
name=name,
version=version,
)
+ yield models.PackageData.from_data(package_data, package_only)
@classmethod
def find_linux_rootfs_root_resource(cls, resource, codebase):
diff --git a/src/packagedcode/freebsd.py b/src/packagedcode/freebsd.py
index c1f5342b562..d170e8f8222 100644
--- a/src/packagedcode/freebsd.py
+++ b/src/packagedcode/freebsd.py
@@ -52,7 +52,7 @@ class CompactManifestHandler(models.DatafileHandler):
documentation_url = 'https://www.freebsd.org/cgi/man.cgi?pkg-create(8)#MANIFEST_FILE_DETAILS'
@classmethod
- def _parse(cls, yaml_data):
+ def _parse(cls, yaml_data, package_only=False):
package_data = models.PackageData(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
@@ -97,7 +97,8 @@ def _parse(cls, yaml_data):
# license_mapper needs multiple fields
license_mapper(yaml_data, package_data)
- cls.populate_license_fields(package_data)
+ if not package_only:
+ cls.populate_license_fields(package_data)
if TRACE:
logger_debug(
@@ -107,7 +108,7 @@ def _parse(cls, yaml_data):
return package_data
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
"""
Yield one or more Package manifest objects given a file ``location`` pointing to a
package archive, manifest or similar.
@@ -115,7 +116,7 @@ def parse(cls, location):
with io.open(location, encoding='utf-8') as loc:
yaml_data = saneyaml.load(loc)
- yield cls._parse(yaml_data)
+ yield cls._parse(yaml_data, package_only)
@staticmethod
def get_license_detections_and_expression(package_data):
diff --git a/src/packagedcode/godeps.py b/src/packagedcode/godeps.py
index 68ff54787ff..c9d15a3934a 100644
--- a/src/packagedcode/godeps.py
+++ b/src/packagedcode/godeps.py
@@ -38,7 +38,7 @@ class GodepsHandler(models.NonAssemblableDatafileHandler):
documentation_url = 'https://github.com/tools/godep'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
godeps = Godep(location)
if godeps.import_path:
@@ -64,7 +64,7 @@ def parse(cls, location):
)
)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
namespace=namespace,
@@ -72,6 +72,7 @@ def parse(cls, location):
primary_language=cls.default_primary_language,
dependencies=dependencies,
)
+ yield models.PackageData.from_data(package_data, package_only)
@classmethod
def assign_package_to_resources(cls, package, resource, codebase, package_adder):
diff --git a/src/packagedcode/golang.py b/src/packagedcode/golang.py
index 6075c713e86..68651fa6cb1 100644
--- a/src/packagedcode/golang.py
+++ b/src/packagedcode/golang.py
@@ -49,7 +49,7 @@ class GoModHandler(BaseGoModuleHandler):
documentation_url = 'https://go.dev/ref/mod'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
gomods = go_mod.parse_gomod(location)
dependencies = []
@@ -89,7 +89,7 @@ def parse(cls, location):
if namespace and name:
repository_homepage_url = f'https://pkg.go.dev/{namespace}/{name}'
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=name,
@@ -100,6 +100,7 @@ def parse(cls, location):
dependencies=dependencies,
primary_language=cls.default_primary_language,
)
+ yield models.PackageData.from_data(package_data, package_only)
class GoSumHandler(BaseGoModuleHandler):
@@ -111,7 +112,7 @@ class GoSumHandler(BaseGoModuleHandler):
documentation_url = 'https://go.dev/ref/mod#go-sum-files'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
gosums = go_mod.parse_gosum(location)
package_dependencies = []
for gosum in gosums:
@@ -126,9 +127,10 @@ def parse(cls, location):
)
)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
dependencies=package_dependencies,
primary_language=cls.default_primary_language,
)
+ yield models.PackageData.from_data(package_data, package_only)
diff --git a/src/packagedcode/haxe.py b/src/packagedcode/haxe.py
index 99f5be4ef7d..630d4e36175 100644
--- a/src/packagedcode/haxe.py
+++ b/src/packagedcode/haxe.py
@@ -45,11 +45,11 @@ class HaxelibJsonHandler(models.DatafileHandler):
documentation_url = 'https://lib.haxe.org/documentation/creating-a-haxelib-package/'
@classmethod
- def _parse(cls, json_data):
+ def _parse(cls, json_data, package_only=False):
name = json_data.get('name')
version = json_data.get('version')
- package_data = models.PackageData(
+ package_mapping = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=name,
@@ -60,6 +60,7 @@ def _parse(cls, json_data):
description=json_data.get('description'),
primary_language=cls.default_primary_language,
)
+ package_data = models.PackageData.from_data(package_mapping, package_only)
if name and version:
download_url = f'https://lib.haxe.org/p/{name}/{version}/download/'
@@ -91,7 +92,7 @@ def _parse(cls, json_data):
return package_data
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
"""
Yield one or more Package manifest objects given a file ``location`` pointing to a
package_data archive, manifest or similar.
@@ -111,4 +112,4 @@ def parse(cls, location):
with io.open(location, encoding='utf-8') as loc:
json_data = json.load(loc)
- yield cls._parse(json_data)
+ yield cls._parse(json_data, package_only)
diff --git a/src/packagedcode/maven.py b/src/packagedcode/maven.py
index b5d521dc44a..1498add8cf0 100644
--- a/src/packagedcode/maven.py
+++ b/src/packagedcode/maven.py
@@ -132,13 +132,14 @@ class JavaJarManifestHandler(MavenBasePackageHandler):
documentation_url = 'https://docs.oracle.com/javase/tutorial/deployment/jar/manifestindex.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
sections = parse_manifest(location)
if sections:
main_section = sections[0]
manifest = get_normalized_java_manifest_data(main_section)
if manifest:
- yield models.PackageData(**manifest,)
+ package_data = dict(**manifest,)
+ yield models.PackageData.from_data(package_data, package_only)
class JavaJarManifestHandlerMixin(models.DatafileHandler):
@@ -206,13 +207,14 @@ def is_datafile(cls, location, filetypes=tuple()):
return True
@classmethod
- def parse(cls, location, base_url='https://repo1.maven.org/maven2'):
+ def parse(cls, location, package_only=False, base_url='https://repo1.maven.org/maven2'):
package_data = parse(
location=location,
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
primary_language=cls.default_primary_language,
base_url=base_url,
+ package_only=package_only,
)
if package_data:
yield package_data
@@ -303,7 +305,7 @@ class MavenPomPropertiesHandler(models.NonAssemblableDatafileHandler):
documentation_url = 'https://maven.apache.org/pom.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
"""
Yield PackageData from a pom.properties file (which is typically side-
by-side with its pom file.)
@@ -313,10 +315,10 @@ def parse(cls, location):
if TRACE:
logger.debug(f'MavenPomPropertiesHandler.parse: properties: {properties!r}')
if properties:
- yield from cls.parse_pom_properties(properties=properties)
+ yield from cls.parse_pom_properties(properties=properties, package_only=package_only)
@classmethod
- def parse_pom_properties(cls, properties):
+ def parse_pom_properties(cls, properties, package_only=False):
namespace = properties.pop("groupId", None)
name = properties.pop("artifactId", None)
version = properties.pop("version", None)
@@ -325,7 +327,7 @@ def parse_pom_properties(cls, properties):
else:
extra_data = {}
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
@@ -334,6 +336,7 @@ def parse_pom_properties(cls, properties):
version=version,
extra_data=extra_data,
)
+ yield models.PackageData.from_data(package_data, package_only)
def build_url(
@@ -1189,6 +1192,7 @@ def parse(
package_type,
primary_language,
base_url='https://repo1.maven.org/maven2',
+ package_only=False,
):
"""
Return Packagedata objects from parsing a Maven pom file at `location` or
@@ -1199,7 +1203,8 @@ def parse(
package_type=package_type,
primary_language=primary_language,
location=location,
- base_url=base_url
+ base_url=base_url,
+ package_only=package_only,
)
if package:
return package
@@ -1212,6 +1217,7 @@ def _parse(
location=None,
text=None,
base_url='https://repo1.maven.org/maven2',
+ package_only=False,
):
"""
Yield Packagedata objects from parsing a Maven pom file at `location` or
@@ -1283,7 +1289,7 @@ def _parse(
))
# FIXME: there are still other data to map in a PackageData
- return MavenPackageData(
+ package_data = dict(
datasource_id=datasource_id,
type=package_type,
primary_language=primary_language,
@@ -1300,6 +1306,7 @@ def _parse(
bug_tracking_url=bug_tracking_url,
**urls,
)
+ return MavenPackageData.from_data(package_data, package_only)
class MavenPackageData(models.PackageData):
diff --git a/src/packagedcode/models.py b/src/packagedcode/models.py
index 2915fa4c8b9..50a181214ba 100644
--- a/src/packagedcode/models.py
+++ b/src/packagedcode/models.py
@@ -718,10 +718,55 @@ class PackageData(IdentifiablePackageData):
repr=True,
)
+ @classmethod
+ def from_data(cls, package_data, package_only=False):
+ """
+ Return PackageData object created out of the package metadata
+ present in `package_data` mapping. Also populate license and
+ copyright holder fields by computing them from extracted license
+ statement and extracted copyright.
- def __attrs_post_init__(self, *args, **kwargs):
- self.populate_license_fields()
- self.populate_holder_field()
+ Skip the license/copyright detection step if `package_only` is True.
+ """
+ if "purl" in package_data:
+ package_data.pop("purl")
+
+ package_data = cls(**package_data)
+
+ if not package_only:
+ package_data.populate_license_fields()
+ package_data.populate_holder_field()
+ else:
+ package_data.normalize_extracted_license_statement()
+
+ return package_data
+
+ @property
+ def can_assemble(self):
+ from packagedcode import HANDLER_BY_DATASOURCE_ID
+ handler = HANDLER_BY_DATASOURCE_ID.get(self.datasource_id)
+ if issubclass(handler, NonAssemblableDatafileHandler):
+ return False
+
+ return True
+
+ def normalize_extracted_license_statement(self):
+ """
+ Normalizes the extracted license statement to a readable
+ YAML string if it was a pythonic object.
+ """
+ if (
+ self.extracted_license_statement and
+ not isinstance(self.extracted_license_statement, str)
+ ):
+ if isinstance(self.extracted_license_statement, dict):
+ self.extracted_license_statement = saneyaml.dump(
+ dict(self.extracted_license_statement.items())
+ )
+ else:
+ self.extracted_license_statement = saneyaml.dump(
+ self.extracted_license_statement
+ )
def populate_holder_field(self):
if not self.copyright:
@@ -781,11 +826,7 @@ def populate_license_fields(self):
f"license_detections: {self.license_detections}"
)
- if self.extracted_license_statement and not isinstance(self.extracted_license_statement, str):
- if isinstance(self.extracted_license_statement, dict):
- self.extracted_license_statement = saneyaml.dump(dict(self.extracted_license_statement.items()))
- else:
- self.extracted_license_statement = saneyaml.dump(self.extracted_license_statement)
+ self.normalize_extracted_license_statement()
def update_purl_fields(self, package_data, replace=False):
@@ -1024,13 +1065,16 @@ def is_datafile(cls, location, filetypes=tuple(), _bare_filename=False):
return any(ft in actual_type for ft in filetypes)
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
"""
Yield one or more PackageData objects given a package data file at
``location``.
Subclasses must implement and are responsible for returning proper
computed license fields and list of resources and files.
+
+ If `package_only`, skip the license/copyright detection on extracted
+ license/copyright data.
"""
raise NotImplementedError
@@ -1491,8 +1535,6 @@ def __attrs_post_init__(self, *args, **kwargs):
if not self.package_uid:
self.package_uid = build_package_uid(self.purl)
- self.populate_license_fields()
-
def to_dict(self):
return super().to_dict(with_details=False)
@@ -1504,7 +1546,7 @@ def to_package_data(self):
return PackageData.from_dict(mapping)
@classmethod
- def from_package_data(cls, package_data, datafile_path):
+ def from_package_data(cls, package_data, datafile_path, package_only=False):
"""
Return a Package from a ``package_data`` PackageData object
or mapping. Or None.
@@ -1528,7 +1570,16 @@ def from_package_data(cls, package_data, datafile_path):
if not license_match['from_file']:
license_match['from_file'] = datafile_path
- return cls.from_dict(package_data_mapping)
+ package = cls.from_dict(package_data_mapping)
+
+ if not package.package_uid:
+ package.package_uid = build_package_uid(package.purl)
+
+ if not package_only:
+ package.populate_license_fields()
+ package.populate_holder_field()
+
+ return package
@classmethod
def from_dict(cls, mapping):
diff --git a/src/packagedcode/msi.py b/src/packagedcode/msi.py
index 31deea3040b..5ce1ba534ed 100644
--- a/src/packagedcode/msi.py
+++ b/src/packagedcode/msi.py
@@ -124,6 +124,7 @@ def create_package_data_from_msiinfo_results(
msiinfo_results,
datasource_id='msi_installer',
package_type='msi',
+ package_only=False,
):
"""
Return PackageData from a mapping of `msiinfo_results`
@@ -150,7 +151,7 @@ def create_package_data_from_msiinfo_results(
description = msiinfo_results.pop('Comments', '')
keywords = msiinfo_results.pop('Keywords', [])
- return models.PackageData(
+ package_data = dict(
datasource_id=datasource_id,
type=package_type,
name=name,
@@ -160,11 +161,14 @@ def create_package_data_from_msiinfo_results(
keywords=keywords,
extra_data=msiinfo_results
)
+ return models.PackageData.from_data(package_data, package_only)
-def msi_parse(location,
+def msi_parse(
+ location,
datasource_id='msi_installer',
package_type='msi',
+ package_only=False,
):
"""
Return PackageData from ``location``
@@ -175,6 +179,7 @@ def msi_parse(location,
msiinfo_results=info,
datasource_id=datasource_id,
package_type=package_type,
+ package_only=package_only,
)
else:
return models.PackageData(
@@ -192,5 +197,5 @@ class MsiInstallerHandler(models.DatafileHandler):
documentation_url = 'https://docs.microsoft.com/en-us/windows/win32/msi/windows-installer-portal'
@classmethod
- def parse(cls, location):
- yield msi_parse(location)
+ def parse(cls, location, package_only=False):
+ yield msi_parse(location, package_only)
diff --git a/src/packagedcode/npm.py b/src/packagedcode/npm.py
index c831aa8e855..18ad27d7564 100644
--- a/src/packagedcode/npm.py
+++ b/src/packagedcode/npm.py
@@ -185,7 +185,7 @@ class NpmPackageJsonHandler(BaseNpmHandler):
documentation_url = 'https://docs.npmjs.com/cli/v8/configuring-npm/package-json'
@classmethod
- def _parse(cls, json_data):
+ def _parse(cls, json_data, package_only=False):
name = json_data.get('name')
version = json_data.get('version')
homepage_url = json_data.get('homepage', '')
@@ -200,7 +200,7 @@ def _parse(cls, json_data):
namespace, name = split_scoped_package_name(name)
urls = get_urls(namespace, name, version)
- package = models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
@@ -211,6 +211,7 @@ def _parse(cls, json_data):
homepage_url=homepage_url,
**urls,
)
+ package = models.PackageData.from_data(package_data, package_only)
vcs_revision = json_data.get('gitHead') or None
# mapping of top level package.json items to a function accepting as
@@ -249,7 +250,8 @@ def _parse(cls, json_data):
lics = json_data.get('licenses')
package = licenses_mapper(lic, lics, package)
- package.populate_license_fields()
+ if not package_only:
+ package.populate_license_fields()
if TRACE:
logger_debug(f'NpmPackageJsonHandler: parse: package: {package.to_dict()}')
@@ -257,17 +259,17 @@ def _parse(cls, json_data):
return package
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with io.open(location, encoding='utf-8') as loc:
json_data = json.load(loc)
- yield cls._parse(json_data)
+ yield cls._parse(json_data, package_only)
class BaseNpmLockHandler(BaseNpmHandler):
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with io.open(location, encoding='utf-8') as loc:
package_data = json.load(loc)
@@ -280,7 +282,7 @@ def parse(cls, location):
extra_data = dict(lockfile_version=lockfile_version)
# this is the top level element that we return
- root_package_data = models.PackageData(
+ root_package_mapping = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
@@ -290,6 +292,7 @@ def parse(cls, location):
extra_data=extra_data,
**get_urls(root_ns, root_name, root_version)
)
+ root_package_data = models.PackageData.from_data(root_package_mapping, package_only)
# https://docs.npmjs.com/cli/v8/configuring-npm/package-lock-json#lockfileversion
if lockfile_version == 1:
@@ -359,7 +362,7 @@ def parse(cls, location):
integrity = dep_data.get('integrity')
misc.update(get_algo_hexsum(integrity).items())
- resolved_package = models.PackageData(
+ resolved_package_mapping = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
@@ -369,6 +372,7 @@ def parse(cls, location):
extracted_license_statement=extracted_license_statement,
**misc,
)
+ resolved_package = models.PackageData.from_data(resolved_package_mapping, package_only)
# these are paths t the root of the installed package in v2
if dep:
resolved_package.file_references = [models.FileReference(path=dep)],
@@ -490,7 +494,7 @@ def is_datafile(cls, location, filetypes=tuple()):
return super().is_datafile(location, filetypes=filetypes) and is_yarn_v2(location)
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
"""
Parse a bew yarn.lock v2 YAML format which looks like this:
@@ -545,12 +549,13 @@ def parse(cls, location):
)
top_dependencies.append(dependency)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
dependencies=top_dependencies,
)
+ yield models.PackageData.from_data(package_data, package_only)
class YarnLockV1Handler(BaseNpmHandler):
@@ -569,7 +574,7 @@ def is_datafile(cls, location, filetypes=tuple()):
return super().is_datafile(location, filetypes=filetypes) and not is_yarn_v2(location)
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
"""
Parse a classic yarn.lock format which looks like this:
"@babel/core@^7.1.0", "@babel/core@^7.3.4":
@@ -657,7 +662,7 @@ def parse(cls, location):
misc.update(get_algo_hexsum(integrity).items())
# we create a resolve package with the details
- resolved_package_data = models.PackageData(
+ resolved_package_mapping = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
namespace=ns,
@@ -666,6 +671,7 @@ def parse(cls, location):
primary_language=cls.default_primary_language,
**misc,
)
+ resolved_package_data = models.PackageData.from_data(resolved_package_mapping, package_only)
# we add the sub-deps to the resolved package
for subns, subname, subconstraint in sub_dependencies:
@@ -701,12 +707,13 @@ def parse(cls, location):
)
dependencies.append(dep)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
dependencies=dependencies,
)
+ yield models.PackageData.from_data(package_data, package_only)
def get_checksum_and_url(url):
diff --git a/src/packagedcode/nuget.py b/src/packagedcode/nuget.py
index dee1afb39d5..de584038d48 100644
--- a/src/packagedcode/nuget.py
+++ b/src/packagedcode/nuget.py
@@ -106,7 +106,7 @@ class NugetNuspecHandler(models.DatafileHandler):
documentation_url = 'https://docs.microsoft.com/en-us/nuget/reference/nuspec'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with open(location, 'rb') as loc:
parsed = xmltodict.parse(loc)
@@ -163,7 +163,7 @@ def parse(cls, location):
elif 'licenseUrl' in nuspec:
extracted_license_statement = nuspec.get('licenseUrl')
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=name,
@@ -177,4 +177,5 @@ def parse(cls, location):
vcs_url=vcs_url,
**urls,
)
+ yield models.PackageData.from_data(package_data, package_only)
diff --git a/src/packagedcode/opam.py b/src/packagedcode/opam.py
index 94e2f4942d7..f2fde2e017e 100644
--- a/src/packagedcode/opam.py
+++ b/src/packagedcode/opam.py
@@ -31,7 +31,7 @@ def get_package_root(cls, resource, codebase):
return resource.parent(codebase)
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
opams = parse_opam(location)
package_dependencies = []
@@ -90,7 +90,7 @@ def parse(cls, location):
)
)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=name,
@@ -111,6 +111,7 @@ def parse(cls, location):
repository_homepage_url=repository_homepage_url,
primary_language=cls.default_primary_language
)
+ yield models.PackageData.from_data(package_data, package_only)
@classmethod
def assign_package_to_resources(cls, package, resource, codebase, package_adder):
diff --git a/src/packagedcode/phpcomposer.py b/src/packagedcode/phpcomposer.py
index b729e0bfc25..d217e5283d4 100644
--- a/src/packagedcode/phpcomposer.py
+++ b/src/packagedcode/phpcomposer.py
@@ -58,7 +58,7 @@ class PhpComposerJsonHandler(BasePhpComposerHandler):
documentation_url = 'https://getcomposer.org/doc/04-schema.md'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
"""
Yield one or more Package manifest objects given a file ``location``
pointing to a package archive, manifest or similar.
@@ -69,7 +69,7 @@ def parse(cls, location):
with io.open(location, encoding='utf-8') as loc:
package_json = json.load(loc)
- yield build_package_data(package_json)
+ yield build_package_data(package_json, package_only)
def get_repository_homepage_url(namespace, name):
@@ -86,7 +86,7 @@ def get_api_data_url(namespace, name):
return f'https://packagist.org/p/packages/{name}.json'
-def build_package_data(package_data):
+def build_package_data(package_data, package_only=False):
# Note: A composer.json without name and description is not a usable PHP
# composer package. Name and description fields are required but only for
@@ -103,7 +103,7 @@ def build_package_data(package_data):
else:
ns, _, name = ns_name.rpartition('/')
- package = models.PackageData(
+ package_mapping = dict(
datasource_id=PhpComposerJsonHandler.datasource_id,
type=PhpComposerJsonHandler.default_package_type,
namespace=ns,
@@ -112,6 +112,7 @@ def build_package_data(package_data):
api_data_url=get_api_data_url(ns, name),
primary_language=PhpComposerJsonHandler.default_primary_language,
)
+ package = models.PackageData.from_data(package_mapping, package_only)
# mapping of top level composer.json items to the Package object field name
plain_fields = [
@@ -157,7 +158,9 @@ def build_package_data(package_data):
vendor_mapper(package)
# Per https://getcomposer.org/doc/04-schema.md#license this is an expression
- package.populate_license_fields()
+ if not package_only:
+ package.populate_license_fields()
+
return package
@@ -170,16 +173,16 @@ class PhpComposerLockHandler(BasePhpComposerHandler):
documentation_url = 'https://getcomposer.org/doc/01-basic-usage.md#commit-your-composer-lock-file-to-version-control'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with io.open(location, encoding='utf-8') as loc:
package_data = json.load(loc)
packages = [
- build_package_data(p)
+ build_package_data(p, package_only)
for p in package_data.get('packages', [])
]
packages_dev = [
- build_package_data(p)
+ build_package_data(p, package_only)
for p in package_data.get('packages-dev', [])
]
@@ -192,12 +195,13 @@ def parse(cls, location):
for p in packages_dev
]
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
dependencies=required_deps + required_dev_deps
)
+ yield models.PackageData.from_data(package_data, package_only)
for package in packages + packages_dev:
yield package
diff --git a/src/packagedcode/plugin_package.py b/src/packagedcode/plugin_package.py
index b8b1b84be29..b1487022c40 100644
--- a/src/packagedcode/plugin_package.py
+++ b/src/packagedcode/plugin_package.py
@@ -112,7 +112,6 @@ def get_available_package_parsers(docs=False):
return all_data_packages
-
@scan_impl
class PackageScanner(ScanPlugin):
"""
@@ -161,7 +160,20 @@ class PackageScanner(ScanPlugin):
help_group=SCAN_GROUP,
sort_order=21,
),
-
+ PluggableCommandLineOption(
+ (
+ '--package-only',
+ ),
+ is_flag=True,
+ default=False,
+ conflicting_options=['license', 'summary', 'package', 'system_package'],
+ help=(
+ 'Scan for system and application package data and skip '
+ 'license/copyright detection and top-level package creation.'
+ ),
+ help_group=SCAN_GROUP,
+ sort_order=22,
+ ),
PluggableCommandLineOption(
('--list-packages',),
is_flag=True,
@@ -172,10 +184,10 @@ class PackageScanner(ScanPlugin):
),
]
- def is_enabled(self, package, system_package, **kwargs):
- return package or system_package
+ def is_enabled(self, package, system_package, package_only, **kwargs):
+ return package or system_package or package_only
- def get_scanner(self, package=True, system_package=False, **kwargs):
+ def get_scanner(self, package=True, system_package=False, package_only=False, **kwargs):
"""
Return a scanner callable to scan a file for package data.
"""
@@ -185,9 +197,10 @@ def get_scanner(self, package=True, system_package=False, **kwargs):
get_package_data,
application=package,
system=system_package,
+ package_only=package_only,
)
- def process_codebase(self, codebase, strip_root=False, **kwargs):
+ def process_codebase(self, codebase, strip_root=False, package_only=False, **kwargs):
"""
Populate the ``codebase`` top level ``packages`` and ``dependencies``
with package and dependency instances, assembling parsed package data
@@ -196,6 +209,11 @@ def process_codebase(self, codebase, strip_root=False, **kwargs):
Also perform additional package license detection that depends on either
file license detection or the package detections.
"""
+ # If we only want purls, we want to skip both the package
+ # assembly and the extra package license detection steps
+ if package_only:
+ return
+
has_licenses = hasattr(codebase.root, 'license_detections')
# These steps add proper license detections to package_data and hence
diff --git a/src/packagedcode/pubspec.py b/src/packagedcode/pubspec.py
index a6abd5b8642..df18d409ba4 100644
--- a/src/packagedcode/pubspec.py
+++ b/src/packagedcode/pubspec.py
@@ -60,11 +60,11 @@ class DartPubspecYamlHandler(BaseDartPubspecHandler):
documentation_url = 'https://dart.dev/tools/pub/pubspec'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with open(location) as inp:
pubspec_data = saneyaml.load(inp.read())
- package_data = build_package(pubspec_data)
+ package_data = build_package(pubspec_data, package_only)
if package_data:
yield package_data
@@ -78,18 +78,19 @@ class DartPubspecLockHandler(BaseDartPubspecHandler):
documentation_url = 'https://web.archive.org/web/20220330081004/https://gpalma.pt/blog/what-is-the-pubspec-lock/'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with open(location) as inp:
locks_data = saneyaml.load(inp.read())
dependencies = list(collect_locks(locks_data))
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
dependencies=dependencies
)
+ yield models.PackageData.from_data(package_data, package_only)
def collect_locks(locks_data):
@@ -238,7 +239,7 @@ def build_dep(name, version, scope, is_runtime=True, is_optional=False):
return dep
-def build_package(pubspec_data):
+def build_package(pubspec_data, package_only=False):
"""
Return a package object from a package data mapping or None
"""
@@ -315,7 +316,7 @@ def add_to_extra_if_present(_key):
add_to_extra_if_present('executables')
add_to_extra_if_present('publish_to')
- return models.PackageData(
+ package_data = dict(
datasource_id=DartPubspecYamlHandler.datasource_id,
type=DartPubspecYamlHandler.default_primary_language,
primary_language=DartPubspecYamlHandler.default_primary_language,
@@ -333,3 +334,4 @@ def add_to_extra_if_present(_key):
api_data_url=api_data_url,
repository_download_url=repository_download_url,
)
+ return models.PackageData.from_data(package_data, package_only)
diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py
index d90dacbabef..33f2ec394ea 100644
--- a/src/packagedcode/pypi.py
+++ b/src/packagedcode/pypi.py
@@ -79,11 +79,12 @@ class PythonEggPkgInfoFile(models.DatafileHandler):
documentation_url = 'https://peps.python.org/pep-0376/'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
yield parse_metadata(
location=location,
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
+ package_only=package_only,
)
@classmethod
@@ -103,11 +104,12 @@ class PythonEditableInstallationPkgInfoFile(models.DatafileHandler):
documentation_url = 'https://peps.python.org/pep-0376/'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
yield parse_metadata(
location=location,
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
+ package_only=package_only,
)
@classmethod
@@ -320,11 +322,12 @@ def is_datafile(cls, location):
)
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
yield parse_metadata(
location=location,
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
+ package_only=package_only,
)
@@ -337,11 +340,12 @@ class PythonInstalledWheelMetadataFile(models.DatafileHandler):
documentation_url = 'https://packaging.python.org/en/latest/specifications/core-metadata/'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
yield parse_metadata(
location=location,
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
+ package_only=package_only,
)
@classmethod
@@ -457,7 +461,7 @@ class PyprojectTomlHandler(models.NonAssemblableDatafileHandler):
META_DIR_SUFFIXES = '.dist-info', '.egg-info', 'EGG-INFO',
-def parse_metadata(location, datasource_id, package_type):
+def parse_metadata(location, datasource_id, package_type, package_only=False):
"""
Return a PackageData object from a PKG-INFO or METADATA file at ``location``
which is a path string or pathlib.Path-like object (including a possible zip
@@ -487,7 +491,7 @@ def parse_metadata(location, datasource_id, package_type):
file_references = list(get_file_references(dist))
- return models.PackageData(
+ package_data = dict(
datasource_id=datasource_id,
type=package_type,
primary_language='Python',
@@ -502,6 +506,7 @@ def parse_metadata(location, datasource_id, package_type):
extra_data=extra_data,
**urls,
)
+ return models.PackageData.from_data(package_data, package_only)
def urlsafe_b64decode(data):
@@ -551,7 +556,7 @@ class PypiWheelHandler(models.DatafileHandler):
documentation_url = 'https://peps.python.org/pep-0427/'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with zipfile.ZipFile(location) as zf:
for path in ZipPath(zf).iterdir():
if not path.name.endswith(META_DIR_SUFFIXES):
@@ -564,6 +569,7 @@ def parse(cls, location):
location=metapath,
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
+ package_only=package_only,
)
@@ -577,7 +583,7 @@ class PypiEggHandler(models.DatafileHandler):
documentation_url = 'https://web.archive.org/web/20210604075235/http://peak.telecommunity.com/DevCenter/PythonEggs'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with zipfile.ZipFile(location) as zf:
for path in ZipPath(zf).iterdir():
if not path.name.endswith(META_DIR_SUFFIXES):
@@ -591,6 +597,7 @@ def parse(cls, location):
location=metapath,
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
+ package_only=package_only,
)
@@ -610,7 +617,7 @@ def is_datafile(cls, location, filetypes=tuple()):
return True
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
# FIXME: add dependencies
try:
@@ -622,7 +629,7 @@ def parse(cls, location):
version = sdist.version
urls, extra_data = get_urls(metainfo=sdist, name=name, version=version)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
@@ -635,6 +642,7 @@ def parse(cls, location):
extra_data=extra_data,
**urls,
)
+ yield models.PackageData.from_data(package_data, package_only)
class PythonSetupPyHandler(BaseExtractedPythonLayout):
@@ -646,7 +654,7 @@ class PythonSetupPyHandler(BaseExtractedPythonLayout):
documentation_url = 'https://docs.python.org/3.11/distutils/setupscript.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
setup_args = get_setup_py_args(location)
# it may be legit to have a name-less package?
@@ -664,7 +672,7 @@ def parse(cls, location):
python_requires = get_setup_py_python_requires(setup_args)
extra_data.update(python_requires)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
@@ -678,6 +686,7 @@ def parse(cls, location):
extra_data=extra_data,
**urls,
)
+ yield models.PackageData.from_data(package_data, package_only)
class ResolvedPurl(NamedTuple):
@@ -694,7 +703,7 @@ class BaseDependencyFileHandler(models.DatafileHandler):
"""
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
file_name = fileutils.file_name(location)
dependency_type = get_dparse2_supported_file_name(file_name)
@@ -705,12 +714,13 @@ def parse(cls, location):
location=location,
file_name=dependency_type,
)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
dependencies=dependencies,
)
+ yield models.PackageData.from_data(package_data, package_only)
class SetupCfgHandler(BaseExtractedPythonLayout):
@@ -722,7 +732,7 @@ class SetupCfgHandler(BaseExtractedPythonLayout):
documentation_url = 'https://peps.python.org/pep-0390/'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
metadata = {}
parser = ConfigParser()
@@ -801,7 +811,7 @@ def parse(cls, location):
extracted_license_statement = ''
extracted_license_statement += f" license_files: {license_file_references}"
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=metadata.get('name'),
@@ -812,6 +822,7 @@ def parse(cls, location):
dependencies=dependent_packages,
extracted_license_statement=extracted_license_statement,
)
+ yield models.PackageData.from_data(package_data, package_only)
@classmethod
def parse_reqs(cls, reqs, scope):
@@ -873,7 +884,7 @@ class PipfileLockHandler(BaseDependencyFileHandler):
documentation_url = 'https://github.com/pypa/pipfile'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with open(location) as f:
content = f.read()
@@ -890,13 +901,14 @@ def parse(cls, location):
file_name='Pipfile.lock',
)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
sha256=sha256,
dependencies=dependent_packages,
)
+ yield models.PackageData.from_data(package_data, package_only)
class PipRequirementsFileHandler(BaseDependencyFileHandler):
@@ -919,15 +931,16 @@ class PipRequirementsFileHandler(BaseDependencyFileHandler):
documentation_url = 'https://pip.pypa.io/en/latest/reference/requirements-file-format/'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
dependencies, extra_data = get_requirements_txt_dependencies(location=location)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
primary_language=cls.default_primary_language,
dependencies=dependencies,
extra_data=extra_data,
)
+ yield models.PackageData.from_data(package_data, package_only)
# TODO: enable nested load
diff --git a/src/packagedcode/readme.py b/src/packagedcode/readme.py
index 63741e86648..a03886929ba 100644
--- a/src/packagedcode/readme.py
+++ b/src/packagedcode/readme.py
@@ -55,11 +55,11 @@ class ReadmeHandler(models.NonAssemblableDatafileHandler):
documentation_url = ''
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with open(location, encoding='utf-8') as loc:
readme_manifest = loc.read()
- package_data = build_package(readme_manifest)
+ package_data = build_package(readme_manifest, package_only)
if not package_data.name:
# If no name was detected for the Package, then we use the basename
@@ -71,7 +71,7 @@ def parse(cls, location):
yield package_data
-def build_package(readme_manifest):
+def build_package(readme_manifest, package_only=False):
"""
Return a Package object from a readme_manifest mapping (from a
README.chromium file or similar) or None.
@@ -104,5 +104,8 @@ def build_package(readme_manifest):
continue
setattr(package, package_key, value)
- package.populate_license_fields()
+ if not package_only:
+ package.populate_license_fields()
+ package.populate_holder_field()
+
return package
diff --git a/src/packagedcode/recognize.py b/src/packagedcode/recognize.py
index 5b629c01588..9dd7c643482 100644
--- a/src/packagedcode/recognize.py
+++ b/src/packagedcode/recognize.py
@@ -44,6 +44,7 @@ def recognize_package_data(
location,
application=True,
system=False,
+ package_only=False,
):
"""
Return a list of Package objects if any package_data were recognized for
@@ -55,18 +56,24 @@ def recognize_package_data(
if not filetype.is_file(location):
return []
- assert application or system
- if application and system:
+ assert application or system or package_only
+ if package_only or (application and system):
datafile_handlers = ALL_DATAFILE_HANDLERS
elif application:
datafile_handlers = APPLICATION_PACKAGE_DATAFILE_HANDLERS
elif system:
datafile_handlers = SYSTEM_PACKAGE_DATAFILE_HANDLERS
- return list(_parse(location, datafile_handlers=datafile_handlers))
+
+ return list(_parse(
+ location=location,
+ package_only=package_only,
+ datafile_handlers=datafile_handlers,
+ ))
def _parse(
location,
+ package_only=False,
datafile_handlers=APPLICATION_PACKAGE_DATAFILE_HANDLERS,
):
"""
@@ -87,7 +94,7 @@ def _parse(
logger_debug(f'_parse:.is_datafile: {location}')
try:
- for parsed in handler.parse(location):
+ for parsed in handler.parse(location=location, package_only=package_only):
if TRACE:
logger_debug(f' _parse: parsed: {parsed!r}')
yield parsed
diff --git a/src/packagedcode/rpm.py b/src/packagedcode/rpm.py
index 50be812978c..a66501281b9 100644
--- a/src/packagedcode/rpm.py
+++ b/src/packagedcode/rpm.py
@@ -124,7 +124,7 @@ def to_string(self):
class BaseRpmInstalledDatabaseHandler(models.DatafileHandler):
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
# we receive the location of the Package database file and we need to
# scan the parent which is the directory that contains the rpmdb
loc_path = Path(location)
@@ -136,6 +136,7 @@ def parse(cls, location):
location=xmlish_loc,
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
+ package_only=package_only,
)
# TODO: package_data.namespace = cls.default_package_namespace
return package_data
@@ -277,7 +278,7 @@ class RpmArchiveHandler(models.DatafileHandler):
documentation_url = 'https://en.wikipedia.org/wiki/RPM_Package_Manager'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
rpm_tags = get_rpm_tags(location, include_desc=True)
if TRACE: logger_debug('recognize: rpm_tags', rpm_tags)
@@ -354,7 +355,7 @@ def parse(cls, location):
)
logger_debug('recognize: data to create a package:\n', data)
- package = models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
# TODO: namespace=cls.default_package_namespace,
@@ -368,9 +369,9 @@ def parse(cls, location):
)
if TRACE:
- logger_debug('recognize: created package:\n', package)
+ logger_debug('recognize: created package:\n', name)
- yield package
+ yield models.PackageData.from_data(package_data, package_only)
ALGO_BY_ID = {
diff --git a/src/packagedcode/rpm_installed.py b/src/packagedcode/rpm_installed.py
index 5d01436d2e1..8e6a12261c5 100644
--- a/src/packagedcode/rpm_installed.py
+++ b/src/packagedcode/rpm_installed.py
@@ -36,7 +36,7 @@ def logger_debug(*args):
return logger.debug(' '.join(isinstance(a, str) and a or repr(a) for a in args))
-def parse_rpm_xmlish(location, datasource_id, package_type):
+def parse_rpm_xmlish(location, datasource_id, package_type, package_only=False):
"""
Yield PackageData built from an RPM XML'ish file at ``location``. This is a file
created with the rpm CLI with the xml query option.
@@ -58,6 +58,7 @@ def parse_rpm_xmlish(location, datasource_id, package_type):
rpm_tags=tags,
datasource_id=datasource_id,
package_type=package_type,
+ package_only=package_only,
)
@@ -133,7 +134,7 @@ def collect_tags(raw_tags):
yield name, value_type, value
-def build_package(rpm_tags, datasource_id, package_type, package_namespace=None):
+def build_package(rpm_tags, datasource_id, package_type, package_namespace=None, package_only=False):
"""
Return a PackageData object from an ``rpm_tags`` iterable of (name,
value_type, value) tuples.
@@ -157,8 +158,12 @@ def build_package(rpm_tags, datasource_id, package_type, package_namespace=None)
except Exception as e:
raise Exception(value, converted) from e
converted.update(handled)
+
+ current_filerefs = converted.get("current_filerefs", None)
+ if current_filerefs:
+ converted.pop("current_filerefs")
- package_data = models.PackageData.from_dict(converted)
+ package_data = models.PackageData.from_data(converted, package_only)
return package_data
################################################################################
@@ -183,7 +188,10 @@ def handler(value, **kwargs):
def size_handler(value, **kwargs):
- return {'size': int(value)}
+ if not value == '0':
+ return {'size': int(value)}
+ else:
+ return {'size': None}
def arch_handler(value, **kwargs):
diff --git a/src/packagedcode/rubygems.py b/src/packagedcode/rubygems.py
index 6b77d6945d6..704005c0d21 100644
--- a/src/packagedcode/rubygems.py
+++ b/src/packagedcode/rubygems.py
@@ -41,12 +41,13 @@ class GemArchiveHandler(models.DatafileHandler):
)
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
metadata = extract_gem_metadata(location)
metadata = saneyaml.load(metadata)
yield build_rubygem_package_data(
gem_data=metadata,
datasource_id=cls.datasource_id,
+ package_only=package_only,
)
@@ -84,13 +85,14 @@ class GemMetadataArchiveExtractedHandler(models.DatafileHandler):
)
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with open(location, 'rb') as met:
metadata = met.read()
metadata = saneyaml.load(metadata)
yield build_rubygem_package_data(
gem_data=metadata,
datasource_id=cls.datasource_id,
+ package_only=package_only,
)
@classmethod
@@ -129,7 +131,7 @@ class GemspecHandler(models.DatafileHandler):
documentation_url = 'https://guides.rubygems.org/specification-reference/'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
gemspec = spec.parse_spec(
location=location,
package_type=cls.default_package_type,
@@ -152,7 +154,7 @@ def parse(cls, location):
urls = get_urls(name=name, version=version)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=name,
@@ -166,6 +168,7 @@ def parse(cls, location):
dependencies=dependencies,
**urls
)
+ yield models.PackageData.from_data(package_data, package_only)
class GemspecInExtractedGemHandler(GemspecHandler):
datasource_id = 'gemspec_extracted'
@@ -234,7 +237,7 @@ class GemfileLockHandler(BaseGemProjectHandler):
documentation_url = 'https://bundler.io/man/gemfile.5.html'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
gemfile_lock = GemfileLockParser(location)
all_gems = list(gemfile_lock.all_gems.values())
if not all_gems:
@@ -258,7 +261,7 @@ def parse(cls, location):
]
urls = get_urls(primary_gem.name, primary_gem.version)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
primary_language=cls.default_primary_language,
type=cls.default_package_type,
@@ -267,6 +270,7 @@ def parse(cls, location):
dependencies=deps,
**urls
)
+ yield models.PackageData.from_data(package_data, package_only)
else:
deps = [
models.DependentPackage(
@@ -284,12 +288,13 @@ def parse(cls, location):
) for gem in all_gems
]
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
dependencies=deps,
primary_language=cls.default_primary_language,
)
+ yield models.PackageData.from_data(package_data, package_only)
class GemfileLockInExtractedGemHandler(GemfileLockHandler):
@@ -417,7 +422,7 @@ def extract_gem_metadata(location):
fileutils.delete(extract_loc)
-def build_rubygem_package_data(gem_data, datasource_id):
+def build_rubygem_package_data(gem_data, datasource_id, package_only=False):
"""
Return a PackageData for ``datasource_id`` built from a Gem `gem_data`
mapping or None. The ``gem_data`` can come from a .gemspec or .gem/metadata.
@@ -461,7 +466,7 @@ def build_rubygem_package_data(gem_data, datasource_id):
dependencies = get_dependencies(gem_data.get('dependencies'))
file_references = get_file_references(metadata.get('files'))
- package_data = models.PackageData(
+ package_mapping = dict(
datasource_id=datasource_id,
type=GemArchiveHandler.default_package_type,
primary_language=GemArchiveHandler.default_primary_language,
@@ -477,6 +482,7 @@ def build_rubygem_package_data(gem_data, datasource_id):
dependencies=dependencies,
**urls,
)
+ package_data = models.PackageData.from_data(package_mapping, package_only)
# we can have one singular or a plural list of authors
authors = gem_data.get('authors') or []
diff --git a/src/packagedcode/win_pe.py b/src/packagedcode/win_pe.py
index a51ceb9496e..ce040eaedfe 100644
--- a/src/packagedcode/win_pe.py
+++ b/src/packagedcode/win_pe.py
@@ -276,7 +276,7 @@ def is_datafile(cls, location, filetypes=tuple()):
return True
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
infos = pe_info(location)
version = get_first(
@@ -328,7 +328,7 @@ def parse(cls, location):
parties = [Party(type=party_org, role='author', name=cname)]
homepage_url = get_first(infos, 'URL', 'WWW')
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=name,
@@ -340,3 +340,4 @@ def parse(cls, location):
parties=parties,
homepage_url=homepage_url,
)
+ yield models.PackageData.from_data(package_data, package_only)
diff --git a/src/packagedcode/win_reg.py b/src/packagedcode/win_reg.py
index e4e59cd3b1e..aac41691f01 100644
--- a/src/packagedcode/win_reg.py
+++ b/src/packagedcode/win_reg.py
@@ -77,6 +77,7 @@ def get_installed_dotnet_versions_from_hive(
datasource_id,
package_type,
registry_path='\\Microsoft\\NET Framework Setup\\NDP',
+ package_only=False,
):
"""
Yield PackageData for the installed versions of .NET framework from the
@@ -90,6 +91,7 @@ def get_installed_dotnet_versions_from_hive(
registry_tree=registry_tree,
datasource_id=datasource_id,
package_type=package_type,
+ package_only=package_only,
)
@@ -97,6 +99,7 @@ def get_installed_dotnet_versions_from_regtree(
registry_tree,
datasource_id,
package_type,
+ package_only=False,
):
"""
Yield PackageData for the installed versions of .NET framework from a
@@ -122,13 +125,14 @@ def get_installed_dotnet_versions_from_regtree(
if key == 'InstallPath':
file_references.append(models.FileReference(path=value))
- yield models.PackageData(
+ package_data = dict(
datasource_id=datasource_id,
type=package_type,
name='microsoft-dot-net-framework',
version=version,
file_references=file_references,
)
+ yield models.PackageData.from_data(package_data, package_only)
def get_installed_windows_programs_from_hive(
@@ -136,6 +140,7 @@ def get_installed_windows_programs_from_hive(
datasource_id,
package_type,
registry_path='\\Microsoft\\Windows\\CurrentVersion\\Uninstall',
+ package_only=False,
):
"""
Yield installed Windows PackageData from a Windows registry file at
@@ -151,6 +156,7 @@ def get_installed_windows_programs_from_hive(
registry_tree=registry_tree,
datasource_id=datasource_id,
package_type=package_type,
+ package_only=package_only,
)
@@ -158,6 +164,7 @@ def get_installed_windows_programs_from_regtree(
registry_tree,
datasource_id,
package_type,
+ package_only=False,
):
"""
Yield installed Windows PackageData from a Windows ``registry_tree``.
@@ -213,7 +220,7 @@ def get_installed_windows_programs_from_regtree(
if uninstall_string:
file_references.append(models.FileReference(path=uninstall_string))
- yield models.PackageData(
+ package_data = dict(
datasource_id=datasource_id,
type=package_type,
name=name,
@@ -222,12 +229,14 @@ def get_installed_windows_programs_from_regtree(
homepage_url=homepage_url,
file_references=file_references,
)
+ yield models.PackageData.from_data(package_data, package_only)
def get_packages_from_registry_from_hive(
location,
datasource_id,
package_type,
+ package_only=False,
):
"""
Yield PackageData for Installed Windows Programs from the Windows registry
@@ -238,6 +247,7 @@ def get_packages_from_registry_from_hive(
datasource_id=datasource_id,
package_type=package_type,
registry_path='\\Microsoft\\Windows\\CurrentVersion\\Uninstall',
+ package_only=package_only,
)
yield from get_installed_windows_programs_from_hive(
@@ -245,6 +255,7 @@ def get_packages_from_registry_from_hive(
datasource_id=datasource_id,
package_type=package_type,
registry_path='\\Wow6432Node\\Microsoft\\Windows\\CurrentVersion\\Uninstall',
+ package_only=package_only,
)
yield from get_installed_dotnet_versions_from_hive(
@@ -252,10 +263,11 @@ def get_packages_from_registry_from_hive(
datasource_id=datasource_id,
package_type=package_type,
registry_path='\\Microsoft\\NET Framework Setup\\NDP',
+ package_only=package_only,
)
-def get_installed_packages(root_dir, is_container=True):
+def get_installed_packages(root_dir, is_container=True, package_only=False):
"""
Yield PackageData for Installed Windows Programs for every detected
installed program from Windows registry hive files found in well known
@@ -280,7 +292,7 @@ def get_installed_packages(root_dir, is_container=True):
for software_reg_loc, root_prefix in root_prefixes_by_software_reg_locations.items():
if not os.path.exists(software_reg_loc):
continue
- for package in get_packages_from_registry_from_hive(software_reg_loc):
+ for package in get_packages_from_registry_from_hive(software_reg_loc, package_only):
package.populate_installed_files(root_dir, root_prefix=root_prefix)
yield package
@@ -342,11 +354,12 @@ class BaseRegInstalledProgramHandler(models.DatafileHandler):
root_path_relative_to_datafile_path = None
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
yield from get_packages_from_registry_from_hive(
location=location,
datasource_id=cls.datasource_id,
package_type=cls.default_package_type,
+ package_only=package_only,
)
@classmethod
diff --git a/src/packagedcode/windows.py b/src/packagedcode/windows.py
index 09d62f4c2d9..6fdb07815e3 100644
--- a/src/packagedcode/windows.py
+++ b/src/packagedcode/windows.py
@@ -20,7 +20,7 @@ class MicrosoftUpdateManifestHandler(models.NonAssemblableDatafileHandler):
description = 'Microsoft Update Manifest .mum file'
@classmethod
- def parse(cls, location):
+ def parse(cls, location, package_only=False):
with open(location , 'rb') as loc:
parsed = xmltodict.parse(loc)
@@ -47,7 +47,7 @@ def parse(cls, location):
)
)
- yield models.PackageData(
+ package_data = dict(
datasource_id=cls.datasource_id,
type=cls.default_package_type,
name=name,
@@ -57,3 +57,4 @@ def parse(cls, location):
parties=parties,
copyright=copyrght,
)
+ yield models.PackageData.from_data(package_data, package_only)
diff --git a/src/scancode/api.py b/src/scancode/api.py
index 7d3edbf1516..36cc4bc2ed8 100644
--- a/src/scancode/api.py
+++ b/src/scancode/api.py
@@ -248,20 +248,21 @@ def get_licenses(
SCANCODE_DEBUG_PACKAGE_API = os.environ.get('SCANCODE_DEBUG_PACKAGE_API', False)
-def _get_package_data(location, application=True, system=False, **kwargs):
+def _get_package_data(location, application=True, system=False, package_only=False, **kwargs):
"""
Return a mapping of package manifest information detected in the file at ``location``.
Include ``application`` packages (such as pypi) and/or ``system`` packages.
Note that all exceptions are caught if there are any errors while parsing a
package manifest.
"""
- assert application or system
+ assert application or system or package_only
from packagedcode.recognize import recognize_package_data
try:
return recognize_package_data(
location=location,
application=application,
- system=system
+ system=system,
+ package_only=package_only,
) or []
except Exception as e:
@@ -291,7 +292,7 @@ def get_package_info(location, **kwargs):
return dict(packages=[p.to_dict() for p in packages])
-def get_package_data(location, application=True, system=False, **kwargs):
+def get_package_data(location, application=True, system=False, package_only=False, **kwargs):
"""
Return a mapping of package manifest information detected in the file at
`location`.
@@ -304,6 +305,7 @@ def get_package_data(location, application=True, system=False, **kwargs):
location=location,
application=application,
system=system,
+ package_only=package_only,
**kwargs,
) or []
diff --git a/tests/packagedcode/data/bower/scan-package-only-expected.json b/tests/packagedcode/data/bower/scan-package-only-expected.json
new file mode 100644
index 00000000000..e739c38fa0c
--- /dev/null
+++ b/tests/packagedcode/data/bower/scan-package-only-expected.json
@@ -0,0 +1,113 @@
+{
+ "packages": [],
+ "dependencies": [],
+ "files": [
+ {
+ "path": "scan",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "scan/bower.json",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "bower",
+ "namespace": null,
+ "name": "John Doe",
+ "version": null,
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": null,
+ "description": "Physics-like animations for pretty particles",
+ "release_date": null,
+ "parties": [
+ {
+ "type": null,
+ "role": "author",
+ "name": "Betty Beta ",
+ "email": null,
+ "url": null
+ },
+ {
+ "type": null,
+ "role": "author",
+ "name": "John Doe",
+ "email": "john@doe.com",
+ "url": "http://johndoe.com"
+ }
+ ],
+ "keywords": [
+ "motion",
+ "physics",
+ "particles"
+ ],
+ "homepage_url": null,
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": "- MIT\n- Apache 2.0\n- BSD-3-Clause\n",
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [
+ {
+ "purl": "pkg:bower/get-size",
+ "extracted_requirement": "~1.2.2",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:bower/eventEmitter",
+ "extracted_requirement": "~4.2.11",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:bower/qunit",
+ "extracted_requirement": "~1.16.0",
+ "scope": "devDependencies",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ }
+ ],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "bower_json",
+ "purl": "pkg:bower/John%20Doe"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests/packagedcode/data/build/buck/end2end-expected.json b/tests/packagedcode/data/build/buck/end2end-expected.json
index 70f7b321317..2b169491ec3 100644
--- a/tests/packagedcode/data/build/buck/end2end-expected.json
+++ b/tests/packagedcode/data/build/buck/end2end-expected.json
@@ -256,9 +256,7 @@
"other_license_expression": null,
"other_license_expression_spdx": null,
"other_license_detections": [],
- "extracted_license_statement": [
- "LICENSE"
- ],
+ "extracted_license_statement": "- LICENSE\n",
"notice_text": null,
"source_packages": [],
"file_references": [],
diff --git a/tests/packagedcode/data/build_gradle/end2end/build.gradle-package-only-expected.json b/tests/packagedcode/data/build_gradle/end2end/build.gradle-package-only-expected.json
new file mode 100644
index 00000000000..264afa7d09b
--- /dev/null
+++ b/tests/packagedcode/data/build_gradle/end2end/build.gradle-package-only-expected.json
@@ -0,0 +1,97 @@
+{
+ "packages": [],
+ "dependencies": [],
+ "files": [
+ {
+ "path": "build.gradle",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "maven",
+ "namespace": null,
+ "name": null,
+ "version": null,
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": null,
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [
+ {
+ "purl": "pkg:maven/com.google/guava@1.0",
+ "extracted_requirement": "1.0",
+ "scope": "api",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:maven/org.apache/commons@1.0",
+ "extracted_requirement": "1.0",
+ "scope": "usageDependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:maven/org.jacoco.ant@0.7.4.201502262128",
+ "extracted_requirement": "0.7.4.201502262128",
+ "scope": "",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:maven/org.jacoco.agent@0.7.4.201502262128",
+ "extracted_requirement": "0.7.4.201502262128",
+ "scope": "",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ }
+ ],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "build_gradle",
+ "purl": null
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests/packagedcode/data/cargo/scan-package-only.expected.json b/tests/packagedcode/data/cargo/scan-package-only.expected.json
new file mode 100644
index 00000000000..5355bf80b74
--- /dev/null
+++ b/tests/packagedcode/data/cargo/scan-package-only.expected.json
@@ -0,0 +1,411 @@
+{
+ "packages": [],
+ "dependencies": [],
+ "files": [
+ {
+ "path": "scan",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "scan/Cargo.lock",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "cargo",
+ "namespace": null,
+ "name": null,
+ "version": null,
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "Rust",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [
+ {
+ "purl": "pkg:cargo/ahash@0.7.4",
+ "extracted_requirement": "0.7.4",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/autocfg@1.0.1",
+ "extracted_requirement": "1.0.1",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/cfg-if@0.1.10",
+ "extracted_requirement": "0.1.10",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/cfg-if@1.0.0",
+ "extracted_requirement": "1.0.0",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/chrono@0.4.19",
+ "extracted_requirement": "0.4.19",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/getrandom@0.2.3",
+ "extracted_requirement": "0.2.3",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/hashbrown@0.11.2",
+ "extracted_requirement": "0.11.2",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/libc@0.2.97",
+ "extracted_requirement": "0.2.97",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/log@0.4.14",
+ "extracted_requirement": "0.4.14",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/memory_units@0.4.0",
+ "extracted_requirement": "0.4.0",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/num-integer@0.1.44",
+ "extracted_requirement": "0.1.44",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/num-traits@0.2.14",
+ "extracted_requirement": "0.2.14",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/once_cell@1.8.0",
+ "extracted_requirement": "1.8.0",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/proxy-wasm@0.1.4",
+ "extracted_requirement": "0.1.4",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/time@0.1.43",
+ "extracted_requirement": "0.1.43",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/version_check@0.9.3",
+ "extracted_requirement": "0.9.3",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/wasi@0.10.2%2Bwasi-snapshot-preview1",
+ "extracted_requirement": "0.10.2+wasi-snapshot-preview1",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/wee_alloc@0.4.5",
+ "extracted_requirement": "0.4.5",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/winapi@0.3.9",
+ "extracted_requirement": "0.3.9",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/winapi-i686-pc-windows-gnu@0.4.0",
+ "extracted_requirement": "0.4.0",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:cargo/winapi-x86_64-pc-windows-gnu@0.4.0",
+ "extracted_requirement": "0.4.0",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ }
+ ],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "cargo_lock",
+ "purl": null
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "scan/Cargo.toml",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "cargo",
+ "namespace": null,
+ "name": "apple-xar",
+ "version": "0.4.0-pre",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "Rust",
+ "description": "XAR archive reading and writing",
+ "release_date": null,
+ "parties": [
+ {
+ "type": "person",
+ "role": "author",
+ "name": "Gregory Szorc",
+ "email": "gregory.szorc@gmail.com",
+ "url": null
+ }
+ ],
+ "keywords": [],
+ "homepage_url": "https://github.com/indygreg/PyOxidizer",
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": "https://github.com/indygreg/PyOxidizer.git",
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": "MPL-2.0",
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": "https://crates.io/crates/apple-xar",
+ "repository_download_url": "https://crates.io/api/v1/crates/apple-xar/0.4.0-pre/download",
+ "api_data_url": "https://crates.io/api/v1/crates/apple-xar",
+ "datasource_id": "cargo_toml",
+ "purl": "pkg:cargo/apple-xar@0.4.0-pre"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "scan/dac",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "scan/dac/Cargo.toml",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "cargo",
+ "namespace": null,
+ "name": "daachorse",
+ "version": "0.4.1",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "Rust",
+ "description": "Daachorse: Double-Array Aho-Corasick",
+ "release_date": null,
+ "parties": [
+ {
+ "type": "person",
+ "role": "author",
+ "name": "Koichi Akabe",
+ "email": "vbkaisetsu@gmail.com",
+ "url": null
+ },
+ {
+ "type": "person",
+ "role": "author",
+ "name": "Shunsuke Kanda",
+ "email": "shnsk.knd@gmail.com",
+ "url": null
+ }
+ ],
+ "keywords": [],
+ "homepage_url": "https://github.com/daac-tools/daachorse",
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": "https://github.com/daac-tools/daachorse",
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": "MIT OR Apache-2.0",
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {
+ "workspace": {
+ "members": [
+ "bench",
+ "daacfind"
+ ]
+ }
+ },
+ "dependencies": [],
+ "repository_homepage_url": "https://crates.io/crates/daachorse",
+ "repository_download_url": "https://crates.io/api/v1/crates/daachorse/0.4.1/download",
+ "api_data_url": "https://crates.io/api/v1/crates/daachorse",
+ "datasource_id": "cargo_toml",
+ "purl": "pkg:cargo/daachorse@0.4.1"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests/packagedcode/data/chef/package-only.scan.expected.json b/tests/packagedcode/data/chef/package-only.scan.expected.json
new file mode 100644
index 00000000000..eb2362c2108
--- /dev/null
+++ b/tests/packagedcode/data/chef/package-only.scan.expected.json
@@ -0,0 +1,151 @@
+{
+ "packages": [],
+ "dependencies": [],
+ "files": [
+ {
+ "path": "package",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "package/metadata.json",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "chef",
+ "namespace": null,
+ "name": "301",
+ "version": "0.1.0",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "Ruby",
+ "description": "Installs/Configures 301",
+ "release_date": null,
+ "parties": [
+ {
+ "type": null,
+ "role": "maintainer",
+ "name": "Mark Wilkerson",
+ "email": "mark@segfawlt.net",
+ "url": null
+ }
+ ],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": "https://supermarket.chef.io/cookbooks/301/versions/0.1.0/download",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": "MIT",
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [
+ {
+ "purl": "pkg:chef/nodejs",
+ "extracted_requirement": ">= 0.0.0",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ }
+ ],
+ "repository_homepage_url": "https://supermarket.chef.io/cookbooks/301/versions/0.1.0/",
+ "repository_download_url": "https://supermarket.chef.io/cookbooks/301/versions/0.1.0/download",
+ "api_data_url": "https://supermarket.chef.io/api/v1/cookbooks/301/versions/0.1.0",
+ "datasource_id": "chef_cookbook_metadata_json",
+ "purl": "pkg:chef/301@0.1.0"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "package/metadata.rb",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "chef",
+ "namespace": null,
+ "name": "301",
+ "version": "0.1.0",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "Ruby",
+ "description": "Installs/Configures 301",
+ "release_date": null,
+ "parties": [
+ {
+ "type": null,
+ "role": "maintainer",
+ "name": "Mark Wilkerson",
+ "email": "mark@segfawlt.net",
+ "url": null
+ }
+ ],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": "https://supermarket.chef.io/cookbooks/301/versions/0.1.0/download",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": "MIT",
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [
+ {
+ "purl": "pkg:chef/nodejs",
+ "extracted_requirement": null,
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ }
+ ],
+ "repository_homepage_url": "https://supermarket.chef.io/cookbooks/301/versions/0.1.0/",
+ "repository_download_url": "https://supermarket.chef.io/cookbooks/301/versions/0.1.0/download",
+ "api_data_url": "https://supermarket.chef.io/api/v1/cookbooks/301/versions/0.1.0",
+ "datasource_id": "chef_cookbook_metadata_rb",
+ "purl": "pkg:chef/301@0.1.0"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests/packagedcode/data/cocoapods/assemble/solo/RxDataSources-package-only.podspec-expected.json b/tests/packagedcode/data/cocoapods/assemble/solo/RxDataSources-package-only.podspec-expected.json
new file mode 100644
index 00000000000..93bbebb32e2
--- /dev/null
+++ b/tests/packagedcode/data/cocoapods/assemble/solo/RxDataSources-package-only.podspec-expected.json
@@ -0,0 +1,64 @@
+{
+ "packages": [],
+ "dependencies": [],
+ "files": [
+ {
+ "path": "RxDataSources.podspec",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "cocoapods",
+ "namespace": null,
+ "name": "RxDataSources",
+ "version": "4.0.1",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "Objective-C",
+ "description": "This is a collection of reactive data sources for UITableView and UICollectionView.\n\nIt enables creation of animated data sources for table an collection views in just a couple of lines of code.\n\n```swift\nlet data: Observable = ...\n\nlet dataSource = RxTableViewSectionedAnimatedDataSource()\ndataSource.cellFactory = { (tv, ip, i) in\n let cell = tv.dequeueReusableCell(withIdentifier: \"Cell\") ?? UITableViewCell(style:.Default, reuseIdentifier: \"Cell\")\n cell.textLabel!.text = \"\\(i)\"\n return cell\n}\n\n// animated\ndata\n .bind(to: animatedTableView.rx.items(dataSource: dataSource))\n .disposed(by: disposeBag)\n\n// normal reload\ndata\n .bind(to: tableView.rx.items(dataSource: dataSource))\n .disposed(by: disposeBag)\n```",
+ "release_date": null,
+ "parties": [
+ {
+ "type": "person",
+ "role": "author",
+ "name": "Krunoslav Zaher ",
+ "email": "krunoslav.zaher@gmail.com",
+ "url": null
+ }
+ ],
+ "keywords": [],
+ "homepage_url": "https://github.com/RxSwiftCommunity/RxDataSources",
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": "https://github.com/RxSwiftCommunity/RxDataSources/issues/",
+ "code_view_url": "https://github.com/RxSwiftCommunity/RxDataSources/tree/4.0.1",
+ "vcs_url": "https://github.com/RxSwiftCommunity/RxDataSources.git",
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": "MIT",
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": "https://cocoapods.org/pods/RxDataSources",
+ "repository_download_url": "https://github.com/RxSwiftCommunity/RxDataSources/archive/refs/tags/4.0.1.zip",
+ "api_data_url": "https://raw.githubusercontent.com/CocoaPods/Specs/blob/master/Specs/9/a/4/RxDataSources/4.0.1/RxDataSources.podspec.json",
+ "datasource_id": "cocoapods_podspec",
+ "purl": "pkg:cocoapods/RxDataSources@4.0.1"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests/packagedcode/data/debian/basic/status-package-only.expected b/tests/packagedcode/data/debian/basic/status-package-only.expected
new file mode 100644
index 00000000000..9266be9acd2
--- /dev/null
+++ b/tests/packagedcode/data/debian/basic/status-package-only.expected
@@ -0,0 +1,348 @@
+[
+ {
+ "type": "deb",
+ "namespace": "ubuntu",
+ "name": "libncurses5",
+ "version": "6.1-1ubuntu1.18.04",
+ "qualifiers": {
+ "arch": "amd64"
+ },
+ "subpath": null,
+ "primary_language": null,
+ "description": "shared libraries for terminal handling\n The ncurses library routines are a terminal-independent method of\n updating character screens with reasonable optimization.\n .\n This package contains the shared libraries necessary to run programs\n compiled with ncurses.",
+ "release_date": null,
+ "parties": [
+ {
+ "type": null,
+ "role": "maintainer",
+ "name": "Ubuntu Developers",
+ "email": "ubuntu-devel-discuss@lists.ubuntu.com",
+ "url": null
+ }
+ ],
+ "keywords": [
+ "libs"
+ ],
+ "homepage_url": "https://invisible-island.net/ncurses/",
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [
+ "pkg:deb/ubuntu/ncurses"
+ ],
+ "file_references": [],
+ "extra_data": {
+ "multi_arch": "same"
+ },
+ "dependencies": [],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "debian_installed_status_db",
+ "purl": "pkg:deb/ubuntu/libncurses5@6.1-1ubuntu1.18.04?arch=amd64"
+ },
+ {
+ "type": "deb",
+ "namespace": "ubuntu",
+ "name": "libcom-err2",
+ "version": "1.44.1-1ubuntu1.1",
+ "qualifiers": {
+ "arch": "amd64"
+ },
+ "subpath": null,
+ "primary_language": null,
+ "description": "common error description library\n libcomerr is an attempt to present a common error-handling mechanism to\n manipulate the most common form of error code in a fashion that does not\n have the problems identified with mechanisms commonly in use.",
+ "release_date": null,
+ "parties": [
+ {
+ "type": null,
+ "role": "maintainer",
+ "name": "Ubuntu Developers",
+ "email": "ubuntu-devel-discuss@lists.ubuntu.com",
+ "url": null
+ }
+ ],
+ "keywords": [
+ "libs"
+ ],
+ "homepage_url": "http://e2fsprogs.sourceforge.net",
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [
+ "pkg:deb/ubuntu/e2fsprogs"
+ ],
+ "file_references": [],
+ "extra_data": {
+ "multi_arch": "same"
+ },
+ "dependencies": [],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "debian_installed_status_db",
+ "purl": "pkg:deb/ubuntu/libcom-err2@1.44.1-1ubuntu1.1?arch=amd64"
+ },
+ {
+ "type": "deb",
+ "namespace": "ubuntu",
+ "name": "libapt-pkg5.0",
+ "version": "1.6.11",
+ "qualifiers": {
+ "arch": "amd64"
+ },
+ "subpath": null,
+ "primary_language": null,
+ "description": "package management runtime library\n This library provides the common functionality for searching and\n managing packages as well as information about packages.\n Higher-level package managers can depend upon this library.\n .\n This includes:\n * retrieval of information about packages from multiple sources\n * retrieval of packages and all dependent packages\n needed to satisfy a request either through an internal\n solver or by interfacing with an external one\n * authenticating the sources and validating the retrieved data\n * installation and removal of packages in the system\n * providing different transports to retrieve data over cdrom, ftp,\n http(s), rsh as well as an interface to add more transports like\n tor+http(s) (apt-transport-tor).",
+ "release_date": null,
+ "parties": [
+ {
+ "type": null,
+ "role": "maintainer",
+ "name": "Ubuntu Developers",
+ "email": "ubuntu-devel-discuss@lists.ubuntu.com",
+ "url": null
+ }
+ ],
+ "keywords": [
+ "libs"
+ ],
+ "homepage_url": null,
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [
+ "pkg:deb/ubuntu/apt"
+ ],
+ "file_references": [],
+ "extra_data": {
+ "multi_arch": "same"
+ },
+ "dependencies": [],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "debian_installed_status_db",
+ "purl": "pkg:deb/ubuntu/libapt-pkg5.0@1.6.11?arch=amd64"
+ },
+ {
+ "type": "deb",
+ "namespace": "ubuntu",
+ "name": "libaudit1",
+ "version": "1:2.8.2-1ubuntu1",
+ "qualifiers": {
+ "arch": "amd64"
+ },
+ "subpath": null,
+ "primary_language": null,
+ "description": "Dynamic library for security auditing\n The audit-libs package contains the dynamic libraries needed for\n applications to use the audit framework. It is used to monitor systems for\n security related events.",
+ "release_date": null,
+ "parties": [
+ {
+ "type": null,
+ "role": "maintainer",
+ "name": "Ubuntu Developers",
+ "email": "ubuntu-devel-discuss@lists.ubuntu.com",
+ "url": null
+ }
+ ],
+ "keywords": [
+ "libs"
+ ],
+ "homepage_url": "https://people.redhat.com/sgrubb/audit/",
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [
+ "pkg:deb/ubuntu/audit"
+ ],
+ "file_references": [],
+ "extra_data": {
+ "multi_arch": "same"
+ },
+ "dependencies": [],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "debian_installed_status_db",
+ "purl": "pkg:deb/ubuntu/libaudit1@1:2.8.2-1ubuntu1?arch=amd64"
+ },
+ {
+ "type": "deb",
+ "namespace": "ubuntu",
+ "name": "perl-base",
+ "version": "5.26.1-6ubuntu0.3",
+ "qualifiers": {
+ "arch": "amd64"
+ },
+ "subpath": null,
+ "primary_language": null,
+ "description": "minimal Perl system\n Perl is a scripting language used in many system scripts and utilities.\n .\n This package provides a Perl interpreter and the small subset of the\n standard run-time library required to perform basic tasks. For a full\n Perl installation, install \"perl\" (and its dependencies, \"perl-modules-5.26\"\n and \"perl-doc\").",
+ "release_date": null,
+ "parties": [
+ {
+ "type": null,
+ "role": "maintainer",
+ "name": "Ubuntu Developers",
+ "email": "ubuntu-devel-discuss@lists.ubuntu.com",
+ "url": null
+ }
+ ],
+ "keywords": [
+ "perl"
+ ],
+ "homepage_url": "http://dev.perl.org/perl5/",
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [
+ "pkg:deb/ubuntu/perl"
+ ],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "debian_installed_status_db",
+ "purl": "pkg:deb/ubuntu/perl-base@5.26.1-6ubuntu0.3?arch=amd64"
+ },
+ {
+ "type": "deb",
+ "namespace": "ubuntu",
+ "name": "libudev1",
+ "version": "237-3ubuntu10.22",
+ "qualifiers": {
+ "arch": "amd64"
+ },
+ "subpath": null,
+ "primary_language": null,
+ "description": "libudev shared library\n This library provides access to udev device information.",
+ "release_date": null,
+ "parties": [
+ {
+ "type": null,
+ "role": "maintainer",
+ "name": "Ubuntu Developers",
+ "email": "ubuntu-devel-discuss@lists.ubuntu.com",
+ "url": null
+ }
+ ],
+ "keywords": [
+ "libs"
+ ],
+ "homepage_url": "https://www.freedesktop.org/wiki/Software/systemd",
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [
+ "pkg:deb/ubuntu/systemd"
+ ],
+ "file_references": [],
+ "extra_data": {
+ "multi_arch": "same"
+ },
+ "dependencies": [],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "debian_installed_status_db",
+ "purl": "pkg:deb/ubuntu/libudev1@237-3ubuntu10.22?arch=amd64"
+ }
+]
\ No newline at end of file
diff --git a/tests/packagedcode/data/freebsd/basic/+COMPACT_MANIFEST-package-only.expected b/tests/packagedcode/data/freebsd/basic/+COMPACT_MANIFEST-package-only.expected
new file mode 100644
index 00000000000..9d7a4937eaf
--- /dev/null
+++ b/tests/packagedcode/data/freebsd/basic/+COMPACT_MANIFEST-package-only.expected
@@ -0,0 +1,62 @@
+[
+ {
+ "type": "freebsd",
+ "namespace": null,
+ "name": "dmidecode",
+ "version": "2.12",
+ "qualifiers": {
+ "arch": "freebsd:10:x86:64",
+ "origin": "sysutils/dmidecode"
+ },
+ "subpath": null,
+ "primary_language": null,
+ "description": "Dmidecode is a tool or dumping a computer's DMI (some say SMBIOS) table\ncontents in a human-readable format. The output contains a description of the\nsystem's hardware components, as well as other useful pieces of information\nsuch as serial numbers and BIOS revision.\n\nWWW: http://www.nongnu.org/dmidecode/",
+ "release_date": null,
+ "parties": [
+ {
+ "type": "person",
+ "role": "maintainer",
+ "name": null,
+ "email": "anders@FreeBSD.org",
+ "url": null
+ }
+ ],
+ "keywords": [
+ "sysutils"
+ ],
+ "homepage_url": "http://www.nongnu.org/dmidecode/",
+ "download_url": "https://pkg.freebsd.org/freebsd:10:x86:64/latest/All/dmidecode-2.12.txz",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": "https://svnweb.freebsd.org/ports/head/sysutils/dmidecode",
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": {
+ "licenses": [
+ "GPLv2"
+ ],
+ "licenselogic": "single"
+ },
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "freebsd_compact_manifest",
+ "purl": "pkg:freebsd/dmidecode@2.12?arch=freebsd:10:x86:64&origin=sysutils/dmidecode"
+ }
+]
\ No newline at end of file
diff --git a/tests/packagedcode/data/maven_misc/assemble/johnzon-jsonb-1.2.11-package-only-expected.json b/tests/packagedcode/data/maven_misc/assemble/johnzon-jsonb-1.2.11-package-only-expected.json
new file mode 100644
index 00000000000..f62caf2832a
--- /dev/null
+++ b/tests/packagedcode/data/maven_misc/assemble/johnzon-jsonb-1.2.11-package-only-expected.json
@@ -0,0 +1,395 @@
+{
+ "packages": [],
+ "dependencies": [],
+ "files": [
+ {
+ "path": "johnzon-jsonb-1.2.11",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/META-INF",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/META-INF/DEPENDENCIES",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/META-INF/LICENSE",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/META-INF/MANIFEST.MF",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "jar",
+ "namespace": "org.apache.johnzon",
+ "name": "Johnzon :: JSON-B Implementation",
+ "version": "1.2.11",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": null,
+ "description": "Apache Johnzon is an implementation of JSR-353 (JavaTM API for JSON Processing).",
+ "release_date": null,
+ "parties": [
+ {
+ "role": "vendor",
+ "name": "The Apache Software Foundation"
+ }
+ ],
+ "keywords": [],
+ "homepage_url": "http://johnzon.apache.org/johnzon-jsonb",
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": "http://www.apache.org/licenses/LICENSE-2.0.txt",
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {
+ "documentation_url": "http://johnzon.apache.org/"
+ },
+ "dependencies": [],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "java_jar_manifest",
+ "purl": "pkg:jar/org.apache.johnzon/Johnzon%20::%20JSON-B%20Implementation@1.2.11"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/META-INF/NOTICE",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/META-INF/maven",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/META-INF/maven/org.apache.johnzon",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/META-INF/maven/org.apache.johnzon/johnzon-jsonb",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/META-INF/maven/org.apache.johnzon/johnzon-jsonb/pom.properties",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "maven",
+ "namespace": "org.apache.johnzon",
+ "name": "johnzon-jsonb",
+ "version": "1.2.11",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "Java",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "maven_pom_properties",
+ "purl": "pkg:maven/org.apache.johnzon/johnzon-jsonb@1.2.11"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/META-INF/maven/org.apache.johnzon/johnzon-jsonb/pom.xml",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "maven",
+ "namespace": "org.apache.johnzon",
+ "name": "johnzon-jsonb",
+ "version": "1.2.11",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "Java",
+ "description": "Johnzon :: JSON-B Implementation",
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [
+ "pkg:maven/org.apache.johnzon/johnzon-jsonb@1.2.11?classifier=sources"
+ ],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [
+ {
+ "purl": "pkg:maven/org.apache.geronimo.specs/geronimo-annotation_1.3_spec@1.1",
+ "extracted_requirement": "1.1",
+ "scope": "provided",
+ "is_runtime": true,
+ "is_optional": true,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:maven/org.apache.geronimo.specs/geronimo-jaxrs_2.1_spec@1.1",
+ "extracted_requirement": "1.1",
+ "scope": "provided",
+ "is_runtime": true,
+ "is_optional": true,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:maven/org.apache.geronimo.specs/geronimo-jcdi_2.0_spec@1.1",
+ "extracted_requirement": "1.1",
+ "scope": "provided",
+ "is_runtime": true,
+ "is_optional": true,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:maven/org.apache.geronimo.specs/geronimo-jsonb_1.0_spec",
+ "extracted_requirement": null,
+ "scope": "provided",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:maven/org.apache.johnzon/johnzon-mapper@1.2.11",
+ "extracted_requirement": "1.2.11",
+ "scope": "compile",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:maven/org.apache.cxf/cxf-rt-rs-client",
+ "extracted_requirement": "${cxf.version}",
+ "scope": "test",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:maven/org.apache.cxf/cxf-rt-frontend-jaxrs",
+ "extracted_requirement": "${cxf.version}",
+ "scope": "test",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:maven/org.apache.cxf/cxf-rt-transports-local",
+ "extracted_requirement": "${cxf.version}",
+ "scope": "test",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:maven/org.apache.geronimo.specs/geronimo-interceptor_1.2_spec@1.0",
+ "extracted_requirement": "1.0",
+ "scope": "test",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:maven/org.apache.geronimo.specs/geronimo-atinject_1.0_spec@1.0",
+ "extracted_requirement": "1.0",
+ "scope": "test",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": true,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:maven/org.apache.openwebbeans/openwebbeans-impl",
+ "extracted_requirement": "${owb.version}",
+ "scope": "test",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ }
+ ],
+ "repository_homepage_url": "https://repo1.maven.org/maven2/org/apache/johnzon/johnzon-jsonb/1.2.11/",
+ "repository_download_url": "https://repo1.maven.org/maven2/org/apache/johnzon/johnzon-jsonb/1.2.11/johnzon-jsonb-1.2.11.jar",
+ "api_data_url": "https://repo1.maven.org/maven2/org/apache/johnzon/johnzon-jsonb/1.2.11/johnzon-jsonb-1.2.11.pom",
+ "datasource_id": "maven_pom",
+ "purl": "pkg:maven/org.apache.johnzon/johnzon-jsonb@1.2.11"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/org",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/org/apache",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/org/apache/johnzon",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/org/apache/johnzon/jaxrs",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/org/apache/johnzon/jaxrs/jsonb",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/org/apache/johnzon/jaxrs/jsonb/jaxrs",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/org/apache/johnzon/jaxrs/jsonb/jaxrs/JsonbJaxrsProvider.class",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/org/apache/johnzon/jsonb",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "johnzon-jsonb-1.2.11/org/apache/johnzon/jsonb/JohnzonBuilder.class",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests/packagedcode/data/npm/scan-nested/scan-package-only.expected.json b/tests/packagedcode/data/npm/scan-nested/scan-package-only.expected.json
new file mode 100644
index 00000000000..ff9ce0e7f18
--- /dev/null
+++ b/tests/packagedcode/data/npm/scan-nested/scan-package-only.expected.json
@@ -0,0 +1,952 @@
+{
+ "packages": [],
+ "dependencies": [],
+ "files": [
+ {
+ "path": "scan",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "scan/node_modules",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "scan/node_modules/sequelize",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "scan/node_modules/sequelize/package-lock.json",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "npm",
+ "namespace": "",
+ "name": "sequelize",
+ "version": "3.30.2",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "JavaScript",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {
+ "lockfile_version": 1
+ },
+ "dependencies": [
+ {
+ "purl": "pkg:npm/ansi-regex@2.1.1",
+ "extracted_requirement": "2.1.1",
+ "scope": "devDependencies",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": true,
+ "resolved_package": {
+ "type": "npm",
+ "namespace": "",
+ "name": "ansi-regex",
+ "version": "2.1.1",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "JavaScript",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
+ "size": null,
+ "sha1": "c3b33ab5ee360d86e0e628f0468ae7ef27d654df",
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [
+ [
+ {
+ "path": "ansi-regex",
+ "size": 0,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ }
+ ]
+ ],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": "https://www.npmjs.com/package/ansi-regex",
+ "repository_download_url": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
+ "api_data_url": "https://registry.npmjs.org/ansi-regex/2.1.1",
+ "datasource_id": "npm_package_lock_json",
+ "purl": "pkg:npm/ansi-regex@2.1.1"
+ },
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:npm/to-fast-properties@1.0.3",
+ "extracted_requirement": "1.0.3",
+ "scope": "devDependencies",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": true,
+ "resolved_package": {
+ "type": "npm",
+ "namespace": "",
+ "name": "to-fast-properties",
+ "version": "1.0.3",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "JavaScript",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz",
+ "size": null,
+ "sha1": "b83571fa4d8c25b82e231b06e3a3055de4ca1a47",
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [
+ [
+ {
+ "path": "to-fast-properties",
+ "size": 0,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ }
+ ]
+ ],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": "https://www.npmjs.com/package/to-fast-properties",
+ "repository_download_url": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz",
+ "api_data_url": "https://registry.npmjs.org/to-fast-properties/1.0.3",
+ "datasource_id": "npm_package_lock_json",
+ "purl": "pkg:npm/to-fast-properties@1.0.3"
+ },
+ "extra_data": {}
+ }
+ ],
+ "repository_homepage_url": "https://www.npmjs.com/package/sequelize",
+ "repository_download_url": "https://registry.npmjs.org/sequelize/-/sequelize-3.30.2.tgz",
+ "api_data_url": "https://registry.npmjs.org/sequelize/3.30.2",
+ "datasource_id": "npm_package_lock_json",
+ "purl": "pkg:npm/sequelize@3.30.2"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "scan/node_modules/sequelize/package.json",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "npm",
+ "namespace": null,
+ "name": "sequelize",
+ "version": "3.30.2",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "JavaScript",
+ "description": "Multi dialect ORM for Node.JS/io.js",
+ "release_date": null,
+ "parties": [
+ {
+ "type": "person",
+ "role": "author",
+ "name": "Sascha Depold",
+ "email": "sascha@depold.com",
+ "url": null
+ },
+ {
+ "type": "person",
+ "role": "contributor",
+ "name": "Sascha Depold",
+ "email": "sascha@depold.com",
+ "url": null
+ },
+ {
+ "type": "person",
+ "role": "contributor",
+ "name": "Jan Aagaard Meier",
+ "email": "janzeh@gmail.com\njmei@itu.dk",
+ "url": null
+ },
+ {
+ "type": "person",
+ "role": "contributor",
+ "name": "Daniel Durante",
+ "email": "me@danieldurante.com",
+ "url": null
+ },
+ {
+ "type": "person",
+ "role": "contributor",
+ "name": "Mick Hansen",
+ "email": "mick.kasper.hansen@gmail.com",
+ "url": null
+ }
+ ],
+ "keywords": [
+ "mysql",
+ "nodejs",
+ "object relational mapper"
+ ],
+ "homepage_url": null,
+ "download_url": "https://registry.npmjs.org/sequelize/-/sequelize-3.30.2.tgz",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": "https://github.com/sequelize/sequelize/issues",
+ "code_view_url": null,
+ "vcs_url": "git+https://github.com/sequelize/sequelize.git",
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": [
+ "MIT"
+ ],
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [
+ {
+ "purl": "pkg:npm/bluebird",
+ "extracted_requirement": "^3.3.4",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:npm/wkx",
+ "extracted_requirement": "0.2.0",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:npm/chai",
+ "extracted_requirement": "^3.5.0",
+ "scope": "devDependencies",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:npm/chai-as-promised",
+ "extracted_requirement": "^5.1.0",
+ "scope": "devDependencies",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:npm/watchr",
+ "extracted_requirement": "~2.4.11",
+ "scope": "devDependencies",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ }
+ ],
+ "repository_homepage_url": "https://www.npmjs.com/package/sequelize",
+ "repository_download_url": "https://registry.npmjs.org/sequelize/-/sequelize-3.30.2.tgz",
+ "api_data_url": "https://registry.npmjs.org/sequelize/3.30.2",
+ "datasource_id": "npm_package_json",
+ "purl": "pkg:npm/sequelize@3.30.2"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "scan/node_modules/sequelize/yarn.lock",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "npm",
+ "namespace": null,
+ "name": null,
+ "version": null,
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "JavaScript",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [
+ {
+ "purl": "pkg:npm/abbrev@1.0.9",
+ "extracted_requirement": "1",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {
+ "type": "npm",
+ "namespace": "",
+ "name": "abbrev",
+ "version": "1.0.9",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "JavaScript",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz",
+ "size": null,
+ "sha1": "91b4792588a7738c25f35dd6f63752a2f8776135",
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": "https://www.npmjs.com/package/abbrev",
+ "repository_download_url": "https://registry.npmjs.org/abbrev/-/abbrev-1.0.9.tgz",
+ "api_data_url": "https://registry.npmjs.org/abbrev/1.0.9",
+ "datasource_id": "yarn_lock_v1",
+ "purl": "pkg:npm/abbrev@1.0.9"
+ },
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:npm/xdg-basedir@2.0.0",
+ "extracted_requirement": "^2.0.0",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {
+ "type": "npm",
+ "namespace": "",
+ "name": "xdg-basedir",
+ "version": "2.0.0",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "JavaScript",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-2.0.0.tgz",
+ "size": null,
+ "sha1": "edbc903cc385fc04523d966a335504b5504d1bd2",
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [
+ {
+ "purl": "pkg:npm/os-homedir",
+ "extracted_requirement": "^1.0.0",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ }
+ ],
+ "repository_homepage_url": "https://www.npmjs.com/package/xdg-basedir",
+ "repository_download_url": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-2.0.0.tgz",
+ "api_data_url": "https://registry.npmjs.org/xdg-basedir/2.0.0",
+ "datasource_id": "yarn_lock_v1",
+ "purl": "pkg:npm/xdg-basedir@2.0.0"
+ },
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:npm/xtend@4.0.1",
+ "extracted_requirement": "^4.0.0 ~4.0.1",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {
+ "type": "npm",
+ "namespace": "",
+ "name": "xtend",
+ "version": "4.0.1",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "JavaScript",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz",
+ "size": null,
+ "sha1": "a5c6d532be656e23db820efb943a1f04998d63af",
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": "https://www.npmjs.com/package/xtend",
+ "repository_download_url": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz",
+ "api_data_url": "https://registry.npmjs.org/xtend/4.0.1",
+ "datasource_id": "yarn_lock_v1",
+ "purl": "pkg:npm/xtend@4.0.1"
+ },
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:npm/yallist@2.0.0",
+ "extracted_requirement": "^2.0.0",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {
+ "type": "npm",
+ "namespace": "",
+ "name": "yallist",
+ "version": "2.0.0",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "JavaScript",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": "https://registry.yarnpkg.com/yallist/-/yallist-2.0.0.tgz",
+ "size": null,
+ "sha1": "306c543835f09ee1a4cb23b7bce9ab341c91cdd4",
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": "https://www.npmjs.com/package/yallist",
+ "repository_download_url": "https://registry.npmjs.org/yallist/-/yallist-2.0.0.tgz",
+ "api_data_url": "https://registry.npmjs.org/yallist/2.0.0",
+ "datasource_id": "yarn_lock_v1",
+ "purl": "pkg:npm/yallist@2.0.0"
+ },
+ "extra_data": {}
+ }
+ ],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "yarn_lock_v1",
+ "purl": null
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "scan/package-lock.json",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "npm",
+ "namespace": "",
+ "name": "lodash.clonedeep",
+ "version": "4.3.2",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "JavaScript",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {
+ "lockfile_version": 1
+ },
+ "dependencies": [
+ {
+ "purl": "pkg:npm/ms@2.0.0",
+ "extracted_requirement": "2.0.0",
+ "scope": "devDependencies",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": true,
+ "resolved_package": {
+ "type": "npm",
+ "namespace": "",
+ "name": "ms",
+ "version": "2.0.0",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "JavaScript",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "size": null,
+ "sha1": "5608aeadfc00be6c2901df5f9861788de0d597c8",
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [
+ [
+ {
+ "path": "ms",
+ "size": 0,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ }
+ ]
+ ],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": "https://www.npmjs.com/package/ms",
+ "repository_download_url": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
+ "api_data_url": "https://registry.npmjs.org/ms/2.0.0",
+ "datasource_id": "npm_package_lock_json",
+ "purl": "pkg:npm/ms@2.0.0"
+ },
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:npm/regenerator-runtime@0.11.0",
+ "extracted_requirement": "0.11.0",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": true,
+ "resolved_package": {
+ "type": "npm",
+ "namespace": "",
+ "name": "regenerator-runtime",
+ "version": "0.11.0",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "JavaScript",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.0.tgz",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": "fda03490b7916f937d2b47787f0ee8a046c8fb10def83283e3df4442aca01aa792f0ddf1b68d79a74f6e636c63ee2c4ff35b0d3d7bd12e701022a7554ad81bd4",
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [
+ [
+ {
+ "path": "regenerator-runtime",
+ "size": 0,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ }
+ ]
+ ],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": "https://www.npmjs.com/package/regenerator-runtime",
+ "repository_download_url": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.11.0.tgz",
+ "api_data_url": "https://registry.npmjs.org/regenerator-runtime/0.11.0",
+ "datasource_id": "npm_package_lock_json",
+ "purl": "pkg:npm/regenerator-runtime@0.11.0"
+ },
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:npm/to-fast-properties@1.0.3",
+ "extracted_requirement": "1.0.3",
+ "scope": "devDependencies",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": true,
+ "resolved_package": {
+ "type": "npm",
+ "namespace": "",
+ "name": "to-fast-properties",
+ "version": "1.0.3",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "JavaScript",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz",
+ "size": null,
+ "sha1": "b83571fa4d8c25b82e231b06e3a3055de4ca1a47",
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [
+ [
+ {
+ "path": "to-fast-properties",
+ "size": 0,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ }
+ ]
+ ],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": "https://www.npmjs.com/package/to-fast-properties",
+ "repository_download_url": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz",
+ "api_data_url": "https://registry.npmjs.org/to-fast-properties/1.0.3",
+ "datasource_id": "npm_package_lock_json",
+ "purl": "pkg:npm/to-fast-properties@1.0.3"
+ },
+ "extra_data": {}
+ }
+ ],
+ "repository_homepage_url": "https://www.npmjs.com/package/lodash.clonedeep",
+ "repository_download_url": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.3.2.tgz",
+ "api_data_url": "https://registry.npmjs.org/lodash.clonedeep/4.3.2",
+ "datasource_id": "npm_package_lock_json",
+ "purl": "pkg:npm/lodash.clonedeep@4.3.2"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "scan/package.json",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "npm",
+ "namespace": null,
+ "name": "lodash.clonedeep",
+ "version": "4.3.2",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "JavaScript",
+ "description": "The lodash method `_.cloneDeep` exported as a module.",
+ "release_date": null,
+ "parties": [
+ {
+ "type": "person",
+ "role": "author",
+ "name": "John-David Dalton",
+ "email": "john.david.dalton@gmail.com",
+ "url": "http://allyoucanleet.com/"
+ },
+ {
+ "type": "person",
+ "role": "contributor",
+ "name": "John-David Dalton",
+ "email": "john.david.dalton@gmail.com",
+ "url": "http://allyoucanleet.com/"
+ },
+ {
+ "type": "person",
+ "role": "contributor",
+ "name": "Blaine Bublitz",
+ "email": "blaine.bublitz@gmail.com",
+ "url": "https://github.com/phated"
+ },
+ {
+ "type": "person",
+ "role": "contributor",
+ "name": "Mathias Bynens",
+ "email": "mathias@qiwi.be",
+ "url": "https://mathiasbynens.be/"
+ }
+ ],
+ "keywords": [
+ "lodash-modularized",
+ "clonedeep"
+ ],
+ "homepage_url": "https://lodash.com/",
+ "download_url": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.3.2.tgz",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": "https://github.com/lodash/lodash",
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": [
+ "MIT"
+ ],
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [
+ {
+ "purl": "pkg:npm/lodash._baseclone",
+ "extracted_requirement": "~4.5.0",
+ "scope": "dependencies",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ }
+ ],
+ "repository_homepage_url": "https://www.npmjs.com/package/lodash.clonedeep",
+ "repository_download_url": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.3.2.tgz",
+ "api_data_url": "https://registry.npmjs.org/lodash.clonedeep/4.3.2",
+ "datasource_id": "npm_package_json",
+ "purl": "pkg:npm/lodash.clonedeep@4.3.2"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests/packagedcode/data/nuget/Castle.Core.nuspec-package-only.json.expected b/tests/packagedcode/data/nuget/Castle.Core.nuspec-package-only.json.expected
new file mode 100644
index 00000000000..4e64d83d267
--- /dev/null
+++ b/tests/packagedcode/data/nuget/Castle.Core.nuspec-package-only.json.expected
@@ -0,0 +1,190 @@
+[
+ {
+ "type": "nuget",
+ "namespace": null,
+ "name": "Castle.Core",
+ "version": "4.2.1",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": null,
+ "description": "Castle Core, including DynamicProxy, Logging Abstractions and DictionaryAdapter",
+ "release_date": null,
+ "parties": [
+ {
+ "type": null,
+ "role": "author",
+ "name": "Castle Project Contributors",
+ "email": null,
+ "url": null
+ },
+ {
+ "type": null,
+ "role": "owner",
+ "name": "Castle Project Contributors",
+ "email": null,
+ "url": null
+ }
+ ],
+ "keywords": [],
+ "homepage_url": "http://www.castleproject.org/",
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": "git+https://github.com/castleproject/Core",
+ "copyright": "Copyright (c) 2004-2017 Castle Project - http://www.castleproject.org/",
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": "http://www.apache.org/licenses/LICENSE-2.0.html",
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [
+ {
+ "purl": "pkg:nuget/NETStandard.Library",
+ "extracted_requirement": "1.6.1",
+ "scope": "dependency",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "framework": ".NETStandard1.3",
+ "exclude": "Build,Analyzers"
+ }
+ },
+ {
+ "purl": "pkg:nuget/System.Diagnostics.TraceSource",
+ "extracted_requirement": "4.3.0",
+ "scope": "dependency",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "framework": ".NETStandard1.3",
+ "exclude": "Build,Analyzers"
+ }
+ },
+ {
+ "purl": "pkg:nuget/System.Reflection.TypeExtensions",
+ "extracted_requirement": "4.3.0",
+ "scope": "dependency",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "framework": ".NETStandard1.3",
+ "exclude": "Build,Analyzers"
+ }
+ },
+ {
+ "purl": "pkg:nuget/System.Xml.XmlDocument",
+ "extracted_requirement": "4.3.0",
+ "scope": "dependency",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "framework": ".NETStandard1.3",
+ "exclude": "Build,Analyzers"
+ }
+ },
+ {
+ "purl": "pkg:nuget/System.Dynamic.Runtime",
+ "extracted_requirement": "4.3.0",
+ "scope": "dependency",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "framework": ".NETStandard1.3",
+ "exclude": "Build,Analyzers"
+ }
+ },
+ {
+ "purl": "pkg:nuget/System.Reflection",
+ "extracted_requirement": "4.3.0",
+ "scope": "dependency",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "framework": ".NETStandard1.3",
+ "exclude": "Build,Analyzers"
+ }
+ },
+ {
+ "purl": "pkg:nuget/System.Reflection.Emit",
+ "extracted_requirement": "4.3.0",
+ "scope": "dependency",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "framework": ".NETStandard1.3",
+ "exclude": "Build,Analyzers"
+ }
+ },
+ {
+ "purl": "pkg:nuget/System.Collections.Specialized",
+ "extracted_requirement": "4.3.0",
+ "scope": "dependency",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "framework": ".NETStandard1.3",
+ "exclude": "Build,Analyzers"
+ }
+ },
+ {
+ "purl": "pkg:nuget/System.ComponentModel",
+ "extracted_requirement": "4.3.0",
+ "scope": "dependency",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "framework": ".NETStandard1.3",
+ "exclude": "Build,Analyzers"
+ }
+ },
+ {
+ "purl": "pkg:nuget/System.ComponentModel.TypeConverter",
+ "extracted_requirement": "4.3.0",
+ "scope": "dependency",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "framework": ".NETStandard1.3",
+ "exclude": "Build,Analyzers"
+ }
+ }
+ ],
+ "repository_homepage_url": "https://www.nuget.org/packages/Castle.Core/4.2.1",
+ "repository_download_url": "https://www.nuget.org/api/v2/package/Castle.Core/4.2.1",
+ "api_data_url": "https://api.nuget.org/v3/registration3/castle.core/4.2.1.json",
+ "datasource_id": "nuget_nupsec",
+ "purl": "pkg:nuget/Castle.Core@4.2.1"
+ }
+]
\ No newline at end of file
diff --git a/tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-only-expected.json b/tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-only-expected.json
new file mode 100644
index 00000000000..57199d08212
--- /dev/null
+++ b/tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-only-expected.json
@@ -0,0 +1,821 @@
+{
+ "packages": [],
+ "dependencies": [],
+ "files": [
+ {
+ "path": "AUTHORS.txt",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "LICENSE.txt",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "MANIFEST.in",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "NEWS.rst",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "PKG-INFO",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "pypi",
+ "namespace": null,
+ "name": "pip",
+ "version": "22.0.4",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "Python",
+ "description": "The PyPA recommended tool for installing Python packages.\npip - The Python Package Installer\n==================================\n\n.. image:: https://img.shields.io/pypi/v/pip.svg\n :target: https://pypi.org/project/pip/\n\n.. image:: https://readthedocs.org/projects/pip/badge/?version=latest\n :target: https://pip.pypa.io/en/latest\n\npip is the `package installer`_ for Python. You can use pip to install packages from the `Python Package Index`_ and other indexes.\n\nPlease take a look at our documentation for how to install and use pip:\n\n* `Installation`_\n* `Usage`_\n\nWe release updates regularly, with a new version every 3 months. Find more details in our documentation:\n\n* `Release notes`_\n* `Release process`_\n\nIn pip 20.3, we've `made a big improvement to the heart of pip`_; `learn more`_. We want your input, so `sign up for our user experience research studies`_ to help us do it right.\n\n**Note**: pip 21.0, in January 2021, removed Python 2 support, per pip's `Python 2 support policy`_. Please migrate to Python 3.\n\nIf you find bugs, need help, or want to talk to the developers, please use our mailing lists or chat rooms:\n\n* `Issue tracking`_\n* `Discourse channel`_\n* `User IRC`_\n\nIf you want to get involved head over to GitHub to get the source code, look at our development documentation and feel free to jump on the developer mailing lists and chat rooms:\n\n* `GitHub page`_\n* `Development documentation`_\n* `Development mailing list`_\n* `Development IRC`_\n\nCode of Conduct\n---------------\n\nEveryone interacting in the pip project's codebases, issue trackers, chat\nrooms, and mailing lists is expected to follow the `PSF Code of Conduct`_.\n\n.. _package installer: https://packaging.python.org/guides/tool-recommendations/\n.. _Python Package Index: https://pypi.org\n.. _Installation: https://pip.pypa.io/en/stable/installation/\n.. _Usage: https://pip.pypa.io/en/stable/\n.. _Release notes: https://pip.pypa.io/en/stable/news.html\n.. _Release process: https://pip.pypa.io/en/latest/development/release-process/\n.. _GitHub page: https://github.com/pypa/pip\n.. _Development documentation: https://pip.pypa.io/en/latest/development\n.. _made a big improvement to the heart of pip: https://pyfound.blogspot.com/2020/11/pip-20-3-new-resolver.html\n.. _learn more: https://pip.pypa.io/en/latest/user_guide/#changes-to-the-pip-dependency-resolver-in-20-3-2020\n.. _sign up for our user experience research studies: https://pyfound.blogspot.com/2020/03/new-pip-resolver-to-roll-out-this-year.html\n.. _Python 2 support policy: https://pip.pypa.io/en/latest/development/release-process/#python-2-support\n.. _Issue tracking: https://github.com/pypa/pip/issues\n.. _Discourse channel: https://discuss.python.org/c/packaging\n.. _Development mailing list: https://mail.python.org/mailman3/lists/distutils-sig.python.org/\n.. _User IRC: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa\n.. _Development IRC: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa-dev\n.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md",
+ "release_date": null,
+ "parties": [
+ {
+ "type": "person",
+ "role": "author",
+ "name": "The pip developers",
+ "email": "distutils-sig@python.org",
+ "url": null
+ }
+ ],
+ "keywords": [
+ "Development Status :: 5 - Production/Stable",
+ "Intended Audience :: Developers",
+ "Topic :: Software Development :: Build Tools",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3 :: Only",
+ "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: Implementation :: CPython",
+ "Programming Language :: Python :: Implementation :: PyPy"
+ ],
+ "homepage_url": "https://pip.pypa.io/",
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": "https://github.com/pypa/pip",
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": "license: MIT\nclassifiers:\n - 'License :: OSI Approved :: MIT License'\n",
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {
+ "Documentation": "https://pip.pypa.io",
+ "Changelog": "https://pip.pypa.io/en/stable/news/"
+ },
+ "dependencies": [],
+ "repository_homepage_url": "https://pypi.org/project/pip",
+ "repository_download_url": "https://pypi.org/packages/source/p/pip/pip-22.0.4.tar.gz",
+ "api_data_url": "https://pypi.org/pypi/pip/22.0.4/json",
+ "datasource_id": "pypi_sdist_pkginfo",
+ "purl": "pkg:pypi/pip@22.0.4"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "README.rst",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "docs",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "docs/requirements.txt",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "pypi",
+ "namespace": null,
+ "name": null,
+ "version": null,
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "Python",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [
+ {
+ "purl": "pkg:pypi/sphinx",
+ "extracted_requirement": "sphinx~=4.2,!=4.4.0",
+ "scope": "install",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "is_editable": false,
+ "link": null,
+ "hash_options": [],
+ "is_constraint": false,
+ "is_archive": null,
+ "is_wheel": false,
+ "is_url": null,
+ "is_vcs_url": null,
+ "is_name_at_url": false,
+ "is_local_path": null
+ }
+ },
+ {
+ "purl": "pkg:pypi/towncrier",
+ "extracted_requirement": "towncrier",
+ "scope": "install",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "is_editable": false,
+ "link": null,
+ "hash_options": [],
+ "is_constraint": false,
+ "is_archive": null,
+ "is_wheel": false,
+ "is_url": null,
+ "is_vcs_url": null,
+ "is_name_at_url": false,
+ "is_local_path": null
+ }
+ },
+ {
+ "purl": "pkg:pypi/furo",
+ "extracted_requirement": "furo",
+ "scope": "install",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "is_editable": false,
+ "link": null,
+ "hash_options": [],
+ "is_constraint": false,
+ "is_archive": null,
+ "is_wheel": false,
+ "is_url": null,
+ "is_vcs_url": null,
+ "is_name_at_url": false,
+ "is_local_path": null
+ }
+ },
+ {
+ "purl": "pkg:pypi/myst-parser",
+ "extracted_requirement": "myst_parser",
+ "scope": "install",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "is_editable": false,
+ "link": null,
+ "hash_options": [],
+ "is_constraint": false,
+ "is_archive": null,
+ "is_wheel": false,
+ "is_url": null,
+ "is_vcs_url": null,
+ "is_name_at_url": false,
+ "is_local_path": null
+ }
+ },
+ {
+ "purl": "pkg:pypi/sphinx-copybutton",
+ "extracted_requirement": "sphinx-copybutton",
+ "scope": "install",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "is_editable": false,
+ "link": null,
+ "hash_options": [],
+ "is_constraint": false,
+ "is_archive": null,
+ "is_wheel": false,
+ "is_url": null,
+ "is_vcs_url": null,
+ "is_name_at_url": false,
+ "is_local_path": null
+ }
+ },
+ {
+ "purl": "pkg:pypi/sphinx-inline-tabs",
+ "extracted_requirement": "sphinx-inline-tabs",
+ "scope": "install",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "is_editable": false,
+ "link": null,
+ "hash_options": [],
+ "is_constraint": false,
+ "is_archive": null,
+ "is_wheel": false,
+ "is_url": null,
+ "is_vcs_url": null,
+ "is_name_at_url": false,
+ "is_local_path": null
+ }
+ },
+ {
+ "purl": "pkg:pypi/sphinxcontrib-towncrier",
+ "extracted_requirement": "sphinxcontrib-towncrier>=0.2.0a0",
+ "scope": "install",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "is_editable": false,
+ "link": null,
+ "hash_options": [],
+ "is_constraint": false,
+ "is_archive": null,
+ "is_wheel": false,
+ "is_url": null,
+ "is_vcs_url": null,
+ "is_name_at_url": false,
+ "is_local_path": null
+ }
+ },
+ {
+ "purl": null,
+ "extracted_requirement": ".",
+ "scope": "install",
+ "is_runtime": true,
+ "is_optional": false,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {
+ "is_editable": false,
+ "link": ".",
+ "hash_options": [],
+ "is_constraint": false,
+ "is_archive": false,
+ "is_wheel": null,
+ "is_url": false,
+ "is_vcs_url": false,
+ "is_name_at_url": false,
+ "is_local_path": true
+ }
+ }
+ ],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "pip_requirements",
+ "purl": null
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "pyproject.toml",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "pypi",
+ "namespace": null,
+ "name": null,
+ "version": null,
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "Python",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "pypi_pyproject_toml",
+ "purl": null
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "setup.cfg",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "pypi",
+ "namespace": null,
+ "name": null,
+ "version": null,
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "Python",
+ "description": null,
+ "release_date": null,
+ "parties": [],
+ "keywords": [],
+ "homepage_url": null,
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "pypi_setup_cfg",
+ "purl": null
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "setup.py",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "pypi",
+ "namespace": null,
+ "name": "pip",
+ "version": "22.0.4",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "Python",
+ "description": "The PyPA recommended tool for installing Python packages.",
+ "release_date": null,
+ "parties": [
+ {
+ "type": "person",
+ "role": "author",
+ "name": "The pip developers",
+ "email": "distutils-sig@python.org",
+ "url": null
+ }
+ ],
+ "keywords": [
+ "Development Status :: 5 - Production/Stable",
+ "Intended Audience :: Developers",
+ "Topic :: Software Development :: Build Tools",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3 :: Only",
+ "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: Implementation :: CPython",
+ "Programming Language :: Python :: Implementation :: PyPy"
+ ],
+ "homepage_url": "https://pip.pypa.io/",
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": "https://github.com/pypa/pip",
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": "license: MIT\nclassifiers:\n - 'License :: OSI Approved :: MIT License'\n",
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {
+ "Documentation": "https://pip.pypa.io",
+ "Changelog": "https://pip.pypa.io/en/stable/news/",
+ "python_requires": ">=3.7"
+ },
+ "dependencies": [],
+ "repository_homepage_url": "https://pypi.org/project/pip",
+ "repository_download_url": "https://pypi.org/packages/source/p/pip/pip-22.0.4.tar.gz",
+ "api_data_url": "https://pypi.org/pypi/pip/22.0.4/json",
+ "datasource_id": "pypi_setup_py",
+ "purl": "pkg:pypi/pip@22.0.4"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "src",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "src/pip",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "src/pip.egg-info",
+ "type": "directory",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "src/pip.egg-info/PKG-INFO",
+ "type": "file",
+ "package_data": [
+ {
+ "type": "pypi",
+ "namespace": null,
+ "name": "pip",
+ "version": "22.0.4",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "Python",
+ "description": "The PyPA recommended tool for installing Python packages.\npip - The Python Package Installer\n==================================\n\n.. image:: https://img.shields.io/pypi/v/pip.svg\n :target: https://pypi.org/project/pip/\n\n.. image:: https://readthedocs.org/projects/pip/badge/?version=latest\n :target: https://pip.pypa.io/en/latest\n\npip is the `package installer`_ for Python. You can use pip to install packages from the `Python Package Index`_ and other indexes.\n\nPlease take a look at our documentation for how to install and use pip:\n\n* `Installation`_\n* `Usage`_\n\nWe release updates regularly, with a new version every 3 months. Find more details in our documentation:\n\n* `Release notes`_\n* `Release process`_\n\nIn pip 20.3, we've `made a big improvement to the heart of pip`_; `learn more`_. We want your input, so `sign up for our user experience research studies`_ to help us do it right.\n\n**Note**: pip 21.0, in January 2021, removed Python 2 support, per pip's `Python 2 support policy`_. Please migrate to Python 3.\n\nIf you find bugs, need help, or want to talk to the developers, please use our mailing lists or chat rooms:\n\n* `Issue tracking`_\n* `Discourse channel`_\n* `User IRC`_\n\nIf you want to get involved head over to GitHub to get the source code, look at our development documentation and feel free to jump on the developer mailing lists and chat rooms:\n\n* `GitHub page`_\n* `Development documentation`_\n* `Development mailing list`_\n* `Development IRC`_\n\nCode of Conduct\n---------------\n\nEveryone interacting in the pip project's codebases, issue trackers, chat\nrooms, and mailing lists is expected to follow the `PSF Code of Conduct`_.\n\n.. _package installer: https://packaging.python.org/guides/tool-recommendations/\n.. _Python Package Index: https://pypi.org\n.. _Installation: https://pip.pypa.io/en/stable/installation/\n.. _Usage: https://pip.pypa.io/en/stable/\n.. _Release notes: https://pip.pypa.io/en/stable/news.html\n.. _Release process: https://pip.pypa.io/en/latest/development/release-process/\n.. _GitHub page: https://github.com/pypa/pip\n.. _Development documentation: https://pip.pypa.io/en/latest/development\n.. _made a big improvement to the heart of pip: https://pyfound.blogspot.com/2020/11/pip-20-3-new-resolver.html\n.. _learn more: https://pip.pypa.io/en/latest/user_guide/#changes-to-the-pip-dependency-resolver-in-20-3-2020\n.. _sign up for our user experience research studies: https://pyfound.blogspot.com/2020/03/new-pip-resolver-to-roll-out-this-year.html\n.. _Python 2 support policy: https://pip.pypa.io/en/latest/development/release-process/#python-2-support\n.. _Issue tracking: https://github.com/pypa/pip/issues\n.. _Discourse channel: https://discuss.python.org/c/packaging\n.. _Development mailing list: https://mail.python.org/mailman3/lists/distutils-sig.python.org/\n.. _User IRC: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa\n.. _Development IRC: https://kiwiirc.com/nextclient/#ircs://irc.libera.chat:+6697/pypa-dev\n.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md",
+ "release_date": null,
+ "parties": [
+ {
+ "type": "person",
+ "role": "author",
+ "name": "The pip developers",
+ "email": "distutils-sig@python.org",
+ "url": null
+ }
+ ],
+ "keywords": [
+ "Development Status :: 5 - Production/Stable",
+ "Intended Audience :: Developers",
+ "Topic :: Software Development :: Build Tools",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 3",
+ "Programming Language :: Python :: 3 :: Only",
+ "Programming Language :: Python :: 3.7",
+ "Programming Language :: Python :: 3.8",
+ "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: Implementation :: CPython",
+ "Programming Language :: Python :: Implementation :: PyPy"
+ ],
+ "homepage_url": "https://pip.pypa.io/",
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": "https://github.com/pypa/pip",
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": "license: MIT\nclassifiers:\n - 'License :: OSI Approved :: MIT License'\n",
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [
+ {
+ "path": "AUTHORS.txt",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "LICENSE.txt",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "MANIFEST.in",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "NEWS.rst",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "README.rst",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "pyproject.toml",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "setup.cfg",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "setup.py",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "src/pip/__init__.py",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "src/pip/__main__.py",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "src/pip/py.typed",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "src/pip.egg-info/PKG-INFO",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "src/pip.egg-info/SOURCES.txt",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "src/pip.egg-info/dependency_links.txt",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "src/pip.egg-info/entry_points.txt",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "src/pip.egg-info/not-zip-safe",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ },
+ {
+ "path": "src/pip.egg-info/top_level.txt",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "extra_data": {}
+ }
+ ],
+ "extra_data": {
+ "Documentation": "https://pip.pypa.io",
+ "Changelog": "https://pip.pypa.io/en/stable/news/"
+ },
+ "dependencies": [],
+ "repository_homepage_url": "https://pypi.org/project/pip",
+ "repository_download_url": "https://pypi.org/packages/source/p/pip/pip-22.0.4.tar.gz",
+ "api_data_url": "https://pypi.org/pypi/pip/22.0.4/json",
+ "datasource_id": "pypi_editable_egg_pkginfo",
+ "purl": "pkg:pypi/pip@22.0.4"
+ }
+ ],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "src/pip.egg-info/SOURCES.txt",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "src/pip.egg-info/dependency_links.txt",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "src/pip.egg-info/entry_points.txt",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "src/pip.egg-info/not-zip-safe",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "src/pip.egg-info/top_level.txt",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "src/pip/__init__.py",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "src/pip/__main__.py",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ },
+ {
+ "path": "src/pip/py.typed",
+ "type": "file",
+ "package_data": [],
+ "for_packages": [],
+ "scan_errors": []
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tests/packagedcode/data/rpm/header/libproxy-bin-0.3.0-4.el6_3.x86_64.rpm-package-only-expected.json b/tests/packagedcode/data/rpm/header/libproxy-bin-0.3.0-4.el6_3.x86_64.rpm-package-only-expected.json
new file mode 100644
index 00000000000..0c26c429ed8
--- /dev/null
+++ b/tests/packagedcode/data/rpm/header/libproxy-bin-0.3.0-4.el6_3.x86_64.rpm-package-only-expected.json
@@ -0,0 +1,54 @@
+[
+ {
+ "type": "rpm",
+ "namespace": null,
+ "name": "libproxy-bin",
+ "version": "0.3.0-4.el6_3",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": null,
+ "description": "Binary to test libproxy\nThe libproxy-bin package contains the proxy binary for libproxy",
+ "release_date": null,
+ "parties": [
+ {
+ "type": null,
+ "role": "vendor",
+ "name": "CentOS",
+ "email": null,
+ "url": null
+ }
+ ],
+ "keywords": [],
+ "homepage_url": "http://code.google.com/p/libproxy/",
+ "download_url": null,
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": "LGPLv2+",
+ "notice_text": null,
+ "source_packages": [
+ "pkg:rpm/libproxy@0.3.0-4.el6_3?arch=src"
+ ],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [],
+ "repository_homepage_url": null,
+ "repository_download_url": null,
+ "api_data_url": null,
+ "datasource_id": "rpm_archive",
+ "purl": "pkg:rpm/libproxy-bin@0.3.0-4.el6_3"
+ }
+]
\ No newline at end of file
diff --git a/tests/packagedcode/data/rubygems/gemspec/oj.gemspec-package-only.expected.json b/tests/packagedcode/data/rubygems/gemspec/oj.gemspec-package-only.expected.json
new file mode 100644
index 00000000000..50ba9bb793c
--- /dev/null
+++ b/tests/packagedcode/data/rubygems/gemspec/oj.gemspec-package-only.expected.json
@@ -0,0 +1,100 @@
+[
+ {
+ "type": "gem",
+ "namespace": null,
+ "name": "oj",
+ "version": "::Oj::VERSION",
+ "qualifiers": {},
+ "subpath": null,
+ "primary_language": "Ruby",
+ "description": "A fast JSON parser and serializer.\nThe fastest JSON parser and object serializer.",
+ "release_date": null,
+ "parties": [
+ {
+ "type": "person",
+ "role": "author",
+ "name": "Peter Ohler",
+ "email": null,
+ "url": null
+ },
+ {
+ "type": "person",
+ "role": "author",
+ "name": null,
+ "email": "peter@ohler.com",
+ "url": null
+ }
+ ],
+ "keywords": [],
+ "homepage_url": "http://www.ohler.com/oj",
+ "download_url": "https://rubygems.org/downloads/oj-::Oj::VERSION.gem",
+ "size": null,
+ "sha1": null,
+ "md5": null,
+ "sha256": null,
+ "sha512": null,
+ "bug_tracking_url": null,
+ "code_view_url": null,
+ "vcs_url": null,
+ "copyright": null,
+ "holder": null,
+ "declared_license_expression": null,
+ "declared_license_expression_spdx": null,
+ "license_detections": [],
+ "other_license_expression": null,
+ "other_license_expression_spdx": null,
+ "other_license_detections": [],
+ "extracted_license_statement": null,
+ "notice_text": null,
+ "source_packages": [],
+ "file_references": [],
+ "extra_data": {},
+ "dependencies": [
+ {
+ "purl": "pkg:gem/rake-compiler",
+ "extracted_requirement": ">= 0.9, < 2.0",
+ "scope": "development",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:gem/minitest",
+ "extracted_requirement": "~> 5",
+ "scope": "development",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:gem/test-unit",
+ "extracted_requirement": "~> 3.0",
+ "scope": "development",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ },
+ {
+ "purl": "pkg:gem/wwtd",
+ "extracted_requirement": "~> 0",
+ "scope": "development",
+ "is_runtime": false,
+ "is_optional": true,
+ "is_resolved": false,
+ "resolved_package": {},
+ "extra_data": {}
+ }
+ ],
+ "repository_homepage_url": "https://rubygems.org/gems/oj/versions/::Oj::VERSION",
+ "repository_download_url": "https://rubygems.org/downloads/oj-::Oj::VERSION.gem",
+ "api_data_url": "https://rubygems.org/api/v2/rubygems/oj/versions/::Oj::VERSION.json",
+ "datasource_id": "gemspec",
+ "purl": "pkg:gem/oj@::Oj::VERSION"
+ }
+]
\ No newline at end of file
diff --git a/tests/packagedcode/packages_test_utils.py b/tests/packagedcode/packages_test_utils.py
index a989135f009..ba25c20b3fb 100644
--- a/tests/packagedcode/packages_test_utils.py
+++ b/tests/packagedcode/packages_test_utils.py
@@ -49,6 +49,7 @@ def check_packages_data(
must_exist=True,
remove_uuid=True,
regen=REGEN_TEST_FIXTURES,
+ package_only=False,
):
"""
Helper to test a list of package_data objects against an expected JSON file.
@@ -66,7 +67,8 @@ def check_packages_data(
for package_uid in resource.for_packages
]
resource.for_packages = normalized_package_uids
- populate_license_fields(package_data)
+ if not package_only:
+ package_data.populate_license_fields()
results.append(package_data.to_dict())
check_result_equals_expected_json(
@@ -78,6 +80,7 @@ def check_packages_data(
def populate_license_fields(package_data):
if package_data.extracted_license_statement and not package_data.declared_license_expression:
+
from packagedcode import HANDLER_BY_DATASOURCE_ID
handler = HANDLER_BY_DATASOURCE_ID[package_data.datasource_id]
handler.populate_license_fields(package_data)
diff --git a/tests/packagedcode/test_alpine.py b/tests/packagedcode/test_alpine.py
index 435a5c96b42..43cb257b673 100644
--- a/tests/packagedcode/test_alpine.py
+++ b/tests/packagedcode/test_alpine.py
@@ -53,14 +53,14 @@ def test_parse_alpine_installed_db_full(self):
def test_scan_system_package_end_to_end_installed_alpine(self):
test_dir = self.extract_test_tar('alpine/rootfs/alpine-rootfs.tar.xz')
test_dir = os.path.join(test_dir, 'alpine-rootfs')
- expected_file = self.get_test_loc('alpine/rootfs/alpine-rootfs.tar.xz-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('alpine/rootfs/alpine-rootfs.tar.xz-expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--system-package', test_dir, '--json-pp', result_file])
check_json_scan(expected_file, result_file, regen=REGEN_TEST_FIXTURES)
def test_can_scan_installed_system_package_in_alpine_container_layer(self):
test_dir = self.extract_test_tar('alpine/alpine-container-layer.tar.xz')
- expected_file = self.get_test_loc('alpine/alpine-container-layer.tar.xz-scan-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('alpine/alpine-container-layer.tar.xz-scan-expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--system-package', test_dir, '--json-pp', result_file])
check_json_scan(expected_file, result_file, regen=REGEN_TEST_FIXTURES)
@@ -68,7 +68,7 @@ def test_can_scan_installed_system_package_in_alpine_container_layer(self):
def test_can_get_installed_system_packages_with_license_from_alpine_container_layer(self):
from packagedcode.plugin_package import get_installed_packages
test_dir = self.extract_test_tar('alpine/alpine-container-layer.tar.xz')
- expected_file = self.get_test_loc('alpine/alpine-container-layer.tar.xz-get-installed-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('alpine/alpine-container-layer.tar.xz-get-installed-expected.json')
results = list(get_installed_packages(test_dir))
self.check_packages_data(results, expected_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
diff --git a/tests/packagedcode/test_bower.py b/tests/packagedcode/test_bower.py
index e2185dfa1cb..cd3337af2a2 100644
--- a/tests/packagedcode/test_bower.py
+++ b/tests/packagedcode/test_bower.py
@@ -12,6 +12,8 @@
from packagedcode import bower
from packages_test_utils import PackageTester
from scancode_config import REGEN_TEST_FIXTURES
+from scancode.cli_test_utils import check_json_scan
+from scancode.cli_test_utils import run_scan_click
class TestBower(PackageTester):
@@ -40,11 +42,15 @@ def test_parse_bower_json_author_objects(self):
self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)
def test_end2end_bower_scan_is_moved_to_parent(self):
- from scancode.cli_test_utils import check_json_scan
- from scancode.cli_test_utils import run_scan_click
-
test_file = self.get_test_loc('bower/scan')
expected_file = self.get_test_loc('bower/scan-expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json-pp', result_file])
check_json_scan(expected_file, result_file, regen=REGEN_TEST_FIXTURES)
+
+ def test_end2end_bower_scan_is_moved_to_parent_package_only(self):
+ test_file = self.get_test_loc('bower/scan')
+ expected_file = self.get_test_loc('bower/scan-package-only-expected.json')
+ result_file = self.get_temp_file('results.json')
+ run_scan_click(['--package-only', test_file, '--json-pp', result_file])
+ check_json_scan(expected_file, result_file, regen=REGEN_TEST_FIXTURES)
diff --git a/tests/packagedcode/test_build.py b/tests/packagedcode/test_build.py
index 3d0f438e551..147a075b15b 100644
--- a/tests/packagedcode/test_build.py
+++ b/tests/packagedcode/test_build.py
@@ -85,48 +85,46 @@ def test_BuckPackage_recognize_with_license(self):
def test_MetadataBzl_parse(self):
test_file = self.get_test_loc('metadatabzl/METADATA.bzl')
- result_packages = build.BuckMetadataBzlHandler.parse(test_file)
- expected_packages = [
- models.PackageData(
- datasource_id=build.BuckMetadataBzlHandler.datasource_id,
- type='github',
- name='example',
- version='0.0.1',
- extracted_license_statement=['BSD-3-Clause'],
- parties=[
- models.Party(
- type=models.party_org,
- name='oss_foundation',
- role='maintainer'
- )
- ],
- homepage_url='https://github.com/example/example',
- ),
- ]
+ result_packages = build.BuckMetadataBzlHandler.parse(test_file, package_only=True)
+ package_data = dict(
+ datasource_id=build.BuckMetadataBzlHandler.datasource_id,
+ type='github',
+ name='example',
+ version='0.0.1',
+ extracted_license_statement=['BSD-3-Clause'],
+ parties=[
+ models.Party(
+ type=models.party_org,
+ name='oss_foundation',
+ role='maintainer'
+ )
+ ],
+ homepage_url='https://github.com/example/example',
+ )
+ expected_packages = [models.PackageData.from_data(package_data=package_data, package_only=True)]
compare_package_results(expected_packages, result_packages)
def test_MetadataBzl_recognize_new_format(self):
test_file = self.get_test_loc('metadatabzl/new-format/METADATA.bzl')
- result_packages = build.BuckMetadataBzlHandler.parse(test_file)
- expected_packages = [
- models.PackageData(
- datasource_id=build.BuckMetadataBzlHandler.datasource_id,
- type='github',
- name='example/example',
- version='0.0.1',
- extracted_license_statement='BSD-3-Clause',
- parties=[
- models.Party(
- type=models.party_org,
- name='example_org',
- role='maintainer'
- )
- ],
- download_url='',
- sha1='',
- homepage_url='https://github.com/example/example',
- vcs_url='https://github.com/example/example.git',
- extra_data=dict(vcs_commit_hash="deadbeef")
- )
- ]
+ result_packages = build.BuckMetadataBzlHandler.parse(test_file, package_only=True)
+ package_data = dict(
+ datasource_id=build.BuckMetadataBzlHandler.datasource_id,
+ type='github',
+ name='example/example',
+ version='0.0.1',
+ extracted_license_statement='BSD-3-Clause',
+ parties=[
+ models.Party(
+ type=models.party_org,
+ name='example_org',
+ role='maintainer'
+ )
+ ],
+ download_url='',
+ sha1='',
+ homepage_url='https://github.com/example/example',
+ vcs_url='https://github.com/example/example.git',
+ extra_data=dict(vcs_commit_hash="deadbeef")
+ )
+ expected_packages = [models.PackageData.from_data(package_data=package_data, package_only=True)]
compare_package_results(expected_packages, result_packages)
diff --git a/tests/packagedcode/test_build_gradle.py b/tests/packagedcode/test_build_gradle.py
index d22a473f1b4..90f0fa750c9 100644
--- a/tests/packagedcode/test_build_gradle.py
+++ b/tests/packagedcode/test_build_gradle.py
@@ -30,6 +30,13 @@ def test_end2end_scan_can_detect_build_gradle(self):
run_scan_click(['--package', test_file, '--json-pp', result_file])
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
+ def test_end2end_scan_can_detect_build_gradle_package_only(self):
+ test_file = self.get_test_loc('build_gradle/end2end/build.gradle')
+ expected_file = self.get_test_loc('build_gradle/end2end/build.gradle-package-only-expected.json')
+ result_file = self.get_temp_file('results.json')
+ run_scan_click(['--package-only', test_file, '--json-pp', result_file])
+ check_json_scan(expected_file, result_file, regen=REGEN_TEST_FIXTURES)
+
def check_gradle_parse(location):
packages_data = build_gradle.BuildGradleHandler.parse(location)
diff --git a/tests/packagedcode/test_cargo.py b/tests/packagedcode/test_cargo.py
index 5c25f13502f..ad7cbe9e631 100644
--- a/tests/packagedcode/test_cargo.py
+++ b/tests/packagedcode/test_cargo.py
@@ -109,16 +109,25 @@ def test_parse_cargo_lock_sample5(self):
def test_scan_cli_works(self):
test_file = self.get_test_loc('cargo/scan')
- expected_file = self.get_test_loc('cargo/scan.expected.json', must_exist=False)
+ expected_file = self.get_test_loc('cargo/scan.expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json', result_file])
check_json_scan(
expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES
)
+ def test_scan_cli_works_package_only(self):
+ test_file = self.get_test_loc('cargo/scan')
+ expected_file = self.get_test_loc('cargo/scan-package-only.expected.json')
+ result_file = self.get_temp_file('results.json')
+ run_scan_click(['--package-only', test_file, '--json', result_file])
+ check_json_scan(
+ expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES
+ )
+
def test_scan_works_on_cargo_workspace(self):
test_file = self.get_test_loc('cargo/cargo-with-workspace')
- expected_file = self.get_test_loc('cargo/cargo-with-workspace.expected.json', must_exist=False)
+ expected_file = self.get_test_loc('cargo/cargo-with-workspace.expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', '--license', test_file, '--json', result_file])
check_json_scan(
diff --git a/tests/packagedcode/test_chef.py b/tests/packagedcode/test_chef.py
index bfa3d3a8808..ddfe47dc218 100644
--- a/tests/packagedcode/test_chef.py
+++ b/tests/packagedcode/test_chef.py
@@ -132,9 +132,18 @@ def test_build_package_code_view_url_and_bug_tracking_url(self):
def test_scan_cli_works(self):
test_file = self.get_test_loc('chef/package/')
- expected_file = self.get_test_loc('chef/package.scan.expected.json', must_exist=False)
+ expected_file = self.get_test_loc('chef/package.scan.expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json', result_file])
check_json_scan(
expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES
)
+
+ def test_scan_cli_works_package_only(self):
+ test_file = self.get_test_loc('chef/package/')
+ expected_file = self.get_test_loc('chef/package-only.scan.expected.json')
+ result_file = self.get_temp_file('results.json')
+ run_scan_click(['--package-only', test_file, '--json', result_file])
+ check_json_scan(
+ expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES
+ )
diff --git a/tests/packagedcode/test_cocoapods.py b/tests/packagedcode/test_cocoapods.py
index 0403671c393..0d1407b38d8 100644
--- a/tests/packagedcode/test_cocoapods.py
+++ b/tests/packagedcode/test_cocoapods.py
@@ -46,19 +46,19 @@ def test_cocoapods_can_parse_kmmWebSocket(self):
def test_cocoapods_can_parse_LoadingShimmer(self):
test_file = self.get_test_loc('cocoapods/podspec/LoadingShimmer.podspec')
- expected_loc = self.get_test_loc('cocoapods/podspec/LoadingShimmer.podspec.expected.json', must_exist=False)
+ expected_loc = self.get_test_loc('cocoapods/podspec/LoadingShimmer.podspec.expected.json')
packages = PodspecHandler.parse(test_file)
self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES)
def test_cocoapods_can_parse_nanopb(self):
test_file = self.get_test_loc('cocoapods/podspec/nanopb.podspec')
- expected_loc = self.get_test_loc('cocoapods/podspec/nanopb.podspec.expected.json', must_exist=False)
+ expected_loc = self.get_test_loc('cocoapods/podspec/nanopb.podspec.expected.json')
packages = PodspecHandler.parse(test_file)
self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES)
def test_cocoapods_can_parse_PayTabsSDK(self):
test_file = self.get_test_loc('cocoapods/podspec/PayTabsSDK.podspec')
- expected_loc = self.get_test_loc('cocoapods/podspec/PayTabsSDK.podspec.expected.json', must_exist=False)
+ expected_loc = self.get_test_loc('cocoapods/podspec/PayTabsSDK.podspec.expected.json')
packages = PodspecHandler.parse(test_file)
self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES)
@@ -160,49 +160,56 @@ class TestCocoapodsEndToEndAssemble(PackageTester):
def test_cocoapods_can_assemble_with_single_podspec(self):
test_file = self.get_test_loc('cocoapods/assemble/single-podspec')
- expected_file = self.get_test_loc('cocoapods/assemble/single-podspec-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('cocoapods/assemble/single-podspec-expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json', result_file])
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
def test_cocoapods_can_assemble_with_multiple_podspec(self):
test_file = self.get_test_loc('cocoapods/assemble/multiple-podspec')
- expected_file = self.get_test_loc('cocoapods/assemble/multiple-podspec-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('cocoapods/assemble/multiple-podspec-expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json', result_file])
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
def test_cocoapods_can_assemble_with_solo_podspec(self):
test_file = self.get_test_loc('cocoapods/assemble/solo/RxDataSources.podspec')
- expected_file = self.get_test_loc('cocoapods/assemble/solo/RxDataSources.podspec-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('cocoapods/assemble/solo/RxDataSources.podspec-expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json', result_file])
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
+ def test_cocoapods_can_assemble_with_solo_podspec_package_only(self):
+ test_file = self.get_test_loc('cocoapods/assemble/solo/RxDataSources.podspec')
+ expected_file = self.get_test_loc('cocoapods/assemble/solo/RxDataSources-package-only.podspec-expected.json')
+ result_file = self.get_temp_file('results.json')
+ run_scan_click(['--package-only', test_file, '--json', result_file])
+ check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
+
def test_cocoapods_can_assemble_with_solo_podfile(self):
test_file = self.get_test_loc('cocoapods/assemble/solo/Podfile')
- expected_file = self.get_test_loc('cocoapods/assemble/solo/Podfile-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('cocoapods/assemble/solo/Podfile-expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json', result_file])
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
def test_cocoapods_can_assemble_with_solo_podfile_lock(self):
test_file = self.get_test_loc('cocoapods/assemble/solo/Podfile.lock')
- expected_file = self.get_test_loc('cocoapods/assemble/solo/Podfile.lock-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('cocoapods/assemble/solo/Podfile.lock-expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json', result_file])
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
def test_cocoapods_can_assemble_with_many_podspecs_podfile_and_podfile_lock(self):
test_file = self.get_test_loc('cocoapods/assemble/many-podspecs')
- expected_file = self.get_test_loc('cocoapods/assemble/many-podspecs-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('cocoapods/assemble/many-podspecs-expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json', result_file])
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
def test_cocoapods_can_assemble_with_many_podspecs_podfile_and_podfile_lock_with_license(self):
test_file = self.get_test_loc('cocoapods/assemble/many-podspecs')
- expected_file = self.get_test_loc('cocoapods/assemble/many-podspecs-with-license-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('cocoapods/assemble/many-podspecs-with-license-expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', '--license', test_file, '--json', result_file])
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
diff --git a/tests/packagedcode/test_debian.py b/tests/packagedcode/test_debian.py
index 928a29e4f2f..d185a5c8c53 100644
--- a/tests/packagedcode/test_debian.py
+++ b/tests/packagedcode/test_debian.py
@@ -34,7 +34,7 @@ def test_scan_system_package_end_to_end_installed_debian_basic_rootfs(self):
def test_can_scan_system_package_installed_debian_with_license_from_container_layer(self):
test_dir = self.extract_test_tar('debian/debian-container-layer.tar.xz')
- expected_file = self.get_test_loc('debian/debian-container-layer.tar.xz.scan-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('debian/debian-container-layer.tar.xz.scan-expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--system-package', test_dir, '--json-pp', result_file])
check_json_scan(expected_file, result_file, regen=REGEN_TEST_FIXTURES)
@@ -42,7 +42,7 @@ def test_can_scan_system_package_installed_debian_with_license_from_container_la
def test_can_get_installed_system_packages_with_license_from_debian_container_layer(self):
from packagedcode.plugin_package import get_installed_packages
test_dir = self.extract_test_tar('debian/debian-container-layer.tar.xz')
- expected_file = self.get_test_loc('debian/debian-container-layer.tar.xz.get-installed-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('debian/debian-container-layer.tar.xz.get-installed-expected.json')
results = list(get_installed_packages(test_dir))
self.check_packages_data(results, expected_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
@@ -70,6 +70,15 @@ def test_parse_status_file_basic(self):
packages = list(debian.DebianInstalledStatusDatabaseHandler.parse(test_file))
self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES)
+ def test_parse_status_file_basic_package_only(self):
+ test_file = self.get_test_loc('debian/basic/status')
+ expected_loc = self.get_test_loc('debian/basic/status-package-only.expected')
+ # specify ubuntu distro as this was the source of the test `status` file
+ packages = list(debian.DebianInstalledStatusDatabaseHandler.parse(
+ location=test_file, package_only=True,
+ ))
+ self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES, package_only=True)
+
def test_parse_status_file_with_source_packages(self):
test_file = self.get_test_loc('debian/status-with-source/status')
expected_loc = self.get_test_loc('debian/status-with-source/status.expected')
@@ -85,7 +94,7 @@ def test_parse_status_file_perl_error(self):
@skipIf(on_windows, 'File names cannot contain colons on Windows')
def test_scan_system_package_end_to_end_installed_debian(self):
test_dir = self.extract_test_tar('debian/end-to-end.tgz')
- expected_file = self.get_test_loc('debian/end-to-end.tgz.expected.json', must_exist=False)
+ expected_file = self.get_test_loc('debian/end-to-end.tgz.expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--system-package', test_dir, '--json-pp', result_file])
check_json_scan(expected_file, result_file, regen=REGEN_TEST_FIXTURES)
@@ -137,7 +146,7 @@ def test_multi_arch_is_same(self):
package_type=debian.DebianInstalledMd5sumFilelistHandler.default_package_type,
)
- expected_loc = self.get_test_loc('debian/libatk-adaptor-amd64.md5sums.expected.json', must_exist=False)
+ expected_loc = self.get_test_loc('debian/libatk-adaptor-amd64.md5sums.expected.json')
self.check_packages_data(results, expected_loc, must_exist=False, regen=REGEN_TEST_FIXTURES)
def test_parse_debian_files_list_no_arch(self):
@@ -148,7 +157,7 @@ def test_parse_debian_files_list_no_arch(self):
package_type=debian.DebianInstalledMd5sumFilelistHandler.default_package_type,
)
- expected_loc = self.get_test_loc('debian/files-md5sums/mokutil.md5sums.expected.json', must_exist=False)
+ expected_loc = self.get_test_loc('debian/files-md5sums/mokutil.md5sums.expected.json')
self.check_packages_data(results, expected_loc, must_exist=False, regen=REGEN_TEST_FIXTURES)
@skipIf(on_windows, 'File names cannot contain colons on Windows')
@@ -167,7 +176,7 @@ def test_parse_debian_files_list_with_arch(self):
package_type=debian.DebianInstalledMd5sumFilelistHandler.default_package_type,
)
- expected_loc = self.get_test_loc('debian/files-md5sums/mokutil-amd64.md5sums.expected.json', must_exist=False)
+ expected_loc = self.get_test_loc('debian/files-md5sums/mokutil-amd64.md5sums.expected.json')
self.check_packages_data(results, expected_loc, must_exist=False, regen=REGEN_TEST_FIXTURES)
def test_build_package_data_from_package_filename_deb_does_not_crash_on_version(self):
diff --git a/tests/packagedcode/test_freebsd.py b/tests/packagedcode/test_freebsd.py
index 70280b89376..80b1d7c4ff3 100644
--- a/tests/packagedcode/test_freebsd.py
+++ b/tests/packagedcode/test_freebsd.py
@@ -58,6 +58,12 @@ def test_parse_basic(self):
package = freebsd.CompactManifestHandler.parse(test_file)
self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)
+ def test_parse_basic_package_only(self):
+ test_file = self.get_test_loc('freebsd/basic/+COMPACT_MANIFEST')
+ expected_loc = self.get_test_loc('freebsd/basic/+COMPACT_MANIFEST-package-only.expected')
+ package = freebsd.CompactManifestHandler.parse(location=test_file, package_only=True)
+ self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES, package_only=True)
+
def test_parse_not_yaml(self):
test_file = self.get_test_loc('freebsd/not_yaml/+COMPACT_MANIFEST')
try:
diff --git a/tests/packagedcode/test_maven.py b/tests/packagedcode/test_maven.py
index 62349097ce4..0f5d64aed57 100644
--- a/tests/packagedcode/test_maven.py
+++ b/tests/packagedcode/test_maven.py
@@ -236,6 +236,13 @@ def test_maven_assembly_with_pom_and_manifest(self):
run_scan_click(['--package', '--license', '--license-diagnostics', '--processes', '-1', test_dir, '--json', result_file])
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
+ def test_maven_assembly_with_pom_and_manifest_package_only(self):
+ test_dir = self.get_test_loc('maven_misc/assemble/johnzon-jsonb-1.2.11')
+ result_file = self.get_temp_file('json')
+ expected_file = self.get_test_loc('maven_misc/assemble/johnzon-jsonb-1.2.11-package-only-expected.json')
+ run_scan_click(['--package-only', '--processes', '-1', test_dir, '--json', result_file])
+ check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
+
def test_maven_assembly_with_pom_and_jar_manifest(self):
test_dir = self.get_test_loc('maven_misc/assemble/numbers-1.7.4')
result_file = self.get_temp_file('json')
diff --git a/tests/packagedcode/test_npm.py b/tests/packagedcode/test_npm.py
index 4bf9877011d..85c80c46c48 100644
--- a/tests/packagedcode/test_npm.py
+++ b/tests/packagedcode/test_npm.py
@@ -347,7 +347,7 @@ def test_vcs_repository_mapper_handles_version_on_gh(self):
def test_npm_get_package_resources(self):
test_file = self.get_test_loc('npm/get_package_resources')
- expected_file = self.get_test_loc('npm/get_package_resources.scan.expected.json', must_exist=False)
+ expected_file = self.get_test_loc('npm/get_package_resources.scan.expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json', result_file])
check_json_scan(
@@ -356,16 +356,25 @@ def test_npm_get_package_resources(self):
def test_scan_cli_works(self):
test_file = self.get_test_loc('npm/scan-nested/scan')
- expected_file = self.get_test_loc('npm/scan-nested/scan.expected.json', must_exist=False)
+ expected_file = self.get_test_loc('npm/scan-nested/scan.expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json', result_file])
check_json_scan(
expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES
)
+ def test_scan_cli_works_package_only(self):
+ test_file = self.get_test_loc('npm/scan-nested/scan')
+ expected_file = self.get_test_loc('npm/scan-nested/scan-package-only.expected.json')
+ result_file = self.get_temp_file('results.json')
+ run_scan_click(['--package-only', test_file, '--json', result_file])
+ check_json_scan(
+ expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES
+ )
+
def test_npm_electron_scan(self):
test_file = self.get_test_loc('npm/electron/package')
- expected_file = self.get_test_loc('npm/electron/package.expected.json', must_exist=False)
+ expected_file = self.get_test_loc('npm/electron/package.expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json', result_file])
check_json_scan(
@@ -374,7 +383,7 @@ def test_npm_electron_scan(self):
def test_npm_scan_with_private_package_json(self):
test_file = self.get_test_loc('npm/private/package.json')
- expected_file = self.get_test_loc('npm/private/scan.expected.json', must_exist=False)
+ expected_file = self.get_test_loc('npm/private/scan.expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json', result_file])
check_json_scan(
@@ -383,7 +392,7 @@ def test_npm_scan_with_private_package_json(self):
def test_npm_scan_with_private_package_json_and_yarn_lock(self):
test_file = self.get_test_loc('npm/private-and-yarn/theia/')
- expected_file = self.get_test_loc('npm/private-and-yarn/scan.expected.json', must_exist=False)
+ expected_file = self.get_test_loc('npm/private-and-yarn/scan.expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json', result_file])
check_json_scan(
diff --git a/tests/packagedcode/test_nuget.py b/tests/packagedcode/test_nuget.py
index 37298a45c7e..647b3d1024e 100644
--- a/tests/packagedcode/test_nuget.py
+++ b/tests/packagedcode/test_nuget.py
@@ -56,3 +56,9 @@ def test_parse_as_package(self):
package = nuget.NugetNuspecHandler.parse(test_file)
expected_loc = self.get_test_loc('nuget/Castle.Core.nuspec.json.expected')
self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)
+
+ def test_parse_as_package_only(self):
+ test_file = self.get_test_loc('nuget/Castle.Core.nuspec')
+ package = nuget.NugetNuspecHandler.parse(location=test_file, package_only=True)
+ expected_loc = self.get_test_loc('nuget/Castle.Core.nuspec-package-only.json.expected')
+ self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES, package_only=True)
diff --git a/tests/packagedcode/test_package_models.py b/tests/packagedcode/test_package_models.py
index 98a3606093d..d518bd60f84 100644
--- a/tests/packagedcode/test_package_models.py
+++ b/tests/packagedcode/test_package_models.py
@@ -76,7 +76,7 @@ def test_Package_creation_and_dump(self):
assert list(pd.to_dict().items()) == expected
def test_Package_simple(self):
- package = PackageData(
+ package_mapping = dict(
datasource_id = 'rpm_archive',
type='rpm',
name='Sample',
@@ -86,8 +86,20 @@ def test_Package_simple(self):
vcs_url='git+https://somerepo.com/that.git',
extracted_license_statement='apache-2.0',
)
+ package_data = PackageData.from_data(package_data=package_mapping, package_only=False)
expected_loc = 'models/simple-expected.json'
- self.check_package_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)
+ self.check_package_data(package_data, expected_loc, regen=REGEN_TEST_FIXTURES)
+
+ def test_PackageData_model_can_assemble(self):
+ package_jar = models.PackageData(
+ type='maven', name='this', version='23', datasource_id="java_jar",
+ )
+ package_pom = models.PackageData(
+ type='maven', name='this', version='23', datasource_id="maven_pom",
+ )
+
+ assert not package_jar.can_assemble
+ assert package_pom.can_assemble
def test_Package_model_qualifiers_are_serialized_as_mappings(self):
package = models.PackageData(
@@ -108,7 +120,7 @@ def test_Package_model_qualifiers_are_converted_to_mappings(self):
assert package.qualifiers == dict(this='that')
def test_Package_full(self):
- package = PackageData(
+ package_mapping = dict(
type='rpm',
datasource_id = 'rpm_archive',
namespace='fedora',
@@ -136,8 +148,9 @@ def test_Package_full(self):
notice_text='licensed under the apacche 2.0 \nlicense',
source_packages=["pkg:maven/aspectj/aspectjtools@1.5.4?classifier=sources"],
)
+ package_data = PackageData.from_data(package_data=package_mapping, package_only=False)
expected_loc = 'models/full-expected.json'
- self.check_package_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)
+ self.check_package_data(package_data, expected_loc, regen=REGEN_TEST_FIXTURES)
def test_package_data_datasource_id_are_unique(self):
"""
@@ -177,11 +190,12 @@ def test_package_data_file_patterns_are_tuples(self):
def test_add_to_package(self):
test_loc = self.get_test_loc('npm/electron')
- test_package = models.Package(
+ test_package_data = dict(
type='npm',
name='electron',
version='3.1.11',
)
+ test_package = models.Package.from_data(test_package_data)
test_package_uid = test_package.package_uid
test_codebase = Codebase(
location=test_loc,
@@ -241,12 +255,13 @@ def test_create_package_not_handled_by_packagedcode(self):
'gpl',
'GNU General Public License version 2.0 (GPLv2)',
]
- package = PackageData(
+ package_mapping = dict(
type='sourceforge',
name='openstunts',
copyright='Copyright (c) openstunts project',
extracted_license_statement=extracted_license_statement,
)
+ package = PackageData.from_data(package_data=package_mapping, package_only=False)
# Test generated fields
assert package.purl == 'pkg:sourceforge/openstunts'
assert package.holder == 'openstunts project'
diff --git a/tests/packagedcode/test_plugin_package.py b/tests/packagedcode/test_plugin_package.py
index bbe2216d4a3..2594b4fbd64 100644
--- a/tests/packagedcode/test_plugin_package.py
+++ b/tests/packagedcode/test_plugin_package.py
@@ -166,14 +166,14 @@ def test_package_command_scan_mum(self):
def test_package_command_scan_pubspec_package(self):
test_dir = self.get_test_loc('pubspec/specs/authors-pubspec.yaml')
result_file = self.get_temp_file('json')
- expected_file = self.get_test_loc('plugin/pubspec-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('plugin/pubspec-expected.json')
run_scan_click(['--package', '--strip-root', '--processes', '-1', test_dir, '--json', result_file])
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
def test_package_command_scan_pubspec_lock_package(self):
test_dir = self.get_test_loc('pubspec/locks/dart-pubspec.lock')
result_file = self.get_temp_file('json')
- expected_file = self.get_test_loc('plugin/pubspec-lock-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('plugin/pubspec-lock-expected.json')
run_scan_click(['--package', '--strip-root', '--processes', '-1', test_dir, '--json', result_file])
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
@@ -227,6 +227,42 @@ def test_package_list_command(self, regen=REGEN_TEST_FIXTURES):
ef.write(result.output)
assert result.output == open(expected_file).read()
+ def test_plugin_package_only_fails_with_license_scan(self):
+ test_dir = self.get_test_loc('maven2')
+ result_file = self.get_temp_file('json')
+ try:
+ run_scan_click(['--package-only', '--license', test_dir, '--json', result_file])
+ raise Exception("This SCAN should raise an AssertionError for conflicting CLI options")
+ except AssertionError:
+ pass
+
+ def test_plugin_package_only_fails_with_summary_scan(self):
+ test_dir = self.get_test_loc('maven2')
+ result_file = self.get_temp_file('json')
+ try:
+ run_scan_click(['--package-only', '--summary', '--classify', test_dir, '--json', result_file])
+ raise Exception("This SCAN should raise an AssertionError for conflicting CLI options")
+ except AssertionError:
+ pass
+
+ def test_plugin_package_only_fails_with_package_scan(self):
+ test_dir = self.get_test_loc('maven2')
+ result_file = self.get_temp_file('json')
+ try:
+ run_scan_click(['--package-only', '--package', test_dir, '--json', result_file])
+ raise Exception("This SCAN should raise an AssertionError for conflicting CLI options")
+ except AssertionError:
+ pass
+
+ def test_plugin_package_only_fails_with_system_package_scan(self):
+ test_dir = self.get_test_loc('maven2')
+ result_file = self.get_temp_file('json')
+ try:
+ run_scan_click(['--package-only', '--system-package', test_dir, '--json', result_file])
+ raise Exception("This SCAN should raise an AssertionError for conflicting CLI options")
+ except AssertionError:
+ pass
+
def test_system_package_get_installed_packages(self):
test_dir = self.extract_test_tar('debian/basic-rootfs.tar.gz')
expected_file = self.get_test_loc('plugin/get_installed_packages-expected.json')
diff --git a/tests/packagedcode/test_pubspec.py b/tests/packagedcode/test_pubspec.py
index b87e5e18070..f0a5b6d5e25 100644
--- a/tests/packagedcode/test_pubspec.py
+++ b/tests/packagedcode/test_pubspec.py
@@ -30,7 +30,7 @@ def test_pubspec_yaml_is_package_data_file(self):
def test_parse_lock(self):
test_loc = self.get_test_loc('pubspec/mini-pubspec.lock')
- expected_loc = self.get_test_loc('pubspec/mini-pubspec.lock-expected.json', must_exist=False)
+ expected_loc = self.get_test_loc('pubspec/mini-pubspec.lock-expected.json')
package_data = pubspec.DartPubspecLockHandler.parse(test_loc)
self.check_packages_data(package_data, expected_loc, regen=REGEN_TEST_FIXTURES)
diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py
index 5020e28913b..b38e7e5b5ad 100644
--- a/tests/packagedcode/test_pypi.py
+++ b/tests/packagedcode/test_pypi.py
@@ -39,10 +39,17 @@ def test_package_scan_pypi_end_to_end_full_with_license(self):
run_scan_click(['--package', '--license', '--strip-root', '--processes', '-1', test_dir, '--json', result_file])
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
+ def test_package_scan_pypi_end_to_end_package_only(self):
+ test_dir = self.get_test_loc('pypi/source-package/pip-22.0.4/')
+ result_file = self.get_temp_file('json')
+ expected_file = self.get_test_loc('pypi/source-package/pip-22.0.4-pypi-package-only-expected.json')
+ run_scan_click(['--package-only', '--strip-root', '--processes', '-1', test_dir, '--json', result_file])
+ check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
+
def test_package_scan_pypi_setup_py_end_to_end(self):
test_dir = self.get_test_loc('pypi/source-package/pip-22.0.4/setup.py')
result_file = self.get_temp_file('json')
- expected_file = self.get_test_loc('pypi/source-package/pip-22.0.4-pypi-package-setup-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('pypi/source-package/pip-22.0.4-pypi-package-setup-expected.json')
run_scan_click(['--package', '--strip-root', '--processes', '-1', test_dir, '--json', result_file])
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
@@ -266,14 +273,14 @@ def test_parse_metadata_unpacked_sdist_metadata_v21(self):
def test_can_parse_solo_metadata_from_command_line(self):
test_file = self.get_test_loc('pypi/solo-metadata/PKG-INFO')
- expected_file = self.get_test_loc('pypi/solo-metadata/expected.json', must_exist=False)
+ expected_file = self.get_test_loc('pypi/solo-metadata/expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json', result_file])
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
def test_parse_metadata_prefer_pkg_info_from_egg_info_from_command_line(self):
test_file = self.get_test_loc('pypi/unpacked_sdist/prefer-egg-info-pkg-info/celery')
- expected_file = self.get_test_loc('pypi/unpacked_sdist/prefer-egg-info-pkg-info/celery-expected.json', must_exist=False)
+ expected_file = self.get_test_loc('pypi/unpacked_sdist/prefer-egg-info-pkg-info/celery-expected.json')
result_file = self.get_temp_file('results.json')
run_scan_click(['--package', test_file, '--json', result_file])
check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES)
@@ -539,25 +546,25 @@ class TestPyPiSetupPyNames(PackageTester):
def test_parse_setup_py_with_name(self):
test_file = self.get_test_loc('pypi/setup.py-name-or-no-name/with_name-setup.py')
package = pypi.PythonSetupPyHandler.parse(test_file)
- expected_loc = self.get_test_loc('pypi/setup.py-name-or-no-name/with_name-setup.py.expected.json', must_exist=False)
+ expected_loc = self.get_test_loc('pypi/setup.py-name-or-no-name/with_name-setup.py.expected.json')
self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)
def test_parse_setup_py_without_name(self):
test_file = self.get_test_loc('pypi/setup.py-name-or-no-name/without_name-setup.py')
package = pypi.PythonSetupPyHandler.parse(test_file)
- expected_loc = self.get_test_loc('pypi/setup.py-name-or-no-name/without_name-setup.py.expected.json', must_exist=False)
+ expected_loc = self.get_test_loc('pypi/setup.py-name-or-no-name/without_name-setup.py.expected.json')
self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)
def test_get_setup_py_args_with_name(self):
test_file = self.get_test_loc('pypi/setup.py-name-or-no-name/with_name-setup.py')
kwargs = pypi.get_setup_py_args(test_file)
- expected_loc = self.get_test_loc('pypi/setup.py-name-or-no-name/with_name-setup.py.args.expected.json', must_exist=False)
+ expected_loc = self.get_test_loc('pypi/setup.py-name-or-no-name/with_name-setup.py.args.expected.json')
check_result_equals_expected_json(kwargs, expected_loc, regen=REGEN_TEST_FIXTURES)
def test_get_setup_py_args_without_name(self):
test_file = self.get_test_loc('pypi/setup.py-name-or-no-name/without_name-setup.py')
kwargs = pypi.get_setup_py_args(test_file)
- expected_loc = self.get_test_loc('pypi/setup.py-name-or-no-name/without_name-setup.py.args.expected.json', must_exist=False)
+ expected_loc = self.get_test_loc('pypi/setup.py-name-or-no-name/without_name-setup.py.args.expected.json')
check_result_equals_expected_json(kwargs, expected_loc, regen=REGEN_TEST_FIXTURES)
diff --git a/tests/packagedcode/test_rpm.py b/tests/packagedcode/test_rpm.py
index 56f4bfc5022..1baa3762a85 100644
--- a/tests/packagedcode/test_rpm.py
+++ b/tests/packagedcode/test_rpm.py
@@ -33,6 +33,17 @@ def test_parse_to_package(self):
)
check_result_equals_expected_json(result, expected_loc, regen=REGEN_TEST_FIXTURES)
+ def test_parse_to_package_only(self):
+ test_file = self.get_test_loc('rpm/header/libproxy-bin-0.3.0-4.el6_3.x86_64.rpm')
+ package_datas = rpm.RpmArchiveHandler.parse(location=test_file, package_only=True)
+ result = [pd.to_dict() for pd in package_datas]
+
+ expected_loc = self.get_test_loc(
+ 'rpm/header/libproxy-bin-0.3.0-4.el6_3.x86_64.rpm-package-only-expected.json',
+ must_exist=False,
+ )
+ check_result_equals_expected_json(result, expected_loc, regen=REGEN_TEST_FIXTURES)
+
def test_pyrpm_basic(self):
test_file = self.get_test_loc('rpm/header/python-glc-0.7.1-1.src.rpm')
from packagedcode.pyrpm import RPM
diff --git a/tests/packagedcode/test_rubygems.py b/tests/packagedcode/test_rubygems.py
index 4dc8f26e1d1..af55f355ab7 100644
--- a/tests/packagedcode/test_rubygems.py
+++ b/tests/packagedcode/test_rubygems.py
@@ -66,6 +66,12 @@ def test_rubygems_oj_gemspec(self):
packages = rubygems.GemspecHandler.parse(test_file)
self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES)
+ def test_rubygems_oj_gemspec_package_only(self):
+ test_file = self.get_test_loc('rubygems/gemspec/oj.gemspec')
+ expected_loc = self.get_test_loc('rubygems/gemspec/oj.gemspec-package-only.expected.json')
+ packages = rubygems.GemspecHandler.parse(location=test_file, package_only=True)
+ self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES)
+
def test_rubygems_rubocop_gemspec(self):
test_file = self.get_test_loc('rubygems/gemspec/rubocop.gemspec')
expected_loc = self.get_test_loc('rubygems/gemspec/rubocop.gemspec.expected.json')
diff --git a/tests/scancode/data/help/help.txt b/tests/scancode/data/help/help.txt
index 3076354074c..6def63b8893 100644
--- a/tests/scancode/data/help/help.txt
+++ b/tests/scancode/data/help/help.txt
@@ -12,6 +12,8 @@ Options:
-p, --package Scan for application package and dependency
manifests, lockfiles and related data.
--system-package Scan for installed system package databases.
+ --package-only Scan for system and application package data and skip
+ license/copyright detection and top-level package creation.
-c, --copyright Scan for copyrights.
other scans: