diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index a6d84c1e56e..b507d1efb42 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -168,6 +168,7 @@ # pypi.PypiSdistArchiveHandler, pypi.PypiWheelHandler, pypi.PyprojectTomlHandler, + pypi.PoetryPyprojectTomlHandler, pypi.PythonEditableInstallationPkgInfoFile, pypi.PythonEggPkgInfoFile, pypi.PythonInstalledWheelMetadataFile, diff --git a/src/packagedcode/debian.py b/src/packagedcode/debian.py index c6dd5420b1a..ff1d618fd20 100644 --- a/src/packagedcode/debian.py +++ b/src/packagedcode/debian.py @@ -21,6 +21,7 @@ from packagedcode import models from packagedcode.utils import get_ancestor +from packagedcode.utils import parse_maintainer_name_email """ Handle Debian package archives, control files and installed databases. @@ -653,20 +654,20 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No maintainer = debian_data.get('maintainer') if maintainer: - maintainer_name, maintainer_email = parse_debian_maintainers(maintainer) + maintainer_name, maintainer_email = parse_maintainer_name_email(maintainer) party = models.Party(role='maintainer', name=maintainer_name, email=maintainer_email) parties.append(party) orig_maintainer = debian_data.get('original_maintainer') if orig_maintainer: - maintainer_name, maintainer_email = parse_debian_maintainers(orig_maintainer) + maintainer_name, maintainer_email = parse_maintainer_name_email(orig_maintainer) party = models.Party(role='maintainer', name=maintainer_name, email=maintainer_email) parties.append(party) uploaders = debian_data.get('uploaders') if uploaders: for uploader in uploaders.split(", "): - uploader_name, uploader_email = parse_debian_maintainers(uploader) + uploader_name, uploader_email = parse_maintainer_name_email(uploader) party = models.Party(role='uploader', name=uploader_name, email=uploader_email) parties.append(party) @@ -736,26 +737,6 @@ def build_package_data(debian_data, datasource_id, package_type='deb', distro=No return models.PackageData.from_data(package_data, package_only) -def parse_debian_maintainers(maintainer): - """ - Get name and email values from a debian maintainer string. - - Example string: - Debian systemd Maintainers - """ - email_wrappers = ["<", ">"] - has_email = "@" in maintainer and all([ - True - for char in email_wrappers - if char in maintainer - ]) - if not has_email: - return maintainer, None - - name, _, email = maintainer.rpartition("<") - return name.rstrip(" "), email.rstrip(">") - - def populate_debian_namespace(packages): """ For an iterable of debian `packages`, populate the diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index 33f2ec394ea..445a0562969 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -27,6 +27,7 @@ import packvers as packaging import pip_requirements_parser import pkginfo2 +import toml from commoncode import fileutils from commoncode.fileutils import as_posixpath from commoncode.resource import Resource @@ -38,6 +39,7 @@ from packagedcode import models from packagedcode.utils import build_description +from packagedcode.utils import parse_maintainer_name_email from packagedcode.utils import yield_dependencies_from_package_data from packagedcode.utils import yield_dependencies_from_package_resource @@ -149,7 +151,7 @@ def assemble(cls, package_data, resource, codebase, package_adder): datafile_name_patterns = ( 'Pipfile.lock', 'Pipfile', - ) + PipRequirementsFileHandler.path_patterns + ) + PipRequirementsFileHandler.path_patterns + PyprojectTomlHandler.path_patterns # TODO: we want PKG-INFO first, then (setup.py, setup.cfg), then pyproject.toml for poetry # then we have the rest of the lock files (pipfile, pipfile.lock, etc.) @@ -448,14 +450,119 @@ def get_resource_for_path(path, root, codebase): return root -# FIXME: Implement me -class PyprojectTomlHandler(models.NonAssemblableDatafileHandler): +class PyprojectTomlHandler(BaseExtractedPythonLayout): datasource_id = 'pypi_pyproject_toml' path_patterns = ('*pyproject.toml',) default_package_type = 'pypi' default_primary_language = 'Python' description = 'Python pyproject.toml' - documentation_url = 'https://peps.python.org/pep-0621/' + documentation_url = 'https://packaging.python.org/en/latest/specifications/pyproject-toml/' + + @classmethod + def is_datafile(cls, location, filetypes=tuple()): + return ( + super().is_datafile(location, filetypes=filetypes) + and not is_poetry_pyproject_toml(location) + ) + + @classmethod + def parse(cls, location, package_only=False): + package_data = toml.load(location, _dict=dict) + project_data = package_data.get("project") + if not project_data: + return + + name = project_data.get('name') + version = project_data.get('version') + description = project_data.get('description') or '' + description = description.strip() + + urls, extra_data = get_urls(metainfo=project_data, name=name, version=version) + + extracted_license_statement, license_file = get_declared_license(project_data) + if license_file: + extra_data['license_file'] = license_file + + package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + primary_language='Python', + name=name, + version=version, + extracted_license_statement=extracted_license_statement, + description=description, + keywords=get_keywords(project_data), + parties=get_pyproject_toml_parties(project_data), + extra_data=extra_data, + **urls, + ) + yield models.PackageData.from_data(package_data, package_only) + + +def is_poetry_pyproject_toml(location): + with open(location, 'r') as file: + data = file.read() + + if "tool.poetry" in data: + return True + else: + return False + + +class PoetryPyprojectTomlHandler(BaseExtractedPythonLayout): + datasource_id = 'pypi_poetry_pyproject_toml' + path_patterns = ('*pyproject.toml',) + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'Python poetry pyproject.toml' + documentation_url = 'https://packaging.python.org/en/latest/specifications/pyproject-toml/' + + @classmethod + def is_datafile(cls, location, filetypes=tuple()): + return ( + super().is_datafile(location, filetypes=filetypes) + and is_poetry_pyproject_toml(location) + ) + + @classmethod + def parse(cls, location, package_only=False): + toml_data = toml.load(location, _dict=dict) + + tool_data = toml_data.get('tool') + if not tool_data: + return + + poetry_data = tool_data.get('poetry') + if not poetry_data: + return + + name = poetry_data.get('name') + version = poetry_data.get('version') + description = poetry_data.get('description') or '' + description = description.strip() + + urls, extra_data = get_urls(metainfo=poetry_data, name=name, version=version, poetry=True) + + extracted_license_statement, license_file = get_declared_license(poetry_data) + if license_file: + extra_data['license_file'] = license_file + + package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + primary_language='Python', + name=name, + version=version, + extracted_license_statement=extracted_license_statement, + description=description, + keywords=get_keywords(poetry_data), + parties=get_pyproject_toml_parties(poetry_data), + extra_data=extra_data, + **urls, + ) + yield models.PackageData.from_data(package_data, package_only) + + META_DIR_SUFFIXES = '.dist-info', '.egg-info', 'EGG-INFO', @@ -487,8 +594,11 @@ def parse_metadata(location, datasource_id, package_type, package_only=False): urls, extra_data = get_urls(metainfo=meta, name=name, version=version) - dependencies = get_dist_dependencies(dist) + extracted_license_statement, license_file = get_declared_license(metainfo=meta) + if license_file: + extra_data['license_file'] = license_file + dependencies = get_dist_dependencies(dist) file_references = list(get_file_references(dist)) package_data = dict( @@ -497,7 +607,7 @@ def parse_metadata(location, datasource_id, package_type, package_only=False): primary_language='Python', name=name, version=version, - extracted_license_statement=get_declared_license(meta), + extracted_license_statement=extracted_license_statement, description=get_description(metainfo=meta, location=str(location)), keywords=get_keywords(meta), parties=get_parties(meta), @@ -628,6 +738,9 @@ def parse(cls, location, package_only=False): name = sdist.name version = sdist.version urls, extra_data = get_urls(metainfo=sdist, name=name, version=version) + extracted_license_statement, license_file = get_declared_license(metainfo=sdist) + if license_file: + extra_data['license_file'] = license_file package_data = dict( datasource_id=cls.datasource_id, @@ -636,7 +749,7 @@ def parse(cls, location, package_only=False): name=name, version=version, description=get_description(sdist, location=location), - extracted_license_statement=get_declared_license(sdist), + extracted_license_statement=extracted_license_statement, keywords=get_keywords(sdist), parties=get_parties(sdist), extra_data=extra_data, @@ -672,6 +785,10 @@ def parse(cls, location, package_only=False): python_requires = get_setup_py_python_requires(setup_args) extra_data.update(python_requires) + extracted_license_statement, license_file = get_declared_license(metainfo=setup_args) + if license_file: + extra_data['license_file'] = license_file + package_data = dict( datasource_id=cls.datasource_id, type=cls.default_package_type, @@ -680,7 +797,7 @@ def parse(cls, location, package_only=False): version=version, description=get_description(setup_args), parties=get_setup_parties(setup_args), - extracted_license_statement=get_declared_license(setup_args), + extracted_license_statement=extracted_license_statement, dependencies=dependencies, keywords=get_keywords(setup_args), extra_data=extra_data, @@ -1146,15 +1263,23 @@ def get_legacy_description(location): def get_declared_license(metainfo): """ - Return a mapping of declared license information found in a ``metainfo`` - object or mapping. + Return a mapping of declared license information and license file name + found in a ``metainfo`` package data mapping. """ declared_license = {} # TODO: We should make the declared license as it is, this should be # updated in scancode to parse a pure string lic = get_attribute(metainfo, 'License') + + license_file = None + if lic and 'file' in lic: + license_file = lic.pop('file') + if lic and not lic == 'UNKNOWN': - declared_license['license'] = lic + if 'text' in lic: + declared_license['license'] = lic.get('text') + else: + declared_license['license'] = lic license_classifiers, _ = get_classifiers(metainfo) if license_classifiers: @@ -1163,7 +1288,7 @@ def get_declared_license(metainfo): if not declared_license: declared_license = None - return declared_license + return declared_license, license_file def get_classifiers(metainfo): @@ -1175,6 +1300,7 @@ def get_classifiers(metainfo): classifiers = ( get_attribute(metainfo, 'Classifier', multiple=True) or get_attribute(metainfo, 'Classifiers', multiple=True) + or get_attribute(metainfo, 'classifiers', multiple=True) ) if not classifiers: return [], [] @@ -1218,7 +1344,6 @@ def get_parties( author_email_key='Author-email', maintainer_key='Maintainer', maintainer_email_key='Maintainer-email', - ): """ Return a list of parties found in a ``metainfo`` object or mapping. @@ -1265,6 +1390,47 @@ def get_setup_parties(setup_kwargs): ) +def get_pyproject_toml_parties(metainfo): + + parties = [] + authors = metainfo.get('authors') or [] + for author in authors: + add_pyproject_toml_party( + parties=parties, + party=author, + role='author', + ) + + maintainers = metainfo.get('maintainers') or [] + for maintainer in maintainers: + add_pyproject_toml_party( + parties=parties, + party=maintainer, + role='maintainer', + ) + + return parties + + +def add_pyproject_toml_party(parties, party, role): + + if type(party) is str: + name, email = parse_maintainer_name_email(party) + parties.append(models.Party( + type=models.party_person, + name=name, + role=role, + email=email, + )) + else: + parties.append(models.Party( + type=models.party_person, + name=party.get('name'), + role=role, + email=party.get('email'), + )) + + def get_setup_py_python_requires(setup_args): """ Return a mapping of {python_requires: value} or an empty mapping found in a @@ -1643,7 +1809,7 @@ def get_pypi_urls(name, version, **kwargs): ) -def get_urls(metainfo, name, version): +def get_urls(metainfo, name, version, poetry=False): """ Return a mapping of standard URLs and a mapping of extra-data URls for URLs of this package: @@ -1706,11 +1872,18 @@ def add_url(_url, _utype=None, _attribute=None): ) add_url(homepage_url, _attribute='homepage_url') - project_urls = ( - get_attribute(metainfo, 'Project-URL', multiple=True) - or get_attribute(metainfo, 'project_urls') - or [] - ) + if poetry: + url_fields = ["homepage", "repository", "documentation"] + project_urls = {} + for url_field in url_fields: + project_urls[url_field] = metainfo.get(url_field) + else: + project_urls = ( + get_attribute(metainfo, 'Project-URL', multiple=True) + or get_attribute(metainfo, 'project_urls') + or get_attribute(metainfo, 'urls') + or [] + ) if isinstance(project_urls, list): # these come from METADATA and we convert them back to a mapping diff --git a/src/packagedcode/utils.py b/src/packagedcode/utils.py index a954b882d3d..6cd8381643d 100644 --- a/src/packagedcode/utils.py +++ b/src/packagedcode/utils.py @@ -207,6 +207,26 @@ def yield_dependencies_from_package_data(package_data, datafile_path, package_ui ) +def parse_maintainer_name_email(maintainer): + """ + Get name and email values from a author/maintainer string. + + Example string: + Debian systemd Maintainers + """ + email_wrappers = ["<", ">"] + has_email = "@" in maintainer and all([ + True + for char in email_wrappers + if char in maintainer + ]) + if not has_email: + return maintainer, None + + name, _, email = maintainer.rpartition("<") + return name.rstrip(" "), email.rstrip(">") + + def yield_dependencies_from_package_resource(resource, package_uid=None): """ Yield a Dependency for each dependency from each package from``resource.package_data`` diff --git a/tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests-with-license.json b/tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests-with-license.json index 8d6ebf9f0cc..10b4f755518 100644 --- a/tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests-with-license.json +++ b/tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests-with-license.json @@ -118,7 +118,7 @@ "identifier": "mit-cacd5c0c-204a-85c2-affc-e4c125b2492a", "license_expression": "mit", "license_expression_spdx": "MIT", - "detection_count": 3, + "detection_count": 2, "reference_matches": [ { "license_expression": "mit", @@ -463,72 +463,7 @@ { "path": "pyproject.toml", "type": "file", - "package_data": [ - { - "type": "pypi", - "namespace": null, - "name": null, - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": "Python", - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": null, - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "holder": null, - "declared_license_expression": "mit", - "declared_license_expression_spdx": "MIT", - "license_detections": [ - { - "license_expression": "mit", - "license_expression_spdx": "MIT", - "matches": [ - { - "license_expression": "mit", - "spdx_license_expression": "MIT", - "from_file": "pypi-with-test-manifests/LICENSE", - "start_line": 3, - "end_line": 19, - "matcher": "2-aho", - "score": 100.0, - "matched_length": 161, - "match_coverage": 100.0, - "rule_relevance": 100, - "rule_identifier": "mit.LICENSE", - "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.LICENSE" - } - ], - "identifier": "mit-cacd5c0c-204a-85c2-affc-e4c125b2492a" - } - ], - "other_license_expression": null, - "other_license_expression_spdx": null, - "other_license_detections": [], - "extracted_license_statement": null, - "notice_text": null, - "source_packages": [], - "file_references": [], - "extra_data": {}, - "dependencies": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null, - "datasource_id": "pypi_pyproject_toml", - "purl": null - } - ], + "package_data": [], "for_packages": [ "pkg:pypi/setuptools@58.2.0?uuid=fixed-uid-done-for-testing-5642512d1758" ], diff --git a/tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests.json b/tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests.json index 2d8dc56bbc1..75773a62657 100644 --- a/tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests.json +++ b/tests/packagedcode/data/instance/python-package-instance-expected-with-test-manifests.json @@ -254,50 +254,7 @@ { "path": "pyproject.toml", "type": "file", - "package_data": [ - { - "type": "pypi", - "namespace": null, - "name": null, - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": "Python", - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": null, - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "holder": null, - "declared_license_expression": null, - "declared_license_expression_spdx": null, - "license_detections": [], - "other_license_expression": null, - "other_license_expression_spdx": null, - "other_license_detections": [], - "extracted_license_statement": null, - "notice_text": null, - "source_packages": [], - "file_references": [], - "extra_data": {}, - "dependencies": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null, - "datasource_id": "pypi_pyproject_toml", - "purl": null - } - ], + "package_data": [], "for_packages": [ "pkg:pypi/setuptools@58.2.0?uuid=fixed-uid-done-for-testing-5642512d1758" ], diff --git a/tests/packagedcode/data/plugin/help.txt b/tests/packagedcode/data/plugin/help.txt index 839a91fcefc..41ca37f2fa5 100755 --- a/tests/packagedcode/data/plugin/help.txt +++ b/tests/packagedcode/data/plugin/help.txt @@ -692,9 +692,16 @@ Package type: pypi description: PyPI extracted egg PKG-INFO path_patterns: '*/EGG-INFO/PKG-INFO' -------------------------------------------- +Package type: pypi + datasource_id: pypi_poetry_pyproject_toml + documentation URL: https://packaging.python.org/en/latest/specifications/pyproject-toml/ + primary language: Python + description: Python poetry pyproject.toml + path_patterns: '*pyproject.toml' +-------------------------------------------- Package type: pypi datasource_id: pypi_pyproject_toml - documentation URL: https://peps.python.org/pep-0621/ + documentation URL: https://packaging.python.org/en/latest/specifications/pyproject-toml/ primary language: Python description: Python pyproject.toml path_patterns: '*pyproject.toml' diff --git a/tests/packagedcode/data/pypi/pyproject-toml/poetry/gerapy-pyproject.toml-expected.json b/tests/packagedcode/data/pypi/pyproject-toml/poetry/gerapy-pyproject.toml-expected.json new file mode 100644 index 00000000000..401de07be85 --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/poetry/gerapy-pyproject.toml-expected.json @@ -0,0 +1,103 @@ +[ + { + "type": "pypi", + "namespace": null, + "name": "gerapy", + "version": "0.9.13", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "Distributed Crawler Management Framework Based on Scrapy, Scrapyd, Scrapyd-Client, Scrapyd-API, Django and Vue.js.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Germey", + "email": "cqc@cuiqingcai.com", + "url": null + } + ], + "keywords": [ + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy" + ], + "homepage_url": "https://github.com/Gerapy/Gerapy", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": "mit", + "declared_license_expression_spdx": "MIT", + "license_detections": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "spdx_license_expression": "MIT", + "from_file": null, + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", + "score": 100.0, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", + "rule_url": null, + "matched_text": "MIT" + } + ], + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf" + }, + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "spdx_license_expression": "MIT", + "from_file": null, + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 5, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "pypi_mit_license.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/pypi_mit_license.RULE", + "matched_text": "- 'License :: OSI Approved :: MIT License'" + } + ], + "identifier": "mit-24a5293c-14d7-5403-efac-1a8b7532c0e8" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "license: MIT\nclassifiers:\n - 'License :: OSI Approved :: MIT License'\n", + "notice_text": null, + "source_packages": [], + "file_references": [], + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": "https://pypi.org/project/gerapy", + "repository_download_url": "https://pypi.org/packages/source/g/gerapy/gerapy-0.9.13.tar.gz", + "api_data_url": "https://pypi.org/pypi/gerapy/0.9.13/json", + "datasource_id": "pypi_poetry_pyproject_toml", + "purl": "pkg:pypi/gerapy@0.9.13" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pyproject-toml/poetry/gerapy/pyproject.toml b/tests/packagedcode/data/pypi/pyproject-toml/poetry/gerapy/pyproject.toml new file mode 100644 index 00000000000..6014013301a --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/poetry/gerapy/pyproject.toml @@ -0,0 +1,56 @@ +# Copied from https://github.com/Gerapy/Gerapy/blob/e0fdd816e495ff52682b34f3136bc785e960263a/pyproject.toml + +[tool.poetry] +name = "gerapy" +version = "0.9.13" +description = "Distributed Crawler Management Framework Based on Scrapy, Scrapyd, Scrapyd-Client, Scrapyd-API, Django and Vue.js." +authors = ["Germey "] +license = "MIT" +readme = "README.md" +homepage = "https://github.com/Gerapy/Gerapy" +include = [ + "LICENSE" +] +classifiers = [ + # Trove classifiers - https://packaging.python.org/specifications/core-metadata/#metadata-classifier + # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy' +] + +[tool.poetry.dependencies] +python = "^3.7" +apscheduler = ">=3.5.1,<=3.7.0" +cryptography = ">=2.8" +django = ">=2.2,<3.0" +django-cors-headers = ">=3.2.0,<=3.7.0" +django-apscheduler = ">=0.3.0,<=0.6.0" +furl = ">=2.1.0" +jinja2 = ">=2.11.3" +scrapy = ">=2.7.1" +scrapy-redis = ">=0.6.8" +scrapy-splash = ">=0.7.2" +python-scrapyd-api = ">=2.1.2" +redis = ">=2.10.5" +requests = ">=2.20.0" +pymongo = ">=3.9.0" +pymysql = ">=0.7.10" +pyquery = ">=1.2.17" +beautifulsoup4 = ">=4.7.0" +djangorestframework = ">=3.11.2" +websocket = ">=0.2.1" +pyppeteer = ">=0.0.25" +uberegg = "^0.1.1" + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +gerapy = "gerapy.cmd:cmd" diff --git a/tests/packagedcode/data/pypi/pyproject-toml/poetry/gino-pyproject.toml-expected.json b/tests/packagedcode/data/pypi/pyproject-toml/poetry/gino-pyproject.toml-expected.json new file mode 100644 index 00000000000..28836dd85f7 --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/poetry/gino-pyproject.toml-expected.json @@ -0,0 +1,127 @@ +[ + { + "type": "pypi", + "namespace": null, + "name": "gino", + "version": "1.1.0-rc.1", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "GINO Is Not ORM - a Python asyncio ORM on SQLAlchemy core.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Fantix King", + "email": "fantix.king@gmail.com", + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": "Tony Wang", + "email": "tony@initialcommit.net", + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": "Fantix King", + "email": "fantix.king@gmail.com", + "url": null + } + ], + "keywords": [ + "orm", + "asyncio", + "sqlalchemy", + "asyncpg", + "python3", + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9" + ], + "homepage_url": "https://python-gino.org", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://github.com/python-gino/gino", + "copyright": null, + "holder": null, + "declared_license_expression": "bsd-new", + "declared_license_expression_spdx": "BSD-3-Clause", + "license_detections": [ + { + "license_expression": "bsd-new", + "license_expression_spdx": "BSD-3-Clause", + "matches": [ + { + "license_expression": "bsd-new", + "spdx_license_expression": "BSD-3-Clause", + "from_file": null, + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "bsd-new_10.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/bsd-new_10.RULE", + "matched_text": "BSD-3-Clause" + } + ], + "identifier": "bsd_new-50fa5753-f24d-ec04-33a1-36bb8ac0492c" + }, + { + "license_expression": "bsd-new", + "license_expression_spdx": "BSD-3-Clause", + "matches": [ + { + "license_expression": "bsd-new", + "spdx_license_expression": "BSD-3-Clause", + "from_file": null, + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 99.0, + "matched_length": 5, + "match_coverage": 100.0, + "rule_relevance": 99, + "rule_identifier": "pypi_bsd_license.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/pypi_bsd_license.RULE", + "matched_text": "- 'License :: OSI Approved :: BSD License'" + } + ], + "identifier": "bsd_new-f4e99f86-00ab-18d9-a65d-a3a12767dcf5" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "license: BSD-3-Clause\nclassifiers:\n - 'License :: OSI Approved :: BSD License'\n", + "notice_text": null, + "source_packages": [], + "file_references": [], + "extra_data": { + "documentation": "https://python-gino.org/docs/" + }, + "dependencies": [], + "repository_homepage_url": "https://pypi.org/project/gino", + "repository_download_url": "https://pypi.org/packages/source/g/gino/gino-1.1.0-rc.1.tar.gz", + "api_data_url": "https://pypi.org/pypi/gino/1.1.0-rc.1/json", + "datasource_id": "pypi_poetry_pyproject_toml", + "purl": "pkg:pypi/gino@1.1.0-rc.1" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pyproject-toml/poetry/gino/pyproject.toml b/tests/packagedcode/data/pypi/pyproject-toml/poetry/gino/pyproject.toml new file mode 100644 index 00000000000..c0ed6395366 --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/poetry/gino/pyproject.toml @@ -0,0 +1,81 @@ +# From https://github.com/python-gino/gino/blob/6218b902cc8a31198f1a0b42d67e689140b68c8b/pyproject.toml +# with unessential parts deleted + +[tool.poetry] +name = "gino" +version = "1.1.0-rc.1" +description = "GINO Is Not ORM - a Python asyncio ORM on SQLAlchemy core." +license = "BSD-3-Clause" +authors = ["Fantix King "] +maintainers = ["Tony Wang ", "Fantix King "] +readme = "README.rst" +homepage = "https://python-gino.org" +repository = "https://github.com/python-gino/gino" +documentation = "https://python-gino.org/docs/" +keywords = ["orm", "asyncio", "sqlalchemy", "asyncpg", "python3"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", +] + +[tool.poetry.dependencies] +python = "^3.6" +SQLAlchemy = ">=1.3,<1.4" + +# drivers +asyncpg = { version = ">=0.18,<1.0", optional = true } +aiomysql = "^0.0.22" + +# compatibility +contextvars = { version = "^2.4", python = "<3.7" } +importlib_metadata = { version = "^2.0.0", python = "<3.8" } + +# extensions +gino-starlette = { version = "^0.1.1", optional = true, python = "^3.6" } +gino-aiohttp = {version = "^0.2.0", optional = true, python = "^3.6"} +gino-tornado = { version = "^0.1.0", optional = true, python = "^3.5.2" } +gino-sanic = { version = "^0.1.0", optional = true, python = "^3.6" } +gino-quart = { version = "^0.1.0", optional = true, python = "^3.7" } + +[tool.poetry.extras] +postgresql = ["asyncpg"] +postgres = ["asyncpg"] +pg = ["asyncpg"] +asyncpg = ["asyncpg"] +mysql = ["aiomysql"] +aiomysql = ["aiomysql"] +starlette = ["gino-starlette"] +aiohttp = ["gino-aiohttp"] +tornado = ["gino-tornado"] +sanic = ["gino-sanic"] +quart = ["gino-quart"] + +[tool.poetry.dev-dependencies] +psycopg2-binary = "^2.9.3" +click = "^8.0.3" + +# tests +pytest = "^7.0.1" +pytest-asyncio = "^0.16.0" +pytest-mock = "^3.6.0" +pytest-cov = "^3.0.0" +black = { version = "^22.1.0", python = ">=3.6.2" } +mypy = "^0.931" + +# docs +sphinx = "^4.3.0" +sphinx-rtd-theme = "^1.0.0" +sphinxcontrib-apidoc = "^0.3.0" +sphinx-autobuild = "^2021.3.14" +sphinx-intl = {extras = ["transifex"], version = "^2.0.1"} + +[build-system] +requires = ["poetry>=1.0"] +build-backend = "poetry.masonry.api" diff --git a/tests/packagedcode/data/pypi/pyproject-toml/standard/apache-airflow-pyclient-pyproject.toml-expected.json b/tests/packagedcode/data/pypi/pyproject-toml/standard/apache-airflow-pyclient-pyproject.toml-expected.json new file mode 100644 index 00000000000..8549cc9c015 --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/standard/apache-airflow-pyclient-pyproject.toml-expected.json @@ -0,0 +1,95 @@ +[ + { + "type": "pypi", + "namespace": null, + "name": "apache-airflow-client", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "Apache Airflow API (Stable)", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Apache Software Foundation", + "email": "dev@airflow.apache.org", + "url": null + } + ], + "keywords": [ + "Apache Airflow API (Stable)", + "OpenAPI", + "OpenAPI-Generator", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Framework :: Apache Airflow", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Monitoring" + ], + "homepage_url": "https://airflow.apache.org/", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/apache/airflow-client-python/issues", + "code_view_url": "https://github.com/apache/airflow/clients/python", + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": "apache-2.0", + "declared_license_expression_spdx": "Apache-2.0", + "license_detections": [ + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "matches": [ + { + "license_expression": "apache-2.0", + "spdx_license_expression": "Apache-2.0", + "from_file": null, + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 95.0, + "matched_length": 6, + "match_coverage": 100.0, + "rule_relevance": 95, + "rule_identifier": "pypi_apache_no-version.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/pypi_apache_no-version.RULE", + "matched_text": "- 'License :: OSI Approved :: Apache Software License'" + } + ], + "identifier": "apache_2_0-e267f9d9-ae62-e9c9-9cc2-8cd0a1e4928f" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "classifiers:\n - 'License :: OSI Approved :: Apache Software License'\n", + "notice_text": null, + "source_packages": [], + "file_references": [], + "extra_data": { + "Changelog": "https://github.com/apache/airflow-client-python/blob/main/CHANGELOG.md", + "Documentation": "https://airflow.apache.org/docs/apache-airflow/stable/stable-rest-api-ref.html", + "Download": "https://archive.apache.org/dist/airflow/clients/python/" + }, + "dependencies": [], + "repository_homepage_url": "https://pypi.org/project/apache-airflow-client", + "repository_download_url": null, + "api_data_url": "https://pypi.org/pypi/apache-airflow-client/json", + "datasource_id": "pypi_pyproject_toml", + "purl": "pkg:pypi/apache-airflow-client" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pyproject-toml/standard/apache-airflow-pyclient/pyproject.toml b/tests/packagedcode/data/pypi/pyproject-toml/standard/apache-airflow-pyclient/pyproject.toml new file mode 100644 index 00000000000..54132fc3aee --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/standard/apache-airflow-pyclient/pyproject.toml @@ -0,0 +1,97 @@ +# Copied from https://raw.githubusercontent.com/apache/airflow/d4bdffc45cd2e55783bba1e7442c346aef7ca573/clients/python/pyproject.toml + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling==1.24.2"] +build-backend = "hatchling.build" + +[project] +name = "apache-airflow-client" +dynamic = ["version"] +description = "Apache Airflow API (Stable)" +readme = "README.md" +license-files.globs = ["LICENSE", "NOTICE"] +requires-python = "~=3.8" +authors = [ + { name = "Apache Software Foundation", email = "dev@airflow.apache.org" }, +] +keywords = [ + "Apache Airflow API (Stable)", + "OpenAPI", + "OpenAPI-Generator", +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Framework :: Apache Airflow", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Monitoring", +] + +dependencies = [ + "python-dateutil", + "urllib3 >= 1.25.3", +] + +[project.urls] +"Bug Tracker" = "https://github.com/apache/airflow-client-python/issues" +Changelog = "https://github.com/apache/airflow-client-python/blob/main/CHANGELOG.md" +Documentation = "https://airflow.apache.org/docs/apache-airflow/stable/stable-rest-api-ref.html" +Download = "https://archive.apache.org/dist/airflow/clients/python/" +Homepage = "https://airflow.apache.org/" +"Source Code" = "https://github.com/apache/airflow/clients/python" + +[tool.hatch.envs.test] +dependencies = [ + "pytest-cov>=2.8.1", + "urllib3 >= 1.25.3", +] + +[tool.hatch.envs.test.scripts] +run-coverage = "pytest test" +run = "run-coverage --no-cov" + +[[tool.hatch.envs.test.matrix]] +python = ["3.8", "3.9", "3.10", "3.11"] + +[tool.hatch.version] +path = "./version.txt" +pattern = "^(?P.+)$" + +[tool.hatch.build.targets.sdist] +artifacts = [ + "/airflow_client", + "/docs", + "/test", + "v1.yaml", +] +include = [ + "version.txt", + "INSTALL", + "README.md", +] + diff --git a/tests/packagedcode/data/pypi/pyproject-toml/standard/apache-airflow-pyproject.toml-expected.json b/tests/packagedcode/data/pypi/pyproject-toml/standard/apache-airflow-pyproject.toml-expected.json new file mode 100644 index 00000000000..95226beab54 --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/standard/apache-airflow-pyproject.toml-expected.json @@ -0,0 +1,109 @@ +[ + { + "type": "pypi", + "namespace": null, + "name": "apache-airflow", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "Programmatically author, schedule and monitor data pipelines", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Apache Software Foundation", + "email": "dev@airflow.apache.org", + "url": null + }, + { + "type": "person", + "role": "maintainer", + "name": "Apache Software Foundation", + "email": "dev@airflow.apache.org", + "url": null + } + ], + "keywords": [ + "airflow", + "orchestration", + "workflow", + "dag", + "pipelines", + "automation", + "data", + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Framework :: Apache Airflow", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Monitoring" + ], + "homepage_url": "https://airflow.apache.org/", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": "https://github.com/apache/airflow/issues", + "code_view_url": "https://github.com/apache/airflow", + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": "apache-2.0", + "declared_license_expression_spdx": "Apache-2.0", + "license_detections": [ + { + "license_expression": "apache-2.0", + "license_expression_spdx": "Apache-2.0", + "matches": [ + { + "license_expression": "apache-2.0", + "spdx_license_expression": "Apache-2.0", + "from_file": null, + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 95.0, + "matched_length": 6, + "match_coverage": 100.0, + "rule_relevance": 95, + "rule_identifier": "pypi_apache_no-version.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/pypi_apache_no-version.RULE", + "matched_text": "- 'License :: OSI Approved :: Apache Software License'" + } + ], + "identifier": "apache_2_0-e267f9d9-ae62-e9c9-9cc2-8cd0a1e4928f" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "classifiers:\n - 'License :: OSI Approved :: Apache Software License'\n", + "notice_text": null, + "source_packages": [], + "file_references": [], + "extra_data": { + "Documentation": "https://airflow.apache.org/docs/", + "Downloads": "https://archive.apache.org/dist/airflow/", + "Release Notes": "https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html", + "Slack Chat": "https://s.apache.org/airflow-slack", + "Twitter": "https://twitter.com/ApacheAirflow", + "YouTube": "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + }, + "dependencies": [], + "repository_homepage_url": "https://pypi.org/project/apache-airflow", + "repository_download_url": null, + "api_data_url": "https://pypi.org/pypi/apache-airflow/json", + "datasource_id": "pypi_pyproject_toml", + "purl": "pkg:pypi/apache-airflow" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pyproject-toml/standard/apache-airflow/pyproject.toml b/tests/packagedcode/data/pypi/pyproject-toml/standard/apache-airflow/pyproject.toml new file mode 100644 index 00000000000..13f8a69ab12 --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/standard/apache-airflow/pyproject.toml @@ -0,0 +1,100 @@ +# copied from https://github.com/apache/airflow/blob/e5bf91b7afeb585e9e70c1708fa290c39ce8094e/pyproject.toml + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +# build dependencies should be fixed - including all transitive dependencies. This way we can ensure +# reproducibility of the build and make sure that any future releases of any dependencies will not +# break the build of released airflow sources in the future. +# The dependencies can be automatically upgraded by running: +# pre-commit run --hook-stage manual update-build-dependencies --all-files +requires = [ + "GitPython==3.1.43", + "gitdb==4.0.11", + "hatchling==1.24.2", + "packaging==24.0", + "pathspec==0.12.1", + "pluggy==1.5.0", + "smmap==5.0.1", + "tomli==2.0.1; python_version < '3.11'", + "trove-classifiers==2024.4.10", +] +build-backend = "hatchling.build" + +[project] +name = "apache-airflow" +description = "Programmatically author, schedule and monitor data pipelines" +readme = { file = "generated/PYPI_README.md", content-type = "text/markdown" } +license-files.globs = ["LICENSE", "3rd-party-licenses/*.txt"] +requires-python = "~=3.8,<3.13" +authors = [ + { name = "Apache Software Foundation", email = "dev@airflow.apache.org" }, +] +maintainers = [ + { name = "Apache Software Foundation", email="dev@airflow.apache.org" }, +] +keywords = [ "airflow", "orchestration", "workflow", "dag", "pipelines", "automation", "data" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Framework :: Apache Airflow", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Monitoring", +] + +dynamic = ["version", "optional-dependencies", "dependencies"] + +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# !!! YOU MIGHT BE SURPRISED NOT SEEING THE DEPENDENCIES AS `project.dependencies` !!!!!!!!! +# !!! AND EXTRAS AS `project.optional-dependencies` !!!!!!!!! +# !!! THEY ARE marked as `dynamic` GENERATED by `hatch_build.py` !!!!!!!!! +# !!! SEE COMMENTS BELOW TO FIND WHERE DEPENDENCIES ARE MAINTAINED !!!!!!!!! +# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# +# !!!!!! Those providers are defined in `hatch_build.py` and should be maintained there !!!!!!! +# +# Those extras are available as regular core airflow extras - they install optional features of Airflow. +# +# START CORE EXTRAS HERE +# +# aiobotocore, apache-atlas, apache-webhdfs, async, cgroups, cloudpickle, deprecated-api, github- +# enterprise, google-auth, graphviz, kerberos, ldap, leveldb, otel, pandas, password, pydantic, +# rabbitmq, s3fs, saml, sentry, statsd, uv, virtualenv +# +# END CORE EXTRAS HERE + +[project.scripts] +airflow = "airflow.__main__:main" +[project.urls] +"Bug Tracker" = "https://github.com/apache/airflow/issues" +Documentation = "https://airflow.apache.org/docs/" +Downloads = "https://archive.apache.org/dist/airflow/" +Homepage = "https://airflow.apache.org/" +"Release Notes" = "https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Source Code" = "https://github.com/apache/airflow" +Twitter = "https://twitter.com/ApacheAirflow" +YouTube = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" diff --git a/tests/packagedcode/data/pypi/pyproject-toml/standard/attrs-pyproject.toml-expected.json b/tests/packagedcode/data/pypi/pyproject-toml/standard/attrs-pyproject.toml-expected.json new file mode 100644 index 00000000000..4d7686688f4 --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/standard/attrs-pyproject.toml-expected.json @@ -0,0 +1,117 @@ +[ + { + "type": "pypi", + "namespace": null, + "name": "attrs", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "Classes Without Boilerplate", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "Hynek Schlawack", + "email": "hs@ox.cx", + "url": null + } + ], + "keywords": [ + "class", + "attribute", + "boilerplate", + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Typing :: Typed" + ], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://github.com/python-attrs/attrs", + "copyright": null, + "holder": null, + "declared_license_expression": "mit", + "declared_license_expression_spdx": "MIT", + "license_detections": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "spdx_license_expression": "MIT", + "from_file": null, + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", + "score": 100.0, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", + "rule_url": null, + "matched_text": "MIT" + } + ], + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf" + }, + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "spdx_license_expression": "MIT", + "from_file": null, + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 5, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "pypi_mit_license.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/pypi_mit_license.RULE", + "matched_text": "- 'License :: OSI Approved :: MIT License'" + } + ], + "identifier": "mit-24a5293c-14d7-5403-efac-1a8b7532c0e8" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "license: MIT\nclassifiers:\n - 'License :: OSI Approved :: MIT License'\n", + "notice_text": null, + "source_packages": [], + "file_references": [], + "extra_data": { + "Documentation": "https://www.attrs.org/", + "Changelog": "https://www.attrs.org/en/stable/changelog.html", + "Funding": "https://github.com/sponsors/hynek", + "Tidelift": "https://tidelift.com/subscription/pkg/pypi-attrs?utm_source=pypi-attrs&utm_medium=pypi" + }, + "dependencies": [], + "repository_homepage_url": "https://pypi.org/project/attrs", + "repository_download_url": null, + "api_data_url": "https://pypi.org/pypi/attrs/json", + "datasource_id": "pypi_pyproject_toml", + "purl": "pkg:pypi/attrs" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pyproject-toml/standard/attrs/pyproject.toml b/tests/packagedcode/data/pypi/pyproject-toml/standard/attrs/pyproject.toml new file mode 100644 index 00000000000..49cef1813d7 --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/standard/attrs/pyproject.toml @@ -0,0 +1,78 @@ +# SPDX-License-Identifier: MIT +# This is copied from https://github.com/python-attrs/attrs/blob/f7f317ae4c3790f23ae027db626593d50e8a4e88/pyproject.toml +# and deleted non-essential parts + +[build-system] +requires = ["hatchling", "hatch-vcs", "hatch-fancy-pypi-readme>=23.2.0"] +build-backend = "hatchling.build" + +[project] +name = "attrs" +authors = [{ name = "Hynek Schlawack", email = "hs@ox.cx" }] +license = "MIT" +requires-python = ">=3.7" +description = "Classes Without Boilerplate" +keywords = ["class", "attribute", "boilerplate"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", + "Typing :: Typed", +] +dependencies = ["importlib_metadata;python_version<'3.8'"] +dynamic = ["version", "readme"] + +[project.optional-dependencies] +tests-mypy = [ + 'pytest-mypy-plugins; platform_python_implementation == "CPython" and python_version >= "3.8"', + # Since the mypy error messages keep changing, we have to keep updating this + # pin. + 'mypy>=1.6,<1.10; platform_python_implementation == "CPython" and python_version >= "3.8"', +] +tests = [ + # For regression test to ensure cloudpickle compat doesn't break. + 'cloudpickle; platform_python_implementation == "CPython"', + "hypothesis", + "pympler", + # 4.3.0 dropped last use of `convert` + "pytest>=4.3.0", + "pytest-xdist[psutil]", + "attrs[tests-mypy]", +] +cov = [ + "attrs[tests]", + # Ensure coverage is new enough for `source_pkgs`. + "coverage[toml]>=5.3", +] +docs = [ + "furo", + "myst-parser", + "sphinx", + "sphinx-notfound-page", + "sphinxcontrib-towncrier", + "towncrier", +] +dev = ["attrs[tests]", "pre-commit"] + +[project.urls] +Documentation = "https://www.attrs.org/" +Changelog = "https://www.attrs.org/en/stable/changelog.html" +GitHub = "https://github.com/python-attrs/attrs" +Funding = "https://github.com/sponsors/hynek" +Tidelift = "https://tidelift.com/subscription/pkg/pypi-attrs?utm_source=pypi-attrs&utm_medium=pypi" + +[tool.black] +line-length = 79 + +[tool.mypy] +pretty = true +disallow_untyped_defs = true +check_untyped_defs = true diff --git a/tests/packagedcode/data/pypi/pyproject-toml/standard/flask-pyproject.toml-expected.json b/tests/packagedcode/data/pypi/pyproject-toml/standard/flask-pyproject.toml-expected.json new file mode 100644 index 00000000000..0142448ac84 --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/standard/flask-pyproject.toml-expected.json @@ -0,0 +1,93 @@ +[ + { + "type": "pypi", + "namespace": null, + "name": "Flask", + "version": "3.1.0.dev", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "A simple framework for building complex web applications.", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "maintainer", + "name": "Pallets", + "email": "contact@palletsprojects.com", + "url": null + } + ], + "keywords": [ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Framework :: Flask", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Topic :: Internet :: WWW/HTTP :: Dynamic Content", + "Topic :: Internet :: WWW/HTTP :: WSGI", + "Topic :: Internet :: WWW/HTTP :: WSGI :: Application", + "Topic :: Software Development :: Libraries :: Application Frameworks", + "Typing :: Typed" + ], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": "https://github.com/pallets/flask/", + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": "bsd-new", + "declared_license_expression_spdx": "BSD-3-Clause", + "license_detections": [ + { + "license_expression": "bsd-new", + "license_expression_spdx": "BSD-3-Clause", + "matches": [ + { + "license_expression": "bsd-new", + "spdx_license_expression": "BSD-3-Clause", + "from_file": null, + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 99.0, + "matched_length": 5, + "match_coverage": 100.0, + "rule_relevance": 99, + "rule_identifier": "pypi_bsd_license.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/pypi_bsd_license.RULE", + "matched_text": "- 'License :: OSI Approved :: BSD License'" + } + ], + "identifier": "bsd_new-f4e99f86-00ab-18d9-a65d-a3a12767dcf5" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "classifiers:\n - 'License :: OSI Approved :: BSD License'\n", + "notice_text": null, + "source_packages": [], + "file_references": [], + "extra_data": { + "Donate": "https://palletsprojects.com/donate", + "Documentation": "https://flask.palletsprojects.com/", + "Changes": "https://flask.palletsprojects.com/changes/", + "Chat": "https://discord.gg/pallets", + "license_file": "LICENSE.txt" + }, + "dependencies": [], + "repository_homepage_url": "https://pypi.org/project/Flask", + "repository_download_url": "https://pypi.org/packages/source/F/Flask/Flask-3.1.0.dev.tar.gz", + "api_data_url": "https://pypi.org/pypi/Flask/3.1.0.dev/json", + "datasource_id": "pypi_pyproject_toml", + "purl": "pkg:pypi/flask@3.1.0.dev" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pyproject-toml/standard/flask/LICENSE.txt b/tests/packagedcode/data/pypi/pyproject-toml/standard/flask/LICENSE.txt new file mode 100644 index 00000000000..114234c21d6 --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/standard/flask/LICENSE.txt @@ -0,0 +1,3 @@ +Copyright 2010 Pallets + +license: bsd-new diff --git a/tests/packagedcode/data/pypi/pyproject-toml/standard/flask/pyproject.toml b/tests/packagedcode/data/pypi/pyproject-toml/standard/flask/pyproject.toml new file mode 100644 index 00000000000..3529fbcf709 --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/standard/flask/pyproject.toml @@ -0,0 +1,75 @@ +# Copied from https://github.com/pallets/flask/blob/19610a9e46e0989054ecd0f6be2fcb981013d8eb/pyproject.toml +# with non-essential parts deleted + +[project] +name = "Flask" +version = "3.1.0.dev" +description = "A simple framework for building complex web applications." +readme = "README.md" +license = {file = "LICENSE.txt"} +maintainers = [{name = "Pallets", email = "contact@palletsprojects.com"}] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Web Environment", + "Framework :: Flask", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Topic :: Internet :: WWW/HTTP :: Dynamic Content", + "Topic :: Internet :: WWW/HTTP :: WSGI", + "Topic :: Internet :: WWW/HTTP :: WSGI :: Application", + "Topic :: Software Development :: Libraries :: Application Frameworks", + "Typing :: Typed", +] +requires-python = ">=3.8" +dependencies = [ + "Werkzeug>=3.0.0", + "Jinja2>=3.1.2", + "itsdangerous>=2.1.2", + "click>=8.1.3", + "blinker>=1.6.2", + "importlib-metadata>=3.6.0; python_version < '3.10'", +] + +[project.urls] +Donate = "https://palletsprojects.com/donate" +Documentation = "https://flask.palletsprojects.com/" +Changes = "https://flask.palletsprojects.com/changes/" +Source = "https://github.com/pallets/flask/" +Chat = "https://discord.gg/pallets" + +[project.optional-dependencies] +async = ["asgiref>=3.2"] +dotenv = ["python-dotenv"] + +[project.scripts] +flask = "flask.cli:main" + +[build-system] +requires = ["flit_core<4"] +build-backend = "flit_core.buildapi" + +[tool.flit.module] +name = "flask" + +[tool.flit.sdist] +include = [ + "docs/", + "examples/", + "requirements/", + "tests/", + "CHANGES.rst", + "CONTRIBUTING.rst", + "tox.ini", +] +exclude = [ + "docs/_build/", +] + +[tool.pytest.ini_options] +testpaths = ["tests"] +filterwarnings = [ + "error", +] + diff --git a/tests/packagedcode/data/pypi/pyproject-toml/standard/lc0-pyproject.toml-expected.json b/tests/packagedcode/data/pypi/pyproject-toml/standard/lc0-pyproject.toml-expected.json new file mode 100644 index 00000000000..615c5027693 --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/standard/lc0-pyproject.toml-expected.json @@ -0,0 +1,59 @@ +[ + { + "type": "pypi", + "namespace": null, + "name": "lczero_bindings", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": "Leela Chess Zero Python bindings", + "release_date": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "The LCZero Authors", + "email": null, + "url": null + } + ], + "keywords": [ + "Programming Language :: Python :: 3", + "Topic :: Games/Entertainment :: Board Games", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Environment :: GPU" + ], + "homepage_url": "https://github.com/LeelaChessZero/lc0", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "extra_data": { + "license_file": "COPYING" + }, + "dependencies": [], + "repository_homepage_url": "https://pypi.org/project/lczero_bindings", + "repository_download_url": "https://pypi.org/packages/source/l/lczero_bindings/lczero_bindings-0.1.0.tar.gz", + "api_data_url": "https://pypi.org/pypi/lczero_bindings/0.1.0/json", + "datasource_id": "pypi_pyproject_toml", + "purl": "pkg:pypi/lczero-bindings@0.1.0" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pyproject-toml/standard/lc0/COPYING b/tests/packagedcode/data/pypi/pyproject-toml/standard/lc0/COPYING new file mode 100644 index 00000000000..cb2594e6936 --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/standard/lc0/COPYING @@ -0,0 +1 @@ +GNU GENERAL PUBLIC LICENSE Version 3 \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pyproject-toml/standard/lc0/pyproject.toml b/tests/packagedcode/data/pypi/pyproject-toml/standard/lc0/pyproject.toml new file mode 100644 index 00000000000..1b9ee74da0d --- /dev/null +++ b/tests/packagedcode/data/pypi/pyproject-toml/standard/lc0/pyproject.toml @@ -0,0 +1,23 @@ +# copied from https://github.com/LeelaChessZero/lc0/blob/6df87ad33ab2ebe12abb70ddbd7dde9c4ea242c6/pyproject.toml + +[build-system] +requires = ["meson-python"] +build-backend = "mesonpy" + +[project] +name = "lczero_bindings" +version = "0.1.0" +description = "Leela Chess Zero Python bindings" +authors = [{ name = "The LCZero Authors" }] +license = {file = "COPYING"} +readme = "README.md" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "Topic :: Games/Entertainment :: Board Games", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Environment :: GPU" +] + +[project.urls] +homepage = "https://github.com/LeelaChessZero/lc0" diff --git a/tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-expected.json b/tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-expected.json index c6a9cff51f1..cfd788ea037 100644 --- a/tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-expected.json +++ b/tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-expected.json @@ -519,16 +519,38 @@ { "type": "pypi", "namespace": null, - "name": null, + "name": "pip", "version": null, "qualifiers": {}, "subpath": null, "primary_language": "Python", - "description": null, + "description": "The PyPA recommended tool for installing Python packages.", "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "The pip developers", + "email": "distutils-sig@python.org", + "url": null + } + ], + "keywords": [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy" + ], + "homepage_url": "https://pip.pypa.io/", "download_url": null, "size": null, "sha1": null, @@ -536,27 +558,75 @@ "sha256": null, "sha512": null, "bug_tracking_url": null, - "code_view_url": null, + "code_view_url": "https://github.com/pypa/pip", "vcs_url": null, "copyright": null, "holder": null, - "declared_license_expression": null, - "declared_license_expression_spdx": null, - "license_detections": [], + "declared_license_expression": "mit", + "declared_license_expression_spdx": "MIT", + "license_detections": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "spdx_license_expression": "MIT", + "from_file": "pip-22.0.4/pyproject.toml", + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", + "score": 100.0, + "matched_length": 1, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", + "rule_url": null, + "matched_text": "MIT" + } + ], + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf" + }, + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "spdx_license_expression": "MIT", + "from_file": "pip-22.0.4/pyproject.toml", + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 5, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "pypi_mit_license.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/pypi_mit_license.RULE", + "matched_text": "- 'License :: OSI Approved :: MIT License'" + } + ], + "identifier": "mit-24a5293c-14d7-5403-efac-1a8b7532c0e8" + } + ], "other_license_expression": null, "other_license_expression_spdx": null, "other_license_detections": [], - "extracted_license_statement": null, + "extracted_license_statement": "license: MIT\nclassifiers:\n - 'License :: OSI Approved :: MIT License'\n", "notice_text": null, "source_packages": [], "file_references": [], - "extra_data": {}, + "extra_data": { + "Documentation": "https://pip.pypa.io", + "Changelog": "https://pip.pypa.io/en/stable/news/" + }, "dependencies": [], - "repository_homepage_url": null, + "repository_homepage_url": "https://pypi.org/project/pip", "repository_download_url": null, - "api_data_url": null, + "api_data_url": "https://pypi.org/pypi/pip/json", "datasource_id": "pypi_pyproject_toml", - "purl": null + "purl": "pkg:pypi/pip" } ], "for_packages": [ diff --git a/tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-only-expected.json b/tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-only-expected.json index 57199d08212..942a7d36d34 100644 --- a/tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-only-expected.json +++ b/tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-only-expected.json @@ -344,16 +344,38 @@ { "type": "pypi", "namespace": null, - "name": null, + "name": "pip", "version": null, "qualifiers": {}, "subpath": null, "primary_language": "Python", - "description": null, + "description": "The PyPA recommended tool for installing Python packages.", "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "The pip developers", + "email": "distutils-sig@python.org", + "url": null + } + ], + "keywords": [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy" + ], + "homepage_url": "https://pip.pypa.io/", "download_url": null, "size": null, "sha1": null, @@ -361,7 +383,7 @@ "sha256": null, "sha512": null, "bug_tracking_url": null, - "code_view_url": null, + "code_view_url": "https://github.com/pypa/pip", "vcs_url": null, "copyright": null, "holder": null, @@ -371,17 +393,20 @@ "other_license_expression": null, "other_license_expression_spdx": null, "other_license_detections": [], - "extracted_license_statement": null, + "extracted_license_statement": "license: MIT\nclassifiers:\n - 'License :: OSI Approved :: MIT License'\n", "notice_text": null, "source_packages": [], "file_references": [], - "extra_data": {}, + "extra_data": { + "Documentation": "https://pip.pypa.io", + "Changelog": "https://pip.pypa.io/en/stable/news/" + }, "dependencies": [], - "repository_homepage_url": null, + "repository_homepage_url": "https://pypi.org/project/pip", "repository_download_url": null, - "api_data_url": null, + "api_data_url": "https://pypi.org/pypi/pip/json", "datasource_id": "pypi_pyproject_toml", - "purl": null + "purl": "pkg:pypi/pip" } ], "for_packages": [], diff --git a/tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-with-license-expected.json b/tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-with-license-expected.json index d5172b954d6..cc43e925229 100644 --- a/tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-with-license-expected.json +++ b/tests/packagedcode/data/pypi/source-package/pip-22.0.4-pypi-package-with-license-expected.json @@ -122,7 +122,7 @@ "identifier": "mit-24a5293c-14d7-5403-efac-1a8b7532c0e8", "license_expression": "mit", "license_expression_spdx": "MIT", - "detection_count": 5, + "detection_count": 7, "reference_matches": [ { "license_expression": "mit", @@ -144,7 +144,7 @@ "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf", "license_expression": "mit", "license_expression_spdx": "MIT", - "detection_count": 3, + "detection_count": 4, "reference_matches": [ { "license_expression": "mit", @@ -270,28 +270,6 @@ } ] }, - { - "identifier": "mit-cacd5c0c-204a-85c2-affc-e4c125b2492a", - "license_expression": "mit", - "license_expression_spdx": "MIT", - "detection_count": 2, - "reference_matches": [ - { - "license_expression": "mit", - "license_expression_spdx": "MIT", - "from_file": "pip-22.0.4/LICENSE.txt", - "start_line": 3, - "end_line": 20, - "matcher": "2-aho", - "score": 100.0, - "matched_length": 161, - "match_coverage": 100.0, - "rule_relevance": 100, - "rule_identifier": "mit.LICENSE", - "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.LICENSE" - } - ] - }, { "identifier": "mit-6e6256c5-00ca-dcb6-8033-2fc4b6ff86be", "license_expression": "mit", @@ -327,6 +305,28 @@ "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/pypi_mit_license.RULE" } ] + }, + { + "identifier": "mit-cacd5c0c-204a-85c2-affc-e4c125b2492a", + "license_expression": "mit", + "license_expression_spdx": "MIT", + "detection_count": 1, + "reference_matches": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "from_file": "pip-22.0.4/LICENSE.txt", + "start_line": 3, + "end_line": 20, + "matcher": "2-aho", + "score": 100.0, + "matched_length": 161, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "mit.LICENSE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.LICENSE" + } + ] } ], "files": [ @@ -874,16 +874,38 @@ { "type": "pypi", "namespace": null, - "name": null, + "name": "pip", "version": null, "qualifiers": {}, "subpath": null, "primary_language": "Python", - "description": null, + "description": "The PyPA recommended tool for installing Python packages.", "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, + "parties": [ + { + "type": "person", + "role": "author", + "name": "The pip developers", + "email": "distutils-sig@python.org", + "url": null + } + ], + "keywords": [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Topic :: Software Development :: Build Tools", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy" + ], + "homepage_url": "https://pip.pypa.io/", "download_url": null, "size": null, "sha1": null, @@ -891,7 +913,7 @@ "sha256": null, "sha512": null, "bug_tracking_url": null, - "code_view_url": null, + "code_view_url": "https://github.com/pypa/pip", "vcs_url": null, "copyright": null, "holder": null, @@ -905,45 +927,93 @@ { "license_expression": "mit", "spdx_license_expression": "MIT", - "from_file": "pip-22.0.4/LICENSE.txt", - "start_line": 3, - "end_line": 20, - "matcher": "2-aho", + "from_file": "pip-22.0.4/pyproject.toml", + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", "score": 100.0, - "matched_length": 161, + "matched_length": 1, "match_coverage": 100.0, "rule_relevance": 100, - "rule_identifier": "mit.LICENSE", - "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.LICENSE" + "rule_identifier": "spdx-license-identifier-mit-5da48780aba670b0860c46d899ed42a0f243ff06", + "rule_url": null, + "matched_text": "MIT" } ], - "identifier": "mit-cacd5c0c-204a-85c2-affc-e4c125b2492a" + "identifier": "mit-a822f434-d61f-f2b1-c792-8b8cb9e7b9bf" + }, + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "spdx_license_expression": "MIT", + "from_file": "pip-22.0.4/pyproject.toml", + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 5, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "pypi_mit_license.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/pypi_mit_license.RULE", + "matched_text": "- 'License :: OSI Approved :: MIT License'" + } + ], + "identifier": "mit-24a5293c-14d7-5403-efac-1a8b7532c0e8" } ], "other_license_expression": null, "other_license_expression_spdx": null, "other_license_detections": [], - "extracted_license_statement": null, + "extracted_license_statement": "license: MIT\nclassifiers:\n - 'License :: OSI Approved :: MIT License'\n", "notice_text": null, "source_packages": [], "file_references": [], - "extra_data": {}, + "extra_data": { + "Documentation": "https://pip.pypa.io", + "Changelog": "https://pip.pypa.io/en/stable/news/" + }, "dependencies": [], - "repository_homepage_url": null, + "repository_homepage_url": "https://pypi.org/project/pip", "repository_download_url": null, - "api_data_url": null, + "api_data_url": "https://pypi.org/pypi/pip/json", "datasource_id": "pypi_pyproject_toml", - "purl": null + "purl": "pkg:pypi/pip" } ], "for_packages": [ "pkg:pypi/pip@22.0.4?uuid=fixed-uid-done-for-testing-5642512d1758" ], - "detected_license_expression": null, - "detected_license_expression_spdx": null, - "license_detections": [], + "detected_license_expression": "mit", + "detected_license_expression_spdx": "MIT", + "license_detections": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "matches": [ + { + "license_expression": "mit", + "spdx_license_expression": "MIT", + "from_file": "pip-22.0.4/pyproject.toml", + "start_line": 11, + "end_line": 11, + "matcher": "2-aho", + "score": 100.0, + "matched_length": 5, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "pypi_mit_license.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/pypi_mit_license.RULE" + } + ], + "identifier": "mit-24a5293c-14d7-5403-efac-1a8b7532c0e8" + } + ], "license_clues": [], - "percentage_of_license_text": 0, + "percentage_of_license_text": 2.38, "scan_errors": [] }, { diff --git a/tests/packagedcode/data/pypi/source-package/pip-22.0.4/pyproject.toml b/tests/packagedcode/data/pypi/source-package/pip-22.0.4/pyproject.toml index a02457eeffd..373866fa7f6 100644 --- a/tests/packagedcode/data/pypi/source-package/pip-22.0.4/pyproject.toml +++ b/tests/packagedcode/data/pypi/source-package/pip-22.0.4/pyproject.toml @@ -1,71 +1,57 @@ -[build-system] -requires = ["setuptools", "wheel"] -build-backend = "setuptools.build_meta" - -[tool.towncrier] -# For finding the __version__ -package = "pip" -package_dir = "src" -# For writing into the correct file -filename = "NEWS.rst" -# For finding the news fragments -directory = "news/" +[project] +dynamic = ["version"] + +name = "pip" +description = "The PyPA recommended tool for installing Python packages." +readme = "README.rst" +license = {text = "MIT"} +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Topic :: Software Development :: Build Tools", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] +authors = [ + {name = "The pip developers", email = "distutils-sig@python.org"}, +] -# For rendering properly for this project -issue_format = "`#{issue} `_" -template = "tools/news/template.rst" +# NOTE: requires-python is duplicated in __pip-runner__.py. +# When changing this value, please change the other copy as well. +requires-python = ">=3.8" -# Grouping of entries, within our changelog -type = [ - { name = "Process", directory = "process", showcontent = true }, - { name = "Deprecations and Removals", directory = "removal", showcontent = true }, - { name = "Features", directory = "feature", showcontent = true }, - { name = "Bug Fixes", directory = "bugfix", showcontent = true }, - { name = "Vendored Libraries", directory = "vendor", showcontent = true }, - { name = "Improved Documentation", directory = "doc", showcontent = true }, - { name = "Trivial Changes", directory = "trivial", showcontent = false }, -] +[project.scripts] +pip = "pip._internal.cli.main:main" +pip3 = "pip._internal.cli.main:main" -[tool.vendoring] -destination = "src/pip/_vendor/" -requirements = "src/pip/_vendor/vendor.txt" -namespace = "pip._vendor" +[project.urls] +Homepage = "https://pip.pypa.io/" +Documentation = "https://pip.pypa.io" +Source = "https://github.com/pypa/pip" +Changelog = "https://pip.pypa.io/en/stable/news/" -protected-files = ["__init__.py", "README.rst", "vendor.txt"] -patches-dir = "tools/vendoring/patches" +[build-system] +# The lower bound is for . +requires = ["setuptools>=67.6.1", "wheel"] +build-backend = "setuptools.build_meta" -[tool.vendoring.transformations] -substitute = [ - # pkg_resource's vendored packages are directly vendored in pip. - { match='pkg_resources\.extern', replace="pip._vendor" }, - { match='from \.extern', replace="from pip._vendor" }, - { match='''\('pygments\.lexers\.''', replace="('pip._vendor.pygments.lexers." }, -] -drop = [ - # contains unnecessary scripts - "bin/", - # interpreter and OS specific msgpack libs - "msgpack/*.so", - # unneeded parts of setuptools - "easy_install.py", - "setuptools", - "pkg_resources/_vendor/", - "pkg_resources/extern/", - # trim vendored pygments styles and lexers - "pygments/styles/[!_]*.py", - '^pygments/lexers/(?!python|__init__|_mapping).*\.py$', - # trim rich's markdown support - "rich/markdown.py", -] +[tool.setuptools] +package-dir = {"" = "src"} +include-package-data = false -[tool.vendoring.typing-stubs] -six = ["six.__init__", "six.moves.__init__", "six.moves.configparser"] -distro = [] +[tool.setuptools.dynamic] +version = {attr = "pip.__version__"} -[tool.vendoring.license.directories] -setuptools = "pkg_resources" +[tool.setuptools.packages.find] +where = ["src"] +exclude = ["contrib", "docs", "tests*", "tasks"] -[tool.vendoring.license.fallback-urls] -CacheControl = "https://raw.githubusercontent.com/ionrock/cachecontrol/v0.12.6/LICENSE.txt" -distlib = "https://bitbucket.org/pypa/distlib/raw/master/LICENSE.txt" -webencodings = "https://github.com/SimonSapin/python-webencodings/raw/master/LICENSE" diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py index b38e7e5b5ad..1bf786b11fc 100644 --- a/tests/packagedcode/test_pypi.py +++ b/tests/packagedcode/test_pypi.py @@ -295,6 +295,60 @@ def test_parse_metadata_prefer_pkg_info_from_egg_info_from_command_line(self): assert path == 'celery/celery.egg-info/PKG-INFO' +class TestPyprojectTomlFileHandler(PackageTester): + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_is_pyproject_toml_standard(self): + test_file = self.get_test_loc('pypi/pyproject-toml/standard/attrs/pyproject.toml') + assert pypi.PyprojectTomlHandler.is_datafile(test_file) + + def test_parse_pyproject_toml_standard_attrs(self): + test_file = self.get_test_loc('pypi/pyproject-toml/standard/attrs/pyproject.toml') + package = pypi.PyprojectTomlHandler.parse(test_file) + expected_loc = self.get_test_loc('pypi/pyproject-toml/standard/attrs-pyproject.toml-expected.json') + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + def test_parse_pyproject_toml_standard_apache_airflow(self): + test_file = self.get_test_loc('pypi/pyproject-toml/standard/apache-airflow/pyproject.toml') + package = pypi.PyprojectTomlHandler.parse(test_file) + expected_loc = self.get_test_loc('pypi/pyproject-toml/standard/apache-airflow-pyproject.toml-expected.json') + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + def test_parse_pyproject_toml_standard_apache_airflow_client(self): + test_file = self.get_test_loc('pypi/pyproject-toml/standard/apache-airflow-pyclient/pyproject.toml') + package = pypi.PyprojectTomlHandler.parse(test_file) + expected_loc = self.get_test_loc('pypi/pyproject-toml/standard/apache-airflow-pyclient-pyproject.toml-expected.json') + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + def test_parse_pyproject_toml_standard_flask(self): + test_file = self.get_test_loc('pypi/pyproject-toml/standard/flask/pyproject.toml') + package = pypi.PyprojectTomlHandler.parse(test_file) + expected_loc = self.get_test_loc('pypi/pyproject-toml/standard/flask-pyproject.toml-expected.json') + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + def test_parse_pyproject_toml_standard_lc0(self): + test_file = self.get_test_loc('pypi/pyproject-toml/standard/lc0/pyproject.toml') + package = pypi.PyprojectTomlHandler.parse(test_file) + expected_loc = self.get_test_loc('pypi/pyproject-toml/standard/lc0-pyproject.toml-expected.json') + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + def test_is_pyproject_toml_poetry(self): + test_file = self.get_test_loc('pypi/pyproject-toml/poetry/gerapy/pyproject.toml') + assert pypi.PoetryPyprojectTomlHandler.is_datafile(test_file) + + def test_parse_pyproject_toml_poetry_gerapy(self): + test_file = self.get_test_loc('pypi/pyproject-toml/poetry/gerapy/pyproject.toml') + package = pypi.PoetryPyprojectTomlHandler.parse(test_file) + expected_loc = self.get_test_loc('pypi/pyproject-toml/poetry/gerapy-pyproject.toml-expected.json') + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + def test_parse_pyproject_toml_poetry_gino(self): + test_file = self.get_test_loc('pypi/pyproject-toml/poetry/gino/pyproject.toml') + package = pypi.PoetryPyprojectTomlHandler.parse(test_file) + expected_loc = self.get_test_loc('pypi/pyproject-toml/poetry/gino-pyproject.toml-expected.json') + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + class TestPipRequirementsFileHandler(PackageTester): test_data_dir = os.path.join(os.path.dirname(__file__), 'data') diff --git a/tests/summarycode/data/summary/summary_without_holder/summary-without-holder-pypi.expected.json b/tests/summarycode/data/summary/summary_without_holder/summary-without-holder-pypi.expected.json index 72927f9a554..1e168c82cd1 100644 --- a/tests/summarycode/data/summary/summary_without_holder/summary-without-holder-pypi.expected.json +++ b/tests/summarycode/data/summary/summary_without_holder/summary-without-holder-pypi.expected.json @@ -226,28 +226,6 @@ } ] }, - { - "identifier": "mit-cacd5c0c-204a-85c2-affc-e4c125b2492a", - "license_expression": "mit", - "license_expression_spdx": "MIT", - "detection_count": 2, - "reference_matches": [ - { - "license_expression": "mit", - "license_expression_spdx": "MIT", - "from_file": "pip-22.0.4/LICENSE.txt", - "start_line": 3, - "end_line": 20, - "matcher": "2-aho", - "score": 100.0, - "matched_length": 161, - "match_coverage": 100.0, - "rule_relevance": 100, - "rule_identifier": "mit.LICENSE", - "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.LICENSE" - } - ] - }, { "identifier": "mit-3fce6ea2-8abd-6c6b-3ede-a37af7c6efee", "license_expression": "mit", @@ -341,6 +319,28 @@ "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/pypi_mit_license.RULE" } ] + }, + { + "identifier": "mit-cacd5c0c-204a-85c2-affc-e4c125b2492a", + "license_expression": "mit", + "license_expression_spdx": "MIT", + "detection_count": 1, + "reference_matches": [ + { + "license_expression": "mit", + "license_expression_spdx": "MIT", + "from_file": "pip-22.0.4/LICENSE.txt", + "start_line": 3, + "end_line": 20, + "matcher": "2-aho", + "score": 100.0, + "matched_length": 161, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "mit.LICENSE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.LICENSE" + } + ] } ], "files": [ @@ -933,80 +933,15 @@ { "path": "pip-22.0.4/pyproject.toml", "type": "file", - "package_data": [ - { - "type": "pypi", - "namespace": null, - "name": null, - "version": null, - "qualifiers": {}, - "subpath": null, - "primary_language": "Python", - "description": null, - "release_date": null, - "parties": [], - "keywords": [], - "homepage_url": null, - "download_url": null, - "size": null, - "sha1": null, - "md5": null, - "sha256": null, - "sha512": null, - "bug_tracking_url": null, - "code_view_url": null, - "vcs_url": null, - "copyright": null, - "holder": null, - "declared_license_expression": "mit", - "declared_license_expression_spdx": "MIT", - "license_detections": [ - { - "license_expression": "mit", - "license_expression_spdx": "MIT", - "matches": [ - { - "license_expression": "mit", - "spdx_license_expression": "MIT", - "from_file": "pip-22.0.4/LICENSE.txt", - "start_line": 3, - "end_line": 20, - "matcher": "2-aho", - "score": 100.0, - "matched_length": 161, - "match_coverage": 100.0, - "rule_relevance": 100, - "rule_identifier": "mit.LICENSE", - "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.LICENSE" - } - ], - "identifier": "mit-cacd5c0c-204a-85c2-affc-e4c125b2492a" - } - ], - "other_license_expression": null, - "other_license_expression_spdx": null, - "other_license_detections": [], - "extracted_license_statement": null, - "notice_text": null, - "source_packages": [], - "file_references": [], - "extra_data": {}, - "dependencies": [], - "repository_homepage_url": null, - "repository_download_url": null, - "api_data_url": null, - "datasource_id": "pypi_pyproject_toml", - "purl": null - } - ], + "package_data": [], "for_packages": [ "pkg:pypi/pip@22.0.4?uuid=fixed-uid-done-for-testing-5642512d1758" ], "is_legal": false, - "is_manifest": false, + "is_manifest": true, "is_readme": false, - "is_top_level": false, - "is_key_file": false, + "is_top_level": true, + "is_key_file": true, "detected_license_expression": null, "detected_license_expression_spdx": null, "license_detections": [],