diff --git a/CHANGELOG.rst b/CHANGELOG.rst index be9194a8eba..f0e537d2d7f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -48,14 +48,6 @@ Important API changes: column to "path". The "copyright_holder" has been ranmed to "holder" -Development environment changes: -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - - The license cache consistency is not checked anymore when you are using a Git - checkout. The SCANCODE_DEV_MODE tag file has been removed entirely. Use - instead the --reindex-licenses option to rebuild the license index. - - Copyright detection: ~~~~~~~~~~~~~~~~~~~~ @@ -122,7 +114,6 @@ License detection: - Small, two-words matches that overlap the previous or next match by by the word "license" and assimilated are now filtered as false matches. - - The new --licenses-reference option adds a new "licenses_reference" top level attribute to a scan when using the JSON and YAML outputs. This contains all the details and the full text of every licenses seen in a file or @@ -231,6 +222,66 @@ License Clarity Scoring Update - Scoring Weight = -20 +License Clarity Scoring Update +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + - We are moving away from the license clarity scoring defined by ClearlyDefined + in the license clarity score plugin. The previous license clarity scoring + logic produced a score that was misleading, where it would return a low score + when scanning packages due to the stringent scoring criteria. We are now + using more general criteria to get a sense of what provenance information has + been provided and whether or not there is a conflict in licensing between + what licenses were declared at the top-level key files and what licenses have + been detected in the files under the top-level. + + - The license clarity score is a value from 0-100 calculated by combining the + weighted values determined for each of the scoring elements: + + - Declared license: + + - When true, indicates that the software package licensing is documented at + top-level or well-known locations in the software project, typically in a + package manifest, NOTICE, LICENSE, COPYING or README file. + - Scoring Weight = 40 + + - Identification precision: + + - Indicates how well the license statement(s) of the software identify known + licenses that can be designated by precise keys (identifiers) as provided in + a publicly available license list, such as the ScanCode LicenseDB, the SPDX + license list, the OSI license list, or a URL pointing to a specific license + text in a project or organization website. + - Scoring Weight = 40 + + - License texts: + + - License texts are provided to support the declared license expression in + files such as a package manifest, NOTICE, LICENSE, COPYING or README. + - Scoring Weight = 10 + + - Declared copyright: + + - When true, indicates that the software package copyright is documented at + top-level or well-known locations in the software project, typically in a + package manifest, NOTICE, LICENSE, COPYING or README file. + - Scoring Weight = 10 + + - Ambiguous compound licensing: + + - When true, indicates that the software has a license declaration that + makes it difficult to construct a reliable license expression, such as in + the case of multiple licenses where the conjunctive versus disjunctive + relationship is not well defined. + - Scoring Weight = -10 + + - Conflicting license categories: + + - When true, indicates the declared license expression of the software is in + the permissive category, but that other potentially conflicting categories, + such as copyleft and proprietary, have been detected in lower level code. + - Scoring Weight = -20 + + Outputs: ~~~~~~~~ @@ -251,6 +302,7 @@ Changes: - add resource-level attribute `for_packages`. - remove `package-data` attribute `root_path`. + Documentation Update ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -258,6 +310,18 @@ Documentation Update correct minor documentation issues. +Development environment changes: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The license cache consistency is not checked anymore when you are using a Git + checkout. The SCANCODE_DEV_MODE tag file has been removed entirely. Use + instead the --reindex-licenses option to rebuild the license index. + +- We can now regenerate updated test fixtures using the new SCANCODE_REGEN_TEST_FIXTURES + environemnt variable. There is no need to replace the regen=False with regen=True + in the code. + + 30.1.0 - 2021-09-25 -------------------- diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 35af2e2a552..af78237eb7c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,32 +8,25 @@ jobs: ################################################################################ -# These jobs are using VMs and Azure-provided Pythons 3.6 +# These jobs are using VMs and Azure-provided Pythons 3.8 ################################################################################ - template: etc/ci/azure-posix.yml parameters: job_name: core_tests image_name: ubuntu-20.04 - python_versions: ['3.6'] + python_versions: ['3.8'] test_suites: - misc: | + misc_and_scancode: | + # cli tests are launched below on all OSes venv/bin/pytest -n 3 -vvs --test-suite=all \ - --ignore=tests/scancode \ --ignore=tests/cluecode \ - --ignore=tests/licensedcode + --ignore=tests/licensedcode \ + --ignore=tests/scancode/test_cli.py - scancode: | - venv/bin/pytest -vvs --test-suite=all \ - tests/scancode - - cluecode: | + cluecode_license_base: | venv/bin/pytest -n 3 -vvs --test-suite=all \ tests/cluecode \ - --cov=src --cov-report=term - - license_base: | - venv/bin/pytest -n 3 -vvs --test-suite=all \ --ignore=tests/licensedcode/test_zzzz_cache.py \ --ignore=tests/licensedcode/test_detection_datadriven_external.py \ --ignore=tests/licensedcode/test_detection_datadriven1.py \ @@ -42,20 +35,14 @@ jobs: --ignore=tests/licensedcode/test_detection_datadriven4.py \ tests/licensedcode - license_datadriven1: | - venv/bin/pytest -n 3 -vvs --test-suite=all \ - tests/licensedcode/test_detection_datadriven1.py - - license_datadriven2: | + license_datadriven1_2: | venv/bin/pytest -n 3 -vvs --test-suite=all \ + tests/licensedcode/test_detection_datadriven1.py \ tests/licensedcode/test_detection_datadriven2.py - license_datadriven3: | - venv/bin/pytest -n 3 -vvs --test-suite=all \ - tests/licensedcode/test_detection_datadriven3.py - - license_datadriven4: | + license_datadriven3_4: | venv/bin/pytest -n 3 -vvs --test-suite=all \ + tests/licensedcode/test_detection_datadriven3.py \ tests/licensedcode/test_detection_datadriven4.py license_datadriven_ext: | @@ -91,78 +78,90 @@ jobs: venv/bin/pytest -n 3 -vvs --test-suite=all \ tests/licensedcode/test_zzzz_cache.py - - template: etc/ci/azure-posix.yml parameters: - job_name: mac11 - image_name: macOS-11 - python_versions: ['3.7', '3.8', '3.9'] + job_name: ubuntu18_cpython + image_name: ubuntu-18.04 + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] python_architecture: x64 test_suites: all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu20_cpython + image_name: ubuntu-20.04 + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python_architecture: x64 + test_suites: + all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py - template: etc/ci/azure-posix.yml parameters: - job_name: mac1015 + job_name: macos1015_cpython_1 image_name: macos-10.15 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.6', '3.7', '3.8'] python_architecture: x64 test_suites: all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos1015_cpython_2 + image_name: macos-10.15 + python_versions: ['3.9', '3.10'] + python_architecture: x64 + test_suites: + all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py -# - template: etc/ci/azure-posix.yml -# parameters: -# job_name: mac1100 -# image_name: macos-11.0 -# python_versions: ['3.6', '3.7', '3.8', '3.9'] -# python_architecture: x64 -# test_suites: -# all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py - + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos11_cpython + image_name: macos-11 + python_versions: ['3.7', '3.8', '3.9', '3.10'] + python_architecture: x64 + test_suites: + all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py - template: etc/ci/azure-win.yml parameters: - job_name: win2022 - image_name: windows-2022 - python_versions: ['3.7', '3.8', '3.9'] + job_name: win2019_cpython_1 + image_name: windows-2019 + python_versions: ['3.6', '3.7', '3.8'] python_architecture: x64 test_suites: all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py - - template: etc/ci/azure-win.yml parameters: - job_name: win2019 + job_name: win2019_cpython_2 image_name: windows-2019 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.9', '3.10'] python_architecture: x64 test_suites: all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py - - - template: etc/ci/azure-posix.yml + - template: etc/ci/azure-win.yml parameters: - job_name: ubuntu18 - image_name: ubuntu-18.04 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + job_name: win2022_cpython_1 + image_name: windows-2022 + python_versions: ['3.7', '3.8'] python_architecture: x64 test_suites: - all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py + all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py - - template: etc/ci/azure-posix.yml + - template: etc/ci/azure-win.yml parameters: - job_name: ubuntu20 - image_name: ubuntu-20.04 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + job_name: win2022_cpython_2 + image_name: windows-2022 + python_versions: ['3.9', '3.10'] python_architecture: x64 test_suites: - all: venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py + all: venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py ################################################################################ -# Test using many version of Click to work around regressions in API +# Test using many version of Click to work around any regressions in their API ################################################################################ @@ -170,11 +169,11 @@ jobs: parameters: job_name: ubuntu20_test_all_supported_click_versions image_name: ubuntu-20.04 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] python_architecture: x64 test_suites: click_versions: | - for clk_ver in 8.0.1 7.1.2 7.1.1 7.1 6.7; + for clk_ver in 8.0.4 8.0.3 8.0.2 8.0.1 7.1.2 7.1.1 7.1 6.7; do venv/bin/pip install click==$clk_ver; venv/bin/scancode -i samples/ -n3 --json foo.json; @@ -182,7 +181,6 @@ jobs: done - ################################################################################ # These jobs are using containers and their own Python 3.6 ################################################################################ @@ -305,53 +303,88 @@ jobs: # test_suite: venv/bin/pytest -n 2 -vvs -######################################################################## -# RELEASE on 3.6. Also check that we can pip install -######################################################################## -# -# - job: Build_release_archive_py3 -# pool: -# vmImage: ubuntu-18.04 -# steps: -# - checkout: self -# fetchDepth: 10 -# -# - task: UsePythonVersion@0 -# inputs: -# versionSpec: '3.6' -# displayName: 'Install Python 3.6' -# -# - script: ./etc/release/scancode-test-pip-install.sh -# displayName: 'Test pip wheel installation' -# -# - script: ./etc/release/scancode-test-pip-install-editable.sh -# displayName: 'Test pip editable installation' - ################################################################################ # Tests using a plain pip install to get the latest of all wheels ################################################################################ + - template: etc/ci/azure-posix.yml + parameters: + job_name: ubuntu18_cpython_latest_from_pip + image_name: ubuntu-18.04 + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] + test_suites: + all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py + - template: etc/ci/azure-posix.yml parameters: job_name: ubuntu20_cpython_latest_from_pip image_name: ubuntu-20.04 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.6', '3.7', '3.8', '3.9', '3.10'] test_suites: - all: venv/bin/pip install --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py + all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py - - template: etc/ci/azure-win.yml + - template: etc/ci/azure-posix.yml parameters: - job_name: win2019_cpython_latest_from_pip - image_name: windows-2019 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + job_name: macos1015_cpython_latest_from_pip1 + image_name: macos-10.15 + python_versions: ['3.6', '3.7'] test_suites: - all: venv\Scripts\pip install --force-reinstall --upgrade -e .[dev] && venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py + all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py - template: etc/ci/azure-posix.yml parameters: - job_name: macos1015_cpython_latest_from_pip + job_name: macos1015_cpython_latest_from_pip2 image_name: macos-10.15 - python_versions: ['3.6', '3.7', '3.8', '3.9'] + python_versions: ['3.8', '3.9', '3.10'] + test_suites: + all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos11_cpython_latest_from_pip1 + image_name: macos-11 + python_versions: ['3.7', '3.8', ] + test_suites: + all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py + + - template: etc/ci/azure-posix.yml + parameters: + job_name: macos11_cpython_latest_from_pip2 + image_name: macos-11 + python_versions: ['3.9', '3.10'] + test_suites: + all: venv/bin/pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2019_cpython_latest_from_pip1 + image_name: windows-2019 + python_versions: ['3.6', '3.7'] + test_suites: + all: venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2019_cpython_latest_from_pip2 + image_name: windows-2019 + python_versions: ['3.8', '3.9', '3.10'] test_suites: - all: venv/bin/pip install --force-reinstall --upgrade -e .[dev] && venv/bin/pytest -n 2 -vvs tests/scancode/test_cli.py + all: venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2022_cpython_latest_from_pip1 + image_name: windows-2022 + python_versions: ['3.7', '3.8'] + test_suites: + all: venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py + + - template: etc/ci/azure-win.yml + parameters: + job_name: win2022_cpython_latest_from_pip2 + image_name: windows-2022 + python_versions: ['3.9', '3.10'] + test_suites: + all: venv\Scripts\pip install --upgrade-strategy eager --force-reinstall --upgrade -e .[dev] && venv\Scripts\pytest -n 2 -vvs tests\scancode\test_cli.py + diff --git a/configure b/configure index f5a63a4daa6..49fddded1a9 100755 --- a/configure +++ b/configure @@ -16,6 +16,8 @@ set -e # Source this script for initial configuration # Use configure --help for details # +# NOTE: please keep in sync with Windows script configure.bat +# # This script will search for a virtualenv.pyz app in etc/thirdparty/virtualenv.pyz # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default ################################ @@ -27,15 +29,13 @@ CLI_ARGS=$1 # Requirement arguments passed to pip and used by default or with --dev. REQUIREMENTS="--editable . --constraint requirements.txt" -DEV_REQUIREMENTS="--editable .[dev,packages] --constraint requirements.txt --constraint requirements-dev.txt" +DEV_REQUIREMENTS="--editable .[testing,packages] --constraint requirements.txt --constraint requirements-dev.txt" # where we create a virtualenv VIRTUALENV_DIR=venv -# Cleanable files and directories with the --clean option -CLEANABLE=" - build - venv" +# Cleanable files and directories to delete with the --clean option +CLEANABLE="build venv" # extra arguments passed to pip PIP_EXTRA_ARGS=" " @@ -50,16 +50,17 @@ VIRTUALENV_PYZ_URL=https://bootstrap.pypa.io/virtualenv.pyz CFG_ROOT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin + +################################ +# Thirdparty package locations and index handling # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org +# offline mode for scancode installation with no index at all if [ -f "$CFG_ROOT_DIR/thirdparty" ]; then - # offline mode - PIP_EXTRA_ARGS="--no-index --find-links $CFG_ROOT_DIR/thirdparty " + PIP_EXTRA_ARGS="--no-index --find-links $CFG_ROOT_DIR/thirdparty" else - # online mode - PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --index https://thirdparty.aboutcode.org/pypi/simple" + PIP_EXTRA_ARGS="--find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" fi - ################################ # Set the quiet flag to empty if not defined if [[ "$CFG_QUIET" == "" ]]; then @@ -68,56 +69,22 @@ fi ################################ -# find a proper Python to run +# Find a proper Python to run # Use environment variables or a file if available. # Otherwise the latest Python by default. -if [[ "$PYTHON_EXECUTABLE" == "" ]]; then - # check for a file named PYTHON_EXECUTABLE - if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then - PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") - else - PYTHON_EXECUTABLE=python3 +find_python() { + if [[ "$PYTHON_EXECUTABLE" == "" ]]; then + # check for a file named PYTHON_EXECUTABLE + if [ -f "$CFG_ROOT_DIR/PYTHON_EXECUTABLE" ]; then + PYTHON_EXECUTABLE=$(cat "$CFG_ROOT_DIR/PYTHON_EXECUTABLE") + else + PYTHON_EXECUTABLE=python3 + fi fi -fi - - -################################ -cli_help() { - echo An initial configuration script - echo " usage: ./configure [options]" - echo - echo The default is to configure for regular use. Use --dev for development. - echo Use the --init option if starting a new project and the project - echo dependencies are not available on thirdparty.aboutcode.org/pypi/ - echo and requirements.txt and/or requirements-dev.txt has not been generated. - echo - echo The options are: - echo " --clean: clean built and installed files and exit." - echo " --dev: configure the environment for development." - echo " --init: pull dependencies from PyPI. Used when first setting up a project." - echo " --help: display this help message and exit." - echo - echo By default, the python interpreter version found in the path is used. - echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to - echo configure another Python executable interpreter to use. If this is not - echo set, a file named PYTHON_EXECUTABLE containing a single line with the - echo path of the Python executable to use will be checked last. - set +e - exit -} - - -clean() { - # Remove cleanable file and directories and files from the root dir. - echo "* Cleaning ..." - for cln in $CLEANABLE; - do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; - done - set +e - exit } +################################ create_virtualenv() { # create a virtualenv for Python # Note: we do not use the bundled Python 3 "venv" because its behavior and @@ -148,6 +115,7 @@ create_virtualenv() { } +################################ install_packages() { # install requirements in virtualenv # note: --no-build-isolation means that pip/wheel/setuptools will not @@ -164,6 +132,40 @@ install_packages() { } +################################ +cli_help() { + echo An initial configuration script + echo " usage: ./configure [options]" + echo + echo The default is to configure for regular use. Use --dev for development. + echo + echo The options are: + echo " --clean: clean built and installed files and exit." + echo " --dev: configure the environment for development." + echo " --help: display this help message and exit." + echo + echo By default, the python interpreter version found in the path is used. + echo Alternatively, the PYTHON_EXECUTABLE environment variable can be set to + echo configure another Python executable interpreter to use. If this is not + echo set, a file named PYTHON_EXECUTABLE containing a single line with the + echo path of the Python executable to use will be checked last. + set +e + exit +} + + +################################ +clean() { + # Remove cleanable file and directories and files from the root dir. + echo "* Cleaning ..." + for cln in $CLEANABLE; + do rm -rf "${CFG_ROOT_DIR:?}/${cln:?}"; + done + set +e + exit +} + + ################################ # Main command line entry point CFG_REQUIREMENTS=$REQUIREMENTS @@ -174,17 +176,18 @@ while getopts :-: optchar; do -) case "${OPTARG}" in help ) cli_help;; - clean ) clean;; - dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS" && CFG_DEV_MODE=1;; - init ) PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS --extra-index-url https://pypi.org/simple/";; + clean ) find_python && clean;; + dev ) CFG_REQUIREMENTS="$DEV_REQUIREMENTS";; esac;; esac done -PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS $NO_INDEX" +PIP_EXTRA_ARGS="$PIP_EXTRA_ARGS" +find_python create_virtualenv "$VIRTUALENV_DIR" install_packages "$CFG_REQUIREMENTS" . "$CFG_BIN_DIR/activate" + set +e diff --git a/configure.bat b/configure.bat index 89240ce3608..91d55e85301 100644 --- a/configure.bat +++ b/configure.bat @@ -14,6 +14,8 @@ @rem # Source this script for initial configuration @rem # Use configure --help for details +@rem # NOTE: please keep in sync with POSIX script configure + @rem # This script will search for a virtualenv.pyz app in etc\thirdparty\virtualenv.pyz @rem # Otherwise it will download the latest from the VIRTUALENV_PYZ_URL default @rem ################################ @@ -25,7 +27,7 @@ @rem # Requirement arguments passed to pip and used by default or with --dev. set "REQUIREMENTS=--editable . --constraint requirements.txt" -set "DEV_REQUIREMENTS=--editable .[dev] --constraint requirements.txt --constraint requirements-dev.txt" +set "DEV_REQUIREMENTS=--editable .[testing] --constraint requirements.txt --constraint requirements-dev.txt" @rem # where we create a virtualenv set "VIRTUALENV_DIR=venv" @@ -49,12 +51,12 @@ set "CFG_BIN_DIR=%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" @rem ################################ @rem # Thirdparty package locations and index handling -if exist ""%CFG_ROOT_DIR%\thirdparty"" ( - set "PIP_EXTRA_ARGS=--find-links %CFG_ROOT_DIR%\thirdparty " +@rem # Find packages from the local thirdparty directory or from thirdparty.aboutcode.org +@rem # offline mode for scancode installation with no index at all +if exist "%CFG_ROOT_DIR%\thirdparty" ( + set PIP_EXTRA_ARGS=--no-index --find-links "%CFG_ROOT_DIR%\thirdparty" ) - -set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi" & %INDEX_ARG% -@rem ################################ +set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% --find-links https://thirdparty.aboutcode.org/pypi/simple/links.html" @rem ################################ @@ -66,7 +68,6 @@ if not defined CFG_QUIET ( @rem ################################ @rem # Main command line entry point -set CFG_DEV_MODE=0 set "CFG_REQUIREMENTS=%REQUIREMENTS%" set "NO_INDEX=--no-index" @@ -76,43 +77,41 @@ if not "%1" == "" ( if "%1" EQU "--clean" (goto clean) if "%1" EQU "--dev" ( set "CFG_REQUIREMENTS=%DEV_REQUIREMENTS%" - set CFG_DEV_MODE=1 - ) - if "%1" EQU "--init" ( - set "NO_INDEX= " ) shift goto again ) -set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS% %NO_INDEX%" +set "PIP_EXTRA_ARGS=%PIP_EXTRA_ARGS%" @rem ################################ -@rem # find a proper Python to run +@rem # Find a proper Python to run @rem # Use environment variables or a file if available. @rem # Otherwise the latest Python by default. if not defined PYTHON_EXECUTABLE ( @rem # check for a file named PYTHON_EXECUTABLE - if exist ""%CFG_ROOT_DIR%\PYTHON_EXECUTABLE"" ( - set /p PYTHON_EXECUTABLE=<""%CFG_ROOT_DIR%\PYTHON_EXECUTABLE"" + if exist "%CFG_ROOT_DIR%\PYTHON_EXECUTABLE" ( + set /p PYTHON_EXECUTABLE=<"%CFG_ROOT_DIR%\PYTHON_EXECUTABLE" ) else ( set "PYTHON_EXECUTABLE=py" ) ) + +@rem ################################ :create_virtualenv @rem # create a virtualenv for Python @rem # Note: we do not use the bundled Python 3 "venv" because its behavior and @rem # presence is not consistent across Linux distro and sometimes pip is not @rem # included either by default. The virtualenv.pyz app cures all these issues. -if not exist ""%CFG_BIN_DIR%\python.exe"" ( +if not exist "%CFG_BIN_DIR%\python.exe" ( if not exist "%CFG_BIN_DIR%" ( - mkdir %CFG_BIN_DIR% + mkdir "%CFG_BIN_DIR%" ) - if exist ""%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz"" ( + if exist "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ( %PYTHON_EXECUTABLE% "%CFG_ROOT_DIR%\etc\thirdparty\virtualenv.pyz" ^ --wheel embed --pip embed --setuptools embed ^ --seeder pip ^ @@ -120,9 +119,9 @@ if not exist ""%CFG_BIN_DIR%\python.exe"" ( --no-periodic-update ^ --no-vcs-ignore ^ %CFG_QUIET% ^ - %CFG_ROOT_DIR%\%VIRTUALENV_DIR% + "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%" ) else ( - if not exist ""%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz"" ( + if not exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" ( curl -o "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\virtualenv.pyz" %VIRTUALENV_PYZ_URL% if %ERRORLEVEL% neq 0 ( @@ -136,7 +135,7 @@ if not exist ""%CFG_BIN_DIR%\python.exe"" ( --no-periodic-update ^ --no-vcs-ignore ^ %CFG_QUIET% ^ - %CFG_ROOT_DIR%\%VIRTUALENV_DIR% + "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%" ) ) @@ -145,6 +144,7 @@ if %ERRORLEVEL% neq 0 ( ) +@rem ################################ :install_packages @rem # install requirements in virtualenv @rem # note: --no-build-isolation means that pip/wheel/setuptools will not @@ -152,18 +152,21 @@ if %ERRORLEVEL% neq 0 ( @rem # speeds up the installation. @rem # We always have the PEP517 build dependencies installed already. -%CFG_BIN_DIR%\pip install ^ +"%CFG_BIN_DIR%\pip" install ^ --upgrade ^ --no-build-isolation ^ %CFG_QUIET% ^ %PIP_EXTRA_ARGS% ^ %CFG_REQUIREMENTS% + +@rem ################################ +:create_bin_junction @rem # Create junction to bin to have the same directory between linux and windows if exist "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" ( rmdir /s /q "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" ) -mklink /J %CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin %CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts +mklink /J "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\bin" "%CFG_ROOT_DIR%\%VIRTUALENV_DIR%\Scripts" if %ERRORLEVEL% neq 0 ( exit /b %ERRORLEVEL% @@ -173,20 +176,15 @@ exit /b 0 @rem ################################ - :cli_help echo An initial configuration script echo " usage: configure [options]" echo " " echo The default is to configure for regular use. Use --dev for development. - echo Use the --init option if starting a new project and the project - echo dependencies are not available on thirdparty.aboutcode.org/pypi/ - echo and requirements.txt and/or requirements-dev.txt has not been generated. echo " " echo The options are: echo " --clean: clean built and installed files and exit." echo " --dev: configure the environment for development." - echo " --init: pull dependencies from PyPI. Used when first setting up a project." echo " --help: display this help message and exit." echo " " echo By default, the python interpreter version found in the path is used. @@ -197,6 +195,7 @@ exit /b 0 exit /b 0 +@rem ################################ :clean @rem # Remove cleanable file and directories and files from the root dir. echo "* Cleaning ..." diff --git a/docs/scripts/doc8_style_check.sh b/docs/scripts/doc8_style_check.sh old mode 100755 new mode 100644 diff --git a/docs/scripts/sphinx_build_link_check.sh b/docs/scripts/sphinx_build_link_check.sh old mode 100755 new mode 100644 diff --git a/docs/skeleton-usage.rst b/docs/skeleton-usage.rst index 7d16259c743..ad9b9ffe3ec 100644 --- a/docs/skeleton-usage.rst +++ b/docs/skeleton-usage.rst @@ -49,11 +49,12 @@ customizing the skeleton files to your project: .. code-block:: bash - ./configure --init + ./configure This will initialize the virtual environment for the project, pull in the dependencies from PyPI and add them to the virtual environment. + Generating requirements.txt and requirements-dev.txt ---------------------------------------------------- @@ -77,7 +78,7 @@ Replace \ with the version number of the Python being used, for exampl To generate requirements-dev.txt after requirements.txt has been generated: .. code-block:: bash - ./configure --init --dev + ./configure --dev python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site-packages\`` @@ -85,13 +86,14 @@ Note: on Windows, the ``site-packages`` directory is located at ``venv\Lib\site- .. code-block:: bash python .\\etc\\scripts\\gen_requirements.py -s .\\venv\\Lib\\site-packages\\ - .\configure --init --dev + .\configure --dev python .\\etc\\scripts\\gen_requirements_dev.py -s .\\venv\\Lib\\site-packages\\ + Collecting and generating ABOUT files for dependencies ------------------------------------------------------ -Ensure that the dependencies used by ``etc/scripts/bootstrap.py`` are installed: +Ensure that the dependencies used by ``etc/scripts/fetch_thirdparty.py`` are installed: .. code-block:: bash @@ -102,7 +104,7 @@ dependencies as wheels and generate ABOUT files for them: .. code-block:: bash - python etc/scripts/bootstrap.py -r requirements.txt -r requirements-dev.txt --with-deps + python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt There may be issues with the generated ABOUT files, which will have to be corrected. You can check to see if your corrections are valid by running: @@ -122,8 +124,8 @@ Usage after project initialization Once the ``requirements.txt`` and ``requirements-dev.txt`` have been generated and the project dependencies and their ABOUT files have been uploaded to -thirdparty.aboutcode.org/pypi, you can configure the project without using the -``--init`` option. +thirdparty.aboutcode.org/pypi, you can configure the project as needed, typically +when you update dependencies or use a new checkout. If the virtual env for the project becomes polluted, or you would like to remove it, use the ``--clean`` option: @@ -146,12 +148,11 @@ update the dependencies in ``setup.cfg``, then run: .. code-block:: bash ./configure --clean # Remove existing virtual environment - ./configure --init # Create project virtual environment, pull in new dependencies source venv/bin/activate # Ensure virtual environment is activated python etc/scripts/gen_requirements.py -s venv/lib/python/site-packages/ # Regenerate requirements.txt python etc/scripts/gen_requirements_dev.py -s venv/lib/python/site-packages/ # Regenerate requirements-dev.txt pip install -r etc/scripts/requirements.txt # Install dependencies needed by etc/scripts/bootstrap.py - python etc/scripts/bootstrap.py -r requirements.txt -r requirements-dev.txt --with-deps # Collect dependency wheels and their ABOUT files + python etc/scripts/fetch_thirdparty.py -r requirements.txt -r requirements-dev.txt # Collect dependency wheels and their ABOUT files Ensure that the generated ABOUT files are valid, then take the dependency wheels and ABOUT files and upload them to thirdparty.aboutcode.org/pypi. diff --git a/etc/ci/azure-posix.yml b/etc/ci/azure-posix.yml index 7a9acff4b35..9fdc7f1522e 100644 --- a/etc/ci/azure-posix.yml +++ b/etc/ci/azure-posix.yml @@ -13,10 +13,8 @@ jobs: strategy: matrix: - ${{ each pyver in parameters.python_versions }}: ${{ each tsuite in parameters.test_suites }}: - ${{ format('py{0} {1}', pyver, tsuite.key) }}: - python_version: ${{ pyver }} + ${{ tsuite.key }}: test_suite_label: ${{ tsuite.key }} test_suite: ${{ tsuite.value }} @@ -24,19 +22,18 @@ jobs: - checkout: self fetchDepth: 10 - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python_version)' - architecture: '${{ parameters.python_architecture }}' - displayName: 'Install Python $(python_version)' + - ${{ each pyver in parameters.python_versions }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ pyver }}' + architecture: '${{ parameters.python_architecture }}' + displayName: '${{ pyver }} - Install Python' - - script: | - python --version - python3 --version - python$(python_version) --version - echo "python$(python_version)" > PYTHON_EXECUTABLE - ./configure --dev - displayName: 'Run Configure' + - script: | + python${{ pyver }} --version + echo "python${{ pyver }}" > PYTHON_EXECUTABLE + ./configure --clean && ./configure --dev + displayName: '${{ pyver }} - Configure' - - script: $(test_suite) - displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' + - script: $(test_suite) + displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' diff --git a/etc/ci/azure-win.yml b/etc/ci/azure-win.yml index 03d89274e74..26b41116ef2 100644 --- a/etc/ci/azure-win.yml +++ b/etc/ci/azure-win.yml @@ -13,27 +13,27 @@ jobs: strategy: matrix: - ${{ each pyver in parameters.python_versions }}: ${{ each tsuite in parameters.test_suites }}: - ${{ format('py{0} {1}', pyver, tsuite.key) }}: - python_version: ${{ pyver }} + ${{ tsuite.key }}: test_suite_label: ${{ tsuite.key }} test_suite: ${{ tsuite.value }} + steps: - checkout: self fetchDepth: 10 - - task: UsePythonVersion@0 - inputs: - versionSpec: '$(python_version)' - architecture: '${{ parameters.python_architecture }}' - displayName: 'Install Python $(python_version)' + - ${{ each pyver in parameters.python_versions }}: + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ pyver }}' + architecture: '${{ parameters.python_architecture }}' + displayName: '${{ pyver }} - Install Python' - - script: | - python --version - echo | set /p=python> PYTHON_EXECUTABLE - configure --dev - displayName: 'Run Configure' + - script: | + python --version + echo | set /p=python> PYTHON_EXECUTABLE + configure --clean && configure --dev + displayName: '${{ pyver }} - Configure' - - script: $(test_suite) - displayName: 'Run $(test_suite_label) tests with py$(python_version) on ${{ parameters.job_name }}' + - script: $(test_suite) + displayName: '${{ pyver }} - $(test_suite_label) on ${{ parameters.job_name }}' diff --git a/etc/ci/install_sudo.sh b/etc/ci/install_sudo.sh old mode 100755 new mode 100644 diff --git a/etc/ci/macports-ci b/etc/ci/macports-ci old mode 100755 new mode 100644 diff --git a/etc/scripts/README.rst b/etc/scripts/README.rst index d8b00f9813f..edf82e44824 100755 --- a/etc/scripts/README.rst +++ b/etc/scripts/README.rst @@ -15,10 +15,10 @@ Pre-requisites * To generate or update pip requirement files, you need to start with a clean virtualenv as instructed below (This is to avoid injecting requirements - specific to the tools here in the main requirements). + specific to the tools used here in the main requirements). * For other usages, the tools here can run either in their own isolated - virtualenv best or in the the main configured development virtualenv. + virtualenv or in the the main configured development virtualenv. These requireements need to be installed:: pip install --requirement etc/release/requirements.txt @@ -82,45 +82,14 @@ Populate a thirdparty directory with wheels, sources, .ABOUT and license files Scripts ~~~~~~~ -* **fetch_requirements.py** will fetch package wheels, their ABOUT, LICENSE and - NOTICE files to populate a local a thirdparty directory strictly from our - remote repo and using only pinned packages listed in one or more pip - requirements file(s). Fetch only requirements for specific python versions and - operating systems. Optionally fetch the corresponding source distributions. - -* **publish_files.py** will upload/sync a thirdparty directory of files to our - remote repo. Requires a GitHub personal access token. - -* **build_wheels.py** will build a package binary wheel for multiple OS and - python versions. Optionally wheels that contain native code are built - remotely. Dependent wheels are optionally included. Requires Azure credentials - and tokens if building wheels remotely on multiple operatin systems. - -* **fix_thirdparty.py** will fix a thirdparty directory with a best effort to - add missing wheels, sources archives, create or fetch or fix .ABOUT, .NOTICE - and .LICENSE files. Requires Azure credentials and tokens if requesting the - build of missing wheels remotely on multiple operatin systems. +* **fetch_thirdparty.py** will fetch package wheels, source sdist tarballs + and their ABOUT, LICENSE and NOTICE files to populate a local directory from + a list of PyPI simple URLs (typically PyPI.org proper and our self-hosted PyPI) + using pip requirements file(s), specifiers or pre-existing packages files. + Fetch wheels for specific python version and operating system combinations. * **check_thirdparty.py** will check a thirdparty directory for errors. -* **bootstrap.py** will bootstrap a thirdparty directory from a requirements - file(s) to add or build missing wheels, sources archives and create .ABOUT, - .NOTICE and .LICENSE files. Requires Azure credentials and tokens if - requesting the build of missing wheels remotely on multiple operatin systems. - - - -Usage -~~~~~ - -See each command line --help option for details. - -* (TODO) **add_package.py** will add or update a Python package including wheels, - sources and ABOUT files and this for multiple Python version and OSes(for use - with upload_packages.py afterwards) You will need an Azure personal access - token for buidling binaries and an optional DejaCode API key to post and fetch - new package versions there. TODO: explain how we use romp - Upgrade virtualenv app ---------------------- diff --git a/etc/scripts/bootstrap.py b/etc/scripts/bootstrap.py deleted file mode 100755 index 31f2f553ce4..00000000000 --- a/etc/scripts/bootstrap.py +++ /dev/null @@ -1,244 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import itertools - -import click - -import utils_thirdparty -from utils_thirdparty import Environment -from utils_thirdparty import PypiPackage - - -@click.command() -@click.option( - "-r", - "--requirements-file", - type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), - metavar="FILE", - multiple=True, - default=["requirements.txt"], - show_default=True, - help="Path to the requirements file(s) to use for thirdparty packages.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory where wheels are built and " - "sources, ABOUT and LICENSE files fetched.", -) -@click.option( - "-p", - "--python-version", - type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar="PYVER", - default=utils_thirdparty.PYTHON_VERSIONS, - show_default=True, - multiple=True, - help="Python version(s) to use for this build.", -) -@click.option( - "-o", - "--operating-system", - type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar="OS", - default=tuple(utils_thirdparty.PLATFORMS_BY_OS), - multiple=True, - show_default=True, - help="OS(ses) to use for this build: one of linux, mac or windows.", -) -@click.option( - "-l", - "--latest-version", - is_flag=True, - help="Get the latest version of all packages, ignoring version specifiers.", -) -@click.option( - "--sync-dejacode", - is_flag=True, - help="Synchronize packages with DejaCode.", -) -@click.option( - "--with-deps", - is_flag=True, - help="Also include all dependent wheels.", -) -@click.help_option("-h", "--help") -def bootstrap( - requirements_file, - thirdparty_dir, - python_version, - operating_system, - with_deps, - latest_version, - sync_dejacode, - build_remotely=False, -): - """ - Boostrap a thirdparty Python packages directory from pip requirements. - - Fetch or build to THIRDPARTY_DIR all the wheels and source distributions for - the pip ``--requirement-file`` requirements FILE(s). Build wheels compatible - with all the provided ``--python-version`` PYVER(s) and ```--operating_system`` - OS(s) defaulting to all supported combinations. Create or fetch .ABOUT and - .LICENSE files. - - Optionally ignore version specifiers and use the ``--latest-version`` - of everything. - - Sources and wheels are fetched with attempts first from PyPI, then our remote repository. - If missing wheels are built as needed. - """ - # rename variables for clarity since these are lists - requirements_files = requirements_file - python_versions = python_version - operating_systems = operating_system - - # create the environments we need - evts = itertools.product(python_versions, operating_systems) - environments = [Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] - - # collect all packages to process from requirements files - # this will fail with an exception if there are packages we cannot find - - required_name_versions = set() - - for req_file in requirements_files: - nvs = utils_thirdparty.load_requirements(requirements_file=req_file, force_pinned=False) - required_name_versions.update(nvs) - if latest_version: - required_name_versions = set((name, None) for name, _ver in required_name_versions) - - print( - f"PROCESSING {len(required_name_versions)} REQUIREMENTS in {len(requirements_files)} FILES" - ) - - # fetch all available wheels, keep track of missing - # start with local, then remote, then PyPI - - print("==> COLLECTING ALREADY LOCALLY AVAILABLE REQUIRED WHEELS") - # list of all the wheel filenames either pre-existing, fetched or built - # updated as we progress - available_wheel_filenames = [] - - local_packages_by_namever = { - (p.name, p.version): p - for p in utils_thirdparty.get_local_packages(directory=thirdparty_dir) - } - - # list of (name, version, environment) not local and to fetch - name_version_envt_to_fetch = [] - - # start with a local check - for (name, version), envt in itertools.product(required_name_versions, environments): - local_pack = local_packages_by_namever.get( - ( - name, - version, - ) - ) - if local_pack: - supported_wheels = list(local_pack.get_supported_wheels(environment=envt)) - if supported_wheels: - available_wheel_filenames.extend(w.filename for w in supported_wheels) - print( - f"====> No fetch or build needed. " - f"Local wheel already available for {name}=={version} " - f"on os: {envt.operating_system} for Python: {envt.python_version}" - ) - continue - - name_version_envt_to_fetch.append( - ( - name, - version, - envt, - ) - ) - - print(f"==> TRYING TO FETCH #{len(name_version_envt_to_fetch)} REQUIRED WHEELS") - - # list of (name, version, environment) not fetch and to build - name_version_envt_to_build = [] - - # then check if the wheel can be fetched without building from remote and Pypi - for name, version, envt in name_version_envt_to_fetch: - - fetched_fwn = utils_thirdparty.fetch_package_wheel( - name=name, - version=version, - environment=envt, - dest_dir=thirdparty_dir, - ) - - if fetched_fwn: - available_wheel_filenames.append(fetched_fwn) - else: - name_version_envt_to_build.append( - ( - name, - version, - envt, - ) - ) - - # At this stage we have all the wheels we could obtain without building - for name, version, envt in name_version_envt_to_build: - print( - f"====> Need to build wheels for {name}=={version} on os: " - f"{envt.operating_system} for Python: {envt.python_version}" - ) - - packages_and_envts_to_build = [ - (PypiPackage(name, version), envt) for name, version, envt in name_version_envt_to_build - ] - - print(f"==> BUILDING #{len(packages_and_envts_to_build)} MISSING WHEELS") - - package_envts_not_built, wheel_filenames_built = utils_thirdparty.build_missing_wheels( - packages_and_envts=packages_and_envts_to_build, - build_remotely=build_remotely, - with_deps=with_deps, - dest_dir=thirdparty_dir, - ) - if wheel_filenames_built: - available_wheel_filenames.extend(available_wheel_filenames) - - for pack, envt in package_envts_not_built: - print( - f"====> FAILED to build any wheel for {pack.name}=={pack.version} " - f"on os: {envt.operating_system} for Python: {envt.python_version}" - ) - - print(f"==> FETCHING SOURCE DISTRIBUTIONS") - # fetch all sources, keep track of missing - # This is a list of (name, version) - utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) - - print(f"==> FETCHING ABOUT AND LICENSE FILES") - utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) - - ############################################################################ - if sync_dejacode: - print(f"==> SYNC WITH DEJACODE") - # try to fetch from DejaCode any missing ABOUT - # create all missing DejaCode packages - pass - - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) - - -if __name__ == "__main__": - bootstrap() diff --git a/etc/scripts/build_wheels.py b/etc/scripts/build_wheels.py deleted file mode 100755 index 8a28176e6c1..00000000000 --- a/etc/scripts/build_wheels.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "-n", - "--name", - type=str, - metavar="PACKAGE_NAME", - required=True, - help="Python package name to add or build.", -) -@click.option( - "-v", - "--version", - type=str, - default=None, - metavar="VERSION", - help="Python package version to add or build.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory where wheels are built.", -) -@click.option( - "-p", - "--python-version", - type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar="PYVER", - default=utils_thirdparty.PYTHON_VERSIONS, - show_default=True, - multiple=True, - help="Python version to use for this build.", -) -@click.option( - "-o", - "--operating-system", - type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar="OS", - default=tuple(utils_thirdparty.PLATFORMS_BY_OS), - multiple=True, - show_default=True, - help="OS to use for this build: one of linux, mac or windows.", -) -@click.option( - "--build-remotely", - is_flag=True, - help="Build missing wheels remotely.", -) -@click.option( - "--with-deps", - is_flag=True, - help="Also include all dependent wheels.", -) -@click.option( - "--remote-build-log-file", - type=click.Path(writable=True), - default=None, - metavar="LOG-FILE", - help="Path to an optional log file where to list remote builds download URLs. " - "If provided, do not wait for remote builds to complete (and therefore, " - "do not download them either). Instead create a JSON lines log file with " - "one entry for each build suitable to fetch the artifacts at a later time.", -) -@click.option( - "--verbose", - is_flag=True, - help="Provide verbose output.", -) -@click.help_option("-h", "--help") -def build_wheels( - name, - version, - thirdparty_dir, - python_version, - operating_system, - with_deps, - build_remotely, - remote_build_log_file, - verbose, -): - """ - Build to THIRDPARTY_DIR all the wheels for the Python PACKAGE_NAME and - optional VERSION. Build wheels compatible with all the `--python-version` - PYVER(s) and `--operating_system` OS(s). - - Build native wheels remotely if needed when `--build-remotely` and include - all dependencies with `--with-deps`. - """ - utils_thirdparty.add_or_upgrade_built_wheels( - name=name, - version=version, - python_versions=python_version, - operating_systems=operating_system, - dest_dir=thirdparty_dir, - build_remotely=build_remotely, - with_deps=with_deps, - verbose=verbose, - remote_build_log_file=remote_build_log_file, - ) - - -if __name__ == "__main__": - build_wheels() diff --git a/etc/scripts/check_thirdparty.py b/etc/scripts/check_thirdparty.py index 4fea16c5f35..0f04b349ae8 100755 --- a/etc/scripts/check_thirdparty.py +++ b/etc/scripts/check_thirdparty.py @@ -16,17 +16,39 @@ @click.command() @click.option( "-d", - "--thirdparty-dir", + "--dest_dir", type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), required=True, help="Path to the thirdparty directory to check.", ) +@click.option( + "-w", + "--wheels", + is_flag=True, + help="Check missing wheels.", +) +@click.option( + "-s", + "--sdists", + is_flag=True, + help="Check missing source sdists tarballs.", +) @click.help_option("-h", "--help") -def check_thirdparty_dir(thirdparty_dir): +def check_thirdparty_dir( + dest_dir, + wheels, + sdists, +): """ - Check a thirdparty directory for problems. + Check a thirdparty directory for problems and print these on screen. """ - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) + # check for problems + print(f"==> CHECK FOR PROBLEMS") + utils_thirdparty.find_problems( + dest_dir=dest_dir, + report_missing_sources=sdists, + report_missing_wheels=wheels, + ) if __name__ == "__main__": diff --git a/etc/scripts/fetch_built_wheels.py b/etc/scripts/fetch_built_wheels.py deleted file mode 100644 index a78861e97c0..00000000000 --- a/etc/scripts/fetch_built_wheels.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/scancode-toolkit for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "--remote-build-log-file", - type=click.Path(readable=True), - metavar="LOG-FILE", - help="Path to a remote builds log file.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory to save built wheels.", -) -@click.option( - "--no-wait", - is_flag=True, - default=False, - help="Do not wait for build completion.", -) -@click.option( - "--verbose", - is_flag=True, - help="Provide verbose output.", -) -@click.help_option("-h", "--help") -def fetch_remote_wheels( - remote_build_log_file, - thirdparty_dir, - no_wait, - verbose, -): - """ - Fetch to THIRDPARTY_DIR all the wheels built in the LOG-FILE JSON lines - build log file. - """ - utils_thirdparty.fetch_remotely_built_wheels( - remote_build_log_file=remote_build_log_file, - dest_dir=thirdparty_dir, - no_wait=no_wait, - verbose=verbose, - ) - - -if __name__ == "__main__": - fetch_remote_wheels() diff --git a/etc/scripts/fetch_requirements.py b/etc/scripts/fetch_requirements.py deleted file mode 100755 index 9da9ce96c10..00000000000 --- a/etc/scripts/fetch_requirements.py +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import itertools - -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "-r", - "--requirements-file", - type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), - metavar="FILE", - multiple=True, - default=["requirements.txt"], - show_default=True, - help="Path to the requirements file to use for thirdparty packages.", -) -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - metavar="DIR", - default=utils_thirdparty.THIRDPARTY_DIR, - show_default=True, - help="Path to the thirdparty directory.", -) -@click.option( - "-p", - "--python-version", - type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), - metavar="INT", - multiple=True, - default=["36"], - show_default=True, - help="Python version to use for this build.", -) -@click.option( - "-o", - "--operating-system", - type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), - metavar="OS", - multiple=True, - default=["linux"], - show_default=True, - help="OS to use for this build: one of linux, mac or windows.", -) -@click.option( - "-s", - "--with-sources", - is_flag=True, - help="Fetch the corresponding source distributions.", -) -@click.option( - "-a", - "--with-about", - is_flag=True, - help="Fetch the corresponding ABOUT and LICENSE files.", -) -@click.option( - "--allow-unpinned", - is_flag=True, - help="Allow requirements without pinned versions.", -) -@click.option( - "-s", - "--only-sources", - is_flag=True, - help="Fetch only the corresponding source distributions.", -) -@click.option( - "-u", - "--remote-links-url", - type=str, - metavar="URL", - default=utils_thirdparty.REMOTE_LINKS_URL, - show_default=True, - help="URL to a PyPI-like links web site. " "Or local path to a directory with wheels.", -) -@click.help_option("-h", "--help") -def fetch_requirements( - requirements_file, - thirdparty_dir, - python_version, - operating_system, - with_sources, - with_about, - allow_unpinned, - only_sources, - remote_links_url=utils_thirdparty.REMOTE_LINKS_URL, -): - """ - Fetch and save to THIRDPARTY_DIR all the required wheels for pinned - dependencies found in the `--requirement` FILE requirements file(s). Only - fetch wheels compatible with the provided `--python-version` and - `--operating-system`. - Also fetch the corresponding .ABOUT, .LICENSE and .NOTICE files together - with a virtualenv.pyz app. - - Use exclusively wheel not from PyPI but rather found in the PyPI-like link - repo ``remote_links_url`` if this is a URL. Treat this ``remote_links_url`` - as a local directory path to a wheels directory if this is not a a URL. - """ - - # fetch wheels - python_versions = python_version - operating_systems = operating_system - requirements_files = requirements_file - - if not only_sources: - envs = itertools.product(python_versions, operating_systems) - envs = (utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in envs) - - for env, reqf in itertools.product(envs, requirements_files): - - for package, error in utils_thirdparty.fetch_wheels( - environment=env, - requirements_file=reqf, - allow_unpinned=allow_unpinned, - dest_dir=thirdparty_dir, - remote_links_url=remote_links_url, - ): - if error: - print("Failed to fetch wheel:", package, ":", error) - - # optionally fetch sources - if with_sources or only_sources: - - for reqf in requirements_files: - for package, error in utils_thirdparty.fetch_sources( - requirements_file=reqf, - allow_unpinned=allow_unpinned, - dest_dir=thirdparty_dir, - remote_links_url=remote_links_url, - ): - if error: - print("Failed to fetch source:", package, ":", error) - - if with_about: - utils_thirdparty.add_fetch_or_update_about_and_license_files(dest_dir=thirdparty_dir) - utils_thirdparty.find_problems( - dest_dir=thirdparty_dir, - report_missing_sources=with_sources or only_sources, - report_missing_wheels=not only_sources, - ) - - -if __name__ == "__main__": - fetch_requirements() diff --git a/etc/scripts/fetch_thirdparty.py b/etc/scripts/fetch_thirdparty.py new file mode 100755 index 00000000000..22147b20b7b --- /dev/null +++ b/etc/scripts/fetch_thirdparty.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/skeleton for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import itertools +import os +import sys + +import click + +import utils_thirdparty +import utils_requirements + +TRACE = True + + +@click.command() +@click.option( + "-r", + "--requirements", + "requirements_files", + type=click.Path(exists=True, readable=True, path_type=str, dir_okay=False), + metavar="REQUIREMENT-FILE", + multiple=True, + required=False, + help="Path to pip requirements file(s) listing thirdparty packages.", +) +@click.option( + "--spec", + "--specifier", + "specifiers", + type=str, + metavar="SPECIFIER", + multiple=True, + required=False, + help="Thirdparty package name==version specification(s) as in django==1.2.3. " + "With --latest-version a plain package name is also acceptable.", +) +@click.option( + "-l", + "--latest-version", + is_flag=True, + help="Get the latest version of all packages, ignoring any specified versions.", +) +@click.option( + "-d", + "--dest", + "dest_dir", + type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), + metavar="DIR", + default=utils_thirdparty.THIRDPARTY_DIR, + show_default=True, + help="Path to the detsination directory where to save downloaded wheels, " + "sources, ABOUT and LICENSE files..", +) +@click.option( + "-w", + "--wheels", + is_flag=True, + help="Download wheels.", +) +@click.option( + "-s", + "--sdists", + is_flag=True, + help="Download source sdists tarballs.", +) +@click.option( + "-p", + "--python-version", + "python_versions", + type=click.Choice(utils_thirdparty.PYTHON_VERSIONS), + metavar="PYVER", + default=utils_thirdparty.PYTHON_VERSIONS, + show_default=True, + multiple=True, + help="Python version(s) to use for wheels.", +) +@click.option( + "-o", + "--operating-system", + "operating_systems", + type=click.Choice(utils_thirdparty.PLATFORMS_BY_OS), + metavar="OS", + default=tuple(utils_thirdparty.PLATFORMS_BY_OS), + multiple=True, + show_default=True, + help="OS(ses) to use for wheels: one of linux, mac or windows.", +) +@click.option( + "--index-url", + "index_urls", + type=str, + metavar="INDEX", + default=utils_thirdparty.PYPI_INDEXES, + show_default=True, + multiple=True, + help="PyPI index URL(s) to use for wheels and sources, in order of preferences.", +) +@click.help_option("-h", "--help") +def fetch_thirdparty( + requirements_files, + specifiers, + latest_version, + dest_dir, + python_versions, + operating_systems, + wheels, + sdists, + index_urls, +): + """ + Download to --dest-dir THIRDPARTY_DIR the PyPI wheels, source distributions, + and their ABOUT metadata, license and notices files. + + Download the PyPI packages listed in the combination of: + - the pip requirements --requirements REQUIREMENT-FILE(s), + - the pip name==version --specifier SPECIFIER(s) + - any pre-existing wheels or sdsists found in --dest-dir THIRDPARTY_DIR. + + Download wheels with the --wheels option for the ``--python-version`` PYVER(s) + and ``--operating_system`` OS(s) combinations defaulting to all supported combinations. + + Download sdists tarballs with the --sdists option. + + Generate or Download .ABOUT, .LICENSE and .NOTICE files for all the wheels and sources fetched. + + Download wheels and sdists the provided PyPI simple --index-url INDEX(s) URLs. + """ + print(f"COLLECTING REQUIRED NAMES & VERSIONS FROM {dest_dir}") + existing_packages_by_nv = { + (package.name, package.version): package + for package in utils_thirdparty.get_local_packages(directory=dest_dir) + } + + required_name_versions = set(existing_packages_by_nv.keys()) + + for req_file in requirements_files: + nvs = utils_requirements.load_requirements( + requirements_file=req_file, + with_unpinned=latest_version, + ) + required_name_versions.update(nvs) + + for specifier in specifiers: + nv = utils_requirements.get_name_version( + requirement=specifier, + with_unpinned=latest_version, + ) + required_name_versions.add(nv) + + if not required_name_versions: + print("Error: no requirements requested.") + sys.exit(1) + + if not os.listdir(dest_dir) and not (wheels or sdists): + print("Error: one or both of --wheels and --sdists is required.") + sys.exit(1) + + if latest_version: + latest_name_versions = set() + names = set(name for name, _version in sorted(required_name_versions)) + for name in sorted(names): + latests = utils_thirdparty.PypiPackage.sorted( + utils_thirdparty.get_package_versions( + name=name, version=None, index_urls=index_urls + ) + ) + if not latests: + print(f"No distribution found for: {name}") + continue + latest = latests[-1] + latest_name_versions.add((latest.name, latest.version)) + required_name_versions = latest_name_versions + + if TRACE: + print("required_name_versions:", required_name_versions) + + if wheels: + # create the environments matrix we need for wheels + evts = itertools.product(python_versions, operating_systems) + environments = [utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in evts] + + wheels_not_found = {} + sdists_not_found = {} + # iterate over requirements, one at a time + for name, version in sorted(required_name_versions): + nv = name, version + existing_package = existing_packages_by_nv.get(nv) + if wheels: + for environment in environments: + if existing_package: + existing_wheels = list( + existing_package.get_supported_wheels(environment=environment) + ) + else: + existing_wheels = None + + if existing_wheels: + if TRACE: + print( + f"====> Wheels already available: {name}=={version} on: {environment}: {existing_package.wheels!r}" + ) + if all(w.is_pure() for w in existing_wheels): + break + else: + continue + + if TRACE: + print(f"Fetching wheel for: {name}=={version} on: {environment}") + + try: + ( + fetched_wheel_filenames, + existing_wheel_filenames, + ) = utils_thirdparty.download_wheel( + name=name, + version=version, + environment=environment, + dest_dir=dest_dir, + index_urls=index_urls, + ) + if TRACE: + if existing_wheel_filenames: + print( + f" ====> Wheels already available: {name}=={version} on: {environment}" + ) + for whl in existing_wheel_filenames: + print(f" {whl}") + if fetched_wheel_filenames: + print(f" ====> Wheels fetched: {name}=={version} on: {environment}") + for whl in fetched_wheel_filenames: + print(f" {whl}") + + fwfns = fetched_wheel_filenames + existing_wheel_filenames + + if all(utils_thirdparty.Wheel.from_filename(f).is_pure() for f in fwfns): + break + + except utils_thirdparty.DistributionNotFound as e: + wheels_not_found[f"{name}=={version}"] = str(e) + + if sdists: + if existing_package and existing_package.sdist: + if TRACE: + print( + f" ====> Sdist already available: {name}=={version}: {existing_package.sdist!r}" + ) + continue + + if TRACE: + print(f" Fetching sdist for: {name}=={version}") + + try: + fetched = utils_thirdparty.download_sdist( + name=name, + version=version, + dest_dir=dest_dir, + index_urls=index_urls, + ) + + if TRACE: + if not fetched: + print( + f" ====> Sdist already available: {name}=={version} on: {environment}" + ) + else: + print( + f" ====> Sdist fetched: {fetched} for {name}=={version} on: {environment}" + ) + + except utils_thirdparty.DistributionNotFound as e: + sdists_not_found[f"{name}=={version}"] = str(e) + + if wheels and wheels_not_found: + print(f"==> MISSING WHEELS") + for wh in wheels_not_found: + print(f" {wh}") + + if sdists and sdists_not_found: + print(f"==> MISSING SDISTS") + for sd in sdists_not_found: + print(f" {sd}") + + print(f"==> FETCHING OR CREATING ABOUT AND LICENSE FILES") + utils_thirdparty.fetch_abouts_and_licenses(dest_dir=dest_dir) + utils_thirdparty.clean_about_files(dest_dir=dest_dir) + + # check for problems + print(f"==> CHECK FOR PROBLEMS") + utils_thirdparty.find_problems( + dest_dir=dest_dir, + report_missing_sources=sdists, + report_missing_wheels=wheels, + ) + + +if __name__ == "__main__": + fetch_thirdparty() diff --git a/etc/scripts/fix_thirdparty.py b/etc/scripts/fix_thirdparty.py deleted file mode 100755 index 44de810fded..00000000000 --- a/etc/scripts/fix_thirdparty.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# ScanCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/nexB/skeleton for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import click - -import utils_thirdparty - - -@click.command() -@click.option( - "-d", - "--thirdparty-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False), - required=True, - help="Path to the thirdparty directory to fix.", -) -@click.option( - "--build-wheels", - is_flag=True, - help="Build all missing wheels .", -) -@click.option( - "--build-remotely", - is_flag=True, - help="Build missing wheels remotely.", -) -@click.option( - "--remote-build-log-file", - type=click.Path(writable=True), - default=None, - metavar="LOG-FILE", - help="Path to an optional log file where to list remote builds download URLs. " - "If provided, do not wait for remote builds to complete (and therefore, " - "do not download them either). Instead create a JSON lines log file with " - "one entry for each build suitable to fetch the artifacts at a later time.", -) -@click.option( - "--strip-classifiers", - is_flag=True, - help="Remove dangling PyPI classifiers", -) -@click.help_option("-h", "--help") -def fix_thirdparty_dir( - thirdparty_dir, - build_wheels, - build_remotely, - remote_build_log_file, - strip_classifiers, -): - """ - Fix a thirdparty directory of dependent package wheels and sdist. - - Multiple fixes are applied: - - fetch or build missing binary wheels - - fetch missing source distributions - - derive, fetch or add missing ABOUT files - - fetch missing .LICENSE and .NOTICE files - - remove outdated package versions and the ABOUT, .LICENSE and .NOTICE files - - Optionally build missing binary wheels for all supported OS and Python - version combos locally or remotely. - """ - if strip_classifiers: - print("***ADD*** ABOUT AND LICENSES, STRIP CLASSIFIERS") - utils_thirdparty.add_fetch_or_update_about_and_license_files( - dest_dir=thirdparty_dir, - strip_classifiers=strip_classifiers, - ) - else: - print("***FETCH*** MISSING WHEELS") - package_envts_not_fetched = utils_thirdparty.fetch_missing_wheels(dest_dir=thirdparty_dir) - print("***FETCH*** MISSING SOURCES") - src_name_ver_not_fetched = utils_thirdparty.fetch_missing_sources(dest_dir=thirdparty_dir) - - package_envts_not_built = [] - if build_wheels: - print("***BUILD*** MISSING WHEELS") - results = utils_thirdparty.build_missing_wheels( - packages_and_envts=package_envts_not_fetched, - build_remotely=build_remotely, - remote_build_log_file=remote_build_log_file, - dest_dir=thirdparty_dir, - ) - package_envts_not_built, _wheel_filenames_built = results - - print("***ADD*** ABOUT AND LICENSES") - utils_thirdparty.add_fetch_or_update_about_and_license_files( - dest_dir=thirdparty_dir, - strip_classifiers=strip_classifiers, - ) - - # report issues - for name, version in src_name_ver_not_fetched: - print(f"{name}=={version}: Failed to fetch source distribution.") - - for package, envt in package_envts_not_built: - print( - f"{package.name}=={package.version}: Failed to build wheel " - f"on {envt.operating_system} for Python {envt.python_version}" - ) - - print("***FIND PROBLEMS***") - utils_thirdparty.find_problems(dest_dir=thirdparty_dir) - - -if __name__ == "__main__": - fix_thirdparty_dir() diff --git a/etc/scripts/gen_pypi_simple.py b/etc/scripts/gen_pypi_simple.py index 3accdeef99a..8de2b960128 100644 --- a/etc/scripts/gen_pypi_simple.py +++ b/etc/scripts/gen_pypi_simple.py @@ -5,86 +5,46 @@ # Copyright (c) 2010 David Wolever . All rights reserved. # originally from https://github.com/wolever/pip2pi +import hashlib import os import re import shutil - +from collections import defaultdict from html import escape from pathlib import Path +from typing import NamedTuple """ -name: pip compatibility tags -version: 20.3.1 -download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py -copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) -license_expression: mit -notes: the weel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py - -Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +Generate a PyPI simple index froma directory. """ -get_wheel_from_filename = re.compile( - r"""^(?P(?P.+?)-(?P.*?)) - ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) - \.whl)$""", - re.VERBOSE, -).match - -sdist_exts = ( - ".tar.gz", - ".tar.bz2", - ".zip", - ".tar.xz", -) -wheel_ext = ".whl" -app_ext = ".pyz" -dist_exts = sdist_exts + (wheel_ext, app_ext) class InvalidDistributionFilename(Exception): pass -def get_package_name_from_filename(filename, normalize=True): +def get_package_name_from_filename(filename): """ Return the package name extracted from a package ``filename``. Optionally ``normalize`` the name according to distribution name rules. Raise an ``InvalidDistributionFilename`` if the ``filename`` is invalid:: - >>> get_package_name_from_filename("aboutcode_toolkit-5.1.0-py2.py3-none-any.whl") - 'aboutcode-toolkit' - >>> get_package_name_from_filename("boolean.py-3.7-py2.py3-none-any.whl") - 'boolean-py' - >>> get_package_name_from_filename("boolean.py-3.7.tar.gz") - 'boolean-py' >>> get_package_name_from_filename("foo-1.2.3_rc1.tar.gz") 'foo' - >>> get_package_name_from_filename("foo_bar-1.2-py27-none-any.whl") + >>> get_package_name_from_filename("foo-bar-1.2-py27-none-any.whl") 'foo-bar' - >>> get_package_name_from_filename("foo.py-1.2-py27-none-any.whl") - 'foo-py' >>> get_package_name_from_filename("Cython-0.17.2-cp26-none-linux_x86_64.whl") 'cython' >>> get_package_name_from_filename("python_ldap-2.4.19-cp27-none-macosx_10_10_x86_64.whl") 'python-ldap' + >>> get_package_name_from_filename("foo.whl") + Traceback (most recent call last): + ... + InvalidDistributionFilename: ... + >>> get_package_name_from_filename("foo.png") + Traceback (most recent call last): + ... + InvalidFilePackageName: ... """ if not filename or not filename.endswith(dist_exts): raise InvalidDistributionFilename(filename) @@ -132,8 +92,7 @@ def get_package_name_from_filename(filename, normalize=True): if not name: raise InvalidDistributionFilename(filename) - if normalize: - name = normalize_name(name) + name = normalize_name(name) return name @@ -145,20 +104,87 @@ def normalize_name(name): return name and re.sub(r"[-_.]+", "-", name).lower() or name -def normalize_name_plain(name): +def build_per_package_index(pkg_name, packages, base_url): """ - Return a normalized package name, but do not replace dots + Return an HTML document as string representing the index for a package """ - return name and re.sub(r"[-_]+", "-", name).lower() or name + document = [] + header = f""" + + + + Links for {pkg_name} + + """ + document.append(header) + + for package in packages: + document.append(package.simple_index_entry(base_url)) + + footer = """ + +""" + document.append(footer) + return "\n".join(document) -def build_pypi_index(directory): +def build_links_package_index(packages_by_package_name, base_url): + """ + Return an HTML document as string which is a links index of all packages + """ + document = [] + header = f""" + + + Links for all packages + + """ + document.append(header) + + for _name, packages in packages_by_package_name.items(): + for package in packages: + document.append(package.simple_index_entry(base_url)) + + footer = """ + +""" + document.append(footer) + return "\n".join(document) + + +class Package(NamedTuple): + name: str + index_dir: Path + archive_file: Path + checksum: str + + @classmethod + def from_file(cls, name, index_dir, archive_file): + with open(archive_file, "rb") as f: + checksum = hashlib.sha256(f.read()).hexdigest() + return cls( + name=name, + index_dir=index_dir, + archive_file=archive_file, + checksum=checksum, + ) + + def simple_index_entry(self, base_url): + return ( + f' ' + f"{self.archive_file.name}
" + ) + + +def build_pypi_index(directory, base_url="https://thirdparty.aboutcode.org/pypi"): """ Using a ``directory`` directory of wheels and sdists, create the a PyPI simple directory index at ``directory``/simple/ populated with the proper PyPI simple index directory structure crafted using symlinks. WARNING: The ``directory``/simple/ directory is removed if it exists. + NOTE: in addition to the a PyPI simple index.html there is also a links.html + index file generated which is suitable to use with pip's --find-links """ directory = Path(directory) @@ -168,18 +194,15 @@ def build_pypi_index(directory): shutil.rmtree(str(index_dir), ignore_errors=True) index_dir.mkdir(parents=True) + packages_by_package_name = defaultdict(list) + # generate the main simple index.html simple_html_index = [ - "" - "" - "PyPI Simple Index", - '' - '' - "" - "", + "", + "PyPI Simple Index", + '' '', ] - package_names = set() for pkg_file in directory.iterdir(): pkg_filename = pkg_file.name @@ -191,27 +214,99 @@ def build_pypi_index(directory): ): continue - original_name = get_package_name_from_filename(pkg_filename, normalize=False) - pkg_dir_name = normalize_name(original_name) - pkg_link_name = normalize_name_plain(original_name) - - pkg_index_dir = index_dir / pkg_dir_name + pkg_name = get_package_name_from_filename( + filename=pkg_filename, + ) + pkg_index_dir = index_dir / pkg_name pkg_index_dir.mkdir(parents=True, exist_ok=True) pkg_indexed_file = pkg_index_dir / pkg_filename + link_target = Path("../..") / pkg_filename pkg_indexed_file.symlink_to(link_target) - if pkg_link_name not in package_names: - esc_dir = escape(pkg_dir_name) - esc_link = escape(pkg_link_name) + if pkg_name not in packages_by_package_name: + esc_name = escape(pkg_name) + simple_html_index.append(f'{esc_name}
') - simple_html_index.append(f'{esc_link}
') - package_names.add(pkg_link_name) + packages_by_package_name[pkg_name].append( + Package.from_file( + name=pkg_name, + index_dir=pkg_index_dir, + archive_file=pkg_file, + ) + ) + # finalize main index simple_html_index.append("") index_html = index_dir / "index.html" index_html.write_text("\n".join(simple_html_index)) + # also generate the simple index.html of each package, listing all its versions. + for pkg_name, packages in packages_by_package_name.items(): + per_package_index = build_per_package_index( + pkg_name=pkg_name, + packages=packages, + base_url=base_url, + ) + pkg_index_dir = packages[0].index_dir + ppi_html = pkg_index_dir / "index.html" + ppi_html.write_text(per_package_index) + + # also generate the a links.html page with all packages. + package_links = build_links_package_index( + packages_by_package_name=packages_by_package_name, + base_url=base_url, + ) + links_html = index_dir / "links.html" + links_html.write_text(package_links) + + +""" +name: pip-wheel +version: 20.3.1 +download_url: https://github.com/pypa/pip/blob/20.3.1/src/pip/_internal/models/wheel.py +copyright: Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) +license_expression: mit +notes: the wheel name regex is copied from pip-20.3.1 pip/_internal/models/wheel.py + +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" +get_wheel_from_filename = re.compile( + r"""^(?P(?P.+?)-(?P.*?)) + ((-(?P\d[^-]*?))?-(?P.+?)-(?P.+?)-(?P.+?) + \.whl)$""", + re.VERBOSE, +).match + +sdist_exts = ( + ".tar.gz", + ".tar.bz2", + ".zip", + ".tar.xz", +) + +wheel_ext = ".whl" +app_ext = ".pyz" +dist_exts = sdist_exts + (wheel_ext, app_ext) if __name__ == "__main__": import sys diff --git a/etc/scripts/gen_requirements.py b/etc/scripts/gen_requirements.py index 6f17a75f84f..07e26f77126 100755 --- a/etc/scripts/gen_requirements.py +++ b/etc/scripts/gen_requirements.py @@ -8,37 +8,48 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import click +import argparse +import pathlib + import utils_requirements +""" +Utilities to manage requirements files. +NOTE: this should use ONLY the standard library and not import anything else +because this is used for boostrapping with no requirements installed. +""" -@click.command() -@click.option( - "-s", - "--site-packages-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), - required=True, - metavar="DIR", - help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', -) -@click.option( - "-r", - "--requirements-file", - type=click.Path(path_type=str, dir_okay=False), - metavar="FILE", - default="requirements.txt", - show_default=True, - help="Path to the requirements file to update or create.", -) -@click.help_option("-h", "--help") -def gen_requirements(site_packages_dir, requirements_file): - """ + +def gen_requirements(): + description = """ Create or replace the `--requirements-file` file FILE requirements file with all locally installed Python packages.all Python packages found installed in `--site-packages-dir` """ + parser = argparse.ArgumentParser(description=description) + + parser.add_argument( + "-s", + "--site-packages-dir", + dest="site_packages_dir", + type=pathlib.Path, + required=True, + metavar="DIR", + help="Path to the 'site-packages' directory where wheels are installed such as lib/python3.6/site-packages", + ) + parser.add_argument( + "-r", + "--requirements-file", + type=pathlib.Path, + metavar="FILE", + default="requirements.txt", + help="Path to the requirements file to update or create.", + ) + + args = parser.parse_args() + utils_requirements.lock_requirements( - requirements_file=requirements_file, - site_packages_dir=site_packages_dir, + site_packages_dir=args.site_packages_dir, + requirements_file=args.requirements_file, ) diff --git a/etc/scripts/gen_requirements_dev.py b/etc/scripts/gen_requirements_dev.py index ef804554651..12cc06d385a 100755 --- a/etc/scripts/gen_requirements_dev.py +++ b/etc/scripts/gen_requirements_dev.py @@ -8,51 +8,59 @@ # See https://github.com/nexB/skeleton for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import click +import argparse +import pathlib + import utils_requirements +""" +Utilities to manage requirements files. +NOTE: this should use ONLY the standard library and not import anything else +because this is used for boostrapping with no requirements installed. +""" -@click.command() -@click.option( - "-s", - "--site-packages-dir", - type=click.Path(exists=True, readable=True, path_type=str, file_okay=False, resolve_path=True), - required=True, - metavar="DIR", - help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', -) -@click.option( - "-d", - "--dev-requirements-file", - type=click.Path(path_type=str, dir_okay=False), - metavar="FILE", - default="requirements-dev.txt", - show_default=True, - help="Path to the dev requirements file to update or create.", -) -@click.option( - "-r", - "--main-requirements-file", - type=click.Path(path_type=str, dir_okay=False), - default="requirements.txt", - metavar="FILE", - show_default=True, - help="Path to the main requirements file. Its requirements will be excluded " - "from the generated dev requirements.", -) -@click.help_option("-h", "--help") -def gen_dev_requirements(site_packages_dir, dev_requirements_file, main_requirements_file): - """ + +def gen_dev_requirements(): + description = """ Create or overwrite the `--dev-requirements-file` pip requirements FILE with all Python packages found installed in `--site-packages-dir`. Exclude package names also listed in the --main-requirements-file pip requirements FILE (that are assume to the production requirements and therefore to always be present in addition to the development requirements). """ + parser = argparse.ArgumentParser(description=description) + + parser.add_argument( + "-s", + "--site-packages-dir", + type=pathlib.Path, + required=True, + metavar="DIR", + help='Path to the "site-packages" directory where wheels are installed such as lib/python3.6/site-packages', + ) + parser.add_argument( + "-d", + "--dev-requirements-file", + type=pathlib.Path, + metavar="FILE", + default="requirements-dev.txt", + help="Path to the dev requirements file to update or create.", + ) + parser.add_argument( + "-r", + "--main-requirements-file", + type=pathlib.Path, + default="requirements.txt", + metavar="FILE", + help="Path to the main requirements file. Its requirements will be excluded " + "from the generated dev requirements.", + ) + args = parser.parse_args() + utils_requirements.lock_dev_requirements( - dev_requirements_file=dev_requirements_file, - main_requirements_file=main_requirements_file, - site_packages_dir=site_packages_dir, + dev_requirements_file=args.dev_requirements_file, + main_requirements_file=args.main_requirements_file, + site_packages_dir=args.site_packages_dir, ) diff --git a/etc/scripts/licenses/buildcopytests.py b/etc/scripts/licenses/buildcopytests.py index 297dcbdd82e..203c3d98835 100644 --- a/etc/scripts/licenses/buildcopytests.py +++ b/etc/scripts/licenses/buildcopytests.py @@ -25,7 +25,6 @@ TRACE = True - def load_data(location="00-new-copyright-tests.txt"): with io.open(location, encoding="utf-8") as o: data = [l.strip() for l in o.read().splitlines(False)] @@ -71,8 +70,12 @@ def build_dupe_index(): @click.command() -@click.argument('copyrights_file', type=click.Path(), metavar='FILE',) -@click.help_option('-h', '--help') +@click.argument( + "copyrights_file", + type=click.Path(), + metavar="FILE", +) +@click.help_option("-h", "--help") @click.help_option("-h", "--help") def cli(copyrights_file): """ diff --git a/etc/scripts/licenses/buildrules.py b/etc/scripts/licenses/buildrules.py index 6c7007f0188..208f130d5fa 100644 --- a/etc/scripts/licenses/buildrules.py +++ b/etc/scripts/licenses/buildrules.py @@ -159,8 +159,7 @@ def all_rule_by_tokens(): df = f" file://{rule.data_file}" tf = f" file://{rule.text_file}" raise Exception( - f"Failed to to get tokens from rule:: {rule.identifier}\n" - f"{df}\n{tf}" + f"Failed to to get tokens from rule:: {rule.identifier}\n" f"{df}\n{tf}" ) from e return rule_tokens diff --git a/etc/scripts/licenses/synclic.py b/etc/scripts/licenses/synclic.py index 05d11554fc1..bf51099e419 100644 --- a/etc/scripts/licenses/synclic.py +++ b/etc/scripts/licenses/synclic.py @@ -157,7 +157,11 @@ def get_licenses( print(repr(lic)) raise if TRACE: - print(f" Saving fetched license: {lic.key} in :", round(time.time() - start, 1), "s") + print( + f" Saving fetched license: {lic.key} in :", + round(time.time() - start, 1), + "s", + ) print( "Stored %d external licenses in: %r." @@ -485,6 +489,7 @@ def build_license(self, mapping, skip_oddities=True, scancode_licenses=None): text = text.strip() return lic, text + # these licenses are rare commercial license with no text and only a # link or these licenses may be combos of many others or are ignored # because of some weirdness we detect instead each part of the combos @@ -747,9 +752,7 @@ def create_or_update_license(api_url, api_key, lico, update=False): if not response.ok: content = response.content headers = response.headers - raise Exception( - f"Failed to fetch license for {lico.key} at {url}:\n{headers}\n{content}" - ) + raise Exception(f"Failed to fetch license for {lico.key} at {url}:\n{headers}\n{content}") results = response.json().get("results", []) @@ -761,9 +764,7 @@ def create_or_update_license(api_url, api_key, lico, update=False): if not response.ok: content = response.content headers = response.headers - raise Exception( - f"Failed to create license: {lico.key} at {url}:\n{headers}\n{content}" - ) + raise Exception(f"Failed to create license: {lico.key} at {url}:\n{headers}\n{content}") print("Created new license:", lico) created = response.json() @@ -790,9 +791,7 @@ def create_or_update_license(api_url, api_key, lico, update=False): if not response.ok: content = response.content headers = response.headers - raise Exception( - f"Failed to update license: {lico.key} at {url}:\n{headers}\n{content}" - ) + raise Exception(f"Failed to update license: {lico.key} at {url}:\n{headers}\n{content}") new_results = response.json().get("results", []) if TRACE: @@ -1120,12 +1119,14 @@ def synchronize_licenses( # mappings of key -> License scancodes_by_key = scancode_licenses.by_key - if TRACE: start = time.time() + if TRACE: + start = time.time() externals_by_key = external_source.get_licenses( scancode_licenses, commitish=commitish, ) - if TRACE: print("Fetched all externals_by_key licenses in :", int(time.time() - start)) + if TRACE: + print("Fetched all externals_by_key licenses in :", int(time.time() - start)) if use_spdx_key: scancodes_by_key = scancode_licenses.by_spdx_key diff --git a/etc/scripts/licenses/syncspdx.py b/etc/scripts/licenses/syncspdx.py index 974f2a7bde4..b67e6ff2ffb 100644 --- a/etc/scripts/licenses/syncspdx.py +++ b/etc/scripts/licenses/syncspdx.py @@ -47,7 +47,9 @@ def cli( scancode_licenses = synclic.ScanCodeLicenses() sc_by_key = scancode_licenses.by_key - for license_key, spdx_license_key, api_url in deja.fetch_spdx_license_details(scancode_licenses): + for license_key, spdx_license_key, api_url in deja.fetch_spdx_license_details( + scancode_licenses + ): if spdx_license_key: continue if TRACE or dry_run: diff --git a/etc/scripts/misc/openhub_licenses.json b/etc/scripts/misc/openhub_licenses.json deleted file mode 100644 index 89cfda46798..00000000000 --- a/etc/scripts/misc/openhub_licenses.json +++ /dev/null @@ -1,2034 +0,0 @@ -[ - { - "openhub_url": "https://www.openhub.net/licenses/academic", - "name": "Academic Free License v3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Academic_Free_License_v2_0", - "name": "Academic Free License v2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Academic_Free_License_v2_1", - "name": "Academic Free License v2.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/AceStyleBSD", - "name": "Ace Style Limited BSD" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ACE_TAO_CIAO_License", - "name": "ACE/TAO/CIAOLicense" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ack_bsd", - "name": "ACK BSD type Licence" - }, - { - "openhub_url": "https://www.openhub.net/licenses/adaptive", - "name": "Adaptive Public License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/AFL1-1", - "name": "Academic Free License v1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/AFL-1dot2", - "name": "Academic Free License v1.2" - }, - { - "openhub_url": "https://www.openhub.net/licenses/agpl", - "name": "GNU Affero General Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/AGPL3", - "name": "GNU Affero General Public License v3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/AGPL3_or_later", - "name": "GNU Affero General Public License 3.0 or later" - }, - { - "openhub_url": "https://www.openhub.net/licenses/AHWML", - "name": "AHWM License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/al", - "name": "Aciqra License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ANTLR_Software_Rights_Notice", - "name": "ANTLR Software Rights Notice" - }, - { - "openhub_url": "https://www.openhub.net/licenses/aol-omnicheck-oss-license", - "name": "AOL OmniCheck Open Source Software License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/apache", - "name": "Apache License 1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/apache_2", - "name": "Apache License 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/apache_ish", - "name": "Apache-ish License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Apache_License_1_0", - "name": "Apache License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/apache_v2", - "name": "Apache License, Version 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ApapsisFreePublicLicense", - "name": "Apapsis' Free Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/appcore", - "name": "AppCore" - }, - { - "openhub_url": "https://www.openhub.net/licenses/apple_open_source", - "name": "Apple Public Source License 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Apple_Public_Source_License_1_0", - "name": "Apple Public Source License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Apple_Public_Source_License_1_1", - "name": "Apple Public Source License 1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Apple_Public_Source_License_1_2", - "name": "Apple Public Source License 1.2" - }, - { - "openhub_url": "https://www.openhub.net/licenses/apple-webkit-bsd", - "name": "Apple WebKit BSD License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Arev_Fonts_Copyright", - "name": "Arev Fonts Copyright" - }, - { - "openhub_url": "https://www.openhub.net/licenses/AROS_Public_License_version_1_1", - "name": "AROS Public License version 1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/artistic", - "name": "Artistic License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/artistic_gpl", - "name": "Artistic License/GPL" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Artistic_License_2_0", - "name": "Artistic License 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Ascend_Net_License", - "name": "Ascend.Net License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/attribution_assurance", - "name": "Attribution Assurance License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/BASANT45", - "name": "BASANT45" - }, - { - "openhub_url": "https://www.openhub.net/licenses/bcl-0_1", - "name": "Blackhan's Commercial Software Licence v0.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Beerware", - "name": "Beerware" - }, - { - "openhub_url": "https://www.openhub.net/licenses/be-sample-code", - "name": "Be Sample Code License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/BESEN", - "name": "BESEN license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Biopython-License", - "name": "Biopython License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Bitstream_Vera_Fonts_Copyright", - "name": "Bitstream Vera Fonts Copyright" - }, - { - "openhub_url": "https://www.openhub.net/licenses/bittorrent_open_source", - "name": "BitTorrent Open Source License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/blas", - "name": "BLAS License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/boost", - "name": "Boost Software License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Boost_Software_License", - "name": "Boost Software License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/bsd", - "name": "BSD 4-clause (University of California-Specific)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/BSD-2-Clause", - "name": "BSD 2-clause \"FreeBSD\" License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/bsd_2clause_ish", - "name": "BSD 2-clause \"Simplified\" License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/BSD-2-Clause-NetBSD", - "name": "BSD 2-clause \"NetBSD\" License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/BSD-3-Clause", - "name": "BSD 3-clause \"New\" or \"Revised\" License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/BSD-3-clause-EPIC", - "name": "BSD 3-clause EPIC variant" - }, - { - "openhub_url": "https://www.openhub.net/licenses/BSD-4-Clause", - "name": "BSD 4-clause \"Original\" or \"Old\" License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/bsd_ish", - "name": "New BSD License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/BSD_License", - "name": "3-Clause BSD License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/bsd_mit_based_license", - "name": "BSD/MIT based license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/BSD_ModVar", - "name": "BSD Modification Variant" - }, - { - "openhub_url": "https://www.openhub.net/licenses/buildlic", - "name": "BUILDLIC" - }, - { - "openhub_url": "https://www.openhub.net/licenses/by-nc-nd-40", - "name": "Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International" - }, - { - "openhub_url": "https://www.openhub.net/licenses/caossl", - "name": "Computer Associates Open Source Software License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Carnegie_Mellon_Licence", - "name": "Carnegie Mellon Licence" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CB_FreeBSD_License", - "name": "ConceptBase-FreeBSD-License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cbla", - "name": "ClipBucket License Agreement" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC0", - "name": "Creative Commons - Public Domain Dedication " - }, - { - "openhub_url": "https://www.openhub.net/licenses/cc0-1", - "name": "CC0 1.0 Universal" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC0-1-0", - "name": "Creative Commons Zero v1.0 Universal" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CCA4-International", - "name": "Creative Commons Attribution 4.0 International Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC_AttributionNonCommercialShareAlike_3_0", - "name": "CC Attribution-NonCommercial-ShareAlike 3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-1-0", - "name": "Creative Commons Attribution 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cc-by-25", - "name": "Creative Commons Attribution 2.5 Generic" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cc-by-3", - "name": "Creative Commons Attribution 3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cc-by-4", - "name": "Creative Commons Attribution 4.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cc-by-4-0", - "name": "CC Attribution 4.0 International" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-NC-1-0", - "name": "Creative Commons Attribution Non Commercial 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-NC-2-0", - "name": "Creative Commons Attribution Non Commercial 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ccbync25", - "name": "Creative Commons Attribution-NonCommercial 2.5 Generic" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-NC-2-5", - "name": "Creative Commons Attribution Non Commercial 2.5" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC_BY-NC_40", - "name": "Creative Commons Attribution-NonCommercial 4.0 International" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-NC-4-0", - "name": "Creative Commons Attribution Non Commercial 4.0 International" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-NC-ND-1-0", - "name": "Creative Commons Attribution Non Commercial No Derivatives 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-NC-ND-2_0", - "name": "Creative Commons Attribution Non Commercial No Derivatives 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-NC-ND-2-5", - "name": "Creative Commons Attribution Non Commercial No Derivatives 2.5" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cc_by-nc-nd-3_0", - "name": "Creative Commons Attribution-NonCommercial-NoDerivs 3.0 Unported" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-NC-ND-3-0", - "name": "Creative Commons Attribution Non Commercial No Derivatives 3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-NC-SA-1-0", - "name": "Creative Commons Attribution Non Commercial Share Alike 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-NC-SA-20", - "name": "Creative Commons Attribution-NonCommercial-ShareAlike 2.0 Generic" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-NC-SA-2-0", - "name": "Creative Commons Attribution Non Commercial Share Alike 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-NC-SA-2-5", - "name": "Creative Commons Attribution Non Commercial Share Alike 2.5" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cc_by-nc-sa_2-5_ar", - "name": "CC Attribution-NonCommercial-ShareAlike Argentina" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-NC-SA_25PL", - "name": "Creative Commons BY-NC-SA 2.5 Poland" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-NC-SA-3-0", - "name": "Creative Commons Attribution Non Commercial Share Alike 3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC_BY-NC-SA_4-0", - "name": "Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-ND-1-0", - "name": "Creative Commons Attribution No Derivatives 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-ND-2-0", - "name": "Creative Commons Attribution No Derivatives 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-ND-2-5", - "name": "Creative Commons Attribution No Derivatives 2.5" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-SA-1-0", - "name": "Creative Commons Attribution Share Alike 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-SA-2_0", - "name": "Creative Commons Attribution Share Alike 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ccbysa3-0", - "name": "Creative Commons Attribution Share Alike 3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CC-BY-SA-3_0-DE", - "name": "Creative Commons BY-SA 3.0 Germany" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CCBYSA4", - "name": "Creative Commons Attributions-ShareAlike 4.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ccbysa4-0", - "name": "Creative Commons Attribution-ShareAlike 4.0 International License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cecill_b", - "name": "CeCILL-B Free Software License Agreement" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cecill_c", - "name": "CeCILL-C Free Software License Agreement" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CeCILL_Free_Software_License_Agreement_v1_0", - "name": "CeCILL Free Software License Agreement v1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CeCILL_Free_Software_License_Agreement_v1_1", - "name": "CeCILL Free Software License Agreement v1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cecill-v2_1", - "name": "CeCILL Free Software License Agreement v2.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Celtx_Public_License", - "name": "Celtx Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CERN1_2", - "name": "CERN Open Hardware Licence v1.2" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cgic", - "name": "CGIC License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/chromium", - "name": "Chromium" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cip4_software_license_v2", - "name": "CIP4 Software License 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Claros_Public_License", - "name": "Claros Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/click", - "name": "The Click License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cnews", - "name": "C News-like" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CNRI_Open_Source_License_for_Durus-3_7", - "name": "CNRI Open Source License for Durus-3.7" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CNRI-Python", - "name": "CNRI Python License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CNRI-Python-GPL-Compatible", - "name": "CNRI Python Open Source GPL Compatible License Agreement" - }, - { - "openhub_url": "https://www.openhub.net/licenses/coa", - "name": "Caglow Open Agreement" - }, - { - "openhub_url": "https://www.openhub.net/licenses/codechecker", - "name": "CodeChecker License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CodeIgniter", - "name": "EllisLab, Inc. CodeIgniter" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Commercial", - "name": "Commercial license available" - }, - { - "openhub_url": "https://www.openhub.net/licenses/commercial_license", - "name": "Commercial License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/common_development_and_distribution", - "name": "Common Development and Distribution License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Common_Development_and_Distribution_License_1_1", - "name": "Common Development and Distribution License 1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/common_public", - "name": "Common Public License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/compiere_public_license", - "name": "Compiere Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/computer_associates_trusted", - "name": "Computer Associates Trusted Open Source License 1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/copyheart", - "name": "Copyheart" - }, - { - "openhub_url": "https://www.openhub.net/licenses/copyleft-next", - "name": "copyleft-next" - }, - { - "openhub_url": "https://www.openhub.net/licenses/COSL", - "name": "Cougaar Open Source License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cpal", - "name": "Common Public Attribution License 1.0 " - }, - { - "openhub_url": "https://www.openhub.net/licenses/CPL", - "name": "Common Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cpol", - "name": "The Code Project Open License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Crawl_General_Public_Licence", - "name": "Crawl General Public Licence" - }, - { - "openhub_url": "https://www.openhub.net/licenses/creative_commons", - "name": "CC Attr-NC-NDW 3.0 Unported License." - }, - { - "openhub_url": "https://www.openhub.net/licenses/creativecommons-and3", - "name": "Creative Commons Attribution No Derivatives 3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Creative_Commons_Attribution_2_0_Generic", - "name": "Creative Commons Attribution 2.0 Generic" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Creative_Commons_AttributionShare_Alike_2_0_DE", - "name": "Creative Commons Attribution-Share Alike 2.0 DE" - }, - { - "openhub_url": "https://www.openhub.net/licenses/creative_commons_by_3_0_unported", - "name": " Creative Commons Namensnennung 3.0 Unported" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Creative-Commons-BY-NC-3_0", - "name": "Creative Commons Attribution Non Commercial 3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Creative_Commons_BYSA_2_5", - "name": "Creative Commons Attribution Share Alike 2.5" - }, - { - "openhub_url": "https://www.openhub.net/licenses/creativecommons_org-licenses-by-sa-3_0", - "name": "Creative Commons Attribution-ShareAlike 3.0 Unported" - }, - { - "openhub_url": "https://www.openhub.net/licenses/CSIROLicenseGPL", - "name": "CSIRO Open Source Software License (GPL)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/csystem", - "name": "cSystem " - }, - { - "openhub_url": "https://www.openhub.net/licenses/cua_office", - "name": "CUA Office Public License Version 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/cyberneko", - "name": "CyberNeko Software License, Version 1.0 " - }, - { - "openhub_url": "https://www.openhub.net/licenses/dashCommerce_Core_License", - "name": "dashCommerce Core License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/dashCommerce_Store_License", - "name": "dashCommerce Store License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/dbad-license", - "name": "Don't Be a Dick license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ddmenu", - "name": "wirelessbro" - }, - { - "openhub_url": "https://www.openhub.net/licenses/death-and-repudiation", - "name": "Death and Repudiation License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/devblocks_public_license", - "name": "Devblocks Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/DFSL", - "name": "Deutsche Freie Software Lizenz" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Dibi_License_version_1_0", - "name": "Dibi License version 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Distributable", - "name": "Distributable" - }, - { - "openhub_url": "https://www.openhub.net/licenses/DONT_PANIC_License", - "name": "DON'T PANIC License 1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/DrJava_Open_Source_License", - "name": "DrJava Open Source License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/DroidScreen", - "name": "Droid@Screen OpenSource License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ecl2", - "name": "Educational Community License v2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/eclipse", - "name": "Eclipse Public License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/eCos-2-0", - "name": "eCos license version 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/eCos_license", - "name": "eCos license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/edl-v10", - "name": "Eclipse Distribution License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/educational", - "name": "Educational Community License v1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/eGPL", - "name": "Exception General Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/eiffel", - "name": "Eiffel Forum License v1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/eiffel_2", - "name": "Eiffel Forum License v2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/elxis-public-license", - "name": "Elxis Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/EmbargoedSoftwareLicense", - "name": "SCEngine Restriced BSD License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Emgu_CV_Commercial_License", - "name": "Emgu CV Commercial License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/entessa", - "name": "Entessa Public License v1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/EOxServer_Open_License", - "name": "EOxServer Open License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/epics-open", - "name": "EPICS Open License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/EPL2", - "name": "Eclipse Public License 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/epuck_Open_Source_Hardware_LicenseVersion_1_0", - "name": "e-puck Open Source Hardware LicenseVersion 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/erddap", - "name": "ERDDAP NOAA" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Erlang_Public_License", - "name": "Erlang Public License v1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Erlang_Public_License_v1_1", - "name": "Erlang Public License v1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/eu_datagrid", - "name": "Genesis2D License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/EU_DataGrid_Software_License", - "name": "EU DataGrid Software License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/EUPL", - "name": "European Union Public License 1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/EUPL-1-0", - "name": "European Union Public License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/expat", - "name": "Expat" - }, - { - "openhub_url": "https://www.openhub.net/licenses/fair", - "name": "Fair License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/FMIT", - "name": "Feedback MIT License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/fnOrb_OpenSource_License", - "name": "fnOrb OpenSource License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/fpll", - "name": "Falcon Programming Language License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/frameworx", - "name": "Frameworx Open License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Free_Art_License", - "name": "Genesis2D" - }, - { - "openhub_url": "https://www.openhub.net/licenses/freebsd", - "name": "FreeBSD" - }, - { - "openhub_url": "https://www.openhub.net/licenses/FreeBSD_Documentation", - "name": "FreeBSD Documentation license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/freedom", - "name": "Freedom" - }, - { - "openhub_url": "https://www.openhub.net/licenses/FreeImage_Public_License", - "name": "FreeImage Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/FreeOTFE-Licence-v1_1", - "name": "FreeOTFE Licence (v1.1)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/freepointPublicLicense", - "name": "freepoint public license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/freeware", - "name": "Freeware" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ftl", - "name": "FreeType Project License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/geant4", - "name": "Geant4 Software License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/generama_license", - "name": "generama license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GFDL-1-1", - "name": "GNU Free Documentation License v1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GFDL-1-3", - "name": "GNU Free Documentation License v1.3" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Globus-gpl", - "name": "Globus Toolkit Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GNAT_Modified_General_Public_License", - "name": "GNAT Modified General Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/gnu3", - "name": "General Public License v3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/gnuallpermissive", - "name": "GNU All-Permissive License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GNU_Free_Documentation_License", - "name": "GNU Free Documentation License v1.2" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GNU_General_Public_License_1_0", - "name": "GNU General Public License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GNU_General_Public_License_v2_0_only", - "name": "GNU General Public License v2.0 only" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GNUGeneralPublicLicensev2WithFOSSologyException", - "name": "FOSSology Copyright and Licensing" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GNU-GPLv2", - "name": "GNU General Public License version 2" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GodeniTeam_Software_Beta3", - "name": "GodeniTeam Software License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/googleBSD", - "name": "Google BSD License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/gpl", - "name": "GNU General Public License v2.0 or later" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GPL-1-0plus", - "name": "GNU General Public License v1.0 or later" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GPL2", - "name": "GPL 2" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GPL-2-0-with-autoconf-exception", - "name": "GNU General Public License v2.0 w/Autoconf exception" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GPL-2-0-with-bison-exception", - "name": "GNU General Public License v2.0 w/Bison exception" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GPL-2-0-with-classpath-exception", - "name": "GNU General Public License v2.0 w/Classpath exception" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GPL-2_0-with-font-exception", - "name": "GNU General Public License v2.0 w/Font exception" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GPL-2-0-with-freertos-exception", - "name": "GNU General Public License v2.0 w/FreeRTOS exception" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GPL-2-0-with-GCC-exception", - "name": "GNU General Public License v2.0 w/GCC Runtime Library exception" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GPL2wCE", - "name": "GNU General Public License Version 2 with the Classpath Exception" - }, - { - "openhub_url": "https://www.openhub.net/licenses/gpl3", - "name": "GNU General Public License v3.0 only" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GPL_3", - "name": "GPL 3" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GPL-3", - "name": "GNU General Public License v3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GPL-3-0-with-autoconf-exception", - "name": "GNU General Public License v3.0 w/Autoconf exception" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GPL-3-0-with-GCC-exception", - "name": "GNU General Public License v3.0 w/GCC Runtime Library exception" - }, - { - "openhub_url": "https://www.openhub.net/licenses/gpl3_or_later", - "name": "GNU General Public License v3.0 or later" - }, - { - "openhub_url": "https://www.openhub.net/licenses/gpl_classpath", - "name": "GPL + Classpath Exception" - }, - { - "openhub_url": "https://www.openhub.net/licenses/gpl_v3", - "name": "GNU Public License version 3" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GPLv3", - "name": "GNU GENERAL PUBLIC LICENSE" - }, - { - "openhub_url": "https://www.openhub.net/licenses/GPLv3plus", - "name": "GNU General Public License 3 or later" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Grestul_EULA", - "name": "Grestul EULA" - }, - { - "openhub_url": "https://www.openhub.net/licenses/gSOAP-1-3b", - "name": "gSOAP Public License v1.3b" - }, - { - "openhub_url": "https://www.openhub.net/licenses/gustavfilms", - "name": "gustavfilms" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Health_Level7_Public_License_Version_1_0", - "name": "Health Level-7 Public License Version 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Hets", - "name": "Hets License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/historical", - "name": "Historical Permission Notice and Disclaimer" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Honest_Public_License_HPL", - "name": "Honest Public License (HPL)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/hs_license", - "name": "HS License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/htmlarea_license", - "name": "htmlArea License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/htsql-permissive", - "name": "HTSQL Permissive License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/i9", - "name": "i9 License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/i9_license", - "name": "i9_license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ibm_public", - "name": "IBM Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/IJG", - "name": "Independent JPEG Group License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/illinois-charm", - "name": "Illinois Charm++ License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ImageMagick_License", - "name": "ImageMagick License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Indiana_University_Software_Licence", - "name": "Indiana University Software Licence" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Initial_Developers_PUBLIC_LICENSE_Version_1_0", - "name": "Initial Developer's PUBLIC LICENSE Version 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ink_license", - "name": "INK" - }, - { - "openhub_url": "https://www.openhub.net/licenses/intel_open_source", - "name": "Intel Open Source License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/InterBase_Public_License", - "name": "InterBase Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Internet_Systems_Consortium_License", - "name": "Internet Systems Consortium License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ioke", - "name": "Ioke License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/IPA", - "name": "IPA Font License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ipfaces_com_license", - "name": "The iPFaces Commercial License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/IPL-1-0", - "name": "IBM Public License v1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/isc", - "name": "ISC License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/jabber_open_source", - "name": "Jabber Open Source License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/JASIG_License", - "name": "JA-SIG License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Java_Research_License", - "name": "Java Research License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/jayeshdani", - "name": "jayesh" - }, - { - "openhub_url": "https://www.openhub.net/licenses/jBehave_License", - "name": "jBehave License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/jbpl", - "name": "Jared Beer Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/jGuruEurope_CorporateSourceLicense", - "name": "jGuru Europe Corporate Source License, based on The Apache Software License, Version 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/json", - "name": "The JSON License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Justice832", - "name": "Justice832" - }, - { - "openhub_url": "https://www.openhub.net/licenses/KFGQPC", - "name": "King Fahd Glorious Quran Printing Complex License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/kingboard_license", - "name": "Kingboard License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Kohana", - "name": "Kohana License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Krixware_GPL", - "name": "Krixware GPL" - }, - { - "openhub_url": "https://www.openhub.net/licenses/LAL", - "name": "Licence Art Libre" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Latex_Project_Public_License", - "name": "Latex Project Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/LaviLicense", - "name": "Lavi's License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lgpl", - "name": "GNU Library or \"Lesser\" GPL (LGPL)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lgpl-2_0", - "name": "GNU Library General Public License v2 only" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lgpl2_0", - "name": "GNU Library General Public License v2 or later" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lgpl21", - "name": "GNU Lesser General Public License v2.1 only" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lgpl21le", - "name": "GNU LGPL 2.1 + linking exception (FPC/Lazarus)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lgpl21_or_later", - "name": "GNU Lesser General Public License v2.1 or later" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lgpl3", - "name": "GNU Lesser General Public License v3.0 only" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lgpl3dot0", - "name": "GNU Lesser General Public License version 3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/LGPLv2_1", - "name": "GNU Lesser General Public License 2.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/LGPLv2_1-no3-no13", - "name": "GNU LGPL 2.1 - no GPL conversion (Baka)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lgplv3", - "name": "GNU Lesser General Public License Version 3" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lgpl-v3", - "name": "GNU Lesser General Public License v3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lgplv3-static", - "name": "LGPLv3 + static link exception (ZeroMQ)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lgpv3_or_later", - "name": "GNU Lesser General Public License v3.0 or later" - }, - { - "openhub_url": "https://www.openhub.net/licenses/libpng", - "name": "libpng license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/LibRaw", - "name": "LibRaw" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lifish", - "name": "Lifish License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/LightPlusPlusLicense", - "name": "Light++ Copyright License for Non-military Academic and Personal Use" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lighttpd", - "name": "lighttpd revised BSD license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Linkware", - "name": "Linkware" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Lisp_Lesser_General_Public_License_", - "name": "Lisp Lesser General Public License " - }, - { - "openhub_url": "https://www.openhub.net/licenses/LL-FLOSS-Exception", - "name": "Linden Lab Viewer FLOSS License Exception v0.5" - }, - { - "openhub_url": "https://www.openhub.net/licenses/logolyze-bsd", - "name": "LogoLyze (4-clause BSD style)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/LPGL", - "name": "LPGL" - }, - { - "openhub_url": "https://www.openhub.net/licenses/LPPL-1-0", - "name": "LaTeX Project Public License v1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/LPPL-1-1", - "name": "LaTeX Project Public License v1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/LPPL-1-2", - "name": "LaTeX Project Public License v1.2" - }, - { - "openhub_url": "https://www.openhub.net/licenses/LPPL-1-3c", - "name": "LaTeX Project Public License v1.3c" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lucent_plan9", - "name": "Lucent Public License 1.0 (Plan9)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/lucent_public", - "name": "Lucent Public License v1.02" - }, - { - "openhub_url": "https://www.openhub.net/licenses/m6pl", - "name": "matma6 Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/MAME%E2%84%A2_Distribution_License", - "name": "MAMEâ„¢ Distribution License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/maojiaqiu", - "name": "maojiaqiu" - }, - { - "openhub_url": "https://www.openhub.net/licenses/MapServer_License", - "name": "MapServer License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/mattkrusedualmitgpl", - "name": "Matt Kruse Dual MIT and GPL License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/mattkruselicense", - "name": "Matt Kruse License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/mclaren_mgh", - "name": "McLaren_mgh" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Mediajargo", - "name": "handelsvoorwaarden" - }, - { - "openhub_url": "https://www.openhub.net/licenses/megawave2_public_license", - "name": "megawave2 public license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Microsoft_Public_License_", - "name": "Microsoft Public License " - }, - { - "openhub_url": "https://www.openhub.net/licenses/Microsoft_Reciprocal_License", - "name": "Microsoft Reciprocal License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Microsoft_Shared_Source_Permissive_License", - "name": "Microsoft Shared Source Permissive License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/MirOS", - "name": "MirOS Licence" - }, - { - "openhub_url": "https://www.openhub.net/licenses/mit", - "name": "MIT License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/mitgnu_bsd", - "name": "MitGNU BSD License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/mitre", - "name": "mitre" - }, - { - "openhub_url": "https://www.openhub.net/licenses/MMPL", - "name": "Minecraft Mod Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Modelica-1_1", - "name": "The Modelica License 1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Modelica-2", - "name": "The Modelica License 2" - }, - { - "openhub_url": "https://www.openhub.net/licenses/motosoto", - "name": "Motosoto License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Mozillaish_License", - "name": "Mozilla-ish License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/mozilla_public_1", - "name": "Mozilla Public License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/mozilla_public_1_1", - "name": "Mozilla Public License 1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/mozilla_public_2_0", - "name": "Mozilla Public License 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/mozilla_public_2_0_incompatible", - "name": "Mozilla Public License 2.0 (Incompatible with Secondary Licenses)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/mslpl", - "name": "Microsoft Limited Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Ms-PL", - "name": "Microsoft Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/MS-RSL", - "name": "Microsoft Reference Source License (MS-RSL)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Multics", - "name": "Multics License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Multiple_OSI_Approved", - "name": "Multiple OSI Approved" - }, - { - "openhub_url": "https://www.openhub.net/licenses/nally-license", - "name": "Nally License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/nasa_open", - "name": "NASA Open Source Agreement 1.3" - }, - { - "openhub_url": "https://www.openhub.net/licenses/naumen", - "name": "Naumen Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/NavigoSoftwareLicense", - "name": "Navigo Software License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/nethack", - "name": "Nethack General Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Nette_License_version_1_0", - "name": "Nette License version 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/newmat", - "name": "Newmat" - }, - { - "openhub_url": "https://www.openhub.net/licenses/nextcommander", - "name": "NeXT Commander License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/nlpl", - "name": "NO LIMIT PUBLIC LICENSE" - }, - { - "openhub_url": "https://www.openhub.net/licenses/nokia_open_source", - "name": "Nokia Open Source License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/NonProfit_Open_Software_License", - "name": "Non-Profit Open Software License 3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/NOSA", - "name": "NASA OPEN SOURCE AGREEMENT" - }, - { - "openhub_url": "https://www.openhub.net/licenses/npl", - "name": "Nietzsche Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/NTP", - "name": "NTP License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/NUnit_License", - "name": "NUnit License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/oclc_research", - "name": "OCLC Research Public License 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ODbL-1-0", - "name": "ODC Open Database License v1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/OFL-1-0", - "name": "SIL Open Font License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/OFL-1-1", - "name": "SIL Open Font License 1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/old-apple-webkit-bsd", - "name": "Old Apple WebKit BSD License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/olol", - "name": "Open, libertarian and ostentatious license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/openapc", - "name": "OpenAPC Dual License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Openbravo_Public_License", - "name": "Openbravo Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Open_Font_Licence", - "name": "SIL Open Font License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Open_Game_License", - "name": "Open Game License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/open_group_test", - "name": "Open Group Test Suite License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/OpenLDAP_Public_License", - "name": "OpenLDAP Public License v2.8" - }, - { - "openhub_url": "https://www.openhub.net/licenses/OpenORB_License", - "name": "The OpenORB Community Software License, Version 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/openpbs", - "name": "OpenPBS Software License v2.3" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Open_Publication_License", - "name": "Open Publication License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/open_software", - "name": "Open Software License 3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Open_Software_Licence_30", - "name": "Open Software License v. 3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/OpenSolaris_Binary_License", - "name": "OpenSolaris Binary License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/openssl", - "name": "OpenSSL" - }, - { - "openhub_url": "https://www.openhub.net/licenses/OpenSSL_License", - "name": "OpenSSL License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/OpenTag_v1", - "name": "OpenTag License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/opl", - "name": "Open Content License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Oracle_Open_Source_License__Berkeley_DB", - "name": "Oracle Open Source License - Berkeley DB" - }, - { - "openhub_url": "https://www.openhub.net/licenses/OSCL-C", - "name": "Open Source Community License - Type C" - }, - { - "openhub_url": "https://www.openhub.net/licenses/OSDVPublicLicense1_2", - "name": "OSDV Public License 1.2" - }, - { - "openhub_url": "https://www.openhub.net/licenses/OSL-1-0", - "name": "Open Software License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/OSL-2-0", - "name": "Open Software License 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/OSL-2-1", - "name": "Open Software License 2.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/otn_development_and_distribution_license", - "name": "Oracle Technology Network Development and Distribution License Terms" - }, - { - "openhub_url": "https://www.openhub.net/licenses/owl", - "name": "Open Works License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ozplb", - "name": "Australian Public Licence B" - }, - { - "openhub_url": "https://www.openhub.net/licenses/pcinfotools", - "name": "pcinfotools" - }, - { - "openhub_url": "https://www.openhub.net/licenses/PDDL-1-0", - "name": "ODC Public Domain Dedication & License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/PearCMS", - "name": "PearCMS Standards Of Use" - }, - { - "openhub_url": "https://www.openhub.net/licenses/peerproductionlicense", - "name": "Peer Production License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/petsc", - "name": "PETSc" - }, - { - "openhub_url": "https://www.openhub.net/licenses/pgdg", - "name": "PostgreSQL Global Development Group" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Phorum_License", - "name": "Phorum License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/PHP-3-01", - "name": "PHP LIcense v3.01" - }, - { - "openhub_url": "https://www.openhub.net/licenses/php_license", - "name": "PHP License v3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/PicoContainer_license", - "name": "PicoContainer license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/pil", - "name": "PIL Software License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/pilsl", - "name": "PIL Software License " - }, - { - "openhub_url": "https://www.openhub.net/licenses/pircd_license", - "name": "pircd license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/pjsip_Standard_License", - "name": "pjsip Standard License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Plainly_and_Simply_Free", - "name": "Plainly and Simply Free" - }, - { - "openhub_url": "https://www.openhub.net/licenses/pluto", - "name": "The Pluto Public License, Version 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/poetic", - "name": "Poetic License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/postgresql", - "name": "PostgreSQL Licence" - }, - { - "openhub_url": "https://www.openhub.net/licenses/POVRay_Distrution_License", - "name": "POV-Ray Distribution License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/POVRay_End_User_License", - "name": "POV-Ray End User License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/POVRay_Source_License", - "name": "POV-Ray Source License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Proprietary", - "name": "Proprietary and Confidential" - }, - { - "openhub_url": "https://www.openhub.net/licenses/proprietary-nonconfidential", - "name": "Proprietary" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Public_domain_", - "name": "Public Domain" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Public_Domain", - "name": "Public Domain / GPL When Compiled" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Python-2-0", - "name": "Python License 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/python_license", - "name": "Python 2.4.2 license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/python_software_foundation", - "name": "Python Software Foundation License 2.1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/QTail", - "name": "The MIT License (MIT)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/qt_public", - "name": "Q Public License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/qwtlicence", - "name": "Qwt License, Version 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Rage3D_engine_license", - "name": "Rage3D engine license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/rcshell", - "name": "rc shell license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/realnetworks_public_source", - "name": "RealNetworks Public Source License v1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/reciprocal_public", - "name": "Reciprocal Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Remote_Network_Browser_Licence", - "name": "Remote Network Browser Licence" - }, - { - "openhub_url": "https://www.openhub.net/licenses/repoze-public-license", - "name": "Repoze Public LIcense" - }, - { - "openhub_url": "https://www.openhub.net/licenses/revision", - "name": "Revision License for Open Source Software" - }, - { - "openhub_url": "https://www.openhub.net/licenses/RHeCos-1-1", - "name": "Red Hat eCos Public License v1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ricoh_source", - "name": "Ricoh Source Code Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/RISC_OS_Base_License_v1", - "name": "RISC OS Base License v1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Royalsbachl", - "name": "Royal" - }, - { - "openhub_url": "https://www.openhub.net/licenses/RPL", - "name": "Reciprocal Public License 1.5" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Ruby_License", - "name": "Ruby License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/SAX-PD", - "name": "Sax Public Domain Notice" - }, - { - "openhub_url": "https://www.openhub.net/licenses/scala-license", - "name": "Scala License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Scilab_licence", - "name": "Scilab licence" - }, - { - "openhub_url": "https://www.openhub.net/licenses/sdrvdvkc", - "name": "Культурный центр Союза Десантников России" - }, - { - "openhub_url": "https://www.openhub.net/licenses/sendmail", - "name": "Sendmail License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/SFI_Source_Code_license_agreement", - "name": "SFI Source Code license agreement" - }, - { - "openhub_url": "https://www.openhub.net/licenses/SimPL-2-0", - "name": "Simple Public License 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Simple_Machines_Forum_License", - "name": "Simple Machines Forum License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/sisal_license", - "name": "sisal.license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/sleepycat", - "name": "Sleepycat License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/spikevare", - "name": "SpikeVare" - }, - { - "openhub_url": "https://www.openhub.net/licenses/splunkfree", - "name": "Spunk Free License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Spread_Open_Source_License", - "name": "Spread Open Source License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ssleay", - "name": "SSLeay" - }, - { - "openhub_url": "https://www.openhub.net/licenses/sugarcrm_1_1_3", - "name": "SugarCRM Public License v1.1.3" - }, - { - "openhub_url": "https://www.openhub.net/licenses/SummerLightningLLC_OS_RU", - "name": "Summer Lightning Open Source (RU)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/sun_industry_standards", - "name": "Sun Industry Standards Source License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/sun_public", - "name": "Sun Public License v1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/sybase_open_watcom", - "name": "Sybase Open Watcom Public License 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/symbian-example-source-code-license", - "name": "Symbian Example Source Code License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/SynApp2_License_1_01", - "name": "SynApp2 License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/tagcentric_ubiquity_license", - "name": "TagCentric/Ubiquity License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/tapr-OHL", - "name": "The TAPR Open Hardware License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/telesoft-public-license", - "name": "TELESOFT PUBLIC LICENSE, Version 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/terrapin", - "name": "Terrapin open source license (RANCID)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/tethercode-logo-license", - "name": "tethercode logo license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/tgppl", - "name": "Transitive Grace Period Public Licence" - }, - { - "openhub_url": "https://www.openhub.net/licenses/The_3DFX_GLIDE_Source_Code_General_Public_License", - "name": "3DFX GLIDE Source Code General Public License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/The_Angband_License", - "name": "Angband License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/theherorpg", - "name": "The Hero RPG" - }, - { - "openhub_url": "https://www.openhub.net/licenses/The_Kannel_Software_License_Version_1_0", - "name": "The Kannel Software License, Version 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/The_Radiance_Software_License_Version_1_0", - "name": "The Radiance Software License, Version 1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/TMate_Open_Source_License", - "name": "TMate Open Source License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Transitive_Grace_Period_Public_Licence_v1_0", - "name": "Transitive Grace Period Public Licence, v1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/TrueCrypt_Collective_License", - "name": "TrueCrypt Collective License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Ubuntu_Component_license_Policy", - "name": "Ubuntu Component License Policy" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ufl", - "name": "Unrestricted Forever" - }, - { - "openhub_url": "https://www.openhub.net/licenses/UKYL", - "name": "University of Kentucky License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/UmbracoUILicence", - "name": "Umbraco UI Licence" - }, - { - "openhub_url": "https://www.openhub.net/licenses/unicode", - "name": "Unicode Terms of Use" - }, - { - "openhub_url": "https://www.openhub.net/licenses/University_of_Washingtons_Pine_license", - "name": "University of Washington's Pine license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/unlicense", - "name": "Unlicense" - }, - { - "openhub_url": "https://www.openhub.net/licenses/u_of_i_ncsa", - "name": "University of Illinois/NCSA Open Source License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ut2004-bsd", - "name": "UT2004 BSD License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/valve", - "name": "Valve License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Verbatim_copying", - "name": "Verbatim copying" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Vim_License", - "name": "Vim License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/vmime-commercial-license", - "name": "VMime Commercial License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Volition-FS2", - "name": "Volition Freespace2 Non Commercial License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/vovida_software", - "name": "Vovida Software License v1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/w3c", - "name": "W3C License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/W3C_Software_and_Document_Notice_and_License", - "name": "W3C Software and Document Notice and License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/W3C_Software_Notice_and_License", - "name": "W3C Software Notice and License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Wakanda-Commercial-License", - "name": "Wakanda Commercial License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Wakanda-Community-License", - "name": "Wakanda Community License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/whiskeyware", - "name": "Whiskey Ware" - }, - { - "openhub_url": "https://www.openhub.net/licenses/wobblylang", - "name": "Wobbly Language License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/WTFNMFPLv3", - "name": "WTFNMFPLv3" - }, - { - "openhub_url": "https://www.openhub.net/licenses/wtfpl", - "name": "Do What The Fuck You Want To Public License " - }, - { - "openhub_url": "https://www.openhub.net/licenses/WTFPL", - "name": "WTFPL Version 1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/wtfpl_1_1", - "name": "WTFPLv1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/wtfpl_2", - "name": "WTFPL" - }, - { - "openhub_url": "https://www.openhub.net/licenses/WTFPL_2_0", - "name": "Generous License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/wtfpl_v2", - "name": "wtfpl v2" - }, - { - "openhub_url": "https://www.openhub.net/licenses/wx_windows", - "name": "wxWindows Library License v3.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/wxWindowsLibraryLicense", - "name": "wxWindows Library License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/wxWindows_Library_License", - "name": "wxWindows Library License v3.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/XCore", - "name": "XCore, derivative of University of Illinois/NCSA" - }, - { - "openhub_url": "https://www.openhub.net/licenses/xdoclet2_license", - "name": "xdoclet-2 license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/xdoclet-2-plugins-license", - "name": "xdoclet-2-plugins-license" - }, - { - "openhub_url": "https://www.openhub.net/licenses/XFree86_Project_Licence_version_1_1", - "name": "XFree86 License 1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/xinetd", - "name": "xinetd" - }, - { - "openhub_url": "https://www.openhub.net/licenses/x_net", - "name": "X.Net License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/YAML_under_Commercial_License", - "name": "YAML under Commercial License" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ypl", - "name": "Yahoo! Public License v1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/YPL-1-0", - "name": "Yahoo! Public License v1.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/zimbra-public-license-1-3", - "name": "Zimbra Public License v1.3" - }, - { - "openhub_url": "https://www.openhub.net/licenses/zlib_libpng", - "name": "zlib License (aka zlib/libpng)" - }, - { - "openhub_url": "https://www.openhub.net/licenses/zope", - "name": "Zope Public License 2.0" - }, - { - "openhub_url": "https://www.openhub.net/licenses/Zope_Public_License_ZPL_Version_2_1", - "name": "Zope Public License 2.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/zpl", - "name": "Zimbra Public License, Version 1.2" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ZPL-1-1", - "name": "Zope Public License 1.1" - }, - { - "openhub_url": "https://www.openhub.net/licenses/ZXGE_Public_License", - "name": "ZXGE Public License" - } -] \ No newline at end of file diff --git a/etc/scripts/misc/openhub_scraper.py b/etc/scripts/misc/openhub_scraper.py index a788f986d71..688486c0efd 100644 --- a/etc/scripts/misc/openhub_scraper.py +++ b/etc/scripts/misc/openhub_scraper.py @@ -10,73 +10,90 @@ import io import json import os +import time from urllib.request import urlopen from bs4 import BeautifulSoup +LICENSES_LIST_FILE = "openhub_licenses.json" -def write_license_info_to_file(license_list): + +def get_page(page_no): """ - Neatly format all the scrapped license information in the json format and - write it to a file. + Return the text content of an openhub ``page_no`` page number. """ - with io.open("openhub_licenses.json", "wb") as f: - f.write(json.dumps(license_list, indent=4)) + url = f"https://www.openhub.net/licenses?page={page_no}" + print(f"Fetching: {url}") + return urlopen(url).read() -def parse_local_file(): +def fetch_all_licenses(): """ - Parses the given file using 'BeautifulSoup' and returns html content of - that file. + Save list of license and all license texts """ - local_test_file = os.path.join( - os.path.dirname(__file__), "testdata/openhub_scraper/openhub_html.html" - ) - with open(local_test_file, "r") as f: - parsed_page = BeautifulSoup(f.read(), "html.parser") - return parsed_page + if not os.path.exists(LICENSES_LIST_FILE): + licenses = save_licenses(filename=LICENSES_LIST_FILE) + else: + with open(LICENSES_LIST_FILE) as ip: + licenses = json.load(ip) + print() + for lic in licenses: + fetch_and_save_license(lic["url"]) + + +def list_all_licenses(pages_count=18): + """ + Yield mappings of openhub license {name:xxx, url:zzz} extracted from the + all the openhub license pages + """ + for page_no in range(1, pages_count + 1): + time.sleep(5) + page = get_page(page_no) + for lic in list_licenses_on_page(page): + yield lic -def parse_webpage(url, page_no): +def fetch_and_save_license(url, force=False, directory="openhub_licenses"): """ - Parses the given webpage using 'BeautifulSoup' and returns html content of - that webpage. + Return the text for license page at url + """ + filename = url.split("/")[-1] + lic_file = os.path.join(directory, filename) + if not force and os.path.exists(lic_file): + return + + os.makedirs(directory, exist_ok=True) + print(f" Fetching: {url}") + time.sleep(.1) + content = urlopen(url).read() + with open(lic_file, "wb") as of: + of.write(content) + + +def list_licenses_on_page(page): + """ + Yield mappings of openhub license {name:xxx, url:zzz} extracted from the + ``page`` text of an HTML page. """ - page = urlopen(url + page_no) parsed_page = BeautifulSoup(page, "html.parser") - return parsed_page + all_licenses = parsed_page.find(id="license").select( + "table.table-striped.table-condensed.table" + ) + licenses = all_licenses[0].find_all("a", href=True) + for license in licenses: # NOQA + yield dict(url=license["href"], name=license.get_text()) -def extract_openhub_licenses(start_pg, end_pg, write_to_file, parse_website): +def save_licenses(filename=LICENSES_LIST_FILE): """ - Extract openhub license names, their urls, and save it in a json file or - return the list of the license dictionary. + Write to the ``filename`` file the list of scrapped openhub license as JSON. """ - license_dict = {} - license_list = [] - - for i in range(start_pg, end_pg + 1): - if parse_website: - parsed_page = parse_webpage( - url="https://www.openhub.net/licenses?page=", page_no=str(i) - ) - else: - parsed_page = parse_local_file() - all_licenses = parsed_page.find(id="license").select( - "table.table-condensed.table-striped.table" - ) - license_rows = all_licenses[0].find_all("a", href=True) - for license in license_rows: # NOQA - license_dict["openhub_url"] = license["href"] - license_dict["name"] = license.get_text() - license_list.append(license_dict.copy()) - - if write_to_file: - write_license_info_to_file(license_list) - else: - return license_list + licenses = list(list_all_licenses()) + with io.open(filename, "w") as f: + json.dump(licenses, f, indent=2) + return licenses if __name__ == "__main__": - extract_openhub_licenses(start_pg=1, end_pg=17, write_to_file=True, parse_website=True) + fetch_all_licenses() diff --git a/etc/scripts/misc/test_openhub_scraper.py b/etc/scripts/misc/test_openhub_scraper.py index abc52386688..5fa70d24dcf 100644 --- a/etc/scripts/misc/test_openhub_scraper.py +++ b/etc/scripts/misc/test_openhub_scraper.py @@ -8,38 +8,30 @@ # See https://aboutcode.org for more information about nexB OSS projects. # +import os + import openhub_scraper def test_scraping_unicode_and_ascii(): - license_result = openhub_scraper.extract_openhub_licenses( - start_pg=15, end_pg=15, write_to_file=False, parse_website=False + test_file = os.path.join( + os.path.dirname(__file__), "testdata/openhub_html.html" ) - observed_result_1 = ( - item - for item in license_result - if item["name"] == u"Культурный центр Союза Десантников России" - ).next() - expected_result_1 = { - "openhub_url": u"https://www.openhub.net/licenses/sdrvdvkc", - "name": u"Культурный центр Союза Десантников России", - } - assert observed_result_1 == expected_result_1 + with open(test_file, "r") as f: + test_content = f.read() + + licenses = list(openhub_scraper.list_licenses_on_page(test_content)) - observed_result_2 = ( - item for item in license_result if item["name"] == "Sleepycat License" - ).next() - expected_result_2 = { - "openhub_url": "https://www.openhub.net/licenses/sleepycat", + result = [i for i in licenses if i["name"] == "Sleepycat License"] + expected = [{ + "url": "https://www.openhub.net/licenses/sleepycat", "name": "Sleepycat License", - } - assert observed_result_2 == expected_result_2 + }] + assert result == expected - observed_result_3 = ( - item for item in license_result if item["name"] == "Sun Public License v1.0" - ).next() - expected_result_3 = { - "openhub_url": u"https://www.openhub.net/licenses/sun_public", - "name": u"Sun Public License v1.0", - } - assert observed_result_3 == expected_result_3 + result = [i for i in licenses if i["name"] == "Sun Public License v1.0"] + expected = [{ + "url": "https://www.openhub.net/licenses/sun_public", + "name": "Sun Public License v1.0", + }] + assert result == expected diff --git a/etc/scripts/misc/testdata/openhub_html.html b/etc/scripts/misc/testdata/openhub_html.html new file mode 100644 index 00000000000..6efbacb23ab --- /dev/null +++ b/etc/scripts/misc/testdata/openhub_html.html @@ -0,0 +1,160 @@ + + + +Licenses - Open Hub + + + +
+
+ +
+ + + +
+ +
+ + +
+