Merge branch 'nexB:develop' into add_new_licenses

CHANGELOG.rst

@@ -27,6 +27,9 @@ Important API changes:
    that contains each package instance that can be aggregating data from
    multiple manifests for a single package instance.
 
+ - The data structure for HTML output has been changed to include emails and urls under the 
+   "infos" object. Now HTML template will output holders, authors, emails, and 
+   urls into separate tables like "licenses" and "copyrights".
 
 Copyright detection:
 ~~~~~~~~~~~~~~~~~~~~
@@ -43,6 +46,25 @@ Package detection:
  - Add support to track installed files for each Package type.
 
 
+License detection:
+~~~~~~~~~~~~~~~~~~~
+
+- Unknown licenses have a new flag "is_unknown" to identify them
+  beyond just the naming convention of having "unknown" as part of their name.
+
+- Rules that match at least one unknown license have a flag "has_unknown" set
+  in the returned match results.
+
+
+Many thanks to every contributors that made this possible and in particular:
+
+- Akanksha Garg @akugarg
+- Ayan Sinha Mahapatra @AyanSinhaMahapatra
+- Jono Yang @JonoYang
+- Philippe Ombredanne @pombredanne
+
+
+
 v21.8.4
 ---------
 

CONTRIBUTING.rst

@@ -134,25 +134,34 @@ To set up ScanCode for local development:
 
    Run this command to configure ScanCode::
 
-        ./configure
+        ./configure --dev
 
    On Windows use instead::
 
-        configure 
+        configure --dev
 
-   Then run this: `source bin/activate` or `. bin/activate` 
-   (or run `bin\\activate` on Windows)
+   Then activate the virtual environment::
+
+        source bin/activate
+
+        or
+
+        . bin/activate
+
+   On Windows use::
+
+        Scripts\activate
 
    When you create a new terminal/shell to work on ScanCode rerun the activate step.
 
    When you pull new code from git, rerun ./configure
-   
+
 
 5. Now you can make your code changes in your local clone.
    Please create new unit tests for your code. We love tests!
 
 6. When you are done with your changes, run all the tests.
-   Use this command:: 
+   Use this command::
 
         py.test
 
@@ -162,25 +171,25 @@ To set up ScanCode for local development:
 
    If you are running this on a RedHat based OS you may come across this
    failure::
-   
+
        OSError: libbz2.so.1.0: cannot open shared object file: No such file or directory
- 
+
    Try creating a symbolic link to libbz2.so.1.0 to solve this issue::
 
        locate libbz2.so.1.0
        cd <resulting libbz2.so directory>
        sudo ln -s <your version of libbz2.so> libbz2.so.1.0
-      
+
    See `this issue <https://github.com/nexB/scancode-toolkit/issues/443>`_ for more information.
 
 7. Check the status of your local repository before commit, regarding files changed::
-    
+
     git status
 
 
 8. Commit your changes and push your branch to your GitHub fork::
 
-    git add <file-changed-1> <file-changed-2> <file-changed-3> 
+    git add <file-changed-1> <file-changed-2> <file-changed-3>
     git commit -m "Your detailed description of your changes." --signoff
     git push <repository-alias-name> name-of-your-bugfix-or-feature
 
@@ -196,7 +205,7 @@ create a pull request. You can add new commits to your branch as needed.
 For merging, your request would need to:
 
 1. Include unit tests that are passing (run ``py.test``).
-2. Update documentation as needed for new API, functionality etc. 
+2. Update documentation as needed for new API, functionality etc.
 3. Add a note to ``CHANGELOG.rst`` about the changes.
 4. Add your name to ``AUTHORS.rst``.
 

configure

@@ -67,12 +67,17 @@ CFG_BIN_DIR=$CFG_ROOT_DIR/$VIRTUALENV_DIR/bin
 
 ################################
 # scancode-specific: Thirdparty package locations and index handling
-# Do we have thirdparty/files? use the mit.LICENSE as a proxy
-if [[ -f "$CFG_ROOT_DIR/thirdparty/mit.LICENSE" ]]; then
-    LINKS=$CFG_ROOT_DIR/thirdparty
+if [[ "$PYPI_LINKS" == "" ]]; then
+    # Do we have thirdparty/files? use the mit.LICENSE as a proxy
+    if [[ -f "$CFG_ROOT_DIR/thirdparty/mit.LICENSE" ]]; then
+        LINKS=$CFG_ROOT_DIR/thirdparty
+    else
+        LINKS=https://thirdparty.aboutcode.org/pypi
+    fi
 else
-    LINKS=https://thirdparty.aboutcode.org/pypi
+    LINKS=$PYPI_LINKS
 fi
+
 PIP_EXTRA_ARGS="--no-index --find-links $LINKS"
 ################################
 

docs/source/cli-reference/output-format.rst

@@ -177,6 +177,7 @@ following options.
               "short_name": "MIT Old Style",
               "category": "Permissive",
               "is_exception": false,
+              "is_unknown": false,
               "owner": "MIT",
               "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style",
               "text_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style",

docs/source/cli-reference/synopsis.rst

@@ -228,6 +228,7 @@ A sample JSON output for an individual file will look like::
           "short_name": "MIT Old Style",
           "category": "Permissive",
           "is_exception": false,
+          "is_unknown": false,
           "owner": "MIT",
           "homepage_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style",
           "text_url": "http://fedoraproject.org/wiki/Licensing:MIT#Old_Style",

etc/release/fetch_requirements.py

@@ -8,10 +8,11 @@
 # See https://github.com/nexB/scancode-toolkit for support or download.
 # See https://aboutcode.org for more information about nexB OSS projects.
 #
+import itertools
+
 import click
 
 import utils_thirdparty
-import itertools
 
 
 @click.command()
@@ -63,6 +64,15 @@
     is_flag=True,
     help='Fetch only the corresponding source distributions.',
 )
+@click.option('-u', '--remote-links-url',
+    type=str,
+    metavar='URL',
+    default=utils_thirdparty.REMOTE_LINKS_URL,
+    show_default=True,
+    help='URL to a PyPI-like links web site. '
+         'Or local path to a directory with wheels.',
+)
+
 @click.help_option('-h', '--help')
 def fetch_requirements(
     requirements_file,
@@ -73,6 +83,7 @@ def fetch_requirements(
     with_about,
     allow_unpinned,
     only_sources,
+    remote_links_url=utils_thirdparty.REMOTE_LINKS_URL,
 ):
     """
     Fetch and save to THIRDPARTY_DIR all the required wheels for pinned
@@ -82,7 +93,9 @@ def fetch_requirements(
     Also fetch the corresponding .ABOUT, .LICENSE and .NOTICE files together
     with a virtualenv.pyz app.
 
-    Use exclusively our remote repository (and not PyPI).
+    Use exclusively wheel not from PyPI but rather found in the PyPI-like link
+    repo ``remote_links_url`` if this is a URL. Treat this ``remote_links_url``
+    as a local directory path to a wheels directory if this is not a a URL.
     """
 
     # fetch wheels
@@ -93,23 +106,28 @@ def fetch_requirements(
     if not only_sources:
         envs = itertools.product(python_versions, operating_systems)
         envs = (utils_thirdparty.Environment.from_pyver_and_os(pyv, os) for pyv, os in envs)
+
         for env, reqf in itertools.product(envs, requirements_files):
+
             for package, error in utils_thirdparty.fetch_wheels(
                 environment=env,
                 requirements_file=reqf,
                 allow_unpinned=allow_unpinned,
                 dest_dir=thirdparty_dir,
+                remote_links_url=remote_links_url,
             ):
                 if error:
                     print('Failed to fetch wheel:', package, ':', error)
 
     # optionally fetch sources
     if with_sources or only_sources:
+
         for reqf in requirements_files:
             for package, error in utils_thirdparty.fetch_sources(
                 requirements_file=reqf,
                 allow_unpinned=allow_unpinned,
                 dest_dir=thirdparty_dir,
+                remote_links_url=remote_links_url,
             ):
                 if error:
                     print('Failed to fetch source:', package, ':', error)