define process_codebase function for matching references

Signed-off-by: akugarg <[email protected]>

src/licensedcode/plugin_license.py

@@ -10,11 +10,9 @@
 from functools import partial
 
 import attr
-import os
 
 from commoncode import fileutils
 from commoncode.cliutils import PluggableCommandLineOption
-from licensedcode.models import BasicRule, License
 from plugincode.scan import ScanPlugin
 from plugincode.scan import scan_impl
 from commoncode.cliutils import MISC_GROUP
@@ -133,7 +131,6 @@ def match_reference_license(resource, codebase):
     """
     Find instances for any licenses in referenced filenames
     """
-
     if not resource.is_file:
         return
 
@@ -142,37 +139,34 @@ def match_reference_license(resource, codebase):
     if not licenses:
         return 
 
-    location = resource.location
-    from licensedcode import cache
-    from scancode.api import get_licenses
-    idx = cache.get_index()
-    matches = idx.match(
-        location=location, min_score=0)
-
-    for match in matches:
-        ref_files=match.rule.referenced_filenames
-        if len(ref_files) != 0:
-            for i in range(len(ref_files)):
-                if not ref_files[i].startswith('usr/share/common-licenses'):
-                    new_loc=find_reference_file(location,ref_files[i])
-                    if new_loc:
-                        new_lic=get_licenses(new_loc, min_score=0)
-                        licenses.extend(new_lic['licenses'])
-                        license_expressions.extend(new_lic['license_expressions'])
-
+    referenced_licenses = []
+    referenced_license_expressions = []
+    unique_files = []
+
+    for license in licenses:
+        referenced_filenames = license['matched_rule']['referenced_filenames']
+        for referenced_filename in referenced_filenames:
+            if referenced_filename.startswith('usr/share/common-licenses') or referenced_filename in unique_files:
+                continue
+            unique_files.append(referenced_filename)
+            new_resource = find_reference_licenses(referenced_filename=referenced_filename, resource=resource, codebase=codebase)
+            if new_resource:
+                referenced_licenses.extend(new_resource.licenses)
+                referenced_license_expressions.extend(new_resource.license_expressions)
+
+    licenses.extend(referenced_licenses)
+    license_expressions.extend(referenced_license_expressions)
     codebase.save_resource(resource)
     return resource
 
 
-def find_reference_file(location,referenced_filename):
+def find_reference_licenses(referenced_filename, resource, codebase, **kwargs):
     """
-    Searches for the referenced filename in the same directory and 
-    returns the found location
+    Return the resource object for matching referenced-filename given a resource in codebase
     """
-    file_name=referenced_filename
-    par_dir=fileutils.parent_directory(location)
-    for root, dirs, files in os.walk(par_dir):
-      if file_name in files:
-         path_file = os.path.join(root,file_name)
-         return path_file
-
+    parent = resource.parent(codebase)
+
+    for child in parent.children(codebase):
+        file_name = fileutils.file_name(child.path)
+        if referenced_filename == file_name:
+            return child

tests/licensedcode/data/plugin_license/license_reference/scan-ref.expected.json

@@ -30,6 +30,7 @@
           "short_name": "MIT License",
           "category": "Permissive",
           "is_exception": false,
+          "is_unknown": false,
           "owner": "MIT",
           "homepage_url": "http://opensource.org/licenses/mit-license.php",
           "text_url": "http://opensource.org/licenses/mit-license.php",
@@ -46,11 +47,13 @@
             "licenses": [
               "mit"
             ],
+            "referenced_filenames": [],
             "is_license_text": false,
             "is_license_notice": true,
             "is_license_reference": false,
             "is_license_tag": false,
             "is_license_intro": false,
+            "has_unknown": false,
             "matcher": "1-hash",
             "rule_length": 10,
             "matched_length": 10,
@@ -77,6 +80,7 @@
           "short_name": "Unknown License reference",
           "category": "Unstated License",
           "is_exception": false,
+          "is_unknown": true,
           "owner": "Unspecified",
           "homepage_url": null,
           "text_url": "",
@@ -93,11 +97,15 @@
             "licenses": [
               "unknown-license-reference"
             ],
+            "referenced_filenames": [
+              "LICENSE"
+            ],
             "is_license_text": false,
             "is_license_notice": false,
             "is_license_reference": false,
             "is_license_tag": true,
             "is_license_intro": false,
+            "has_unknown": true,
             "matcher": "2-aho",
             "rule_length": 5,
             "matched_length": 5,
@@ -113,6 +121,7 @@
           "short_name": "MIT License",
           "category": "Permissive",
           "is_exception": false,
+          "is_unknown": false,
           "owner": "MIT",
           "homepage_url": "http://opensource.org/licenses/mit-license.php",
           "text_url": "http://opensource.org/licenses/mit-license.php",
@@ -129,17 +138,20 @@
             "licenses": [
               "mit"
             ],
+            "referenced_filenames": [],
             "is_license_text": false,
             "is_license_notice": true,
             "is_license_reference": false,
             "is_license_tag": false,
             "is_license_intro": false,
+            "has_unknown": false,
             "matcher": "1-hash",
             "rule_length": 10,
             "matched_length": 10,
             "match_coverage": 100.0,
             "rule_relevance": 100
-          }
+          },
+          "matched_text": "that is licensed under [MIT](http://opensource.org/licenses/MIT)."
         }
       ],
       "license_expressions": [

tests/licensedcode/data/plugin_license/license_reference/scan-wref.expected.json

@@ -30,6 +30,7 @@
           "short_name": "Unknown License reference",
           "category": "Unstated License",
           "is_exception": false,
+          "is_unknown": true,
           "owner": "Unspecified",
           "homepage_url": null,
           "text_url": "",
@@ -46,11 +47,15 @@
             "licenses": [
               "unknown-license-reference"
             ],
+            "referenced_filenames": [
+              "LICENSE"
+            ],
             "is_license_text": false,
             "is_license_notice": false,
             "is_license_reference": false,
             "is_license_tag": true,
             "is_license_intro": false,
+            "has_unknown": true,
             "matcher": "2-aho",
             "rule_length": 5,
             "matched_length": 5,