Support non-English licenses

Improve in tests. Add a language tag there too.
Fix licenses with incorrect or missing metadata

Reference: #139
Signed-off-by: Philippe Ombredanne <[email protected]>

src/licensedcode/data/licenses/avisynth-c-interface-exception.yml

@@ -5,4 +5,4 @@ category: Copyleft Limited
 owner: Kevin Atkinson
 homepage_url: http://www.kevina.org/avisynth_c/readme.txt
 is_exception: yes
-spdx_license_key: LicenseRef-scancode-avisynth-c-interface-exception
+spdx_license_key: LicenseRef-scancode-avisynth-c-exception

src/licensedcode/data/licenses/broadcom-linking-exception-2.0.yml

@@ -4,7 +4,7 @@ name: Broadcom Linking Exception to GPL 2.0
 category: Copyleft Limited
 owner: Broadcom
 is_exception: yes
-spdx_license_key: LicenseRef-scancode-broadcom-linking-exception-2.0
+spdx_license_key: LicenseRef-scancode-bcm-linking-exception-2.0
 standard_notice: |
     Copyright (c) 2006-2007 Broadcom Corporation
     All Rights Reserved

src/licensedcode/data/licenses/cc-by-nc-3.0-de.yml

@@ -1,8 +1,8 @@
 key: cc-by-nc-3.0-de
 language: de
+short_name: CC-BY-NC-3.0-DE
 category: Free Restricted
 owner: Creative Commons
-short_name: Creative Commons Attribution Non Commercial 3.0 Germany
 name: Creative Commons Attribution Non Commercial 3.0 Germany
 spdx_license_key: CC-BY-NC-3.0-DE
 other_urls:

src/licensedcode/data/licenses/cc-by-nc-nd-2.0-at.yml

@@ -7,3 +7,5 @@ owner: Creative Commons
 homepage_url: https://creativecommons.org/licenses/by-nc-nd/2.0/
 text_urls:
     - https://creativecommons.org/licenses/by-nc-nd/2.0/legalcode.at
+spdx_license_key: LicenseRef-scancode-cc-by-nc-nd-2.0-at
+

src/licensedcode/data/licenses/cc-by-nc-nd-3.0-de.yml

@@ -2,7 +2,7 @@ key: cc-by-nc-nd-3.0-de
 language: de
 category: Free Restricted
 owner: Creative Commons
-short_name: Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany
+short_name: CC-BY-NC-ND-3.0-DE
 name: Creative Commons Attribution Non Commercial No Derivatives 3.0 Germany
 spdx_license_key: CC-BY-NC-ND-3.0-DE
 other_urls:

src/licensedcode/data/licenses/cc-by-nc-sa-2.0-fr.yml

@@ -2,7 +2,7 @@ key: cc-by-nc-sa-2.0-fr
 language: fr
 category: Free Restricted
 owner: Creative Commons
-short_name: Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France
+short_name: CC-BY-NC-SA-2.0-FR
 name: Creative Commons Attribution-NonCommercial-ShareAlike 2.0 France
 spdx_license_key: CC-BY-NC-SA-2.0-FR
 other_urls:

src/licensedcode/data/licenses/cc-by-nc-sa-3.0-de.yml

@@ -2,7 +2,7 @@ key: cc-by-nc-sa-3.0-de
 category: Free Restricted
 owner: Creative Commons
 language: de
-short_name: Creative Commons Attribution Non Commercial Share Alike 3.0 Germany
+short_name: CC-BY-NC-SA-3.0-DE
 name: Creative Commons Attribution Non Commercial Share Alike 3.0 Germany
 spdx_license_key: CC-BY-NC-SA-3.0-DE
 other_urls:

src/licensedcode/data/licenses/cc-by-nd-3.0-de.yml

@@ -2,7 +2,7 @@ key: cc-by-nd-3.0-de
 category: Free Restricted
 owner: Creative Commons
 language: de
-short_name: Creative Commons Attribution No Derivatives 3.0 Germany
+short_name: CC-BY-ND-3.0-DE
 name: Creative Commons Attribution No Derivatives 3.0 Germany
 spdx_license_key: CC-BY-ND-3.0-DE
 other_urls:

src/licensedcode/data/licenses/cc-by-sa-3.0-de.yml

@@ -1,7 +1,7 @@
 key: cc-by-sa-3.0-de
 language: de
 category: Copyleft Limited
-short_name: Creative Commons Attribution Share Alike 3.0 Germany
+short_name: CC-BY-SA-3.0-DE
 name: Creative Commons Attribution Share Alike 3.0 Germany
 spdx_license_key: CC-BY-SA-3.0-DE
 other_urls:

src/licensedcode/data/licenses/cc-gpl-2.0-pt.yml

@@ -7,3 +7,4 @@ owner: Creative Commons
 homepage_url: https://creativecommons.org/licenses/GPL/2.0/
 text_urls:
     - https://creativecommons.org/licenses/GPL/2.0/legalcode.pt
+spdx_license_key: LicenseRef-scancode-cc-gpl-2.0-pt

src/licensedcode/data/licenses/cc-lgpl-2.1-pt.yml

@@ -7,3 +7,5 @@ owner: Creative Commons
 homepage_url: https://creativecommons.org/licenses/LGPL/2.1/
 text_urls:
     - https://creativecommons.org/licenses/LGPL/2.1/legalcode.pt
+spdx_license_key: LicenseRef-scancode-cc-lgpl-2.1-pt
+

src/licensedcode/data/licenses/etalab-2.0-fr.yml

@@ -2,8 +2,8 @@ key: etalab-2.0-fr
 language: fr
 owner: France
 category: Unstated License
-short_name: Etalab Open License 2.0
-name: Etalab Open License 2.0
+short_name: Etalab Open License 2.0 French
+name: Etalab Open License 2.0 French
 spdx_license_key: etalab-2.0
 other_urls:
     - https://github.com/DISIC/politique-de-contribution-open-source/blob/master/LICENSE.pdf

src/licensedcode/data/licenses/etalab-2.0.yml

@@ -1,7 +1,7 @@
 key: etalab-2.0
 language: en
-short_name: Etalab Open License 2.0
-name: Etalab Open License 2.0
+short_name: Etalab Open License 2.0 English
+name: Etalab Open License 2.0 English
 category: Permissive
 owner: DINUM
 homepage_url: https://raw.githubusercontent.com/DISIC/politique-de-contribution-open-source/master/LICENSE

src/licensedcode/data/licenses/nysl-0.9982-jp.yml

@@ -3,6 +3,7 @@ language: jp
 short_name: NYSL 0.9982 JP
 name: NYSL 0.9982 Japanese
 category: Permissive
+spdx_license_key: LicenseRef-scancode-nysl-0.9982-jp
 owner: Kazuhiro Inaba
 homepage_url: http://www.kmonos.net/nysl/index.en.html
 text_urls:

src/licensedcode/data/licenses/ogl-canada-2.0-fr.yml

@@ -1,8 +1,8 @@
 key: ogl-canada-2.0-fr
 language: fr
-short_name: OGL Canada 2.0
+short_name: OGL Canada 2.0 Francais
 name: Licence du gouvernement ouvert Canada 2.0
 category: Permissive
 owner: Canada Government
 homepage_url: https://ouvert.canada.ca/fr/licence-du-gouvernement-ouvert-canada
-
+spdx_license_key: LicenseRef-scancode-ogl-canada-2.0-fr

src/licensedcode/data/licenses/scilab-fr.yml

@@ -6,4 +6,5 @@ category: Proprietary Free
 owner: INRIA/ENPC
 homepage_url: http://web.archive.org/web/20051212214843/http://www.scilab.org/legal/license.html
 text_urls: 
-    - https://directory.fsf.org/wiki/License:Scilab-old
+    - https://directory.fsf.org/wiki/License:Scilab-old
+spdx_license_key: LicenseRef-scancode-scilab-fr

src/licensedcode/data/licenses/scola-fr.yml

@@ -1,7 +1,8 @@
 key: scola-fr
 language: fr
-short_name: Statistics Canada Open Licence Agreement
+short_name: Statistics Canada Open Licence
 name: Entente de licence ouverte de Statistique Canada
 owner:  Statistique Canada
 homepage_url: https://www.statcan.gc.ca/fra/reference/licence
-category: Unstated License
+category: Unstated License
+spdx_license_key: LicenseRef-scancode-scola-fr

src/licensedcode/data/rules/cc-by-3.0-at.yml

@@ -1,3 +1,4 @@
 license_expression: cc-by-3.0-at
-language: de
-is_license_text: yes
+is_license_text: yes
+ignorable_urls:
+    - https://creativecommons.org/

src/licensedcode/data/rules/cc-by-nc-nd-2.0-at.yml

@@ -1,3 +1,4 @@
 license_expression: cc-by-nc-nd-2.0-at
-language: de
 is_license_notice: yes
+ignorable_urls:
+    - https://creativecommons.org/

src/licensedcode/data/rules/cc-by-nc-sa-3.0_zh.yml

@@ -1,3 +1,4 @@
 license_expression: cc-by-nc-sa-3.0
-language: zh
 is_license_notice: yes
+ignorable_urls:
+    - http://creativecommons.org/licenses/by-nc-sa/3.0/deed.zh

src/licensedcode/data/rules/cc-by-nc-sa-4.0_zh.yml

@@ -1,3 +1,5 @@
 license_expression: cc-by-nc-sa-4.0
-language: zh
 is_license_notice: yes
+ignorable_urls:
+    - http://creativecommons.org/licenses/by-nc-sa/4.0
+    - https://github.com/kesenhoo/android-training-course-in-chinese

src/licensedcode/data/rules/gfdl-1.1_fr_gnome_1.yml

@@ -1,5 +1,12 @@
 license_expression: gfdl-1.1
 is_license_notice: yes
-language: fr
-notes: plain text conversion with pandoc of desktop-docs/fdl/fr/index.docbook
- from https://download.gnome.org/sources/gnome-desktop/3.14/gnome-desktop-3.14.2.tar.xz
+notes: plain text conversion with pandoc of desktop-docs/fdl/fr/index.docbook from https://download.gnome.org/sources/gnome-desktop/3.14/gnome-desktop-3.14.2.tar.xz
+ignorable_copyrights:
+    - Copyright (c) ANNEE
+ignorable_holders:
+    - ANNEE
+ignorable_urls:
+    - http://www.gnu.org/copyleft
+    - http://www.gnu.org/copyleft/
+    - http://www.gnu.org/copyleft/gpl.html
+    - http://www.gnu.org/fsf/fsf.html

src/licensedcode/models.py

@@ -531,7 +531,7 @@ def validate(licenses, verbose=False, no_dupe_urls=False):
             if lic.key != lic.key.lower():
                 error('Incorrect license key case. Should be lowercase.')
 
-            if len(lic.key) >= 50:
+            if len(lic.key) > 50:
                 error('key must be 50 characters or less.')
 
             if not lic.short_name:
@@ -605,7 +605,7 @@ def validate(licenses, verbose=False, no_dupe_urls=False):
 
             # SPDX consistency
             if lic.spdx_license_key:
-                if len(lic.spdx_license_key) >= 50:
+                if len(lic.spdx_license_key) > 50:
                     error('spdx_license_key must be 50 characters or less.')
 
                 by_spdx_key[lic.spdx_license_key].append(key)

tests/licensedcode/data/datadriven/external/atarashi/CECILL-C.c.yml

@@ -1,3 +1,3 @@
 license_expressions:
   - cecill-c
-  - cecill-c
+  - cecill-c-en

tests/licensedcode/data/datadriven/external/fossology-licenses/cecill-b.txt.yml

@@ -1,3 +1,3 @@
 license_expressions:
-  - cecill-b
+  - cecill-b-en
 notes: this is a license from fossology license reference CECILL-B (CeCILL-B License) http://www.cecill.info/licences/Licence_CeCILL-B_V1-en.txt

tests/licensedcode/data/datadriven/external/fossology-licenses/cecill-c.txt.yml

@@ -1,3 +1,3 @@
 license_expressions:
-  - cecill-c
+  - cecill-c-en
 notes: this is a license from fossology license reference CECILL-C (CeCILL-C License) http://www.cecill.info/licences/Licence_CeCILL-C_V1-en.txt

tests/licensedcode/data/datadriven/external/fossology-tests/MS/MS-LPL.txt.yml

@@ -1,2 +1,2 @@
 license_expressions:
-  - ms-lpl
+  - ms-office-extensible-file

tests/licensedcode/data/datadriven/external/fossology-tests/SPDX/LAL-1.2.spdx.txt.yml

@@ -1,2 +1,2 @@
 license_expressions:
-  - unknown-spdx
+  - lal-1.2

tests/licensedcode/data/datadriven/external/fossology-tests/SPDX/LAL-1.3.yml

@@ -1,2 +1,2 @@
 license_expressions:
-  - unknown-spdx
+  - lal-1.3

tests/licensedcode/data/datadriven/external/fossology-tests/SPDX/LiLiQ-P-1.1.yml

@@ -1,2 +1,2 @@
 license_expressions:
-  - unknown-spdx
+  - liliq-p-1.1

tests/licensedcode/data/datadriven/external/fossology-tests/SPDX/LiLiQ-R-1.1.yml

@@ -1,2 +1,2 @@
 license_expressions:
-  - unknown-spdx
+  - liliq-r-1.1

tests/licensedcode/data/datadriven/external/fossology-tests/SPDX/LiLiQ-Rplus-1.1.yml

@@ -1,2 +1,2 @@
 license_expressions:
-  - unknown-spdx
+  - liliq-rplus-1.1

tests/licensedcode/data/datadriven/external/glc/CECILL-B.t1.yml

@@ -1,5 +1,5 @@
 license_expressions:
-  - cecill-b
+  - cecill-b-en
 notes: |
   License test derived from a file of the BSD-licensed repository at:
   https://raw.githubusercontent.com/google/licensecheck/v0.3.1/testdata/CECILL-B.t1

tests/licensedcode/data/datadriven/external/glc/CECILL-C.t1.yml

@@ -1,5 +1,5 @@
 license_expressions:
-  - cecill-c
+  - cecill-c-en
 notes: |
   License test derived from a file of the BSD-licensed repository at:
   https://raw.githubusercontent.com/google/licensecheck/v0.3.1/testdata/CECILL-C.t1

tests/licensedcode/data/datadriven/lic1/french_gfdl.docbook.yml

@@ -1,6 +1,7 @@
+language: fr
 license_expressions:
   - gfdl-1.1
   - gfdl-1.1
-  - gfdl-1.1-plus
+  - gfdl-1.1
+  - gfdl-1.1
   - gpl-1.0-plus
-

tests/licensedcode/data/datadriven/lic3/nysl-0.9982.txt.yml

@@ -1,3 +1,4 @@
+language: jp
 license_expressions:
-  - nysl-0.9982
+  - nysl-0.9982-jp
   - nysl-0.9982

tests/licensedcode/data/datadriven/lic3/nysl-0.9982_jp.txt.yml

@@ -1,2 +1,3 @@
+language: jp
 license_expressions:
-  - nysl-0.9982
+  - nysl-0.9982-jp

tests/licensedcode/data/datadriven/lic4/D-FSL-1.0.yml

@@ -1,2 +1,3 @@
+language: de
 license_expressions:
-  - d-fsl-1.0-en
+  - d-fsl-1.0-de

tests/licensedcode/data/datadriven/unknown/cclrc.txt.yml

@@ -1,4 +1,3 @@
 license_expressions:
-  - unknown
-  - warranty-disclaimer
+  - cclrc
 notes: this is a license from fossology license reference CCLRC (CCLRC License) http://www2-pcmdi.llnl.gov/cdat/docs/cdat-license

tests/licensedcode/data/datadriven/unknown/opl-1.0.txt.yml

@@ -1,12 +1,3 @@
 license_expressions:
-  - mpl-1.1
-  - mpl-1.1
-  - unknown-license-reference
-  - unknown
-  - free-unknown
-  - warranty-disclaimer
-  - unknown
-  - unknown-license-reference
-  - generic-trademark
+  - openi-pl-1.0
 notes: this is an mpl-1.1 derivative which is very rare.
-

tests/licensedcode/licensedcode_test_utils.py

@@ -50,6 +50,7 @@ class LicenseTest(object):
     license_expressions = attr.attrib(default=attr.Factory(list))
     notes = attr.attrib(default=None)
     expected_failure = attr.attrib(default=False)
+    language = attr.attrib(default='en')
 
     licensing = Licensing()
 
@@ -67,6 +68,7 @@ def __attrs_post_init__(self, *args, **kwargs):
                 raise Exception(f'Failed to read: file://{self.data_file}', e)
 
             self.license_expressions = data.pop('license_expressions', [])
+            self.language = data.pop('language', 'en')
             self.notes = data.pop('notes', None)
             # True if the test is expected to fail
             self.expected_failure = data.pop('expected_failure', False)
@@ -105,6 +107,8 @@ def to_dict(self):
         dct = {}
         if self.license_expressions:
             dct['license_expressions'] = self.license_expressions
+        if self.language and self.language != 'en':
+            dct['language'] = self.language
         if self.expected_failure:
             dct['expected_failure'] = self.expected_failure
         if self.notes:

tests/licensedcode/test_models.py

@@ -107,7 +107,7 @@ def test_build_rules_from_licenses(self):
 
     def test_validate_license_library(self):
         errors, warnings, infos = models.License.validate(
-            cache.get_licenses_db(),
+            licenses=models.load_licenses(),
             verbose=False,
         )
         assert errors == {}

tests/scancode/data/altpath/copyright.expected.json

@@ -14,7 +14,8 @@
       "message": null,
       "errors": [],
       "extra_data": {
-        "spdx_license_list_version": "3.15",
+        "spdx_license_list_version": "3.16",
+        "OUTDATED": "WARNING: Outdated ScanCode Toolkit version! You are using an outdated version of ScanCode Toolkit: 31.0.0 released on: 2021-09-24. A new version is available with important improvements including bug and security fixes, updated license, copyright and package detection, and improved scanning accuracy. Please download and install the latest version of ScanCode. Visit https://github.com/nexB/scancode-toolkit/releases for details.",
         "files_count": 1
       }
     }