Improve copyright detection

CHANGELOG.rst

@@ -46,6 +46,10 @@ v33.0.0 (next next, roadmap)
     license/exception texts added, and also 1 license was deprecated.
     For more details see https://github.com/aboutcode-org/scancode-toolkit/pull/3897
 
+- New and improved copyright detection with many false positive removed
+  and refined detection added.
+
+
 v32.2.1 - 2024-07-02
 ---------------------
 

etc/scripts/gen_copyright_tests.py

@@ -93,7 +93,7 @@ def create_copyright_tests(
                 content = url
 
             if end_line != 0:
-                content = "".join(content.strip().splitlines()[start_line:end_line])
+                content = "\n".join(content.strip().splitlines()[start_line:end_line])
 
             with open(name, "w") as out:
                 out.write(content)

requirements.txt

@@ -10,7 +10,7 @@ chardet==5.0.0
 charset-normalizer==2.1.0
 click==8.1.7
 colorama==0.4.5
-commoncode==31.2.1
+commoncode==32.0.0
 construct==2.10.68
 container-inspector==31.1.0
 cryptography==42.0.5
@@ -54,7 +54,7 @@ ply==3.11
 publicsuffix2==2.20191221
 pyahocorasick==2.1.0
 pycparser==2.21
-pygmars==0.7.0
+pygmars==0.9.0
 Pygments==2.13.0
 pymaven-patch==0.3.2
 pyparsing==3.0.9

setup-mini.cfg

@@ -70,7 +70,7 @@ install_requires =
     chardet >= 3.0.0
     click >= 6.7, !=7.0
     colorama >= 0.3.9
-    commoncode >= 31.0.3
+    commoncode >= 32.0.0
     container-inspector >= 31.0.0
     debian-inspector >= 31.1.0
     dparse2 >= 0.7.0
@@ -100,7 +100,7 @@ install_requires =
     plugincode >= 32.0.0
     publicsuffix2
     pyahocorasick >= 2.0.0
-    pygmars >= 0.7.0
+    pygmars >= 0.9.0
     pygments
     pymaven_patch >= 0.2.8
     requests >= 2.7.0
@@ -112,7 +112,7 @@ install_requires =
     xmltodict >= 0.11.0
     zipp >= 3.0.0; python_version < "3.9"
     typecode >= 30.0.1
-#    typecode[full] >= 30.0.0
+#    typecode[full] >= 30.0.1
 #    extractcode[full] >= 31.0.0
 
 
@@ -199,6 +199,7 @@ scancode_post_scan =
     filter-clues = cluecode.plugin_filter_clues:RedundantCluesFilter
     consolidate = summarycode.plugin_consolidate:Consolidator
     license-references = licensedcode.licenses_reference:LicenseReference
+    todo = summarycode.todo:AmbiguousDetectionsToDoPlugin
     classify = summarycode.classify_plugin:FileClassifier
 
 

setup.cfg

@@ -70,7 +70,7 @@ install_requires =
     chardet >= 3.0.0
     click >= 6.7, !=7.0
     colorama >= 0.3.9
-    commoncode >= 31.0.3
+    commoncode >= 32.0.0
     container-inspector >= 31.0.0
     debian-inspector >= 31.1.0
     dparse2 >= 0.7.0
@@ -100,7 +100,7 @@ install_requires =
     plugincode >= 32.0.0
     publicsuffix2
     pyahocorasick >= 2.0.0
-    pygmars >= 0.7.0
+    pygmars >= 0.9.0
     pygments
     pymaven_patch >= 0.2.8
     requests >= 2.7.0
@@ -138,9 +138,9 @@ testing =
 
 docs =
     Sphinx == 5.1.0
-    sphinx-rtd-theme >= 0.5.0
-    doc8 >= 0.8.1
+    sphinx_rtd_theme >= 0.5.1
     sphinx-reredirects >= 0.1.2
+    doc8 >= 0.8.1
     sphinx-autobuild
     sphinx-rtd-dark-mode>=1.3.0
     sphinx-copybutton