fix: Remove any YAML front matter from ScanCode license files

ScanCode 32.0.0 started to prepend its `*.LICENSE` files with YAML-encoded metadata, see [1]. This is a hot fix to remove this header, if present, from the license files. A better solution will be implemented later as part of a larger refactoring of license providers. Different ScanCode versions also differ in whether license files come with a final newline or not. Align on not having a final newline to make tests pass either way. [1]: aboutcode-org/scancode-toolkit#3100 Signed-off-by: Sebastian Schuberth <[email protected]>
oss-review-toolkit · Nov 16, 2023 · c2d89f1 · c2d89f1
1 parent a9594cb
commit c2d89f1
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 4 deletions.
diff --git a/plugins/reporters/spdx/src/funTest/assets/spdx-document-reporter-expected-output.spdx.json b/plugins/reporters/spdx/src/funTest/assets/spdx-document-reporter-expected-output.spdx.json
@@ -11,7 +11,7 @@
   "dataLicense" : "CC0-1.0",
   "comment" : "some document comment",
   "hasExtractedLicensingInfos" : [ {
-    "extractedText" : "ASMUS License\n\nDisclaimer and legal rights\n---------------------------\n\nThis file contains bugs. All representations to the contrary are void.\n\nSource code in this file and the accompanying headers and included \nfiles may be distributed free of charge by anyone, as long as full \ncredit is given and any and all liabilities are assumed by the \nrecipient.\n",
+    "extractedText" : "ASMUS License\n\nDisclaimer and legal rights\n---------------------------\n\nThis file contains bugs. All representations to the contrary are void.\n\nSource code in this file and the accompanying headers and included \nfiles may be distributed free of charge by anyone, as long as full \ncredit is given and any and all liabilities are assumed by the \nrecipient.",
     "licenseId" : "LicenseRef-scancode-asmus"
   }, {
     "extractedText" : "To anyone who acknowledges that the file \"sRGB Color Space Profile.icm\" \nis provided \"AS IS\" WITH NO EXPRESS OR IMPLIED WARRANTY:\npermission to use, copy and distribute this file for any purpose is hereby \ngranted without fee, provided that the file is not changed including the HP \ncopyright notice tag, and that the name of Hewlett-Packard Company not be \nused in advertising or publicity pertaining to distribution of the software \nwithout specific, written prior permission. Hewlett-Packard Company makes \nno representations about the suitability of this software for any purpose.",

diff --git a/plugins/reporters/spdx/src/funTest/assets/spdx-document-reporter-expected-output.spdx.yml b/plugins/reporters/spdx/src/funTest/assets/spdx-document-reporter-expected-output.spdx.yml
@@ -15,7 +15,7 @@ hasExtractedLicensingInfos:
     \nThis file contains bugs. All representations to the contrary are void.\n\nSource\
     \ code in this file and the accompanying headers and included \nfiles may be distributed\
     \ free of charge by anyone, as long as full \ncredit is given and any and all\
-    \ liabilities are assumed by the \nrecipient.\n"
+    \ liabilities are assumed by the \nrecipient."
   licenseId: "LicenseRef-scancode-asmus"
 - extractedText: "To anyone who acknowledges that the file \"sRGB Color Space Profile.icm\"\
     \ \nis provided \"AS IS\" WITH NO EXPRESS OR IMPLIED WARRANTY:\npermission to\

diff --git a/utils/spdx/src/main/kotlin/Utils.kt b/utils/spdx/src/main/kotlin/Utils.kt
@@ -130,8 +130,18 @@ fun getLicenseTextReader(
 ): (() -> String)? {
     return if (id.startsWith(LICENSE_REF_PREFIX)) {
         getLicenseTextResource(id)?.let { { it.readText() } }
-            ?: addScanCodeLicenseTextsDir(licenseTextDirectories).firstNotNullOfOrNull {
-                getLicenseTextFile(id, it)?.let { file -> { file.readText() } }
+            ?: addScanCodeLicenseTextsDir(licenseTextDirectories).firstNotNullOfOrNull { dir ->
+                getLicenseTextFile(id, dir)?.let { file ->
+                    {
+                        val lines = file.readLines()
+
+                        // Remove any YAML front matter enclosed by "---" from ScanCode license files.
+                        val licenseLines = lines.takeUnless { it.first() == "---" }
+                            ?: lines.drop(1).dropWhile { it != "---" }.drop(1)
+
+                        licenseLines.joinToString("\n").trim()
+                    }
+                }
             }
     } else {
         SpdxLicense.forId(id.removeSuffix("+"))?.let { { it.text } }