feat: update syft license concept to complex struct (#1743)

this PR makes the following changes to update the underlying license model to have more expressive capabilities
it also provides some guarantee's surrounding the license values themselves

- Licenses are updated from string -> pkg.LicenseSet which contain pkg.License with the following fields:
- original `Value` read by syft
- If it's possible to construct licenses will always have a valid SPDX expression for downstream consumption
- the above is run against a generated list of SPDX license ID to try and find the correct ID
- SPDX concluded vs declared is added to the new struct
- URL source for license is added to the new struct
- Location source is added to the new struct to show where the expression was pulled from

go.mod

@@ -56,6 +56,7 @@ require (
 	github.com/anchore/stereoscope v0.0.0-20230412183729-8602f1afc574
 	github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da
 	github.com/docker/docker v23.0.6+incompatible
+	github.com/github/go-spdx/v2 v2.1.2
 	github.com/go-git/go-billy/v5 v5.4.1
 	github.com/go-git/go-git/v5 v5.6.1
 	github.com/google/go-containerregistry v0.15.1

go.sum

@@ -207,6 +207,8 @@ github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbS
 github.com/gabriel-vasile/mimetype v1.4.0 h1:Cn9dkdYsMIu56tGho+fqzh7XmvY2YyGU0FnbhiOsEro=
 github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/github/go-spdx/v2 v2.1.2 h1:p+Tv0yMgcuO0/vnMe9Qh4tmUgYhI6AsLVlakZ/Sx+DM=
+github.com/github/go-spdx/v2 v2.1.2/go.mod h1:hMCrsFgT0QnCwn7G8gxy/MxMpy67WgZrwFeISTn0o6w=
 github.com/glebarez/go-sqlite v1.20.3 h1:89BkqGOXR9oRmG58ZrzgoY/Fhy5x0M+/WV48U5zVrZ4=
 github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY=
 github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4=

internal/constants.go

@@ -6,5 +6,5 @@ const (
 
 	// JSONSchemaVersion is the current schema version output by the JSON encoder
 	// This is roughly following the "SchemaVer" guidelines for versioning the JSON schema. Please see schema/json/README.md for details on how to increment.
-	JSONSchemaVersion = "7.1.6"
+	JSONSchemaVersion = "8.0.0"
 )

internal/licenses/parser.go

@@ -4,7 +4,10 @@ import (
 	"io"
 
 	"github.com/google/licensecheck"
-	"golang.org/x/exp/slices"
+
+	"github.com/anchore/syft/syft/license"
+	"github.com/anchore/syft/syft/pkg"
+	"github.com/anchore/syft/syft/source"
 )
 
 const (
@@ -13,21 +16,24 @@ const (
 )
 
 // Parse scans the contents of a license file to attempt to determine the type of license it is
-func Parse(reader io.Reader) (licenses []string, err error) {
+func Parse(reader io.Reader, l source.Location) (licenses []pkg.License, err error) {
+	licenses = make([]pkg.License, 0)
 	contents, err := io.ReadAll(reader)
 	if err != nil {
 		return nil, err
 	}
 	cov := licensecheck.Scan(contents)
-
-	if cov.Percent < float64(coverageThreshold) {
-		licenses = append(licenses, unknownLicenseType)
+	if cov.Percent < coverageThreshold {
+		// unknown or no licenses here?
+		return licenses, nil
 	}
+
 	for _, m := range cov.Match {
-		if slices.Contains(licenses, m.ID) {
-			continue
-		}
-		licenses = append(licenses, m.ID)
+		lic := pkg.NewLicenseFromLocations(m.ID, l)
+		lic.Type = license.Concluded
+
+		licenses = append(licenses, lic)
 	}
-	return
+
+	return licenses, nil
 }

internal/stringset.go

@@ -15,8 +15,10 @@ func NewStringSet(start ...string) StringSet {
 }
 
 // Add a string to the set.
-func (s StringSet) Add(i string) {
-	s[i] = struct{}{}
+func (s StringSet) Add(i ...string) {
+	for _, str := range i {
+		s[str] = struct{}{}
+	}
 }
 
 // Remove a string from the set.
@@ -41,3 +43,19 @@ func (s StringSet) ToSlice() []string {
 	sort.Strings(ret)
 	return ret
 }
+
+func (s StringSet) Equals(o StringSet) bool {
+	if len(s) != len(o) {
+		return false
+	}
+	for k := range s {
+		if !o.Contains(k) {
+			return false
+		}
+	}
+	return true
+}
+
+func (s StringSet) Empty() bool {
+	return len(s) < 1
+}

schema/json/generate.go

@@ -78,7 +78,6 @@ func build() *jsonschema.Schema {
 	}
 	documentSchema := reflector.ReflectFromType(reflect.TypeOf(&syftjsonModel.Document{}))
 	metadataSchema := reflector.ReflectFromType(reflect.TypeOf(&artifactMetadataContainer{}))
-
 	// TODO: inject source definitions
 
 	// inject the definitions of all metadatas into the schema definitions