Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support multiple known CPEs in index #2813

Merged
merged 1 commit into from
Apr 25, 2024
Merged

feat: support multiple known CPEs in index #2813

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions internal/task/package_task_factory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,10 +111,10 @@ func NewPackageTask(cfg CatalogingFactoryConfig, c pkg.Cataloger, tags ...string
if cfg.DataGenerationConfig.GenerateCPEs {
// generate CPEs (note: this is excluded from package ID, so is safe to mutate)
// we might have binary classified CPE already with the package so we want to append here
dictionaryCPE, ok := cpe.DictionaryFind(p)
dictionaryCPEs, ok := cpe.DictionaryFind(p)
if ok {
log.Tracef("used CPE dictionary to find CPE for %s package %q: %s", p.Type, p.Name, dictionaryCPE.Attributes.BindToFmtString())
p.CPEs = append(p.CPEs, dictionaryCPE)
log.Tracef("used CPE dictionary to find CPEs for %s package %q: %s", p.Type, p.Name, dictionaryCPEs)
p.CPEs = append(p.CPEs, dictionaryCPEs...)
} else {
p.CPEs = append(p.CPEs, cpe.Generate(p)...)
}
Expand Down
2 changes: 1 addition & 1 deletion syft/pkg/cataloger/common/cpe/create.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,6 @@ func Generate(p pkg.Package) []cpe.CPE {
return cpegenerate.FromPackageAttributes(p)
}

func DictionaryFind(p pkg.Package) (cpe.CPE, bool) {
func DictionaryFind(p pkg.Package) ([]cpe.CPE, bool) {
return cpegenerate.FromDictionaryFind(p)
}
8,980 changes: 6,752 additions & 2,228 deletions syft/pkg/cataloger/internal/cpegenerate/dictionary/data/cpe-index.json
View file
Open in desktop

Large diffs are not rendered by default.

73 changes: 22 additions & 51 deletions syft/pkg/cataloger/internal/cpegenerate/dictionary/index-generator/generate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,16 +167,24 @@ func indexCPEList(list CpeList) *dictionary.Indexed {
return indexed
}

func updateIndex(indexed *dictionary.Indexed, ecosystem string, pkgName string, cpe string) {
if _, exists := indexed.EcosystemPackages[ecosystem]; !exists {
indexed.EcosystemPackages[ecosystem] = make(dictionary.Packages)
}

if indexed.EcosystemPackages[ecosystem][pkgName] == nil {
indexed.EcosystemPackages[ecosystem][pkgName] = dictionary.NewSet()
}

indexed.EcosystemPackages[ecosystem][pkgName].Add(cpe)
}

func addEntryForRustCrate(indexed *dictionary.Indexed, ref string, cpeItemName string) {
// Prune off the non-package-name parts of the URL
ref = strings.TrimPrefix(ref, prefixForRustCrates)
ref = strings.Split(ref, "/")[0]

if _, ok := indexed.EcosystemPackages[dictionary.EcosystemRustCrates]; !ok {
indexed.EcosystemPackages[dictionary.EcosystemRustCrates] = make(dictionary.Packages)
}

indexed.EcosystemPackages[dictionary.EcosystemRustCrates][ref] = cpeItemName
updateIndex(indexed, dictionary.EcosystemRustCrates, ref, cpeItemName)
}

func addEntryForJenkinsPluginGitHub(indexed *dictionary.Indexed, ref string, cpeItemName string) {
Expand All @@ -190,12 +198,7 @@ func addEntryForJenkinsPluginGitHub(indexed *dictionary.Indexed, ref string, cpe
}

ref = strings.TrimSuffix(ref, "-plugin")

if _, ok := indexed.EcosystemPackages[dictionary.EcosystemJenkinsPlugins]; !ok {
indexed.EcosystemPackages[dictionary.EcosystemJenkinsPlugins] = make(dictionary.Packages)
}

indexed.EcosystemPackages[dictionary.EcosystemJenkinsPlugins][ref] = cpeItemName
updateIndex(indexed, dictionary.EcosystemJenkinsPlugins, ref, cpeItemName)
}

func addEntryForJenkinsPlugin(indexed *dictionary.Indexed, ref string, cpeItemName string) {
Expand All @@ -207,35 +210,23 @@ func addEntryForJenkinsPlugin(indexed *dictionary.Indexed, ref string, cpeItemNa
return
}

if _, ok := indexed.EcosystemPackages[dictionary.EcosystemJenkinsPlugins]; !ok {
indexed.EcosystemPackages[dictionary.EcosystemJenkinsPlugins] = make(dictionary.Packages)
}

indexed.EcosystemPackages[dictionary.EcosystemJenkinsPlugins][ref] = cpeItemName
updateIndex(indexed, dictionary.EcosystemJenkinsPlugins, ref, cpeItemName)
}

func addEntryForPyPIPackage(indexed *dictionary.Indexed, ref string, cpeItemName string) {
// Prune off the non-package-name parts of the URL
ref = strings.TrimPrefix(ref, prefixForPyPIPackages)
ref = strings.Split(ref, "/")[0]

if _, ok := indexed.EcosystemPackages[dictionary.EcosystemPyPI]; !ok {
indexed.EcosystemPackages[dictionary.EcosystemPyPI] = make(dictionary.Packages)
}

indexed.EcosystemPackages[dictionary.EcosystemPyPI][ref] = cpeItemName
updateIndex(indexed, dictionary.EcosystemPyPI, ref, cpeItemName)
}

func addEntryForNativeRubyGem(indexed *dictionary.Indexed, ref string, cpeItemName string) {
// Prune off the non-package-name parts of the URL
ref = strings.TrimPrefix(ref, prefixForNativeRubyGems)
ref = strings.Split(ref, "/")[0]

if _, ok := indexed.EcosystemPackages[dictionary.EcosystemRubyGems]; !ok {
indexed.EcosystemPackages[dictionary.EcosystemRubyGems] = make(dictionary.Packages)
}

indexed.EcosystemPackages[dictionary.EcosystemRubyGems][ref] = cpeItemName
updateIndex(indexed, dictionary.EcosystemRubyGems, ref, cpeItemName)
}

func addEntryForRubyGem(indexed *dictionary.Indexed, ref string, cpeItemName string) {
Expand All @@ -244,11 +235,7 @@ func addEntryForRubyGem(indexed *dictionary.Indexed, ref string, cpeItemName str
ref = strings.TrimPrefix(ref, prefixForRubyGemsHTTP)
ref = strings.Split(ref, "/")[0]

if _, ok := indexed.EcosystemPackages[dictionary.EcosystemRubyGems]; !ok {
indexed.EcosystemPackages[dictionary.EcosystemRubyGems] = make(dictionary.Packages)
}

indexed.EcosystemPackages[dictionary.EcosystemRubyGems][ref] = cpeItemName
updateIndex(indexed, dictionary.EcosystemRubyGems, ref, cpeItemName)
}

func addEntryForNPMPackage(indexed *dictionary.Indexed, ref string, cpeItemName string) {
Expand All @@ -257,11 +244,7 @@ func addEntryForNPMPackage(indexed *dictionary.Indexed, ref string, cpeItemName
ref = strings.Split(ref, "?")[0]
ref = strings.TrimPrefix(ref, prefixForNPMPackages)

if _, ok := indexed.EcosystemPackages[dictionary.EcosystemNPM]; !ok {
indexed.EcosystemPackages[dictionary.EcosystemNPM] = make(dictionary.Packages)
}

indexed.EcosystemPackages[dictionary.EcosystemNPM][ref] = cpeItemName
updateIndex(indexed, dictionary.EcosystemNPM, ref, cpeItemName)
}

func phpExtensionPackageFromURLFragment(ref string) string {
Expand Down Expand Up @@ -301,11 +284,7 @@ func addEntryForPHPPearPackage(indexed *dictionary.Indexed, ref string, cpeItemN
return
}

if _, ok := indexed.EcosystemPackages[dictionary.EcosystemPHPPear]; !ok {
indexed.EcosystemPackages[dictionary.EcosystemPHPPear] = make(dictionary.Packages)
}

indexed.EcosystemPackages[dictionary.EcosystemPHPPear][ref] = cpeItemName
updateIndex(indexed, dictionary.EcosystemPHPPear, ref, cpeItemName)
}

func addEntryForPHPPeclPackage(indexed *dictionary.Indexed, ref string, cpeItemName string) {
Expand All @@ -317,11 +296,7 @@ func addEntryForPHPPeclPackage(indexed *dictionary.Indexed, ref string, cpeItemN
return
}

if _, ok := indexed.EcosystemPackages[dictionary.EcosystemPHPPecl]; !ok {
indexed.EcosystemPackages[dictionary.EcosystemPHPPecl] = make(dictionary.Packages)
}

indexed.EcosystemPackages[dictionary.EcosystemPHPPecl][ref] = cpeItemName
updateIndex(indexed, dictionary.EcosystemPHPPecl, ref, cpeItemName)
}

func addEntryForPHPComposerPackage(indexed *dictionary.Indexed, ref string, cpeItemName string) {
Expand All @@ -335,9 +310,5 @@ func addEntryForPHPComposerPackage(indexed *dictionary.Indexed, ref string, cpeI

ref = components[0] + "/" + components[1]

if _, ok := indexed.EcosystemPackages[dictionary.EcosystemPHPComposer]; !ok {
indexed.EcosystemPackages[dictionary.EcosystemPHPComposer] = make(dictionary.Packages)
}

indexed.EcosystemPackages[dictionary.EcosystemPHPComposer][ref] = cpeItemName
updateIndex(indexed, dictionary.EcosystemPHPComposer, ref, cpeItemName)
}
29 changes: 15 additions & 14 deletions syft/pkg/cataloger/internal/cpegenerate/dictionary/index-generator/generate_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"testing"

"github.com/google/go-cmp/cmp"
"github.com/scylladb/go-set/strset"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

Expand Down Expand Up @@ -59,7 +60,7 @@ func Test_addEntryFuncs(t *testing.T) {
expectedIndexed: dictionary.Indexed{
EcosystemPackages: map[string]dictionary.Packages{
dictionary.EcosystemRustCrates: {
"unicycle": "cpe:2.3:a:unicycle_project:unicycle:*:*:*:*:*:rust:*:*",
"unicycle": dictionary.NewSet("cpe:2.3:a:unicycle_project:unicycle:*:*:*:*:*:rust:*:*"),
},
},
},
Expand All @@ -72,7 +73,7 @@ func Test_addEntryFuncs(t *testing.T) {
expectedIndexed: dictionary.Indexed{
EcosystemPackages: map[string]dictionary.Packages{
dictionary.EcosystemJenkinsPlugins: {
"sonarqube": "cpe:2.3:a:sonarsource:sonarqube_scanner:2.7:*:*:*:*:jenkins:*:*",
"sonarqube": dictionary.NewSet("cpe:2.3:a:sonarsource:sonarqube_scanner:2.7:*:*:*:*:jenkins:*:*"),
},
},
},
Expand All @@ -94,7 +95,7 @@ func Test_addEntryFuncs(t *testing.T) {
expectedIndexed: dictionary.Indexed{
EcosystemPackages: map[string]dictionary.Packages{
dictionary.EcosystemJenkinsPlugins: {
"svn-partial-release-mgr": "cpe:2.3:a:jenkins:subversion_partial_release_manager:1.0.1:*:*:*:*:jenkins:*:*",
"svn-partial-release-mgr": dictionary.NewSet("cpe:2.3:a:jenkins:subversion_partial_release_manager:1.0.1:*:*:*:*:jenkins:*:*"),
},
},
},
Expand All @@ -107,7 +108,7 @@ func Test_addEntryFuncs(t *testing.T) {
expectedIndexed: dictionary.Indexed{
EcosystemPackages: map[string]dictionary.Packages{
dictionary.EcosystemPyPI: {
"vault-cli": "cpe:2.3:a:vault-cli_project:vault-cli:*:*:*:*:*:python:*:*",
"vault-cli": dictionary.NewSet("cpe:2.3:a:vault-cli_project:vault-cli:*:*:*:*:*:python:*:*"),
},
},
},
Expand All @@ -120,7 +121,7 @@ func Test_addEntryFuncs(t *testing.T) {
expectedIndexed: dictionary.Indexed{
EcosystemPackages: map[string]dictionary.Packages{
dictionary.EcosystemRubyGems: {
"openssl": "cpe:2.3:a:ruby-lang:openssl:-:*:*:*:*:ruby:*:*",
"openssl": dictionary.NewSet("cpe:2.3:a:ruby-lang:openssl:-:*:*:*:*:ruby:*:*"),
},
},
},
Expand All @@ -133,7 +134,7 @@ func Test_addEntryFuncs(t *testing.T) {
expectedIndexed: dictionary.Indexed{
EcosystemPackages: map[string]dictionary.Packages{
dictionary.EcosystemRubyGems: {
"actionview": "cpe:2.3:a:action_view_project:action_view:*:*:*:*:*:ruby:*:*",
"actionview": dictionary.NewSet("cpe:2.3:a:action_view_project:action_view:*:*:*:*:*:ruby:*:*"),
},
},
},
Expand All @@ -146,7 +147,7 @@ func Test_addEntryFuncs(t *testing.T) {
expectedIndexed: dictionary.Indexed{
EcosystemPackages: map[string]dictionary.Packages{
dictionary.EcosystemRubyGems: {
"rbovirt": "cpe:2.3:a:amos_benari:rbovirt:*:*:*:*:*:ruby:*:*",
"rbovirt": dictionary.NewSet("cpe:2.3:a:amos_benari:rbovirt:*:*:*:*:*:ruby:*:*"),
},
},
},
Expand All @@ -159,7 +160,7 @@ func Test_addEntryFuncs(t *testing.T) {
expectedIndexed: dictionary.Indexed{
EcosystemPackages: map[string]dictionary.Packages{
dictionary.EcosystemNPM: {
"@nubosoftware/node-static": "cpe:2.3:a:\\@nubosoftware\\/node-static_project:\\@nubosoftware\\/node-static:-:*:*:*:*:node.js:*:*",
"@nubosoftware/node-static": dictionary.NewSet("cpe:2.3:a:\\@nubosoftware\\/node-static_project:\\@nubosoftware\\/node-static:-:*:*:*:*:node.js:*:*"),
},
},
},
Expand All @@ -172,7 +173,7 @@ func Test_addEntryFuncs(t *testing.T) {
expectedIndexed: dictionary.Indexed{
EcosystemPackages: map[string]dictionary.Packages{
dictionary.EcosystemPHPPecl: {
"imagick": "cpe:2.3:a:php:imagick:*:*:*:*:*:*:*:*",
"imagick": dictionary.NewSet("cpe:2.3:a:php:imagick:*:*:*:*:*:*:*:*"),
},
},
},
Expand All @@ -185,7 +186,7 @@ func Test_addEntryFuncs(t *testing.T) {
expectedIndexed: dictionary.Indexed{
EcosystemPackages: map[string]dictionary.Packages{
dictionary.EcosystemPHPPecl: {
"memcached": "cpe:2.3:a:php:memcached:*:*:*:*:*:*:*:*",
"memcached": dictionary.NewSet("cpe:2.3:a:php:memcached:*:*:*:*:*:*:*:*"),
},
},
},
Expand All @@ -198,7 +199,7 @@ func Test_addEntryFuncs(t *testing.T) {
expectedIndexed: dictionary.Indexed{
EcosystemPackages: map[string]dictionary.Packages{
dictionary.EcosystemPHPPear: {
"PEAR": "cpe:2.3:a:php:pear:*:*:*:*:*:*:*:*",
"PEAR": dictionary.NewSet("cpe:2.3:a:php:pear:*:*:*:*:*:*:*:*"),
},
},
},
Expand All @@ -211,7 +212,7 @@ func Test_addEntryFuncs(t *testing.T) {
expectedIndexed: dictionary.Indexed{
EcosystemPackages: map[string]dictionary.Packages{
dictionary.EcosystemPHPPear: {
"abcdefg": "cpe:2.3:a:php:abcdefg:*:*:*:*:*:*:*:*",
"abcdefg": dictionary.NewSet("cpe:2.3:a:php:abcdefg:*:*:*:*:*:*:*:*"),
},
},
},
Expand All @@ -224,7 +225,7 @@ func Test_addEntryFuncs(t *testing.T) {
expectedIndexed: dictionary.Indexed{
EcosystemPackages: map[string]dictionary.Packages{
dictionary.EcosystemPHPComposer: {
"frappant/frp-form-answers": "cpe:2.3:a:frappant:forms_export:*:*:*:*:*:*:*:*",
"frappant/frp-form-answers": dictionary.NewSet("cpe:2.3:a:frappant:forms_export:*:*:*:*:*:*:*:*"),
},
},
},
Expand All @@ -239,7 +240,7 @@ func Test_addEntryFuncs(t *testing.T) {

tt.addEntryFunc(indexed, tt.inputRef, tt.inputCpeItemName)

if diff := cmp.Diff(tt.expectedIndexed, *indexed); diff != "" {
if diff := cmp.Diff(tt.expectedIndexed, *indexed, cmp.AllowUnexported(strset.Set{})); diff != "" {
t.Errorf("addEntry* mismatch (-want +got):\n%s", diff)
}
})
Expand Down
Loading
Loading