From 2362c088a4377f2ef34de1fed4cf0f945ce47f8b Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Thu, 13 Jun 2019 13:15:05 +0900 Subject: [PATCH 1/2] refactor(alpine): Query tuning --- db/rdb/alpine.go | 66 ++++++++---------------------------------------- db/rdb/debian.go | 11 +++++--- db/rdb/ubuntu.go | 11 +++++--- 3 files changed, 25 insertions(+), 63 deletions(-) diff --git a/db/rdb/alpine.go b/db/rdb/alpine.go index b7616a74..2608854b 100644 --- a/db/rdb/alpine.go +++ b/db/rdb/alpine.go @@ -76,63 +76,19 @@ func (o *Alpine) InsertOval(root *models.Root, meta models.FetchMeta, driver *go } // GetByPackName select definitions by packName -func (o *Alpine) GetByPackName(driver *gorm.DB, osVer, packName, _ string) ([]models.Definition, error) { - osVer = majorMinor(osVer) - packs := []models.Package{} - err := driver.Where(&models.Package{Name: packName}).Find(&packs).Error +func (o *Alpine) GetByPackName(driver *gorm.DB, osVer, packName, _ string) (defs []models.Definition, err error) { + err = driver.Joins("JOIN roots ON roots.id = definitions.root_id AND roots.family= ? AND roots.os_version = ?", + config.Alpine, majorMinor(osVer)). + Joins("JOIN packages ON packages.definition_id = definitions.id"). + Where("packages.name = ?", packName). + Preload("Cves"). + Preload("Advisory"). + Preload("AffectedPacks"). + Preload("References"). + Find(&defs).Error + if err != nil && err != gorm.ErrRecordNotFound { return nil, err } - - defs := []models.Definition{} - for _, p := range packs { - def := models.Definition{} - err = driver.Where("id = ?", p.DefinitionID).Find(&def).Error - if err != nil && err != gorm.ErrRecordNotFound { - return nil, err - } - - root := models.Root{} - err = driver.Where("id = ?", def.RootID).Find(&root).Error - if err != nil && err != gorm.ErrRecordNotFound { - return nil, err - } - - if root.Family == config.Alpine && root.OSVersion == osVer { - defs = append(defs, def) - } - - for i, def := range defs { - adv := models.Advisory{} - err = driver.Model(&def).Related(&adv, "Advisory").Error - if err != nil && err != gorm.ErrRecordNotFound { - return nil, err - } - - cves := []models.Cve{} - err = driver.Model(&adv).Related(&cves, "Cves").Error - if err != nil && err != gorm.ErrRecordNotFound { - return nil, err - } - - adv.Cves = cves - defs[i].Advisory = adv - - packs := []models.Package{} - err = driver.Model(&def).Related(&packs, "AffectedPacks").Error - if err != nil && err != gorm.ErrRecordNotFound { - return nil, err - } - defs[i].AffectedPacks = packs - - refs := []models.Reference{} - err = driver.Model(&def).Related(&refs, "References").Error - if err != nil && err != gorm.ErrRecordNotFound { - return nil, err - } - defs[i].References = refs - } - } - return defs, nil } diff --git a/db/rdb/debian.go b/db/rdb/debian.go index c43bfa7d..b3bed3ab 100644 --- a/db/rdb/debian.go +++ b/db/rdb/debian.go @@ -110,15 +110,18 @@ func (o *Debian) InsertOval(root *models.Root, meta models.FetchMeta, driver *go } // GetByPackName select definitions by packName -func (o *Debian) GetByPackName(driver *gorm.DB, osVer, packName, _ string) ([]models.Definition, error) { - defs := []models.Definition{} - driver.Joins("JOIN roots ON roots.id = definitions.root_id AND roots.family= ? AND roots.os_version = ?", +func (o *Debian) GetByPackName(driver *gorm.DB, osVer, packName, _ string) (defs []models.Definition, err error) { + err = driver.Joins("JOIN roots ON roots.id = definitions.root_id AND roots.family= ? AND roots.os_version = ?", config.Debian, major(osVer)). Joins("JOIN packages ON packages.definition_id = definitions.id"). Where("packages.name = ?", packName). Preload("Debian"). Preload("AffectedPacks"). Preload("References"). - Find(&defs) + Find(&defs).Error + + if err != nil && err != gorm.ErrRecordNotFound { + return nil, err + } return defs, nil } diff --git a/db/rdb/ubuntu.go b/db/rdb/ubuntu.go index 4be2486a..0c7926d7 100644 --- a/db/rdb/ubuntu.go +++ b/db/rdb/ubuntu.go @@ -86,9 +86,8 @@ func (o *Ubuntu) InsertOval(root *models.Root, meta models.FetchMeta, driver *go } // GetByPackName select definitions by packName -func (o *Ubuntu) GetByPackName(driver *gorm.DB, osVer, packName, _ string) ([]models.Definition, error) { - defs := []models.Definition{} - driver.Joins("JOIN roots ON roots.id = definitions.root_id AND roots.family= ? AND roots.os_version = ?", +func (o *Ubuntu) GetByPackName(driver *gorm.DB, osVer, packName, _ string) (defs []models.Definition, err error) { + err = driver.Joins("JOIN roots ON roots.id = definitions.root_id AND roots.family= ? AND roots.os_version = ?", config.Ubuntu, major(osVer)). Joins("JOIN packages ON packages.definition_id = definitions.id"). Where("packages.name = ?", packName). @@ -96,6 +95,10 @@ func (o *Ubuntu) GetByPackName(driver *gorm.DB, osVer, packName, _ string) ([]mo Preload("Advisory"). Preload("AffectedPacks"). Preload("References"). - Find(&defs) + Find(&defs).Error + + if err != nil && err != gorm.ErrRecordNotFound { + return nil, err + } return defs, nil } From 8b98657de17d7065eda5a171b6d9d39183625dc4 Mon Sep 17 00:00:00 2001 From: Kota Kanbe Date: Thu, 13 Jun 2019 14:32:58 +0900 Subject: [PATCH 2/2] fix(alpine): bug of majorMinor --- db/rdb/alpine.go | 4 ++-- db/rdb/rdb.go | 10 ++++----- db/rdb/rdb_test.go | 53 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 db/rdb/rdb_test.go diff --git a/db/rdb/alpine.go b/db/rdb/alpine.go index 2608854b..80fa2afb 100644 --- a/db/rdb/alpine.go +++ b/db/rdb/alpine.go @@ -78,11 +78,11 @@ func (o *Alpine) InsertOval(root *models.Root, meta models.FetchMeta, driver *go // GetByPackName select definitions by packName func (o *Alpine) GetByPackName(driver *gorm.DB, osVer, packName, _ string) (defs []models.Definition, err error) { err = driver.Joins("JOIN roots ON roots.id = definitions.root_id AND roots.family= ? AND roots.os_version = ?", - config.Alpine, majorMinor(osVer)). + config.Alpine, majorDotMinor(osVer)). Joins("JOIN packages ON packages.definition_id = definitions.id"). Where("packages.name = ?", packName). - Preload("Cves"). Preload("Advisory"). + Preload("Advisory.Cves"). Preload("AffectedPacks"). Preload("References"). Find(&defs).Error diff --git a/db/rdb/rdb.go b/db/rdb/rdb.go index 5677fadd..7dce5414 100644 --- a/db/rdb/rdb.go +++ b/db/rdb/rdb.go @@ -229,7 +229,7 @@ func (d *Driver) InsertFetchMeta(meta models.FetchMeta) error { func (d *Driver) CountDefs(osFamily, osVer string) (int, error) { switch osFamily { case c.Alpine: - osVer = majorMinor(osVer) + osVer = majorDotMinor(osVer) case c.SUSEEnterpriseServer: // SUSE provides OVAL each major.minor case c.Amazon: @@ -255,7 +255,7 @@ func (d *Driver) CountDefs(osFamily, osVer string) (int, error) { func (d *Driver) GetLastModified(osFamily, osVer string) time.Time { switch osFamily { case c.Alpine: - osVer = majorMinor(osVer) + osVer = majorDotMinor(osVer) case c.SUSEEnterpriseServer: // SUSE provides OVAL each major.minor case c.Amazon: @@ -277,12 +277,12 @@ func major(osVer string) (majorVersion string) { return strings.Split(osVer, ".")[0] } -func majorMinor(osVer string) (majorMinorVersion string) { +func majorDotMinor(osVer string) (majorMinorVersion string) { ss := strings.Split(osVer, ".") - if len(ss) == 1 { + if len(ss) < 3 { return osVer } - return strings.Join(ss[:len(ss)-1], ".") + return strings.Join(ss[:2], ".") } // getAmazonLinux2 returns AmazonLinux1 or 2 diff --git a/db/rdb/rdb_test.go b/db/rdb/rdb_test.go new file mode 100644 index 00000000..e120764c --- /dev/null +++ b/db/rdb/rdb_test.go @@ -0,0 +1,53 @@ +package rdb + +import ( + "testing" + + _ "github.com/jinzhu/gorm/dialects/mysql" + _ "github.com/jinzhu/gorm/dialects/postgres" + _ "github.com/jinzhu/gorm/dialects/sqlite" +) + +func Test_majorMinor(t *testing.T) { + type args struct { + osVer string + } + tests := []struct { + name string + args args + wantMajorMinorVersion string + }{ + { + "3", + args{"3"}, + "3", + }, + { + "3.9", + args{"3.9"}, + "3.9", + }, + { + "3.9.2", + args{"3.9.2"}, + "3.9", + }, + { + "3.9.2.2", + args{"3.9.2.2"}, + "3.9", + }, + { + "3.9.2.2", + args{"3.9.2.2"}, + "3.9", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if gotMajorMinorVersion := majorDotMinor(tt.args.osVer); gotMajorMinorVersion != tt.wantMajorMinorVersion { + t.Errorf("majorMinor() = %v, want %v", gotMajorMinorVersion, tt.wantMajorMinorVersion) + } + }) + } +}