-
Notifications
You must be signed in to change notification settings - Fork 44
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
9 changed files
with
722 additions
and
36 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| package cmd | ||
|
|
||
| import ( | ||
| "time" | ||
|
|
||
| "github.com/inconshreveable/log15" | ||
| "github.com/spf13/cobra" | ||
| "github.com/spf13/viper" | ||
| "github.com/vulsio/gost/db" | ||
| "github.com/vulsio/gost/fetcher" | ||
| "github.com/vulsio/gost/models" | ||
| "github.com/vulsio/gost/util" | ||
| "golang.org/x/xerrors" | ||
| ) | ||
|
|
||
| var archCmd = &cobra.Command{ | ||
| Use: "arch", | ||
| Short: "Fetch the CVE information from Arch Linux", | ||
| Long: `Fetch the CVE information from Arch Linux`, | ||
| RunE: fetchArch, | ||
| } | ||
|
|
||
| func init() { | ||
| fetchCmd.AddCommand(archCmd) | ||
| } | ||
|
|
||
| func fetchArch(_ *cobra.Command, _ []string) (err error) { | ||
| if err := util.SetLogger(viper.GetBool("log-to-file"), viper.GetString("log-dir"), viper.GetBool("debug"), viper.GetBool("log-json")); err != nil { | ||
| return xerrors.Errorf("Failed to SetLogger. err: %w", err) | ||
| } | ||
|
|
||
| log15.Info("Initialize Database") | ||
| driver, err := db.NewDB(viper.GetString("dbtype"), viper.GetString("dbpath"), viper.GetBool("debug-sql"), db.Option{}) | ||
| if err != nil { | ||
| if xerrors.Is(err, db.ErrDBLocked) { | ||
| return xerrors.Errorf("Failed to open DB. Close DB connection before fetching. err: %w", err) | ||
| } | ||
| return xerrors.Errorf("Failed to open DB. err: %w", err) | ||
| } | ||
|
|
||
| fetchMeta, err := driver.GetFetchMeta() | ||
| if err != nil { | ||
| return xerrors.Errorf("Failed to get FetchMeta from DB. err: %w", err) | ||
| } | ||
| if fetchMeta.OutDated() { | ||
| return xerrors.Errorf("Failed to Insert CVEs into DB. err: SchemaVersion is old. SchemaVersion: %+v", map[string]uint{"latest": models.LatestSchemaVersion, "DB": fetchMeta.SchemaVersion}) | ||
| } | ||
| // If the fetch fails the first time (without SchemaVersion), the DB needs to be cleaned every time, so insert SchemaVersion. | ||
| if err := driver.UpsertFetchMeta(fetchMeta); err != nil { | ||
| return xerrors.Errorf("Failed to upsert FetchMeta to DB. err: %w", err) | ||
| } | ||
|
|
||
| log15.Info("Fetched all CVEs from Arch Linux") | ||
| advJSONs, err := fetcher.FetchArch() | ||
| if err != nil { | ||
| return xerrors.Errorf("Failed to fetch Arch. err: %w", err) | ||
| } | ||
| advs := models.ConvertArch(advJSONs) | ||
|
|
||
| log15.Info("Fetched", "Advisories", len(advs)) | ||
|
|
||
| log15.Info("Insert Arch Linux CVEs into DB", "db", driver.Name()) | ||
| if err := driver.InsertArch(advs); err != nil { | ||
| return xerrors.Errorf("Failed to insert. dbpath: %s, err: %w", viper.GetString("dbpath"), err) | ||
| } | ||
|
|
||
| fetchMeta.LastFetchedAt = time.Now() | ||
| if err := driver.UpsertFetchMeta(fetchMeta); err != nil { | ||
| return xerrors.Errorf("Failed to upsert FetchMeta to DB. dbpath: %s, err: %w", viper.GetString("dbpath"), err) | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,143 @@ | ||
| package db | ||
|
|
||
| import ( | ||
| "errors" | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
|
|
||
| "github.com/cheggaaa/pb/v3" | ||
| "github.com/spf13/viper" | ||
| "github.com/vulsio/gost/models" | ||
| "golang.org/x/xerrors" | ||
| "gorm.io/gorm" | ||
| ) | ||
|
|
||
| // GetArch : | ||
| func (r *RDBDriver) GetArch(advID string) (*models.ArchADV, error) { | ||
| var a models.ArchADV | ||
| if err := r.conn. | ||
| Preload("Packages"). | ||
| Preload("Issues"). | ||
| Preload("Advisories"). | ||
| Where(&models.ArchADV{Name: advID}). | ||
| First(&a).Error; err != nil { | ||
| if errors.Is(err, gorm.ErrRecordNotFound) { | ||
| return nil, nil | ||
| } | ||
| return nil, xerrors.Errorf("Failed to find first record by %s. err: %w", advID, err) | ||
| } | ||
| return &a, nil | ||
| } | ||
|
|
||
| // GetArchMulti : | ||
| func (r *RDBDriver) GetArchMulti(advIDs []string) (map[string]models.ArchADV, error) { | ||
| m := make(map[string]models.ArchADV) | ||
| for _, id := range advIDs { | ||
| a, err := r.GetArch(id) | ||
| if err != nil { | ||
| return nil, xerrors.Errorf("Failed to get Arch. err: %w", err) | ||
| } | ||
| if a != nil { | ||
| m[id] = *a | ||
| } | ||
| } | ||
| return m, nil | ||
| } | ||
|
|
||
| // InsertArch : | ||
| func (r *RDBDriver) InsertArch(advs []models.ArchADV) error { | ||
| if err := r.deleteAndInsertArch(advs); err != nil { | ||
| return xerrors.Errorf("Failed to insert Arch Advisory data. err: %w", err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (r *RDBDriver) deleteAndInsertArch(advs []models.ArchADV) (err error) { | ||
| bar := pb.StartNew(len(advs)).SetWriter(func() io.Writer { | ||
| if viper.GetBool("log-json") { | ||
| return io.Discard | ||
| } | ||
| return os.Stderr | ||
| }()) | ||
| tx := r.conn.Begin() | ||
|
|
||
| defer func() { | ||
| if err != nil { | ||
| tx.Rollback() | ||
| return | ||
| } | ||
| tx.Commit() | ||
| }() | ||
|
|
||
| // Delete all old records | ||
| for _, table := range []interface{}{models.ArchAdvisory{}, models.ArchIssue{}, models.ArchPackage{}, models.ArchADV{}} { | ||
| if err := tx.Session(&gorm.Session{AllowGlobalUpdate: true}).Delete(table).Error; err != nil { | ||
| return xerrors.Errorf("Failed to delete old records. err: %w", err) | ||
| } | ||
| } | ||
|
|
||
| batchSize := viper.GetInt("batch-size") | ||
| if batchSize < 1 { | ||
| return fmt.Errorf("Failed to set batch-size. err: batch-size option is not set properly") | ||
| } | ||
|
|
||
| for idx := range chunkSlice(len(advs), batchSize) { | ||
| if err = tx.Create(advs[idx.From:idx.To]).Error; err != nil { | ||
| return xerrors.Errorf("Failed to insert. err: %w", err) | ||
| } | ||
| bar.Add(idx.To - idx.From) | ||
| } | ||
| bar.Finish() | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // GetUnfixedAdvsArch : | ||
| func (r *RDBDriver) GetUnfixedAdvsArch(pkgName string) (map[string]models.ArchADV, error) { | ||
| return r.getAdvsArchWithFixStatus(pkgName, "Vulnerable") | ||
| } | ||
|
|
||
| // GetFixedAdvsArch : | ||
| func (r *RDBDriver) GetFixedAdvsArch(pkgName string) (map[string]models.ArchADV, error) { | ||
| return r.getAdvsArchWithFixStatus(pkgName, "Fixed") | ||
| } | ||
|
|
||
| func (r *RDBDriver) getAdvsArchWithFixStatus(pkgName, fixStatus string) (map[string]models.ArchADV, error) { | ||
| var as []models.ArchADV | ||
| if err := r.conn. | ||
| Joins("JOIN arch_packages ON arch_packages.arch_adv_id = arch_advs.id AND arch_packages.name = ?", pkgName). | ||
| Preload("Packages"). | ||
| Preload("Issues"). | ||
| Preload("Advisories"). | ||
| Where(&models.ArchADV{Status: fixStatus}). | ||
| Find(&as).Error; err != nil { | ||
| return nil, xerrors.Errorf("Failed to find advisory by pkgname: %s, fix status: %s. err: %w", pkgName, fixStatus, err) | ||
| } | ||
|
|
||
| m := make(map[string]models.ArchADV) | ||
| for _, a := range as { | ||
| m[a.Name] = a | ||
| } | ||
| return m, nil | ||
| } | ||
|
|
||
| // GetAdvisoriesArch gets AdvisoryID: []CVE IDs | ||
| func (r *RDBDriver) GetAdvisoriesArch() (map[string][]string, error) { | ||
| m := make(map[string][]string) | ||
| var as []models.ArchADV | ||
| // the maximum value of a host parameter number is SQLITE_MAX_VARIABLE_NUMBER, which defaults to 999 for SQLite versions prior to 3.32.0 (2020-05-22) or 32766 for SQLite versions after 3.32.0. | ||
| // https://www.sqlite.org/limits.html Maximum Number Of Host Parameters In A Single SQL Statement | ||
| if err := r.conn.Preload("Issues").FindInBatches(&as, 999, func(_ *gorm.DB, _ int) error { | ||
| for _, a := range as { | ||
| for _, i := range a.Issues { | ||
| m[a.Name] = append(m[a.Name], i.Issue) | ||
| } | ||
| } | ||
| return nil | ||
| }).Error; err != nil { | ||
| return nil, xerrors.Errorf("Failed to find Arch. err: %w", err) | ||
| } | ||
|
|
||
| return m, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.