Notify the difference from the previous scan result (future-architect…

…#392)

add diff option

README.ja.md

@@ -939,6 +939,7 @@ report:
                 [-cvedb-path=/path/to/cve.sqlite3]
                 [-cvedb-url=http://127.0.0.1:1323 or mysql connection string]
                 [-cvss-over=7]
+                [-diff]
                 [-ignore-unscored-cves]
                 [-to-email]
                 [-to-slack]
@@ -986,6 +987,8 @@ report:
         http://cve-dictionary.com:8080 or mysql connection string
   -cvss-over float
         -cvss-over=6.5 means reporting CVSS Score 6.5 and over (default: 0 (means report all))
+  -diff
+        Difference between previous result and current result
   -debug
         debug mode
   -debug-sql

README.md

@@ -950,6 +950,7 @@ report:
                 [-cvedb-path=/path/to/cve.sqlite3]
                 [-cvedb-url=http://127.0.0.1:1323 or mysql connection string]
                 [-cvss-over=7]
+                [-diff]
                 [-ignore-unscored-cves]
                 [-to-email]
                 [-to-slack]
@@ -997,6 +998,8 @@ report:
         http://cve-dictionary.com:8080 or mysql connection string
   -cvss-over float
         -cvss-over=6.5 means reporting CVSS Score 6.5 and over (default: 0 (means report all))
+  -diff
+        Difference between previous result and current result
   -debug
         debug mode
   -debug-sql

commands/report.go

@@ -74,6 +74,8 @@ type ReportCmd struct {
 	azureContainer string
 
 	pipe bool
+
+	diff bool
 }
 
 // Name return subcommand name
@@ -95,6 +97,7 @@ func (*ReportCmd) Usage() string {
 		[-cvedb-path=/path/to/cve.sqlite3]
 		[-cvedb-url=http://127.0.0.1:1323 or mysql connection string]
 		[-cvss-over=7]
+		[-diff]
 		[-ignore-unscored-cves]
 		[-to-email]
 		[-to-slack]
@@ -171,6 +174,11 @@ func (p *ReportCmd) SetFlags(f *flag.FlagSet) {
 		0,
 		"-cvss-over=6.5 means reporting CVSS Score 6.5 and over (default: 0 (means report all))")
 
+	f.BoolVar(&p.diff,
+		"diff",
+		false,
+		fmt.Sprintf("Difference between previous result and current result "))
+
 	f.BoolVar(
 		&p.ignoreUnscoredCves,
 		"ignore-unscored-cves",
@@ -273,11 +281,6 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 	c.Conf.HTTPProxy = p.httpProxy
 
 	c.Conf.Pipe = p.pipe
-	jsonDir, err := jsonDir(f.Args())
-	if err != nil {
-		util.Log.Errorf("Failed to read from JSON: %s", err)
-		return subcommands.ExitFailure
-	}
 
 	c.Conf.FormatXML = p.formatXML
 	c.Conf.FormatJSON = p.formatJSON
@@ -287,6 +290,19 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 	c.Conf.FormatFullText = p.formatFullText
 
 	c.Conf.GZIP = p.gzip
+	c.Conf.Diff = p.diff
+
+	var dir string
+	var err error
+	if p.diff {
+		dir, err = jsonDir([]string{})
+	} else {
+		dir, err = jsonDir(f.Args())
+	}
+	if err != nil {
+		util.Log.Errorf("Failed to read from JSON: %s", err)
+		return subcommands.ExitFailure
+	}
 
 	// report
 	reports := []report.ResultWriter{
@@ -303,7 +319,7 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 
 	if p.toLocalFile {
 		reports = append(reports, report.LocalFileWriter{
-			CurrentDir: jsonDir,
+			CurrentDir: dir,
 		})
 	}
 
@@ -363,7 +379,8 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 		}
 	}
 
-	history, err := loadOneScanHistory(jsonDir)
+	var history models.ScanHistory
+	history, err = loadOneScanHistory(dir)
 	if err != nil {
 		util.Log.Error(err)
 		return subcommands.ExitFailure
@@ -388,8 +405,7 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 				return subcommands.ExitFailure
 			}
 			filled.Lang = c.Conf.Lang
-
-			if err := overwriteJSONFile(jsonDir, *filled); err != nil {
+			if err := overwriteJSONFile(dir, *filled); err != nil {
 				util.Log.Errorf("Failed to write JSON: %s", err)
 				return subcommands.ExitFailure
 			}
@@ -400,10 +416,31 @@ func (p *ReportCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{}
 		}
 	}
 
+	if p.diff {
+		currentHistory := models.ScanHistory{ScanResults: results}
+		previousHistory, err := loadPreviousScanHistory(currentHistory)
+		if err != nil {
+			util.Log.Error(err)
+			return subcommands.ExitFailure
+		}
+
+		history, err = diff(currentHistory, previousHistory)
+		if err != nil {
+			util.Log.Error(err)
+			return subcommands.ExitFailure
+		}
+		results = []models.ScanResult{}
+		for _, r := range history.ScanResults {
+			filled, _ := r.FillCveDetail()
+			results = append(results, *filled)
+		}
+	}
+
 	var res models.ScanResults
 	for _, r := range results {
 		res = append(res, r.FilterByCvssOver())
 	}
+
 	for _, w := range reports {
 		if err := w.Write(res...); err != nil {
 			util.Log.Errorf("Failed to report: %s", err)

commands/util.go

@@ -26,6 +26,7 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+	"time"
 
 	c "github.com/future-architect/vuls/config"
 	"github.com/future-architect/vuls/cveapi"
@@ -121,6 +122,19 @@ func jsonDir(args []string) (string, error) {
 	return dirs[0], nil
 }
 
+// loadOneServerScanResult read JSON data of one server
+func loadOneServerScanResult(jsonFile string) (result models.ScanResult, err error) {
+	var data []byte
+	if data, err = ioutil.ReadFile(jsonFile); err != nil {
+		err = fmt.Errorf("Failed to read %s: %s", jsonFile, err)
+		return
+	}
+	if json.Unmarshal(data, &result) != nil {
+		err = fmt.Errorf("Failed to parse %s: %s", jsonFile, err)
+	}
+	return
+}
+
 // loadOneScanHistory read JSON data
 func loadOneScanHistory(jsonDir string) (scanHistory models.ScanHistory, err error) {
 	var results []models.ScanResult
@@ -130,20 +144,16 @@ func loadOneScanHistory(jsonDir string) (scanHistory models.ScanHistory, err err
 		return
 	}
 	for _, f := range files {
-		if filepath.Ext(f.Name()) != ".json" {
+		if filepath.Ext(f.Name()) != ".json" || strings.HasSuffix(f.Name(), "_diff.json") {
 			continue
 		}
+
 		var r models.ScanResult
-		var data []byte
 		path := filepath.Join(jsonDir, f.Name())
-		if data, err = ioutil.ReadFile(path); err != nil {
-			err = fmt.Errorf("Failed to read %s: %s", path, err)
-			return
-		}
-		if json.Unmarshal(data, &r) != nil {
-			err = fmt.Errorf("Failed to parse %s: %s", path, err)
+		if r, err = loadOneServerScanResult(path); err != nil {
 			return
 		}
+
 		results = append(results, r)
 	}
 	if len(results) == 0 {
@@ -170,14 +180,111 @@ func fillCveInfoFromCveDB(r models.ScanResult) (*models.ScanResult, error) {
 	return r.FillCveDetail()
 }
 
+func loadPreviousScanHistory(current models.ScanHistory) (previous models.ScanHistory, err error) {
+	var dirs jsonDirs
+	if dirs, err = lsValidJSONDirs(); err != nil {
+		return
+	}
+
+	for _, result := range current.ScanResults {
+		for _, dir := range dirs[1:] {
+			var r models.ScanResult
+			path := filepath.Join(dir, result.ServerName+".json")
+			if r, err = loadOneServerScanResult(path); err != nil {
+				continue
+			}
+			if r.Family == result.Family && r.Release == result.Release {
+				previous.ScanResults = append(previous.ScanResults, r)
+				break
+			}
+		}
+	}
+	return previous, nil
+}
+
+func diff(currentHistory, previousHistory models.ScanHistory) (diffHistory models.ScanHistory, err error) {
+	for _, currentResult := range currentHistory.ScanResults {
+		found := false
+		var previousResult models.ScanResult
+		for _, previousResult = range previousHistory.ScanResults {
+			if currentResult.ServerName == previousResult.ServerName {
+				found = true
+				break
+			}
+		}
+
+		if found {
+			currentResult.ScannedCves = getNewCves(previousResult, currentResult)
+
+			currentResult.KnownCves = []models.CveInfo{}
+			currentResult.UnknownCves = []models.CveInfo{}
+
+			currentResult.Packages = models.PackageInfoList{}
+			for _, s := range currentResult.ScannedCves {
+				currentResult.Packages = append(currentResult.Packages, s.Packages...)
+			}
+			currentResult.Packages = currentResult.Packages.UniqByName()
+		}
+
+		diffHistory.ScanResults = append(diffHistory.ScanResults, currentResult)
+	}
+	return diffHistory, err
+}
+
+func getNewCves(previousResult, currentResult models.ScanResult) (newVulninfos []models.VulnInfo) {
+	previousCveIDsSet := map[string]bool{}
+	for _, previousVulnInfo := range previousResult.ScannedCves {
+		previousCveIDsSet[previousVulnInfo.CveID] = true
+	}
+
+	for _, v := range currentResult.ScannedCves {
+		if previousCveIDsSet[v.CveID] {
+			if isCveInfoUpdated(currentResult, previousResult, v.CveID) {
+				newVulninfos = append(newVulninfos, v)
+			}
+		} else {
+			newVulninfos = append(newVulninfos, v)
+		}
+	}
+	return
+}
+
+func isCveInfoUpdated(currentResult, previousResult models.ScanResult, CveID string) bool {
+	type lastModified struct {
+		Nvd time.Time
+		Jvn time.Time
+	}
+
+	previousModifies := lastModified{}
+	for _, c := range previousResult.KnownCves {
+		if CveID == c.CveID {
+			previousModifies.Nvd = c.CveDetail.Nvd.LastModifiedDate
+			previousModifies.Jvn = c.CveDetail.Jvn.LastModifiedDate
+		}
+	}
+
+	currentModifies := lastModified{}
+	for _, c := range currentResult.KnownCves {
+		if CveID == c.CveDetail.CveID {
+			currentModifies.Nvd = c.CveDetail.Nvd.LastModifiedDate
+			currentModifies.Jvn = c.CveDetail.Jvn.LastModifiedDate
+		}
+	}
+	return !currentModifies.Nvd.Equal(previousModifies.Nvd) ||
+		!currentModifies.Jvn.Equal(previousModifies.Jvn)
+}
+
 func overwriteJSONFile(dir string, r models.ScanResult) error {
 	before := c.Conf.FormatJSON
+	beforeDiff := c.Conf.Diff
 	c.Conf.FormatJSON = true
+	c.Conf.Diff = false
 	w := report.LocalFileWriter{CurrentDir: dir}
 	if err := w.Write(r); err != nil {
 		return fmt.Errorf("Failed to write summary report: %s", err)
 	}
 	c.Conf.FormatJSON = before
+	c.Conf.Diff = beforeDiff
 	return nil
 }