diff --git a/README.md b/README.md index c0238788..d0336fd0 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,17 @@ # go-cve-dictionary -HTTP API server to get CVE information stored in SQLite3 on localhost. -Current data sources are NVD(English) and JVN(Japanese). +This is tool to build a local copy of the NVD (National Vulnerabilities Database) [1] +and the Japanese JVN [2], which contain security vulnerabilities according to their +CVE identifiers [3] including exhaustive information and a risk score. The local +copy is generated in sqlite format, and the tool has a server mode for easy querying. + +[1] https://en.wikipedia.org/wiki/National_Vulnerability_Database +[2] https://en.wikipedia.org/wiki/Common_Vulnerabilities_and_Exposures +[3] http://jvndb.jvn.jp/apis/termsofuse.html ## Install requirements -Vuls requires the following packages. +go-cve-dictionary requires the following packages. - sqlite - git @@ -46,18 +52,18 @@ $ sudo chmod 700 /var/log/vuls $ go get github.com/kotakanbe/go-cve-dictionary ``` -Start go-cve-dictionary as server mode. -For the first time, go-cve-dictionary fetches vulnerability data from NVD. +Fetch Vulnerability data from NVD. It takes about 10 minutes (on AWS). ```bash -$ go-cve-dictionary server -... Fetching ... +$ for i in {2002..2016}; do ./go-cve-dictionary fetchnvd -years $i; done +... snip ... $ ls -alh cve.sqlite3 -rw-r--r-- 1 ec2-user ec2-user 7.0M Mar 24 13:20 cve.sqlite3 ``` -Now we has vulnerbility data, So start as server mode again. +Now we have vulnerability data. +Start go-cve-dictionary as server mode. ```bash $ go-cve-dictionary server [Mar 24 15:21:55] INFO Opening DB. datafile: /home/ec2-user/cve.sqlite3 @@ -142,10 +148,85 @@ $ curl http://127.0.0.1:1323/cves/CVE-2014-0160 | jq "." ``` +# Hello Ruby on Rails 4.0.2 + +``` +$ curl -v -H "Accept: application/json" -H "Content-type: application/json" -X POST -d '{"name": "cpe:/a:rubyonrails:ruby_on_rails:4.0.2:-"}' http://localhost:1323/cpes | jq "." +[ + { + "ID": 345, + "CreatedAt": "2016-04-10T10:52:26.196610454+09:00", + "UpdatedAt": "2016-04-10T10:52:26.196610454+09:00", + "DeletedAt": null, + "CveInfoID": 0, + "CveID": "CVE-2016-0751", + "Nvd": { + "ID": 345, + "CreatedAt": "2016-04-10T10:52:26.196853826+09:00", + "UpdatedAt": "2016-04-10T10:52:26.196853826+09:00", + "DeletedAt": null, + "CveDetailID": 345, + "Summary": "actionpack/lib/action_dispatch/http/mime_type.rb in Action Pack in Ruby on Rails before 3.2.22.1, 4.0.x and 4.1.x before 4.1.14.1, 4.2.x before 4.2.5.1, and 5.x before 5.0.0.beta1.1 does not properly restrict use of the MIME type cache, which allows remote attackers to cause a denial of service (memory consumption) via a crafted HTTP Accept header.", + "Score": 5, + "AccessVector": "NETWORK", + "AccessComplexity": "LOW", + "Authentication": "NONE", + "ConfidentialityImpact": "NONE", + "IntegrityImpact": "NONE", + "AvailabilityImpact": "PARTIAL", + "Cpes": null, + "References": [ + { + "ID": 486, + "CreatedAt": "2016-04-10T10:52:26.217958168+09:00", + "UpdatedAt": "2016-04-10T10:52:26.217958168+09:00", + "DeletedAt": null, + "JvnID": 0, + "NvdID": 345, + "Source": "MLIST", + "Link": "https://groups.google.com/forum/message/raw?msg=ruby-security-ann/9oLY_FCzvoc/5CDXbvpYEgAJ" + }, + { + "ID": 487, + "CreatedAt": "2016-04-10T10:52:26.218175571+09:00", + "UpdatedAt": "2016-04-10T10:52:26.218175571+09:00", + "DeletedAt": null, + "JvnID": 0, + "NvdID": 345, + "Source": "MLIST", + "Link": "http://www.openwall.com/lists/oss-security/2016/01/25/9" + } + ], + "PublishedDate": "2016-02-15T21:59:05.877-05:00", + "LastModifiedDate": "2016-03-18T21:02:43.817-04:00" + }, + "Jvn": { + "ID": 0, + "CreatedAt": "0001-01-01T00:00:00Z", + "UpdatedAt": "0001-01-01T00:00:00Z", + "DeletedAt": null, + "CveDetailID": 0, + "Title": "", + "Summary": "", + "JvnLink": "", + "JvnID": "", + "Score": 0, + "Severity": "", + "Vector": "", + "References": null, + "Cpes": null, + "PublishedDate": "0001-01-01T00:00:00Z", + "LastModifiedDate": "0001-01-01T00:00:00Z" + } + }, + ... snip ... +] +``` + # Usage: ``` -./go-cve-dictionary -help +$ go-cve-dictionary -help Usage: go-cve-dictionary Subcommands: @@ -182,17 +263,21 @@ go-cve-dictionary has four subcommands - server Start HTTP server -# Usage: Update NVD Data. +# Usage: Fetch NVD Data. ``` -$ go-cve-dictionary fetchnvd -h +$ ./go-cve-dictionary fetchnvd -help fetchnvd: fetchnvd [-last2y] + [-years] 2015 2016 ... [-dbpath=/path/to/cve.sqlite3] [-debug] [-debug-sql] +For the first time, run the blow command to fetch data for entire period. (It takes about 10 minutes) + $ for i in {2002..2016}; do go-cve-dictionary fetchnvd -years $i ; done + -dbpath string /path/to/sqlite3 (default "$PWD/cve.sqlite3") -debug @@ -201,25 +286,35 @@ fetchnvd: SQL debug mode -last2y Refresh NVD data in the last two years. + -years + Refresh NVD data of specific years. ``` -- Fetch data of the entire period +- Fetch data in the last two years ``` -$ go-cve-dictionary fetchnvd -entire +$ go-cve-dictionary fetchnvd -last2y ``` -- Fetch data last 2 years +- Fetch data of specific years ``` -$ go-cve-dictionary fetchnvd -last2y +$ go-cve-dictionary fetchnvd -years 2002 2003 2016 +``` + +- Fetch NVD data for entire period. ``` +for i in {2002..2016}; do ./go-cve-dictionary fetchnvd -years $i; done + +``` + + ---- -# Usage: Update JVN Data. +# Usage: Fetch JVN Data. ``` -./go-cve-dictionary fetchjvn -h +$ go-cve-dictionary fetchjvn -h fetchjvn: fetchjvn [-dump-path=/path/to/cve.json] @@ -231,7 +326,7 @@ fetchjvn: [-debug-sql] -dbpath string - /path/to/sqlite3 (default "/Users/kotakanbe/go/src/github.com/kotakanbe/go-cve-dictionary/cve.sqlite3") + /path/to/sqlite3 (default "$PWD/cve.sqlite3") -debug debug mode -debug-sql @@ -270,7 +365,7 @@ $ go-cve-dictionary fetchnvd -week # Usage: Run HTTP Server. ``` -./go-cve-dictionary server -h +$ go-cve-dictionary server -h server: server [-bind=127.0.0.1] @@ -282,7 +377,7 @@ server: -bind string HTTP server bind to IP address (default: loop back interface) (default "127.0.0.1") -dbpath string - /path/to/sqlite3 (default : /Users/kotakanbe/go/src/github.com/kotakanbe/go-cve-dictionary/cve.sqlite3) (default "/Users/kotakanbe/go/src/github.com/kotakanbe/go-cve-dictionary/cve.sqlite3") + /path/to/sqlite3 (default : $PWD/cve.sqlite3) -debug debug mode (default: false) -debug-sql @@ -307,8 +402,8 @@ Use job scheduler like Cron (with -last2y option). - How to cross compile ```bash - $ cd /path/to/your/local-git-reporsitory/vuls - $ GOOS=linux GOARCH=amd64 go build -o vuls.amd64 + $ cd /path/to/your/local-git-reporsitory/go-cve-dictionary + $ GOOS=linux GOARCH=amd64 go build -o cvedict.amd64 ``` - Logging @@ -330,7 +425,7 @@ Run with --debug, --sql-debug option. # Authors -kotakanbe ([@kotakanbe](https://twitter.com/kotakanbe)) created vuls and [these fine people](https://github.com/future-architect/vuls/graphs/contributors) have contributed. +kotakanbe ([@kotakanbe](https://twitter.com/kotakanbe)) created go-cve-dictionary and [these fine people](https://github.com/future-architect/go-cve-dictionary/graphs/contributors) have contributed. ---- diff --git a/commands/fetchnvd.go b/commands/fetchnvd.go index 0fd48297..198e27b7 100644 --- a/commands/fetchnvd.go +++ b/commands/fetchnvd.go @@ -3,6 +3,8 @@ package commands import ( "flag" "os" + "strconv" + "time" "github.com/google/subcommands" c "github.com/kotakanbe/go-cve-dictionary/config" @@ -18,7 +20,9 @@ type FetchNvdCmd struct { debugSQL bool dbpath string + last2Y bool + years bool } // Name return subcommand name @@ -29,14 +33,17 @@ func (*FetchNvdCmd) Synopsis() string { return "Fetch Vulnerability dictionary f // Usage return usage func (*FetchNvdCmd) Usage() string { - //TODO return `fetchnvd: fetchnvd [-last2y] + [-years] 2015 2016 ... [-dbpath=/path/to/cve.sqlite3] [-debug] [-debug-sql] +For the first time, run the blow command to fetch data for entire period. (It takes about 10 minutes) + $ for i in {2002..2016}; do go-cve-dictionary fetchnvd -years $i; done + ` } @@ -52,6 +59,9 @@ func (p *FetchNvdCmd) SetFlags(f *flag.FlagSet) { f.BoolVar(&p.last2Y, "last2y", false, "Refresh NVD data in the last two years.") + + f.BoolVar(&p.years, "years", false, + "Refresh NVD data of specific years.") } // Execute execute @@ -65,13 +75,47 @@ func (p *FetchNvdCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface } c.Conf.DBPath = p.dbpath - c.Conf.FetchNvdLast2Y = p.last2Y if !c.Conf.Validate() { return subcommands.ExitUsageError } - entries, err := nvd.FetchFiles() + years := []int{} + thisYear := time.Now().Year() + + switch { + case p.last2Y: + for i := 0; i < 2; i++ { + years = append(years, thisYear-i) + } + case p.years: + if len(f.Args()) == 0 { + log.Errorf("Specify years to fetch (from 2002 to %d)", thisYear) + return subcommands.ExitUsageError + } + for _, arg := range f.Args() { + year, err := strconv.Atoi(arg) + if err != nil || year < 2002 || time.Now().Year() < year { + log.Errorf("Specify years to fetch (from 2002 to %d), arg: %s", thisYear, arg) + return subcommands.ExitUsageError + } + found := false + for _, y := range years { + if y == year { + found = true + break + } + } + if !found { + years = append(years, year) + } + } + default: + log.Errorf("specify -last2y or -years") + return subcommands.ExitUsageError + } + + entries, err := nvd.FetchFiles(years) if err != nil { log.Error(err) return subcommands.ExitFailure diff --git a/commands/server.go b/commands/server.go index cdaedb2f..084f8bba 100644 --- a/commands/server.go +++ b/commands/server.go @@ -9,7 +9,6 @@ import ( c "github.com/kotakanbe/go-cve-dictionary/config" "github.com/kotakanbe/go-cve-dictionary/db" log "github.com/kotakanbe/go-cve-dictionary/log" - "github.com/kotakanbe/go-cve-dictionary/nvd" server "github.com/kotakanbe/go-cve-dictionary/server" "golang.org/x/net/context" ) @@ -52,7 +51,7 @@ func (p *ServerCmd) SetFlags(f *flag.FlagSet) { pwd := os.Getenv("PWD") f.StringVar(&p.dbpath, "dbpath", pwd+"/cve.sqlite3", - fmt.Sprintf("/path/to/sqlite3 (default : %s)", pwd+"/cve.sqlite3")) + fmt.Sprintf("/path/to/sqlite3")) f.StringVar(&p.bind, "bind", @@ -99,21 +98,10 @@ func (p *ServerCmd) Execute(_ context.Context, f *flag.FlagSet, _ ...interface{} } if count == 0 { - log.Info("Fetching vulnerability data from NVD because no NVD data found in DB.") - entries, err := nvd.FetchFiles() - if err != nil { - log.Error(err) - return subcommands.ExitFailure - } - - if err := db.InsertNvd(entries); err != nil { - log.Errorf("Failed to insert. dbpath: %s, err: %s", - c.Conf.DBPath, err) - return subcommands.ExitFailure - } - - // Exit because fetching NVD data costs huge memory and keeping... - log.Info("Success") + log.Info("No Vulnerability data found. Run the below command to fetch data from NVD") + log.Info("") + log.Info(" for i in {2002..2016}; do go-cve-dictionary fetchnvd -years $i ; done") + log.Info("") return subcommands.ExitSuccess } diff --git a/config/config.go b/config/config.go index 3b37c264..45a0f0d4 100644 --- a/config/config.go +++ b/config/config.go @@ -21,8 +21,6 @@ type Config struct { FetchJvnEntire bool FetchJvnPeriodChar string - FetchNvdLast2Y bool - Bind string `valid:"ipv4"` Port string `valid:"port"` diff --git a/nvd/nvd.go b/nvd/nvd.go index 1120593a..67d48c31 100644 --- a/nvd/nvd.go +++ b/nvd/nvd.go @@ -58,8 +58,8 @@ type Link struct { } // FetchFiles Fetch CVE vulnerability informatino from JVN -func FetchFiles() (entries []Entry, err error) { - urls := makeFeedURLs(c.Conf.FetchNvdLast2Y) +func FetchFiles(years []int) (entries []Entry, err error) { + urls := makeFeedURLs(years) nvds, err := fetchFeedFileConcurrently(urls, c.Conf.HTTPProxy) if err != nil { return entries, @@ -71,18 +71,11 @@ func FetchFiles() (entries []Entry, err error) { return entries, nil } -func makeFeedURLs(lastTwoYears bool) (urls []string) { +func makeFeedURLs(years []int) (urls []string) { // http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-2016.xml.gz formatTemplate := "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml.gz" - year := time.Now().Year() - if lastTwoYears { - for i := 0; i < 2; i++ { - urls = append(urls, fmt.Sprintf(formatTemplate, year-i)) - } - } else { - for i := 2002; i < year+1; i++ { - urls = append(urls, fmt.Sprintf(formatTemplate, i)) - } + for _, year := range years { + urls = append(urls, fmt.Sprintf(formatTemplate, year)) } return } diff --git a/nvd/nvd_test.go b/nvd/nvd_test.go index f50b5565..e1ce1718 100644 --- a/nvd/nvd_test.go +++ b/nvd/nvd_test.go @@ -1,43 +1 @@ package nvd - -import ( - "fmt" - "testing" - "time" -) - -func TestMakeFeedURLs(t *testing.T) { - year := time.Now().Year() - format := "http://static.nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%d.xml.gz" - - urls := []string{} - for i := 2002; i < year+1; i++ { - urls = append(urls, fmt.Sprintf(format, i)) - } - - var testdata = []struct { - in bool - urls []string - }{ - { - in: true, - urls: []string{ - fmt.Sprintf(format, year), - fmt.Sprintf(format, year-1), - }, - }, - { - in: false, - urls: urls, - }, - } - - for _, tt := range testdata { - urls := makeFeedURLs(tt.in) - for i := range urls { - if urls[i] != tt.urls[i] { - t.Errorf("expected %s, actual %s", urls[i], tt.urls[i]) - } - } - } -}