From 9956ffab822529c96778fde2a1bc41a5bd970a15 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sun, 25 Feb 2024 21:34:11 +0700 Subject: [PATCH 1/5] feat(threat): concat w/o fmt Signed-off-by: Dwi Siswanto --- threat/var.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/threat/var.go b/threat/var.go index 6c8dc6f9..d6e27ef3 100644 --- a/threat/var.go +++ b/threat/var.go @@ -4,11 +4,11 @@ package threat -import "fmt" - var ( - DbURL = fmt.Sprintf("%s/raw/master/db/db.tar.zst", repoURL) - dbQuery = fmt.Sprintf("checksum=file:%s/raw/master/db/MD5SUMS", repoURL) + dbFile = "db.tar.zst" + DbURL = repoURL + "/raw/master/db/" + dbFile + sumURL = repoURL + "/raw/master/db/MD5SUMS" + dbQuery = "checksum=file:" + sumURL ) var str = map[Threat]string{ From 84ae7983a751950dd91498c46213fda06cecd972 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sun, 25 Feb 2024 21:37:26 +0700 Subject: [PATCH 2/5] feat(threat): add Verify func to checks datasets integrity Signed-off-by: Dwi Siswanto --- go.mod | 1 + go.sum | 2 ++ threat/error.go | 6 +++- threat/verify.go | 92 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 threat/verify.go diff --git a/go.mod b/go.mod index 75c8b0e8..dd7e146f 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.19 require ( github.com/antonmedv/expr v1.12.7 github.com/bitfield/script v0.22.0 + github.com/codingsince1985/checksum v1.3.0 github.com/daniel-hutao/spinlock v0.1.0 github.com/dwisiswant0/clientip v0.3.0 github.com/go-playground/validator/v10 v10.16.0 diff --git a/go.sum b/go.sum index 0f2b4f6a..eb5054e2 100644 --- a/go.sum +++ b/go.sum @@ -218,6 +218,8 @@ github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/codingsince1985/checksum v1.3.0 h1:kqqIqWBwjidGmt/pO4yXCEX+np7HACGx72EB+MkKcVY= +github.com/codingsince1985/checksum v1.3.0/go.mod h1:QfRskdtdWap+gJil8e5obw6I8/cWJ0SwMUACruWDSU8= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/daniel-hutao/spinlock v0.1.0 h1:qk6v2L6mJLUmxzq1eJ5xUIlCh4q0wM+26Qy/KfH5c3U= diff --git a/threat/error.go b/threat/error.go index 7b1acd1c..a2eef9c9 100644 --- a/threat/error.go +++ b/threat/error.go @@ -5,5 +5,9 @@ package threat const ( - errFilepath = "unable to get file path location of given %s threat type" + errFilepath = "unable to get file path location of given %s threat type" + errGetSumFile = "unable to fetch checksum file: %w" + errReadSumFile = "cannot read checksum file: %w" + errChecksum = "cannot perform checksum for '%s' file: %w" + errCorrupted = "threat '%s' datasets is corrupted, expect '%s' got '%s' sum" ) diff --git a/threat/verify.go b/threat/verify.go new file mode 100644 index 00000000..efd1e5f5 --- /dev/null +++ b/threat/verify.go @@ -0,0 +1,92 @@ +package threat + +import ( + "bufio" + "fmt" + "strings" + + "net/http" + "path/filepath" + + "github.com/codingsince1985/checksum" +) + +// Verify checks the integrity of files by comparing their checksums with the +// MD5 sums obtained from a teler-resources repository. +// +// It fetches the MD5 sums, verifies that the fetched data is correct, and then +// checks the checksums of the local files against the obtained MD5 sums. It +// returns true if all checksums match, otherwise returns false along with an +// error if any issues occur during the verification process. +func Verify() (bool, error) { + md5sums, err := fetchMD5Sums() + if err != nil { + return false, err + } + + return verifyChecksums(md5sums) +} + +// fetchMD5Sums retrieves MD5 sums from a remote source and returns them as a +// map where filenames are the keys and MD5 sums are the values. +// +// In case of an error during the retrieval, it returns an error. +func fetchMD5Sums() (map[string]string, error) { + // Initialize a map to store the MD5 sums + md5sums := make(map[string]string) + + resp, err := http.Get(sumURL) + if err != nil { + return md5sums, fmt.Errorf(errGetSumFile, err) + } + defer resp.Body.Close() + + // Create a scanner to read the file content line by line + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + + // Split each line into filename and MD5 sum + parts := strings.Fields(line) + if len(parts) == 2 { + filename, md5 := parts[1], parts[0] + if filename == dbFile { + continue + } + + md5sums[filename] = md5 + } + } + + // Check for errors during scanning + if err := scanner.Err(); err != nil { + return md5sums, fmt.Errorf(errReadSumFile, err) + } + + return md5sums, nil +} + +// verifyChecksums compares the MD5 sums obtained from a remote source +// with the local checksums of the files. It takes a map of filenames to +// MD5 sums as input and returns true if all checksums match, otherwise +// returns false along with an error if any checksums do not match. +func verifyChecksums(md5sums map[string]string) (bool, error) { + for _, threat := range List() { + p, err := threat.Filename(true) + if err != nil { + return false, err + } + + sum, err := checksum.MD5sum(p) + if err != nil { + return false, fmt.Errorf(errChecksum, p, err) + } + + f := filepath.Base(p) + if md5sum := md5sums[f]; sum != md5sum { + return false, fmt.Errorf(errCorrupted, threat.String(), md5sum, sum) + } + } + + return true, nil +} From a3fa96d7c2e02ddfb0cc0bab6f02a198226beb37 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Sun, 25 Feb 2024 21:44:48 +0700 Subject: [PATCH 3/5] feat(threat): add license header Signed-off-by: Dwi Siswanto --- threat/verify.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/threat/verify.go b/threat/verify.go index efd1e5f5..45bda7c6 100644 --- a/threat/verify.go +++ b/threat/verify.go @@ -1,3 +1,7 @@ +// Licensed to Dwi Siswanto under one or more agreements. +// Dwi Siswanto licenses this file to you under the Apache 2.0 License. +// See the LICENSE-APACHE file in the project root for more information. + package threat import ( From df08f7e1c8853f47d1a398e26969ffaae42d8c12 Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Mon, 26 Feb 2024 19:59:24 +0700 Subject: [PATCH 4/5] refactor(threat): update corrupted term to malformed Signed-off-by: Dwi Siswanto --- threat/error.go | 2 +- threat/verify.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/threat/error.go b/threat/error.go index a2eef9c9..c9b33ab5 100644 --- a/threat/error.go +++ b/threat/error.go @@ -9,5 +9,5 @@ const ( errGetSumFile = "unable to fetch checksum file: %w" errReadSumFile = "cannot read checksum file: %w" errChecksum = "cannot perform checksum for '%s' file: %w" - errCorrupted = "threat '%s' datasets is corrupted, expect '%s' got '%s' sum" + errMalformed = "threat '%s' datasets is malformed, expect '%s' got '%s' sum" ) diff --git a/threat/verify.go b/threat/verify.go index 45bda7c6..320104dd 100644 --- a/threat/verify.go +++ b/threat/verify.go @@ -88,7 +88,7 @@ func verifyChecksums(md5sums map[string]string) (bool, error) { f := filepath.Base(p) if md5sum := md5sums[f]; sum != md5sum { - return false, fmt.Errorf(errCorrupted, threat.String(), md5sum, sum) + return false, fmt.Errorf(errMalformed, threat.String(), md5sum, sum) } } From d402b0a94881c321eac08f7153918ecee730bd7e Mon Sep 17 00:00:00 2001 From: Dwi Siswanto Date: Mon, 26 Feb 2024 20:48:07 +0700 Subject: [PATCH 5/5] feat(teler): implement Verify datasets (checksum) Signed-off-by: Dwi Siswanto --- teler.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/teler.go b/teler.go index f0914c1d..fec515ba 100644 --- a/teler.go +++ b/teler.go @@ -443,6 +443,24 @@ func (t *Teler) getResources() error { updated = false } + // Do checksum for threat datasets + if updated { + t.log.Debug("verifying datasets") + verify, err := threat.Verify() + if err != nil { + // Got something error while verifying + updated = false + } + + // Checks if datasets is malformed/corrupted + // + // If not verified, err is defintely not nil. + if !verify { + t.log.Debug(err.Error()) + updated = false + } + } + // Download the datasets of threat ruleset from teler-resources // if threat datasets is not up-to-date, update check is disabled // and in-memory option is true