Merge: + config: add cache size and time settings

Close #947

* commit '4e8473f8a5b04d2b3883ab74323734ef5e976c0e':
  * use dnsproxy v0.19.1
  * dnsfilter: use golibs/cache

dnsfilter/dnsfilter.go

@@ -5,6 +5,8 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha256"
+	"encoding/binary"
+	"encoding/gob"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -18,16 +20,14 @@ import (
 	"github.com/joomcode/errorx"
 
 	"github.com/AdguardTeam/dnsproxy/upstream"
+	"github.com/AdguardTeam/golibs/cache"
 	"github.com/AdguardTeam/golibs/log"
 	"github.com/AdguardTeam/urlfilter"
 	"github.com/bluele/gcache"
 	"github.com/miekg/dns"
 	"golang.org/x/net/publicsuffix"
 )
 
-const defaultCacheSize = 64 * 1024 // in number of elements
-const defaultCacheTime = 30 * time.Minute
-
 const defaultHTTPTimeout = 5 * time.Minute
 const defaultHTTPMaxIdleConnections = 100
 
@@ -68,6 +68,11 @@ type Config struct {
 	SafeBrowsingEnabled bool   `yaml:"safebrowsing_enabled"`
 	ResolverAddress     string // DNS server address
 
+	SafeBrowsingCacheSize uint `yaml:"safebrowsing_cache_size"` // (in bytes)
+	SafeSearchCacheSize   uint `yaml:"safesearch_cache_size"`   // (in bytes)
+	ParentalCacheSize     uint `yaml:"parental_cache_size"`     // (in bytes)
+	CacheTime             uint `yaml:"cache_time"`              // Element's TTL (in minutes)
+
 	Rewrites []RewriteEntry `yaml:"rewrites"`
 
 	// Filtering callback function
@@ -172,9 +177,9 @@ func (r Reason) String() string {
 type dnsFilterContext struct {
 	stats             Stats
 	dialCache         gcache.Cache // "host" -> "IP" cache for safebrowsing and parental control servers
-	safebrowsingCache gcache.Cache
-	parentalCache     gcache.Cache
-	safeSearchCache   gcache.Cache
+	safebrowsingCache cache.Cache
+	parentalCache     cache.Cache
+	safeSearchCache   cache.Cache
 }
 
 var gctx dnsFilterContext // global dnsfilter context
@@ -352,39 +357,52 @@ func matchBlockedServicesRules(host string, svcs []ServiceEntry) Result {
 	return res
 }
 
-func setCacheResult(cache gcache.Cache, host string, res Result) {
-	err := cache.Set(host, res)
+/*
+expire byte[4]
+res Result
+*/
+func (d *Dnsfilter) setCacheResult(cache cache.Cache, host string, res Result) {
+	var buf bytes.Buffer
+
+	expire := uint(time.Now().Unix()) + d.Config.CacheTime*60
+	var exp []byte
+	exp = make([]byte, 4)
+	binary.BigEndian.PutUint32(exp, uint32(expire))
+	_, _ = buf.Write(exp)
+
+	enc := gob.NewEncoder(&buf)
+	err := enc.Encode(res)
 	if err != nil {
-		log.Debug("cache.Set: %s", err)
+		log.Error("gob.Encode(): %s", err)
 		return
 	}
+	_ = cache.Set([]byte(host), buf.Bytes())
 	log.Debug("Stored in cache %p: %s", cache, host)
 }
 
-func getCachedResult(cache gcache.Cache, host string) (result Result, isFound bool) {
-	isFound = false // not found yet
+func getCachedResult(cache cache.Cache, host string) (Result, bool) {
+	data := cache.Get([]byte(host))
+	if data == nil {
+		return Result{}, false
+	}
 
-	// get raw value
-	rawValue, err := cache.Get(host)
-	if err == gcache.KeyNotFoundError {
-		// not a real error, just not found
+	exp := int(binary.BigEndian.Uint32(data[:4]))
+	if exp <= int(time.Now().Unix()) {
+		cache.Del([]byte(host))
 		return Result{}, false
 	}
+
+	var buf bytes.Buffer
+	buf.Write(data[4:])
+	dec := gob.NewDecoder(&buf)
+	r := Result{}
+	err := dec.Decode(&r)
 	if err != nil {
-		// real error
+		log.Debug("gob.Decode(): %s", err)
 		return Result{}, false
 	}
 
-	// since it can be something else, validate that it belongs to proper type
-	cachedValue, ok := rawValue.(Result)
-	if !ok {
-		// this is not our type -- error
-		text := "SHOULD NOT HAPPEN: entry with invalid type was found in lookup cache"
-		log.Println(text)
-		return
-	}
-	isFound = ok
-	return cachedValue, isFound
+	return r, true
 }
 
 // for each dot, hash it and add it to string
@@ -445,7 +463,7 @@ func (d *Dnsfilter) checkSafeSearch(host string) (Result, error) {
 	res := Result{IsFiltered: true, Reason: FilteredSafeSearch}
 	if ip := net.ParseIP(safeHost); ip != nil {
 		res.IP = ip
-		setCacheResult(gctx.safeSearchCache, host, res)
+		d.setCacheResult(gctx.safeSearchCache, host, res)
 		return res, nil
 	}
 
@@ -468,7 +486,7 @@ func (d *Dnsfilter) checkSafeSearch(host string) (Result, error) {
 	}
 
 	// Cache result
-	setCacheResult(gctx.safeSearchCache, host, res)
+	d.setCacheResult(gctx.safeSearchCache, host, res)
 	return res, nil
 }
 
@@ -523,7 +541,7 @@ func (d *Dnsfilter) checkSafeBrowsing(host string) (Result, error) {
 	result, err := d.lookupCommon(host, &gctx.stats.Safebrowsing, true, format, handleBody)
 
 	if err == nil {
-		setCacheResult(gctx.safebrowsingCache, host, result)
+		d.setCacheResult(gctx.safebrowsingCache, host, result)
 	}
 
 	return result, err
@@ -589,7 +607,7 @@ func (d *Dnsfilter) checkParental(host string) (Result, error) {
 	result, err := d.lookupCommon(host, &gctx.stats.Parental, false, format, handleBody)
 
 	if err == nil {
-		setCacheResult(gctx.parentalCache, host, result)
+		d.setCacheResult(gctx.parentalCache, host, result)
 	}
 
 	return result, err
@@ -852,18 +870,30 @@ func (d *Dnsfilter) createCustomDialContext(resolverAddr string) dialFunctionTyp
 func New(c *Config, filters map[int]string) *Dnsfilter {
 
 	if c != nil {
+		cacheConf := cache.Config{
+			EnableLRU: true,
+		}
+
 		// initialize objects only once
+
 		if gctx.safebrowsingCache == nil {
-			gctx.safebrowsingCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build()
+			cacheConf.MaxSize = c.SafeBrowsingCacheSize
+			gctx.safebrowsingCache = cache.New(cacheConf)
 		}
+
 		if gctx.safeSearchCache == nil {
-			gctx.safeSearchCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build()
+			cacheConf.MaxSize = c.SafeSearchCacheSize
+			gctx.safeSearchCache = cache.New(cacheConf)
 		}
+
 		if gctx.parentalCache == nil {
-			gctx.parentalCache = gcache.New(defaultCacheSize).LRU().Expiration(defaultCacheTime).Build()
+			cacheConf.MaxSize = c.ParentalCacheSize
+			gctx.parentalCache = cache.New(cacheConf)
 		}
+
 		if len(c.ResolverAddress) != 0 && gctx.dialCache == nil {
-			gctx.dialCache = gcache.New(maxDialCacheSize).LRU().Expiration(defaultCacheTime).Build()
+			dur := time.Duration(c.CacheTime) * time.Minute
+			gctx.dialCache = gcache.New(maxDialCacheSize).LRU().Expiration(dur).Build()
 		}
 	}
 

dnsfilter/dnsfilter_test.go

@@ -30,13 +30,13 @@ var setts RequestFilteringSettings
 
 func purgeCaches() {
 	if gctx.safebrowsingCache != nil {
-		gctx.safebrowsingCache.Purge()
+		gctx.safebrowsingCache.Clear()
 	}
 	if gctx.parentalCache != nil {
-		gctx.parentalCache.Purge()
+		gctx.parentalCache.Clear()
 	}
 	if gctx.safeSearchCache != nil {
-		gctx.safeSearchCache.Purge()
+		gctx.safeSearchCache.Clear()
 	}
 }
 
@@ -51,6 +51,10 @@ func NewForTest(c *Config, filters map[int]string) *Dnsfilter {
 	setts = RequestFilteringSettings{}
 	setts.FilteringEnabled = true
 	if c != nil {
+		c.SafeBrowsingCacheSize = 1000
+		c.SafeSearchCacheSize = 1000
+		c.ParentalCacheSize = 1000
+		c.CacheTime = 30
 		setts.SafeSearchEnabled = c.SafeSearchEnabled
 		setts.SafeBrowsingEnabled = c.SafeBrowsingEnabled
 		setts.ParentalEnabled = c.ParentalEnabled

dnsforward/dnsforward.go

@@ -100,6 +100,7 @@ type FilteringConfig struct {
 	// Per-client settings can override this configuration.
 	BlockedServices []string `yaml:"blocked_services"`
 
+	CacheSize        uint `yaml:"cache_size"` // DNS cache size (in bytes)
 	dnsfilter.Config `yaml:",inline"`
 }
 
@@ -203,6 +204,7 @@ func (s *Server) startInternal(config *ServerConfig) error {
 		RatelimitWhitelist:       s.conf.RatelimitWhitelist,
 		RefuseAny:                s.conf.RefuseAny,
 		CacheEnabled:             true,
+		CacheSizeBytes:           int(s.conf.CacheSize),
 		Upstreams:                s.conf.Upstreams,
 		DomainsReservedUpstreams: s.conf.DomainsReservedUpstreams,
 		BeforeRequestHandler:     s.beforeRequestHandler,

dnsforward/dnsforward_test.go

@@ -491,6 +491,11 @@ func createTestServer(t *testing.T) *Server {
 	s.conf.FilteringConfig.SafeBrowsingEnabled = true
 	s.conf.Filters = make([]dnsfilter.Filter, 0)
 
+	s.conf.SafeBrowsingCacheSize = 1000
+	s.conf.SafeSearchCacheSize = 1000
+	s.conf.ParentalCacheSize = 1000
+	s.conf.CacheTime = 30
+
 	rules := "||nxdomain.example.org^\n||null.example.org^\n127.0.0.1	host.example.org\n"
 	filter := dnsfilter.Filter{ID: 0, Data: []byte(rules)}
 	s.conf.Filters = append(s.conf.Filters, filter)

go.mod

@@ -3,8 +3,8 @@ module github.com/AdguardTeam/AdGuardHome
 go 1.12
 
 require (
-	github.com/AdguardTeam/dnsproxy v0.18.4
-	github.com/AdguardTeam/golibs v0.1.4
+	github.com/AdguardTeam/dnsproxy v0.19.1
+	github.com/AdguardTeam/golibs v0.2.1
 	github.com/AdguardTeam/urlfilter v0.5.0
 	github.com/NYTimes/gziphandler v1.1.1
 	github.com/bluele/gcache v0.0.0-20190518031135-bc40bd653833
@@ -16,9 +16,9 @@ require (
 	github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414
 	github.com/miekg/dns v1.1.8
 	github.com/sparrc/go-ping v0.0.0-20181106165434-ef3ab45e41b0
-	github.com/stretchr/testify v1.3.0
+	github.com/stretchr/testify v1.4.0
 	golang.org/x/net v0.0.0-20190620200207-3b0461eec859
 	golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0
 	gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477
-	gopkg.in/yaml.v2 v2.2.1
+	gopkg.in/yaml.v2 v2.2.2
 )

go.sum

@@ -1,9 +1,9 @@
-github.com/AdguardTeam/dnsproxy v0.18.4 h1:UjqN83y7uTduPxSJ5Pg7aO92Fgs4MfDDIfjqaK2YujQ=
-github.com/AdguardTeam/dnsproxy v0.18.4/go.mod h1:R0YfjEUyGf7rsQ31gbw0GahraUjxCmH4dUqPEYVy81k=
+github.com/AdguardTeam/dnsproxy v0.19.1 h1:Xyv5qv9UCj2IVB++2YN/DxdwFBdt3lHjD0Mp/bEJS74=
+github.com/AdguardTeam/dnsproxy v0.19.1/go.mod h1:NaulY9i279jZwN8QBbvbZnn5HkrjBgJi4hbFY5nW+Kc=
 github.com/AdguardTeam/golibs v0.1.3 h1:hmapdTtMtIk3T8eQDwTOLdqZLGDKNKk9325uC8z12xg=
 github.com/AdguardTeam/golibs v0.1.3/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko=
-github.com/AdguardTeam/golibs v0.1.4 h1:zZirpC23mZkf9Upasn9nbQ+lqqjcXJnkWdXO80vTu7s=
-github.com/AdguardTeam/golibs v0.1.4/go.mod h1:b0XkhgIcn2TxwX6C5AQMtpIFAgjPehNgxJErWkwA3ko=
+github.com/AdguardTeam/golibs v0.2.1 h1:jGCnbM5UOUq/GrG+8eLN7Y+OTfEo5F/8L0wq3ur2h4E=
+github.com/AdguardTeam/golibs v0.2.1/go.mod h1:caAJ5knSHbR6vV6qfRDgAfXVia4hHgLqeztAY4UX0fw=
 github.com/AdguardTeam/urlfilter v0.5.0 h1:ATzs2Er0BMt7NbZnFJ4UEzt3uIV+rydbQCYqBXNRbJc=
 github.com/AdguardTeam/urlfilter v0.5.0/go.mod h1:6YehXZ8e0Hx2MvqeQWLFom6IkPinm04tNhO1CkwAxmg=
 github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I=
@@ -78,6 +78,8 @@ github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
@@ -114,5 +116,4 @@ gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477 h1:5xUJw+l
 gopkg.in/asaskevich/govalidator.v4 v4.0.0-20160518190739-766470278477/go.mod h1:QDV1vrFSrowdoOba0UM8VJPUZONT7dnfdLsM+GG53Z8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

home/config.go

@@ -211,6 +211,12 @@ func initConfig() {
 		// also change the default config
 		config.DNS.UpstreamDNS = defaultDNS
 	}
+
+	config.DNS.CacheSize = 4 * 1024 * 1024
+	config.DNS.SafeBrowsingCacheSize = 1 * 1024 * 1024
+	config.DNS.SafeSearchCacheSize = 1 * 1024 * 1024
+	config.DNS.ParentalCacheSize = 1 * 1024 * 1024
+	config.DNS.CacheTime = 30
 }
 
 // getConfigFilename returns path to the current config file