Skip to content

Commit

Permalink
groupcache: enable H2C + add benchmarks
Browse files Browse the repository at this point in the history 
Add h2c benchmarks. Somehow h2c is slowed than regular HTTP locally for
me. Probably because these aren't real life benchmarks i.e. establishing
TCP connections locally has no cost. And if there is only one operation
happening at any time then h2 has to do more work hence more CPU usage.

With 500 parallel requests going on at the same time, h2 becomes faster
than (or is at the same level as) h1 while using a minimal number of TCP
connections. Thus, it will work much faster in real life situations. So,
let's enable it.

```
name                                                               time/op
GroupcacheRetrieval/h2c/seq-16                                      227µs ± 1%
GroupcacheRetrieval/h2c/parallel=500-16                            54.8µs ±20%
GroupcacheRetrieval/h1,_max_one_TCP_connection/seq-16               150µs ± 2%
GroupcacheRetrieval/h1,_max_one_TCP_connection/parallel=500-16      146µs ± 1%
GroupcacheRetrieval/h1,_unlimited_TCP_connections/seq-16            145µs ± 3%
GroupcacheRetrieval/h1,_unlimited_TCP_connections/parallel=500-16  52.8µs ± 9%

name                                                               alloc/op
GroupcacheRetrieval/h2c/seq-16                                      183kB ± 0%
GroupcacheRetrieval/h2c/parallel=500-16                             143kB ± 1%
GroupcacheRetrieval/h1,_max_one_TCP_connection/seq-16               161kB ± 0%
GroupcacheRetrieval/h1,_max_one_TCP_connection/parallel=500-16      162kB ± 0%
GroupcacheRetrieval/h1,_unlimited_TCP_connections/seq-16            161kB ± 0%
GroupcacheRetrieval/h1,_unlimited_TCP_connections/parallel=500-16   116kB ± 2%

name                                                               allocs/op
GroupcacheRetrieval/h2c/seq-16                                        283 ± 0%
GroupcacheRetrieval/h2c/parallel=500-16                               256 ± 1%
GroupcacheRetrieval/h1,_max_one_TCP_connection/seq-16                 260 ± 0%
GroupcacheRetrieval/h1,_max_one_TCP_connection/parallel=500-16        262 ± 0%
GroupcacheRetrieval/h1,_unlimited_TCP_connections/seq-16              260 ± 0%
GroupcacheRetrieval/h1,_unlimited_TCP_connections/parallel=500-16     279 ± 1%
```

Signed-off-by: Giedrius Statkevičius <[email protected]>
  • Loading branch information
@GiedriusS
GiedriusS committed Jan 18, 2022
1 parent 4dffcca commit 558ce8a
Show file tree
Hide file tree
Showing 7 changed files with 207 additions and 27 deletions.
1 change: 1 addition & 0 deletions cmd/thanos/store.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ func runStore(
httpserver.WithListen(conf.httpConfig.bindAddress),
httpserver.WithGracePeriod(time.Duration(conf.httpConfig.gracePeriod)),
httpserver.WithTLSConfig(conf.httpConfig.tlsConfig),
httpserver.WithEnableH2C(true), // For groupcache.
)

g.Add(func() error {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ require (
go.uber.org/automaxprocs v1.4.0
go.uber.org/goleak v1.1.12
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
golang.org/x/net v0.0.0-20211020060615-d418f374d309
golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d
golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/text v0.3.7
Expand Down
3 changes: 2 additions & 1 deletion go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1987,8 +1987,9 @@ golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211020060615-d418f374d309 h1:A0lJIi+hcTR6aajJH4YqKWwohY4aW9RO7oRMcdv+HKI=
golang.org/x/net v0.0.0-20211020060615-d418f374d309/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d h1:1n1fc535VhN8SYtD4cDUyNlfpAF2ROMM9+11equK3hs=
golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand Down
9 changes: 9 additions & 0 deletions pkg/cache/groupcache.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ package cache

import (
"context"
"crypto/tls"
"encoding/json"
"io/ioutil"
"net"
"net/http"
"path/filepath"
"strconv"
Expand All @@ -24,6 +26,7 @@ import (
"github.com/thanos-io/thanos/pkg/store/cache/cachekey"
"github.com/vimeo/galaxycache"
galaxyhttp "github.com/vimeo/galaxycache/http"
"golang.org/x/net/http2"
"gopkg.in/yaml.v2"
)

Expand Down Expand Up @@ -93,6 +96,12 @@ func NewGroupcacheWithConfig(logger log.Logger, reg prometheus.Registerer, conf
cfg *CachingBucketConfig) (*Groupcache, error) {
httpProto := galaxyhttp.NewHTTPFetchProtocol(&galaxyhttp.HTTPOptions{
BasePath: basepath,
Transport: &http2.Transport{
AllowHTTP: true,
DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) {
return net.Dial(network, addr)
},
},
})
universe := galaxycache.NewUniverse(httpProto, conf.SelfURL)

Expand Down
200 changes: 176 additions & 24 deletions pkg/cache/groupcache_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package cache

import (
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
Expand All @@ -16,32 +17,43 @@ import (
"github.com/go-kit/log"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/route"
"github.com/thanos-io/thanos/pkg/component"
"github.com/thanos-io/thanos/pkg/discovery/dns"
"github.com/thanos-io/thanos/pkg/model"
"github.com/thanos-io/thanos/pkg/objstore"
"github.com/thanos-io/thanos/pkg/prober"
httpserver "github.com/thanos-io/thanos/pkg/server/http"
"github.com/thanos-io/thanos/pkg/store/cache/cachekey"
"github.com/thanos-io/thanos/pkg/testutil"
galaxyhttp "github.com/vimeo/galaxycache/http"
"golang.org/x/net/http2"
)

// Benchmark retrieval of one key from one groupcache node.
func BenchmarkGroupcacheRetrieval(b *testing.B) {
const basePath = `/_groupcache/`
const groupName = `groupName`
const selfURLH1 = "http://localhost:12345"
const selfURLH2C = "http://localhost:12346"

func TestMain(m *testing.M) {
reg := prometheus.NewRegistry()
router := route.New()

bkt := objstore.NewInMemBucket()
b.Cleanup(func() { bkt.Close() })
defer bkt.Close()

payload := strings.Repeat("foobar", 16*1024/6)

testutil.Ok(b, bkt.Upload(context.Background(), "test", strings.NewReader(payload)))

listener, err := net.Listen("tcp4", "0.0.0.0:0")
testutil.Ok(b, err)

listenerAddr := listener.Addr().String()
port := strings.Split(listenerAddr, ":")[1]
if err := bkt.Upload(context.Background(), "test", strings.NewReader(payload)); err != nil {
fmt.Printf("failed to upload: %s\n", err.Error())
os.Exit(1)
}

selfURL := fmt.Sprintf("http://localhost:%v", port)
httpServer := httpserver.New(log.NewNopLogger(), nil, component.Bucket, &prober.HTTPProbe{},
httpserver.WithListen("0.0.0.0:12345"))
httpServer.Handle("/", router)
httpServerH2C := httpserver.New(log.NewNopLogger(), nil, component.Bucket, &prober.HTTPProbe{},
httpserver.WithListen("0.0.0.0:12346"), httpserver.WithEnableH2C(true))
httpServerH2C.Handle("/", router)

cachingBucketConfig := NewCachingBucketConfig()
cachingBucketConfig.CacheGet("test", nil, func(s string) bool {
Expand All @@ -52,30 +64,170 @@ func BenchmarkGroupcacheRetrieval(b *testing.B) {
log.NewJSONLogger(os.Stderr),
reg,
GroupcacheConfig{
Peers: []string{selfURL},
SelfURL: selfURL,
GroupcacheGroup: "groupName",
Peers: []string{selfURLH1},
SelfURL: selfURLH1,
GroupcacheGroup: groupName,
DNSSDResolver: dns.MiekgdnsResolverType,
DNSInterval: 30 * time.Second,
MaxSize: model.Bytes(16 * 1024),
},
"/_groupcache/",
basePath,
router,
bkt,
cachingBucketConfig,
)
testutil.Ok(b, err)
if err != nil {
fmt.Printf("failed creating group cache: %s\n", err.Error())
os.Exit(1)
}

go func() {
if err = httpServer.ListenAndServe(); err != nil {
fmt.Printf("failed to listen: %s\n", err.Error())
}
}()
go func() {
if err = httpServerH2C.ListenAndServe(); err != nil {
fmt.Printf("failed to listen: %s\n", err.Error())
}
}()

go func() { _ = http.Serve(listener, router) }()
b.Cleanup(func() { listener.Close() })
cachingBucketConfig.SetCacheImplementation(groupCache)

b.ResetTimer()
b.ReportAllocs()
exitVal := m.Run()

httpServer.Shutdown(nil)
os.Exit(exitVal)
}

for i := 0; i < b.N; i++ {
resp, err := http.Get(selfURL + fmt.Sprintf("/_groupcache/groupName/%v", cachekey.BucketCacheKey{Verb: "content", Name: "test"}))
// Benchmark retrieval of one key from one groupcache node.
func BenchmarkGroupcacheRetrieval(b *testing.B) {
b.Run("h2c", func(b *testing.B) {
fetcher := galaxyhttp.NewHTTPFetchProtocol(&galaxyhttp.HTTPOptions{
Transport: &http2.Transport{
AllowHTTP: true,
DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) {
return net.Dial(network, addr)
},
},
BasePath: basePath,
})

f, err := fetcher.NewFetcher(selfURLH2C)
testutil.Ok(b, err)
testutil.Ok(b, resp.Body.Close())
}

b.Run("seq", func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()

for i := 0; i < b.N; i++ {
_, _, err = f.Fetch(context.Background(), groupName, cachekey.BucketCacheKey{Verb: "content", Name: "test"}.String())
testutil.Ok(b, err)
}
})
b.Run("parallel=500", func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()

ch := make(chan struct{})

for i := 0; i < 500; i++ {
go func() {
for range ch {
_, _, err = f.Fetch(context.Background(), groupName, cachekey.BucketCacheKey{Verb: "content", Name: "test"}.String())
testutil.Ok(b, err)
}
}()
}

for i := 0; i < b.N; i++ {
ch <- struct{}{}
}
close(ch)
})
})
b.Run("h1, max one TCP connection", func(b *testing.B) {
fetcher := galaxyhttp.NewHTTPFetchProtocol(&galaxyhttp.HTTPOptions{
BasePath: basePath,
Transport: &http.Transport{
MaxConnsPerHost: 1,
MaxIdleConnsPerHost: 1,
},
})

f, err := fetcher.NewFetcher(selfURLH1)
testutil.Ok(b, err)

b.Run("seq", func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()

for i := 0; i < b.N; i++ {
_, _, err = f.Fetch(context.Background(), groupName, cachekey.BucketCacheKey{Verb: "content", Name: "test"}.String())
testutil.Ok(b, err)
}
})

b.Run("parallel=500", func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()

ch := make(chan struct{})

for i := 0; i < 500; i++ {
go func() {
for range ch {
_, _, err = f.Fetch(context.Background(), groupName, cachekey.BucketCacheKey{Verb: "content", Name: "test"}.String())
testutil.Ok(b, err)
}
}()
}

for i := 0; i < b.N; i++ {
ch <- struct{}{}
}
close(ch)
})
})
b.Run("h1, unlimited TCP connections", func(b *testing.B) {
fetcher := galaxyhttp.NewHTTPFetchProtocol(&galaxyhttp.HTTPOptions{
BasePath: basePath,
Transport: &http.Transport{},
})

f, err := fetcher.NewFetcher(selfURLH1)
testutil.Ok(b, err)

b.Run("seq", func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()

for i := 0; i < b.N; i++ {
_, _, err = f.Fetch(context.Background(), groupName, cachekey.BucketCacheKey{Verb: "content", Name: "test"}.String())
testutil.Ok(b, err)
}
})

b.Run("parallel=500", func(b *testing.B) {
b.ResetTimer()
b.ReportAllocs()

ch := make(chan struct{})

for i := 0; i < 500; i++ {
go func() {
for range ch {
_, _, err = f.Fetch(context.Background(), groupName, cachekey.BucketCacheKey{Verb: "content", Name: "test"}.String())
testutil.Ok(b, err)
}
}()
}

for i := 0; i < b.N; i++ {
ch <- struct{}{}
}
close(ch)
})
})

}
12 changes: 11 additions & 1 deletion pkg/server/http/http.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
toolkit_web "github.com/prometheus/exporter-toolkit/web"
"golang.org/x/net/http2"
"golang.org/x/net/http2/h2c"

"github.com/thanos-io/thanos/pkg/component"
"github.com/thanos-io/thanos/pkg/prober"
Expand Down Expand Up @@ -48,12 +50,20 @@ func New(logger log.Logger, reg *prometheus.Registry, comp component.Component,
registerProbes(mux, prober, logger)
registerProfiler(mux)

var h http.Handler
if options.enableH2C {
h2s := &http2.Server{}
h = h2c.NewHandler(mux, h2s)
} else {
h = mux
}

return &Server{
logger: log.With(logger, "service", "http/server", "component", comp.String()),
comp: comp,
prober: prober,
mux: mux,
srv: &http.Server{Addr: options.listen, Handler: mux},
srv: &http.Server{Addr: options.listen, Handler: h},
opts: options,
}
}
Expand Down
7 changes: 7 additions & 0 deletions pkg/server/http/option.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ type options struct {
listen string
tlsConfigPath string
mux *http.ServeMux
enableH2C bool
}

// Option overrides behavior of Server.
Expand Down Expand Up @@ -48,6 +49,12 @@ func WithTLSConfig(tls string) Option {
})
}

func WithEnableH2C(enableH2C bool) Option {
return optionFunc(func(o *options) {
o.enableH2C = enableH2C
})
}

// WithMux overrides the the server's default mux.
func WithMux(mux *http.ServeMux) Option {
return optionFunc(func(o *options) {
Expand Down

0 comments on commit 558ce8a

Please sign in to comment.